def testNewContentFromTestContentDir(self):
self.assertTrue(
os.path.isdir(TEST_CONTENT_DIR),
"Directory '%s' containing test files does not exist." %
TEST_CONTENT_DIR)
files = os.listdir(TEST_CONTENT_DIR)
print("\n\nFAILEJA", TEST_CONTENT_DIR, files)
self.assertGreater(
len(files), 0, "Directory '%s' containing test files is empty." %
TEST_CONTENT_DIR)
cnt = 0
for filename in files:
cnt += 1
c = Content(caption="New content #%d" % cnt)
full_path = os.path.join(TEST_CONTENT_DIR, str(filename))
c.set_file(str(filename), full_path)
c.set_fileinfo()
c.generate_thumbnail()
c.save()
maintype = c.mimetype.split("/")[0]
print("MIMETYYPPI", c.mimetype, c.preview)
if maintype in ["video", "audio"]:
ffp = content.filetools.FFProbe(c.file.path)
info = content.filetools.fileinfo(c.file.path)
print(info)
if ffp.is_video():
new_video, cmd_str, output = create_videoinstance(
c.file.path)
vi = Videoinstance(content=c)
vi.save()
vi.set_file(new_video, "webm")
vi.set_metadata(info)
vi.save()
print("%s %.1f sec %dx%d pix" %
(vi.mimetype, vi.duration, vi.width, vi.height))
if ffp.is_audio():
new_audio, cmd_str, output = create_audioinstance(
c.file.path)
ai = Audioinstance(content=c)
ai.save()
ai.set_file(new_audio, "ogg")
ai.set_metadata(info)
ai.save()
print("%s %.1f sec" % (ai.mimetype, ai.duration))
self.all_content.append(c)
# self.assertEqual(c.file.path, "sd", c.file.path)
# import time
# time.sleep(20)
self.assertEqual(Content.objects.count(), len(self.all_content),
"1 or more files failed")
def simple_upload(request):
"""
Handles uploaded files
"""
try:
if request.method == 'POST': # If the form has been submitted...
user = request.user if request.user.is_authenticated() else None
form = UploadForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # File was posted with form
c = None
kwargs = {}
kwargs['author'] = user.username.title() if user else 'Anonymous'
# Create a new Egg
e = Egg(**kwargs)
print kwargs
for filefield, tmpname in handle_uploaded_file(request):
print "HANDLING FILE:", filefield, tmpname
c = Content(**kwargs)
originalname = str(request.FILES["file"])
# c.user = request.user # Only authenticated users can use this view
c.set_file(originalname, tmpname) # Save uploaded file to filesystem
c.get_type_instance() # Create thumbnail if it is supported
c.save()
e.content = c
e.uid = c.uid
break # We save only the first file
if c:
c.user = user
c.save()
e.user = user
print "USER", user
if user:
response_status = 200 # Created
else:
response_status = 401 # Unauthorized
e.save()
# We can handle tags after egg has id (it is saved)
response = HttpResponse(status=response_status)
#response.status_code = 201
# FIXME: use reverse()
#return HttpResponseRedirect(reverse('egg api', args=[e.uid]))
response['Location'] = '/fourdnest/api/v1/egg/%s/' % e.uid
return response
else:
response = HttpResponse(status=204)
return response
else:
raise Http404
except Exception, err:
print err
raise
return HttpResponse("Server error", status=500)
def testNewContentSaving(self):
self.assertTrue(
os.path.isdir(TEST_CONTENT_DIR),
f"Directory '{TEST_CONTENT_DIR}' containing test files does not exist."
)
files = os.listdir(TEST_CONTENT_DIR)
self.assertGreater(
len(files), 0,
f"Directory '{TEST_CONTENT_DIR}' containing test files is empty.")
cnt = 0
for filename in files:
full_path = os.path.join(TEST_CONTENT_DIR, str(filename))
if os.path.isfile(full_path) is False:
print(f"Skip {full_path}, not a file")
continue
cnt += 1
c = Content(caption="New content #%d" % cnt)
c.set_file(str(filename), full_path)
c.set_fileinfo()
c.generate_thumbnail()
c.save()
maintype = c.mimetype.split("/")[0]
if maintype in ["video", "audio"]:
ffp = content.filetools.FFProbe(c.file.path)
info = content.filetools.fileinfo(c.file.path)
print(info)
if ffp.is_video():
new_video, cmd_str, output = create_videoinstance(
c.file.path)
vi = Videoinstance(content=c)
vi.save()
vi.set_file(new_video, "webm")
vi.set_metadata(info)
vi.save()
print(
f"{c.mimetype} {c.preview} {vi.mimetype} {vi.duration:.1f} sec {vi.width}x{vi.height} px"
)
if ffp.is_audio():
new_audio, cmd_str, output = create_audioinstance(
c.file.path)
ai = Audioinstance(content=c)
ai.save()
ai.set_file(new_audio, "ogg")
ai.set_metadata(info)
ai.save()
print(
f"{c.mimetype} {c.preview} {ai.mimetype} {ai.duration:.1f} sec"
)
self.all_content.append(c)
def testNewContentSaving(self):
self.assertTrue(os.path.isdir(TESTCONTENT_DIR), "Directory '%s' containing test files does not exist." % TESTCONTENT_DIR)
files = os.listdir(TESTCONTENT_DIR)
self.assertGreater(len(files), 0, "Directory '%s' containing test files is empty." % TESTCONTENT_DIR)
cnt = 0
for filename in files:
cnt += 1
c = Content(caption=u'New content #%d' % cnt)
full_path = os.path.join(TESTCONTENT_DIR, filename)
if not os.path.isfile(full_path):
continue
c.set_file(filename, full_path)
c.set_fileinfo()
c.generate_thumbnail()
c.save()
maintype = c.mimetype.split('/')[0]
print "MIMETYYPPI", c.mimetype, c.preview
if maintype in ['video', 'audio']:
ffp = content.filetools.FFProbe(c.file.path)
info = content.filetools.fileinfo(c.file.path)
print info
#finfo = get_ffmpeg_videoinfo(c.file.path)
#print finfo
if ffp.is_video():
new_video, cmd_str = create_videoinstance(c.file.path)
vi = Videoinstance(content=c)
vi.save()
vi.set_file(new_video, 'webm')
#info = get_videoinfo(get_ffmpeg_videoinfo(vi.file.path))
vi.set_metadata(info)
vi.save()
print u'%s %.1f sec %dx%d pix' % (vi.mimetype, vi.duration, vi.width, vi.height)
if ffp.is_audio():
new_audio, cmd_str = create_audioinstance(c.file.path)
ai = Audioinstance(content=c)
ai.save()
ai.set_file(new_audio, 'ogg')
#info = get_audioinfo(get_ffmpeg_videoinfo(ai.file.path))
#print info
ai.set_metadata(info)
ai.save()
print u'%s %.1f sec' % (ai.mimetype, ai.duration)
#print c.get_type_instance()
#print c.caption
self.all_content.append(c)
#self.assertEqual(c.file.path, "sd", c.file.path)
#import time
#time.sleep(20)
self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def testNewContentFromTestContentDir(self):
self.assertTrue(os.path.isdir(TESTCONTENT_DIR), "Directory '%s' containing test files does not exist." % TESTCONTENT_DIR)
files = os.listdir(TESTCONTENT_DIR)
self.assertGreater(len(files), 0, "Directory '%s' containing test files is empty." % TESTCONTENT_DIR)
cnt = 0
for filename in files:
cnt += 1
c = Content(caption=u'New content #%d' % cnt)
full_path = os.path.join(TESTCONTENT_DIR, filename)
c.set_file(filename, full_path)
c.save()
#print c.get_type_instance()
#print c.caption
self.all_content.append(c)
#self.assertEqual(c.file.path, "sd", c.file.path)
self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def savefiles(msg):
"""
Extract parts from msg (which is an email.message_from_string(str) instance)
and send them to the plok-database.
NOTES:
- uses only the first found email address to assume recipient
"""
part_counter = 1
subject = get_subject(msg)
tos = get_recipient(msg)
msg_id = msg.get('message-id', '')
froms = msg.get_all('from', [])
p = re.compile('([\w\.\-]+)@')
try: # May raise in some cases IndexError: list index out of range
matches = p.findall(froms[0])
sender_nick = matches[0].split(".")[0].title() # Use all before first '.'
except:
print "ERROR: No From header %s" % (msg_id)
return False
print subject, tos, froms, msg_id
if len(tos) == 0:
print "ERROR: No Tos found %s" % (msg_id)
return False
p = re.compile('([\w]+)\.([\w]+)@') # e.g. [email protected]
matches = p.findall(tos[0])
if len(matches) > 0:
user = matches[0][0].title()
key = matches[0][1].lower()
else:
print "ERROR: No user.authkey found from %s %s" % (tos[0], msg_id)
return False
# TODO: replace this with AuthTicket stuff
from django.contrib.auth import authenticate
user = authenticate(authtoken='qwerty123')
print user
parts_not_to_save = ["multipart/mixed",
"multipart/alternative",
"multipart/related",
"text/plain",
]
saved_parts = 0
for part in msg.walk():
part_content_type = part.get_content_type()
if part_content_type in parts_not_to_save:
# print "NOT SAVING", part_content_type
continue
print part_content_type
filename, filedata = handle_part(part)
if filename is None:
print "No filename"
continue
c = Content(
user = user,
#privacy = legal_key_map[key],
caption = subject,
author = sender_nick,
)
c.set_file(filename, filedata)
c.get_type_instance()
c.save()
print c
saved_parts += 1
return saved_parts
def savefiles(msg, simulate):
"""
Extract parts from msg (which is an email.message_from_string(str) instance)
and send them to the database.
NOTES:
- uses only the first found email address to assume recipient
TODO stuff
- reject if From: is empty
"""
part_counter = 1
subject = get_subject(msg)
tos = get_recipient(msg)
#print tos
msg_id = msg.get('message-id', '')
froms = msg.get_all('from', [])
p = re.compile('([\w\.\-]+)@')
try: # May raise in some cases IndexError: list index out of range
matches = p.findall(froms[0])
sender_nick = matches[0].split(
".")[0].title() # Use all before first '.'
except:
print "ERROR: No From header %s" % (msg_id)
return False
if len(tos) == 0:
print "ERROR: No Tos found %s" % (msg_id)
return False
p = re.compile('([\w]+)\.([\w]+)@') # e.g. [email protected]
matches = p.findall(tos[0])
if len(matches) > 0:
username = matches[0][0].title()
key = matches[0][1].lower()
else:
print "ERROR: No user.authkey found from %s %s" % (tos[0], msg_id)
return False
#print "User, key:", username, key
# TODO: replace this with AuthTicket stuff
#from django.contrib.auth import authenticate
#user = authenticate(authtoken='qwerty123')
try:
user = User.objects.get(username=username.lower())
except User.DoesNotExist:
print "User.DoesNotExist !", username
log.warning("User.DoesNotExist: '%s'" % username)
return False
contentgroup = None
if user.albumitgroups.count() > 0:
contentgroup = user.albumitgroups.all()[0]
sourceorg = None
if user.sourceorgs.count() > 0:
sourceorg = user.sourceorgs.all()[0]
photographer_name = sender_nick
photographer = None
if sourceorg and sourceorg.photographers.count() > 0:
p = sourceorg.photographers.all()[0]
photographer_name = u'{} {}'.format(p.firstname, p.lastname)
photographer = p
#privacy = 'PRIVATE'
privacy = 'RESTRICTED'
if key.lower() == 'pub':
privacy = 'PUBLIC'
elif key.lower() == 'res':
privacy = 'RESTRICTED'
else:
log.warning("Privacy part not found: '%s'" % key)
parts_not_to_save = [
"multipart/mixed",
"multipart/alternative",
"multipart/related",
"text/plain",
]
if simulate: # Print lots of debug stuff
print u'=========\nMetadata:\n========='
print u'''Subject: %s\nUsername: %s\nFrom: %s\nTo: %s\nM-id: %s\n(%s)''' % (
subject, user, u','.join(froms), u','.join(tos), msg_id, privacy)
print u'=========\nParts:\n========='
saved_parts = 0
log.info("Walking through message parts")
for part in msg.walk():
part_content_type = part.get_content_type()
filename, filedata = handle_part(part)
if part_content_type in parts_not_to_save or filename is None:
# print "NOT SAVING", part_content_type
log_msg = "Not saving '%s', filename '%s'." % (part_content_type,
filename)
log.info(log_msg)
if simulate: print log_msg # Print lots of debug stuff
continue
#print filedata, type(filedata), len(filedata)
if filedata is None or len(filedata) == 0:
log_msg = "Not saving '%s', filename '%s', file has no data" % (
part_content_type, filename)
log.warning(log_msg)
if simulate:
print log_msg # Print lots of debug stuff
continue
log_msg = u'Saving: %s (%s)' % (filename, part_content_type)
log.info(log_msg)
if simulate:
print log_msg # Print lots of debug stuff
c = Content(
user=user,
privacy=privacy,
caption=subject,
author=photographer_name,
#author=sender_nick,
#group=contentgroup,
)
if simulate is False:
log.info("Saving file %s" % filename)
c.set_file(filename, filedata)
log.info("set_fileinfo()")
c.set_fileinfo()
log.info("c.generate_thumbnail()")
c.generate_thumbnail()
c.save()
saved_parts += 1
log.info("Saving really")
else:
log.info("Not saving, simulate %s" % simulate)
m = Metadata(
content=c,
sourceorg=sourceorg,
photographer=photographer,
group=contentgroup,
caption=subject,
author=photographer_name,
geometry=c.point,
)
m.save()
return saved_parts
def api_upload(request):
"""
Renders the upload form page.
"""
try:
if request.method == 'POST': # If the form has been submitted...
#for header in request.META.keys():
# if header.startswith('HTTP'):
# print header, request.META[header]
#print request.raw_post_data[:1000]
#if request.user.is_authenticated() is False:
# return HttpResponse(status=401)
form = UploadForm(request.POST, request.FILES) # A form bound to the POST data
#validate_authorization(request)
if form.is_valid(): # File was posted with form
c = None
data = dict(request.POST)
jsondata = request.POST.get('data')
if jsondata is None: # Temporary fix for ambiguous parameter name data / metadata
jsondata = request.POST.get('metadata')
if jsondata:
try: # data is a json string containing the same keys as multipart form
data = json.loads(jsondata)
except: # if it was not valid json, use normal post data
data = dict(request.POST)
#raise
SUPPORTED_FIELDS = ['title', 'caption', 'author']
kwargs = {}
for field in SUPPORTED_FIELDS:
kwargs[field] = data.get(field, '')
try:
kwargs['point'] = Point(float(data.get('lon')), float(data.get('lat')))
except:
kwargs['point'] = None
#raise
pass
# Create a new Egg
e = Egg(**kwargs)
print kwargs
response_status = 201 # Created
tags = data.get('tags')
if tags:
tags = [x.lower() for x in tags]
tag_str = ','.join(tags)
else:
tags = []
tag_str = ''
user = c = None
file_md5_sums = []
for filefield, tmpname in handle_uploaded_file(request):
print "HANDLING FILE:", filefield, tmpname
c = Content(**kwargs)
c.keywords = tag_str
originalname = str(request.FILES["file"])
# c.user = request.user # Only authenticated users can use this view
c.set_file(originalname, tmpname) # Save uploaded file to filesystem
digest_maker = hashlib.md5()
with open(c.file.path, 'rb') as f:
buf = f.read(4096)
while buf:
digest_maker.update(buf)
buf = f.read(4096)
file_md5_sums.append(digest_maker.hexdigest())
#print digest_maker.hexdigest()
c.get_type_instance() # Create thumbnail if it is supported
c.save()
# Copy coordinates from content (parsed while saving it if they existed)
e.content = c
if e.point is None and c.point:
e.point = c.point
e.uid = c.uid
break # We save only the first file
# Handle authorization after files are handled
user = validate_4dnest_authorization(request, file_md5_sums)
if c:
c.user = user
c.save()
e.user = user
print "USER", user
if user:
response_status = 201 # Created
else:
response_status = 401 # Unauthorized
e.save()
# We can handle tags after egg has id (it is saved)
#print "LOOPING TAGS"
for tagname in tags:
try:
tag = Tag.objects.get(name=tagname)
#print "Tag old:",
except Tag.DoesNotExist:
tag = Tag(name=tagname)
tag.save()
#print "Tag new:",
#print tagname, tag
e.tags.add(tag)
response = HttpResponse(status=response_status)
#response.status_code = 201
# FIXME: use reverse()
#return HttpResponseRedirect(reverse('egg api', args=[e.uid]))
response['Location'] = '/fourdnest/api/v1/egg/%s/' % e.uid
return response
else:
response = HttpResponse(status=204)
return response
else:
raise Http404
except Exception, err:
print err
raise
return HttpResponse("Server error", status=500)
