def testNewContentFromTestContentDir(self): self.assertTrue( os.path.isdir(TEST_CONTENT_DIR), "Directory '%s' containing test files does not exist." % TEST_CONTENT_DIR) files = os.listdir(TEST_CONTENT_DIR) print("\n\nFAILEJA", TEST_CONTENT_DIR, files) self.assertGreater( len(files), 0, "Directory '%s' containing test files is empty." % TEST_CONTENT_DIR) cnt = 0 for filename in files: cnt += 1 c = Content(caption="New content #%d" % cnt) full_path = os.path.join(TEST_CONTENT_DIR, str(filename)) c.set_file(str(filename), full_path) c.set_fileinfo() c.generate_thumbnail() c.save() maintype = c.mimetype.split("/")[0] print("MIMETYYPPI", c.mimetype, c.preview) if maintype in ["video", "audio"]: ffp = content.filetools.FFProbe(c.file.path) info = content.filetools.fileinfo(c.file.path) print(info) if ffp.is_video(): new_video, cmd_str, output = create_videoinstance( c.file.path) vi = Videoinstance(content=c) vi.save() vi.set_file(new_video, "webm") vi.set_metadata(info) vi.save() print("%s %.1f sec %dx%d pix" % (vi.mimetype, vi.duration, vi.width, vi.height)) if ffp.is_audio(): new_audio, cmd_str, output = create_audioinstance( c.file.path) ai = Audioinstance(content=c) ai.save() ai.set_file(new_audio, "ogg") ai.set_metadata(info) ai.save() print("%s %.1f sec" % (ai.mimetype, ai.duration)) self.all_content.append(c) # self.assertEqual(c.file.path, "sd", c.file.path) # import time # time.sleep(20) self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def simple_upload(request): """ Handles uploaded files """ try: if request.method == 'POST': # If the form has been submitted... user = request.user if request.user.is_authenticated() else None form = UploadForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # File was posted with form c = None kwargs = {} kwargs['author'] = user.username.title() if user else 'Anonymous' # Create a new Egg e = Egg(**kwargs) print kwargs for filefield, tmpname in handle_uploaded_file(request): print "HANDLING FILE:", filefield, tmpname c = Content(**kwargs) originalname = str(request.FILES["file"]) # c.user = request.user # Only authenticated users can use this view c.set_file(originalname, tmpname) # Save uploaded file to filesystem c.get_type_instance() # Create thumbnail if it is supported c.save() e.content = c e.uid = c.uid break # We save only the first file if c: c.user = user c.save() e.user = user print "USER", user if user: response_status = 200 # Created else: response_status = 401 # Unauthorized e.save() # We can handle tags after egg has id (it is saved) response = HttpResponse(status=response_status) #response.status_code = 201 # FIXME: use reverse() #return HttpResponseRedirect(reverse('egg api', args=[e.uid])) response['Location'] = '/fourdnest/api/v1/egg/%s/' % e.uid return response else: response = HttpResponse(status=204) return response else: raise Http404 except Exception, err: print err raise return HttpResponse("Server error", status=500)
def testNewContentSaving(self): self.assertTrue( os.path.isdir(TEST_CONTENT_DIR), f"Directory '{TEST_CONTENT_DIR}' containing test files does not exist." ) files = os.listdir(TEST_CONTENT_DIR) self.assertGreater( len(files), 0, f"Directory '{TEST_CONTENT_DIR}' containing test files is empty.") cnt = 0 for filename in files: full_path = os.path.join(TEST_CONTENT_DIR, str(filename)) if os.path.isfile(full_path) is False: print(f"Skip {full_path}, not a file") continue cnt += 1 c = Content(caption="New content #%d" % cnt) c.set_file(str(filename), full_path) c.set_fileinfo() c.generate_thumbnail() c.save() maintype = c.mimetype.split("/")[0] if maintype in ["video", "audio"]: ffp = content.filetools.FFProbe(c.file.path) info = content.filetools.fileinfo(c.file.path) print(info) if ffp.is_video(): new_video, cmd_str, output = create_videoinstance( c.file.path) vi = Videoinstance(content=c) vi.save() vi.set_file(new_video, "webm") vi.set_metadata(info) vi.save() print( f"{c.mimetype} {c.preview} {vi.mimetype} {vi.duration:.1f} sec {vi.width}x{vi.height} px" ) if ffp.is_audio(): new_audio, cmd_str, output = create_audioinstance( c.file.path) ai = Audioinstance(content=c) ai.save() ai.set_file(new_audio, "ogg") ai.set_metadata(info) ai.save() print( f"{c.mimetype} {c.preview} {ai.mimetype} {ai.duration:.1f} sec" ) self.all_content.append(c)
def testNewContentSaving(self): self.assertTrue(os.path.isdir(TESTCONTENT_DIR), "Directory '%s' containing test files does not exist." % TESTCONTENT_DIR) files = os.listdir(TESTCONTENT_DIR) self.assertGreater(len(files), 0, "Directory '%s' containing test files is empty." % TESTCONTENT_DIR) cnt = 0 for filename in files: cnt += 1 c = Content(caption=u'New content #%d' % cnt) full_path = os.path.join(TESTCONTENT_DIR, filename) if not os.path.isfile(full_path): continue c.set_file(filename, full_path) c.set_fileinfo() c.generate_thumbnail() c.save() maintype = c.mimetype.split('/')[0] print "MIMETYYPPI", c.mimetype, c.preview if maintype in ['video', 'audio']: ffp = content.filetools.FFProbe(c.file.path) info = content.filetools.fileinfo(c.file.path) print info #finfo = get_ffmpeg_videoinfo(c.file.path) #print finfo if ffp.is_video(): new_video, cmd_str = create_videoinstance(c.file.path) vi = Videoinstance(content=c) vi.save() vi.set_file(new_video, 'webm') #info = get_videoinfo(get_ffmpeg_videoinfo(vi.file.path)) vi.set_metadata(info) vi.save() print u'%s %.1f sec %dx%d pix' % (vi.mimetype, vi.duration, vi.width, vi.height) if ffp.is_audio(): new_audio, cmd_str = create_audioinstance(c.file.path) ai = Audioinstance(content=c) ai.save() ai.set_file(new_audio, 'ogg') #info = get_audioinfo(get_ffmpeg_videoinfo(ai.file.path)) #print info ai.set_metadata(info) ai.save() print u'%s %.1f sec' % (ai.mimetype, ai.duration) #print c.get_type_instance() #print c.caption self.all_content.append(c) #self.assertEqual(c.file.path, "sd", c.file.path) #import time #time.sleep(20) self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def testNewContentFromTestContentDir(self): self.assertTrue(os.path.isdir(TESTCONTENT_DIR), "Directory '%s' containing test files does not exist." % TESTCONTENT_DIR) files = os.listdir(TESTCONTENT_DIR) self.assertGreater(len(files), 0, "Directory '%s' containing test files is empty." % TESTCONTENT_DIR) cnt = 0 for filename in files: cnt += 1 c = Content(caption=u'New content #%d' % cnt) full_path = os.path.join(TESTCONTENT_DIR, filename) c.set_file(filename, full_path) c.save() #print c.get_type_instance() #print c.caption self.all_content.append(c) #self.assertEqual(c.file.path, "sd", c.file.path) self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def savefiles(msg): """ Extract parts from msg (which is an email.message_from_string(str) instance) and send them to the plok-database. NOTES: - uses only the first found email address to assume recipient """ part_counter = 1 subject = get_subject(msg) tos = get_recipient(msg) msg_id = msg.get('message-id', '') froms = msg.get_all('from', []) p = re.compile('([\w\.\-]+)@') try: # May raise in some cases IndexError: list index out of range matches = p.findall(froms[0]) sender_nick = matches[0].split(".")[0].title() # Use all before first '.' except: print "ERROR: No From header %s" % (msg_id) return False print subject, tos, froms, msg_id if len(tos) == 0: print "ERROR: No Tos found %s" % (msg_id) return False p = re.compile('([\w]+)\.([\w]+)@') # e.g. [email protected] matches = p.findall(tos[0]) if len(matches) > 0: user = matches[0][0].title() key = matches[0][1].lower() else: print "ERROR: No user.authkey found from %s %s" % (tos[0], msg_id) return False # TODO: replace this with AuthTicket stuff from django.contrib.auth import authenticate user = authenticate(authtoken='qwerty123') print user parts_not_to_save = ["multipart/mixed", "multipart/alternative", "multipart/related", "text/plain", ] saved_parts = 0 for part in msg.walk(): part_content_type = part.get_content_type() if part_content_type in parts_not_to_save: # print "NOT SAVING", part_content_type continue print part_content_type filename, filedata = handle_part(part) if filename is None: print "No filename" continue c = Content( user = user, #privacy = legal_key_map[key], caption = subject, author = sender_nick, ) c.set_file(filename, filedata) c.get_type_instance() c.save() print c saved_parts += 1 return saved_parts
def savefiles(msg, simulate): """ Extract parts from msg (which is an email.message_from_string(str) instance) and send them to the database. NOTES: - uses only the first found email address to assume recipient TODO stuff - reject if From: is empty """ part_counter = 1 subject = get_subject(msg) tos = get_recipient(msg) #print tos msg_id = msg.get('message-id', '') froms = msg.get_all('from', []) p = re.compile('([\w\.\-]+)@') try: # May raise in some cases IndexError: list index out of range matches = p.findall(froms[0]) sender_nick = matches[0].split( ".")[0].title() # Use all before first '.' except: print "ERROR: No From header %s" % (msg_id) return False if len(tos) == 0: print "ERROR: No Tos found %s" % (msg_id) return False p = re.compile('([\w]+)\.([\w]+)@') # e.g. [email protected] matches = p.findall(tos[0]) if len(matches) > 0: username = matches[0][0].title() key = matches[0][1].lower() else: print "ERROR: No user.authkey found from %s %s" % (tos[0], msg_id) return False #print "User, key:", username, key # TODO: replace this with AuthTicket stuff #from django.contrib.auth import authenticate #user = authenticate(authtoken='qwerty123') try: user = User.objects.get(username=username.lower()) except User.DoesNotExist: print "User.DoesNotExist !", username log.warning("User.DoesNotExist: '%s'" % username) return False contentgroup = None if user.albumitgroups.count() > 0: contentgroup = user.albumitgroups.all()[0] sourceorg = None if user.sourceorgs.count() > 0: sourceorg = user.sourceorgs.all()[0] photographer_name = sender_nick photographer = None if sourceorg and sourceorg.photographers.count() > 0: p = sourceorg.photographers.all()[0] photographer_name = u'{} {}'.format(p.firstname, p.lastname) photographer = p #privacy = 'PRIVATE' privacy = 'RESTRICTED' if key.lower() == 'pub': privacy = 'PUBLIC' elif key.lower() == 'res': privacy = 'RESTRICTED' else: log.warning("Privacy part not found: '%s'" % key) parts_not_to_save = [ "multipart/mixed", "multipart/alternative", "multipart/related", "text/plain", ] if simulate: # Print lots of debug stuff print u'=========\nMetadata:\n=========' print u'''Subject: %s\nUsername: %s\nFrom: %s\nTo: %s\nM-id: %s\n(%s)''' % ( subject, user, u','.join(froms), u','.join(tos), msg_id, privacy) print u'=========\nParts:\n=========' saved_parts = 0 log.info("Walking through message parts") for part in msg.walk(): part_content_type = part.get_content_type() filename, filedata = handle_part(part) if part_content_type in parts_not_to_save or filename is None: # print "NOT SAVING", part_content_type log_msg = "Not saving '%s', filename '%s'." % (part_content_type, filename) log.info(log_msg) if simulate: print log_msg # Print lots of debug stuff continue #print filedata, type(filedata), len(filedata) if filedata is None or len(filedata) == 0: log_msg = "Not saving '%s', filename '%s', file has no data" % ( part_content_type, filename) log.warning(log_msg) if simulate: print log_msg # Print lots of debug stuff continue log_msg = u'Saving: %s (%s)' % (filename, part_content_type) log.info(log_msg) if simulate: print log_msg # Print lots of debug stuff c = Content( user=user, privacy=privacy, caption=subject, author=photographer_name, #author=sender_nick, #group=contentgroup, ) if simulate is False: log.info("Saving file %s" % filename) c.set_file(filename, filedata) log.info("set_fileinfo()") c.set_fileinfo() log.info("c.generate_thumbnail()") c.generate_thumbnail() c.save() saved_parts += 1 log.info("Saving really") else: log.info("Not saving, simulate %s" % simulate) m = Metadata( content=c, sourceorg=sourceorg, photographer=photographer, group=contentgroup, caption=subject, author=photographer_name, geometry=c.point, ) m.save() return saved_parts
def api_upload(request): """ Renders the upload form page. """ try: if request.method == 'POST': # If the form has been submitted... #for header in request.META.keys(): # if header.startswith('HTTP'): # print header, request.META[header] #print request.raw_post_data[:1000] #if request.user.is_authenticated() is False: # return HttpResponse(status=401) form = UploadForm(request.POST, request.FILES) # A form bound to the POST data #validate_authorization(request) if form.is_valid(): # File was posted with form c = None data = dict(request.POST) jsondata = request.POST.get('data') if jsondata is None: # Temporary fix for ambiguous parameter name data / metadata jsondata = request.POST.get('metadata') if jsondata: try: # data is a json string containing the same keys as multipart form data = json.loads(jsondata) except: # if it was not valid json, use normal post data data = dict(request.POST) #raise SUPPORTED_FIELDS = ['title', 'caption', 'author'] kwargs = {} for field in SUPPORTED_FIELDS: kwargs[field] = data.get(field, '') try: kwargs['point'] = Point(float(data.get('lon')), float(data.get('lat'))) except: kwargs['point'] = None #raise pass # Create a new Egg e = Egg(**kwargs) print kwargs response_status = 201 # Created tags = data.get('tags') if tags: tags = [x.lower() for x in tags] tag_str = ','.join(tags) else: tags = [] tag_str = '' user = c = None file_md5_sums = [] for filefield, tmpname in handle_uploaded_file(request): print "HANDLING FILE:", filefield, tmpname c = Content(**kwargs) c.keywords = tag_str originalname = str(request.FILES["file"]) # c.user = request.user # Only authenticated users can use this view c.set_file(originalname, tmpname) # Save uploaded file to filesystem digest_maker = hashlib.md5() with open(c.file.path, 'rb') as f: buf = f.read(4096) while buf: digest_maker.update(buf) buf = f.read(4096) file_md5_sums.append(digest_maker.hexdigest()) #print digest_maker.hexdigest() c.get_type_instance() # Create thumbnail if it is supported c.save() # Copy coordinates from content (parsed while saving it if they existed) e.content = c if e.point is None and c.point: e.point = c.point e.uid = c.uid break # We save only the first file # Handle authorization after files are handled user = validate_4dnest_authorization(request, file_md5_sums) if c: c.user = user c.save() e.user = user print "USER", user if user: response_status = 201 # Created else: response_status = 401 # Unauthorized e.save() # We can handle tags after egg has id (it is saved) #print "LOOPING TAGS" for tagname in tags: try: tag = Tag.objects.get(name=tagname) #print "Tag old:", except Tag.DoesNotExist: tag = Tag(name=tagname) tag.save() #print "Tag new:", #print tagname, tag e.tags.add(tag) response = HttpResponse(status=response_status) #response.status_code = 201 # FIXME: use reverse() #return HttpResponseRedirect(reverse('egg api', args=[e.uid])) response['Location'] = '/fourdnest/api/v1/egg/%s/' % e.uid return response else: response = HttpResponse(status=204) return response else: raise Http404 except Exception, err: print err raise return HttpResponse("Server error", status=500)