def connect_all(): """ to be called from models.py """ dbgprint("CONNECTING SIGNALS") upload_done.connect(upload_done_callback) download_start.connect(download_start_callback)
def process_IN_MOVED_FROM(self, event): dbgprint( "INOTIFY: IN_MOVED_FROM", event) # the case that the file was just moved on the same filesystem inside # a user's music folder is VERY COMPLICATED to handle in a way that # just the path of the song in the db is updated. therefore, I just # remove the db entry. fswatch_file_removed.delay(event)
def upload_done_callback(sender, **kwargs): handler = kwargs.pop("handler") request = kwargs.pop("request") try: upload = handler.userUploadStatus except Exception, e: dbgprint("An error occured when determining upload state", e) dbgprint("upload_done_callback: empty file uploaded by:", request.user) return
def fswatch_file_removed(event): try: song = Song.objects.get(path_orig=event.pathname) except Song.DoesNotExist: return try: dbgprint("fswatch_file_removed: deleting song entry", song) song.delete() except CeleryExceptions.MaxRetriesExceededError: # well, whorses will have their trinkets return except Exception, exc: raise fswatch_file_removed.retry(exc=exc, countdown=10)
def download_start_callback(sender, **kwargs): request = kwargs.pop("request") playlist_id = request.POST.get("playlist_id") playlist = Playlist.objects.get(user=request.user, id=playlist_id) upload = Upload(user=request.user, step="preparing", step_status=0) # create file list to_compress = [] for item in playlist.items.all(): path = item.song.path_orig.encode("utf-8") to_compress.append(path) # determine zipfile path zipname = playlist.name.replace(os.path.sep, "_").encode("utf-8") + ".zip" zipdir = os.path.join(settings.FILE_DOWNLOAD_USER_DIR, request.user.username) if not os.path.isdir(zipdir): os.makedirs(zipdir) zippath = os.path.join(zipdir, zipname) if os.path.isfile(zippath): os.remove(zippath) upload.step = "compressing" upload.save() amount = len(playlist.items.all()) processed = 0 newzip = zipfile.ZipFile(zippath, mode="w") for songpath in to_compress: dbgprint("Adding", songpath, type(songpath), os.path.exists(songpath)) newzip.write(songpath) processed += 1 step_status = int(processed * 100 / amount) # amount won't be 0 if step_status % 3 == 0: upload.step_status = step_status upload.save() newzip.close() upload.step = "finished" upload.step_status = 0 upload.save()
def process_IN_CLOSE_WRITE(self, event): dbgprint( "INOTIFY: IN_CLOSE_WRITE", event) fswatch_file_written.delay(event)
def process_IN_MODIFY(self, event): dbgprint( "INOTIFY: IN_MODIFY", event)
def process_IN_MOVED_TO(self, event): # see process_IN_MOVED_FROM dbgprint( "INOTIFY: IN_MOVED_TO", event) fswatch_file_written.delay(event)
def process_IN_DELETE(self, event): dbgprint( "INOTIFY: IN_DELETE", event) fswatch_file_removed.delay(event)
def __init__(self): dbgprint( "INOTIFY ProcessInotifyEvent constructed")
def upload_done(user_id, upload_status_id, uploaded_file_path): #def upload_done(useruppath, userupdir, upload_status_id): user = User.objects.get(id=user_id) upload_status = Upload.objects.get(id=upload_status_id) print("Entering status copying") step_status = 0 # determine user upload dir userupdir = os.path.join( settings.FILE_UPLOAD_USER_DIR, user.username ) # create user's upload dir # if not os.path.isdir(userupdir): # os.makedirs(userupdir) # move temporary file to users's upload dir determine file name and path, # don't overwrite # useruppath = os.path.join(userupdir, uploaded_file_name) # ii = 0 # while os.path.exists(useruppath): # useruppath = os.path.join( # userupdir, # uploaded_file_name + "_" + str(ii) # ) # ii += 1 # print("Copy-to location:", useruppath, type(useruppath)) # shutil.move(uploaded_file_path, uploaded_file_name) # now put the upload content to the user's location # step_status = 0 # processed = 0 # amount = upfile.size # helper.dbgprint("Chunks:", amount) # old_status = 0 # with open(useruppath, 'wb+') as destination: # for chunk in upfile.chunks(): # processed += upfile.DEFAULT_CHUNK_SIZE # destination.write(chunk) # step_status = int(processed*100/amount) # if (old_status < step_status) and (step_status % 2 == 0): # helper.dbgprint("Chunks copied:", step_status, "%") # userUploadStatus.step_status = step_status # userUploadStatus.save() # old_status = step_status # destination.close() # helper.dbgprint("Uploaded file written to", useruppath) ################################# decompress ################################ # determine file type, unzip or untar (.zip, .7z, [.tar].[gz|bz2|xz|lzma]) # put deflates in tmp dir. I will use the user's specific django upload temp # TODO: check for tar bombs (expand root dir, xTB null file images) ############################################################################ upload_status.step = "decompress" upload_status.step_status = 0 upload_status.save() deflates = [] magic_mime = magic.Magic(mime=True) mime = magic_mime.from_file(uploaded_file_path.encode('utf-8')) # Magic has problems with unicode? if "application/zip" == mime: # deflate todeflate = zipfile.ZipFile(uploaded_file_path, 'r') processed = 0 step_status = 0 old_status = 0 amount = len(todeflate.namelist()) for name in todeflate.namelist(): extracted_to = todeflate.extract(name, userupdir) deflates.append(extracted_to) processed += 1 step_status = int(processed*100/amount) if (old_status < step_status) and ((step_status % 3) == 0): old_status = step_status upload_status.step_status = step_status try: upload_status.save() except: pass dbgprint("Deflated", step_status, "%") todeflate.close() else: deflates.append(uploaded_file_path) # Suppose we have decompressed a zip, so there multiple files now, and # their paths will be stored as list in ''deflates''. ############################### structuring ################################ # read tags, determine file paths, move files ############################################################################ upload_status.step = "structuring" upload_status.step_status = 0 upload_status.save() step_status = 0 old_status = 0 amount = len(deflates) processed = 0 for defl in deflates: if not amount == 0: step_status = int(processed*100/amount) else: step_status = 1 processed += 1 if (old_status < step_status) and ((step_status % 2) == 0): dbgprint("Structuring:", step_status, "%") old_status = step_status upload_status.step_status = step_status try: upload_status.save() except: pass # there could be subdirectories in zips deflate list if os.path.isdir(defl): continue # It's only worth to have a look at files with following mimess mime = magic_mime.from_file(defl.encode('utf-8')) if mime == 'application/ogg': f_extension = 'ogg' elif mime == 'audio/mpeg': f_extension = 'mp3' else: dbgprint("Ignoring mime", mime, "on file", defl) continue try: tags = get_tags(defl) except Exception, e: dbgprint("Error reading tags on", defl, e) continue if tags == None: # ignore files that have no tags #dbgprint("File has no tags:", defl) continue # determine target location of deflate based on tags musicuploaddir = os.path.join( settings.MUSIC_PATH, upload_status.user.username, 'uploads' ) if not os.path.isdir(musicuploaddir): os.makedirs(musicuploaddir) # this filedir does not have to be unique if tags['artist']: artist = tags['artist'].replace(os.path.sep, '_') else: artist = "UNKNOWN ARTIST" if tags['album']: album = tags['album'].replace(os.path.sep, '_') else: album = "UNKNOWN ALBUM" filedir = os.path.join( musicuploaddir, #.encode('utf-8'), artist, album ) if not os.path.isdir(filedir): # this could fail if filedir exists and is a file os.makedirs(filedir) if tags['track']: track = str(tags['track']) else: track = '0' if tags ['title']: title = tags['title'].replace(os.path.sep, '_') else: title = "UNKNOWN TITLE" filename = track + ". " + \ title + '.' + \ f_extension filepath = os.path.join(filedir, filename) jj = 0 while os.path.exists(filepath): filename = track + ". " + \ title + str(jj) + '.' + \ f_extension filepath = os.path.join(filedir, filename) jj += 1 #dbgprint("Moving deflate", defl, "to", filepath) shutil.move(defl, filepath)
download_start.connect(download_start_callback) def upload_done_callback(sender, **kwargs): handler = kwargs.pop("handler") request = kwargs.pop("request") try: upload = handler.userUploadStatus except Exception, e: dbgprint("An error occured when determining upload state", e) dbgprint("upload_done_callback: empty file uploaded by:", request.user) return ################################# copying ################################ dbgprint("Entering status copying") step_status = 0 upload.step = "copying" upload.step_status = 0 upload.save() f = handler.file # create user's upload dir uploaddir = os.path.join(settings.FILE_UPLOAD_USER_DIR, request.user.username) if not os.path.isdir(uploaddir): os.makedirs(uploaddir) # move temporary file to users's upload dir determine file name and path, # don't overwrite uploadpath = os.path.join(uploaddir, f.name) ii = 0