def upload_done(user_id, upload_status_id, uploaded_file_path): #def upload_done(useruppath, userupdir, upload_status_id): user = User.objects.get(id=user_id) upload_status = Upload.objects.get(id=upload_status_id) print("Entering status copying") step_status = 0 # determine user upload dir userupdir = os.path.join( settings.FILE_UPLOAD_USER_DIR, user.username ) # create user's upload dir # if not os.path.isdir(userupdir): # os.makedirs(userupdir) # move temporary file to users's upload dir determine file name and path, # don't overwrite # useruppath = os.path.join(userupdir, uploaded_file_name) # ii = 0 # while os.path.exists(useruppath): # useruppath = os.path.join( # userupdir, # uploaded_file_name + "_" + str(ii) # ) # ii += 1 # print("Copy-to location:", useruppath, type(useruppath)) # shutil.move(uploaded_file_path, uploaded_file_name) # now put the upload content to the user's location # step_status = 0 # processed = 0 # amount = upfile.size # helper.dbgprint("Chunks:", amount) # old_status = 0 # with open(useruppath, 'wb+') as destination: # for chunk in upfile.chunks(): # processed += upfile.DEFAULT_CHUNK_SIZE # destination.write(chunk) # step_status = int(processed*100/amount) # if (old_status < step_status) and (step_status % 2 == 0): # helper.dbgprint("Chunks copied:", step_status, "%") # userUploadStatus.step_status = step_status # userUploadStatus.save() # old_status = step_status # destination.close() # helper.dbgprint("Uploaded file written to", useruppath) ################################# decompress ################################ # determine file type, unzip or untar (.zip, .7z, [.tar].[gz|bz2|xz|lzma]) # put deflates in tmp dir. I will use the user's specific django upload temp # TODO: check for tar bombs (expand root dir, xTB null file images) ############################################################################ upload_status.step = "decompress" upload_status.step_status = 0 upload_status.save() deflates = [] magic_mime = magic.Magic(mime=True) mime = magic_mime.from_file(uploaded_file_path.encode('utf-8')) # Magic has problems with unicode? if "application/zip" == mime: # deflate todeflate = zipfile.ZipFile(uploaded_file_path, 'r') processed = 0 step_status = 0 old_status = 0 amount = len(todeflate.namelist()) for name in todeflate.namelist(): extracted_to = todeflate.extract(name, userupdir) deflates.append(extracted_to) processed += 1 step_status = int(processed*100/amount) if (old_status < step_status) and ((step_status % 3) == 0): old_status = step_status upload_status.step_status = step_status try: upload_status.save() except: pass dbgprint("Deflated", step_status, "%") todeflate.close() else: deflates.append(uploaded_file_path) # Suppose we have decompressed a zip, so there multiple files now, and # their paths will be stored as list in ''deflates''. ############################### structuring ################################ # read tags, determine file paths, move files ############################################################################ upload_status.step = "structuring" upload_status.step_status = 0 upload_status.save() step_status = 0 old_status = 0 amount = len(deflates) processed = 0 for defl in deflates: if not amount == 0: step_status = int(processed*100/amount) else: step_status = 1 processed += 1 if (old_status < step_status) and ((step_status % 2) == 0): dbgprint("Structuring:", step_status, "%") old_status = step_status upload_status.step_status = step_status try: upload_status.save() except: pass # there could be subdirectories in zips deflate list if os.path.isdir(defl): continue # It's only worth to have a look at files with following mimess mime = magic_mime.from_file(defl.encode('utf-8')) if mime == 'application/ogg': f_extension = 'ogg' elif mime == 'audio/mpeg': f_extension = 'mp3' else: dbgprint("Ignoring mime", mime, "on file", defl) continue try: tags = get_tags(defl) except Exception, e: dbgprint("Error reading tags on", defl, e) continue if tags == None: # ignore files that have no tags #dbgprint("File has no tags:", defl) continue # determine target location of deflate based on tags musicuploaddir = os.path.join( settings.MUSIC_PATH, upload_status.user.username, 'uploads' ) if not os.path.isdir(musicuploaddir): os.makedirs(musicuploaddir) # this filedir does not have to be unique if tags['artist']: artist = tags['artist'].replace(os.path.sep, '_') else: artist = "UNKNOWN ARTIST" if tags['album']: album = tags['album'].replace(os.path.sep, '_') else: album = "UNKNOWN ALBUM" filedir = os.path.join( musicuploaddir, #.encode('utf-8'), artist, album ) if not os.path.isdir(filedir): # this could fail if filedir exists and is a file os.makedirs(filedir) if tags['track']: track = str(tags['track']) else: track = '0' if tags ['title']: title = tags['title'].replace(os.path.sep, '_') else: title = "UNKNOWN TITLE" filename = track + ". " + \ title + '.' + \ f_extension filepath = os.path.join(filedir, filename) jj = 0 while os.path.exists(filepath): filename = track + ". " + \ title + str(jj) + '.' + \ f_extension filepath = os.path.join(filedir, filename) jj += 1 #dbgprint("Moving deflate", defl, "to", filepath) shutil.move(defl, filepath)
else: step_status = 1 processed += 1 if (old_status < step_status) and ((step_status % 3) == 0): dbgprint("Structuring:", step_status, "%") old_status = step_status upload.step_status = step_status upload.save() # there could be subdirectories in zips deflate list if os.path.isdir(defl): continue try: tags = get_tags(defl) except Exception, e: dbgprint("Error reading tags on", defl, e) continue if tags == None: dbgprint("Deflate has no tags:", defl) continue # determine target location of deflate based on tags musicuploaddir = os.path.join(settings.MUSIC_PATH, request.user.username, "uploads") if not os.path.isdir(musicuploaddir): os.makedirs(musicuploaddir) # this filedir does not have to be unique filedir = os.path.join(