def connect_all():
"""
to be called from models.py
"""
dbgprint("CONNECTING SIGNALS")
upload_done.connect(upload_done_callback)
download_start.connect(download_start_callback)
def process_IN_MOVED_FROM(self, event):
dbgprint( "INOTIFY: IN_MOVED_FROM", event)
# the case that the file was just moved on the same filesystem inside
# a user's music folder is VERY COMPLICATED to handle in a way that
# just the path of the song in the db is updated. therefore, I just
# remove the db entry.
fswatch_file_removed.delay(event)
def upload_done_callback(sender, **kwargs):
handler = kwargs.pop("handler")
request = kwargs.pop("request")
try:
upload = handler.userUploadStatus
except Exception, e:
dbgprint("An error occured when determining upload state", e)
dbgprint("upload_done_callback: empty file uploaded by:", request.user)
return
def fswatch_file_removed(event):
try:
song = Song.objects.get(path_orig=event.pathname)
except Song.DoesNotExist:
return
try:
dbgprint("fswatch_file_removed: deleting song entry", song)
song.delete()
except CeleryExceptions.MaxRetriesExceededError:
# well, whorses will have their trinkets
return
except Exception, exc:
raise fswatch_file_removed.retry(exc=exc, countdown=10)
def download_start_callback(sender, **kwargs):
request = kwargs.pop("request")
playlist_id = request.POST.get("playlist_id")
playlist = Playlist.objects.get(user=request.user, id=playlist_id)
upload = Upload(user=request.user, step="preparing", step_status=0)
# create file list
to_compress = []
for item in playlist.items.all():
path = item.song.path_orig.encode("utf-8")
to_compress.append(path)
# determine zipfile path
zipname = playlist.name.replace(os.path.sep, "_").encode("utf-8") + ".zip"
zipdir = os.path.join(settings.FILE_DOWNLOAD_USER_DIR, request.user.username)
if not os.path.isdir(zipdir):
os.makedirs(zipdir)
zippath = os.path.join(zipdir, zipname)
if os.path.isfile(zippath):
os.remove(zippath)
upload.step = "compressing"
upload.save()
amount = len(playlist.items.all())
processed = 0
newzip = zipfile.ZipFile(zippath, mode="w")
for songpath in to_compress:
dbgprint("Adding", songpath, type(songpath), os.path.exists(songpath))
newzip.write(songpath)
processed += 1
step_status = int(processed * 100 / amount) # amount won't be 0
if step_status % 3 == 0:
upload.step_status = step_status
upload.save()
newzip.close()
upload.step = "finished"
upload.step_status = 0
upload.save()
def process_IN_CLOSE_WRITE(self, event):
dbgprint( "INOTIFY: IN_CLOSE_WRITE", event)
fswatch_file_written.delay(event)
def process_IN_MODIFY(self, event):
dbgprint( "INOTIFY: IN_MODIFY", event)
def process_IN_MOVED_TO(self, event):
# see process_IN_MOVED_FROM
dbgprint( "INOTIFY: IN_MOVED_TO", event)
fswatch_file_written.delay(event)
def process_IN_DELETE(self, event):
dbgprint( "INOTIFY: IN_DELETE", event)
fswatch_file_removed.delay(event)
def __init__(self):
dbgprint( "INOTIFY ProcessInotifyEvent constructed")
def upload_done(user_id, upload_status_id, uploaded_file_path):
#def upload_done(useruppath, userupdir, upload_status_id):
user = User.objects.get(id=user_id)
upload_status = Upload.objects.get(id=upload_status_id)
print("Entering status copying")
step_status = 0
# determine user upload dir
userupdir = os.path.join(
settings.FILE_UPLOAD_USER_DIR,
user.username
)
# create user's upload dir
# if not os.path.isdir(userupdir):
# os.makedirs(userupdir)
# move temporary file to users's upload dir determine file name and path,
# don't overwrite
# useruppath = os.path.join(userupdir, uploaded_file_name)
# ii = 0
# while os.path.exists(useruppath):
# useruppath = os.path.join(
# userupdir,
# uploaded_file_name + "_" + str(ii)
# )
# ii += 1
# print("Copy-to location:", useruppath, type(useruppath))
# shutil.move(uploaded_file_path, uploaded_file_name)
# now put the upload content to the user's location
# step_status = 0
# processed = 0
# amount = upfile.size
# helper.dbgprint("Chunks:", amount)
# old_status = 0
# with open(useruppath, 'wb+') as destination:
# for chunk in upfile.chunks():
# processed += upfile.DEFAULT_CHUNK_SIZE
# destination.write(chunk)
# step_status = int(processed*100/amount)
# if (old_status < step_status) and (step_status % 2 == 0):
# helper.dbgprint("Chunks copied:", step_status, "%")
# userUploadStatus.step_status = step_status
# userUploadStatus.save()
# old_status = step_status
# destination.close()
# helper.dbgprint("Uploaded file written to", useruppath)
################################# decompress ################################
# determine file type, unzip or untar (.zip, .7z, [.tar].[gz|bz2|xz|lzma])
# put deflates in tmp dir. I will use the user's specific django upload temp
# TODO: check for tar bombs (expand root dir, xTB null file images)
############################################################################
upload_status.step = "decompress"
upload_status.step_status = 0
upload_status.save()
deflates = []
magic_mime = magic.Magic(mime=True)
mime = magic_mime.from_file(uploaded_file_path.encode('utf-8')) # Magic has problems with unicode?
if "application/zip" == mime:
# deflate
todeflate = zipfile.ZipFile(uploaded_file_path, 'r')
processed = 0
step_status = 0
old_status = 0
amount = len(todeflate.namelist())
for name in todeflate.namelist():
extracted_to = todeflate.extract(name, userupdir)
deflates.append(extracted_to)
processed += 1
step_status = int(processed*100/amount)
if (old_status < step_status) and ((step_status % 3) == 0):
old_status = step_status
upload_status.step_status = step_status
try:
upload_status.save()
except:
pass
dbgprint("Deflated", step_status, "%")
todeflate.close()
else:
deflates.append(uploaded_file_path)
# Suppose we have decompressed a zip, so there multiple files now, and
# their paths will be stored as list in ''deflates''.
############################### structuring ################################
# read tags, determine file paths, move files
############################################################################
upload_status.step = "structuring"
upload_status.step_status = 0
upload_status.save()
step_status = 0
old_status = 0
amount = len(deflates)
processed = 0
for defl in deflates:
if not amount == 0:
step_status = int(processed*100/amount)
else:
step_status = 1
processed += 1
if (old_status < step_status) and ((step_status % 2) == 0):
dbgprint("Structuring:", step_status, "%")
old_status = step_status
upload_status.step_status = step_status
try:
upload_status.save()
except:
pass
# there could be subdirectories in zips deflate list
if os.path.isdir(defl):
continue
# It's only worth to have a look at files with following mimess
mime = magic_mime.from_file(defl.encode('utf-8'))
if mime == 'application/ogg':
f_extension = 'ogg'
elif mime == 'audio/mpeg':
f_extension = 'mp3'
else:
dbgprint("Ignoring mime", mime, "on file", defl)
continue
try:
tags = get_tags(defl)
except Exception, e:
dbgprint("Error reading tags on", defl, e)
continue
if tags == None:
# ignore files that have no tags
#dbgprint("File has no tags:", defl)
continue
# determine target location of deflate based on tags
musicuploaddir = os.path.join(
settings.MUSIC_PATH,
upload_status.user.username,
'uploads'
)
if not os.path.isdir(musicuploaddir):
os.makedirs(musicuploaddir)
# this filedir does not have to be unique
if tags['artist']: artist = tags['artist'].replace(os.path.sep, '_')
else: artist = "UNKNOWN ARTIST"
if tags['album']: album = tags['album'].replace(os.path.sep, '_')
else: album = "UNKNOWN ALBUM"
filedir = os.path.join(
musicuploaddir, #.encode('utf-8'),
artist,
album
)
if not os.path.isdir(filedir):
# this could fail if filedir exists and is a file
os.makedirs(filedir)
if tags['track']: track = str(tags['track'])
else: track = '0'
if tags ['title']: title = tags['title'].replace(os.path.sep, '_')
else: title = "UNKNOWN TITLE"
filename = track + ". " + \
title + '.' + \
f_extension
filepath = os.path.join(filedir, filename)
jj = 0
while os.path.exists(filepath):
filename = track + ". " + \
title + str(jj) + '.' + \
f_extension
filepath = os.path.join(filedir, filename)
jj += 1
#dbgprint("Moving deflate", defl, "to", filepath)
shutil.move(defl, filepath)
download_start.connect(download_start_callback)
def upload_done_callback(sender, **kwargs):
handler = kwargs.pop("handler")
request = kwargs.pop("request")
try:
upload = handler.userUploadStatus
except Exception, e:
dbgprint("An error occured when determining upload state", e)
dbgprint("upload_done_callback: empty file uploaded by:", request.user)
return
################################# copying ################################
dbgprint("Entering status copying")
step_status = 0
upload.step = "copying"
upload.step_status = 0
upload.save()
f = handler.file
# create user's upload dir
uploaddir = os.path.join(settings.FILE_UPLOAD_USER_DIR, request.user.username)
if not os.path.isdir(uploaddir):
os.makedirs(uploaddir)
# move temporary file to users's upload dir determine file name and path,
# don't overwrite
uploadpath = os.path.join(uploaddir, f.name)
ii = 0
