def convert_file(self, h, path, p, metadata, processor_state, ignore_limit):
f = File.from_hash(h)
if p not in processor_table:
p = 'default'
processor = processor_table[p](path, f, processor_state, ignore_limit)
# Execute the synchronous step.
processor.sync()
# Save compression information
f = File.from_hash(
h
) # Reload file; user might have changed the config vector while processing
f.compression = compression_rate(path, f)
f.metadata = json.dumps(metadata)
f.save()
# Notify frontend: sync step is done.
self.update_state(state="READY")
# Execute the asynchronous step.
processor.important = False
processor. async ()
def process_file(path, h, ignore_limit):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def report(self, id):
rate_limit_update(1, section="report")
if not current_app.debug and rate_limit_exceeded(section="report"):
return {'error': 413}, 413
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
def frame(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if (
request.user_agent.platform in ["android", "iphone", "ipad"]
or "windows phone" in request.user_agent.string.lower()
):
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v["embedded"] = True
v["filename"] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params["embedded"] = True
return render_template("direct.html", **template_params)
def process_file(path, h):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state)
task_result = task.freeze(
) # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def frame(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if request.user_agent.platform in [
'android', 'iphone', 'ipad'
] or 'windows phone' in request.user_agent.string.lower():
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['embedded'] = True
v['filename'] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params['embedded'] = True
return render_template("direct.html", **template_params)
def process_file(path, h):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def process_file(path, h, ignore_limit):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state,
ignore_limit)
task_result = task.freeze(
) # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def report(self, id):
rate_limit_update(1, section="report")
if not current_app.debug and rate_limit_exceeded(section="report"):
return {'error': 413}, 413
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
def fragment(self, id):
klass = RedisObject.klass(id)
if klass is not File:
abort(404)
f = File.from_hash(id)
params = _template_params(f)
params['album'] = True
return render_template(params['fragment'], **params)
def fragment(self, id):
klass = RedisObject.klass(id)
if klass is not File:
abort(404)
f = File.from_hash(id)
params = _template_params(f)
params['album'] = True
return render_template(params['fragment'], **params)
def cleanup(results, path, h):
f = File.from_hash(h)
os.unlink(path)
if f.status in ["internal_error", "error", "timeout", "unrecognised"]:
failed = FailedFile(hash=h, status=f.status) # Create a "failed file" record
failed.save()
delete_file(f)
def status(self, id):
klass = RedisObject.klass(id)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
if f.status in ['done', 'ready']:
return tor_redirect('/' + f.hash)
return render_template("status.html", **_template_params(f))
def status(self, id):
klass = RedisObject.klass(id)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
if f.status in ['done', 'ready']:
return tor_redirect('/' + f.hash)
return render_template("status.html", **_template_params(f))
def _upload_object(result, status):
if status == 200:
return {"hash": result}
else:
resp = {"error": status}
if status == 409:
f = _file_object(File.from_hash(result))
resp[result] = f
resp["hash"] = result
return resp, status
def _upload_object(result, status):
if status == 200:
return {'hash': result}
else:
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(result))
resp[result] = f
resp['hash'] = result
return resp, status
def _upload_object(result, status):
if status == 200:
return {'hash': result}
else:
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(result))
resp[result] = f
resp['hash'] = result
return resp, status
def convert_file(self, h, path, p, extra):
f = File.from_hash(h)
if p not in processor_table:
p = 'default'
processor = processor_table[p](path, f, extra)
# Execute the synchronous step.
processor.sync()
# Save compression information
f = File.from_hash(h) # Reload file; user might have changed the config vector while processing
f.compression = compression_rate(path, f)
f.save()
# Notify frontend: sync step is done.
self.update_state(state="READY")
# Execute the asynchronous step.
processor.important = False
processor.async()
def _upload_f(f, filename):
result = upload(f, filename)
if not isinstance(result, tuple):
return {'hash': result}
else:
h, status = result
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(h))
resp[h] = f
resp['hash'] = h
return resp, status
def status(self, h):
klass = RedisObject.klass(h)
if not klass:
return {'error': 404}, 404
if klass is not File:
return {'error': 415}, 415
f = File.from_hash(h)
ret = {'status': f.status}
if ret['status'] == 'done':
ret[h] = _file_object(f)
ret['hash'] = h
return ret
def get(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
return render_template("view.html", **_template_params(f))
def get(self, id, layout):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
return render_template("view.html", **_template_params(f))
def process_file(path, h):
f = File.from_hash(h)
p, extra = detect(path)
f.processor = p
f.save()
task = convert_file.s(h, path, p, extra)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = task | cleanup.s(path, h)
c.apply_async()
f.taskid = task_result.id
f.save()
def process_file(path, h, ignore_limit):
t = time.time() + 2
while True:
f = File.from_hash(h)
if f or time.time() > t:
break
time.sleep(0.05) # Wait for Redis to catch up
try:
result = detect(path)
processor = result['type']
except Exception as e:
print("failed file:", h, e)
traceback.print_exc()
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
queue = "priority" if processor.startswith("image") else "celery"
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in list(result['flags'].items()):
setattr(f.flags, flag, value)
f.save()
args = [h, path, processor, metadata, processor_state, ignore_limit]
task = signature("mediacrush.tasks.convert_file",
args=args,
options={'queue': queue})
task_result = task.freeze(
) # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def _upload_f(f, filename):
result = upload(f, filename)
if not isinstance(result, tuple):
return {'hash': result}
else:
# It's a tuple which means it was uploaded before or it was rate limited
# jdiez, this is super hacky, please refactor this
h, status = result
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(h))
resp[h] = f
resp['hash'] = h
return resp, status
def direct(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
return render_template("direct.html", **template_params)
def process_file(path, h, ignore_limit):
t = time.time() + 2
while True:
f = File.from_hash(h)
if f or time.time() > t:
break
time.sleep(0.05) # Wait for Redis to catch up
try:
result = detect(path)
processor = result['type'] if result else 'default'
except:
processor = 'default'
finally:
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
queue = "priority" if processor.startswith("image") else "celery"
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
args = [h, path, processor, metadata, processor_state, ignore_limit]
task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue})
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def urlinfo(self):
l = request.form['list']
items = l.split(",") if "," in l else [l]
result = {}
for item in items:
key = _k("url.%s" % item)
h = r.get(key)
if h:
f = File.from_hash(h)
if f:
result[item] = _file_object(f)
else:
result[item] = None
r.delete(key)
else:
result[item] = None
return result
def get(self, id, layout):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
return render_template("view.html", **_template_params(f))
def get(self, id, layout):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
return render_template("view.html", **_template_params(f))
def urlinfo(self):
l = request.form['list']
items = l.split(",") if "," in l else [l]
result = {}
for item in items:
key = _k("url.%s" % item)
h = r.get(key)
if h:
f = File.from_hash(h)
if f:
result[item] = _file_object(f)
else:
result[item] = None
r.delete(key)
else:
result[item] = None
return result
def frame(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['embedded'] = True
v['filename'] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params['embedded'] = True
return render_template("direct.html", **template_params)
def direct(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if request.user_agent.platform in ['android', 'iphone', 'ipad'] or 'windows phone' in request.user_agent.string.lower():
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
return render_template("direct.html", **template_params)
def get(self, id, layout):
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
if not f.processor:
# TODO: Better error page
return render_template("error.html", error="Unable to detect file type")
return render_template("view.html", **_template_params(f))
def status(self, h):
klass = RedisObject.klass(h)
if not klass:
return {'error': 404}, 404
if klass is FailedFile:
ff = klass.from_hash(h)
return {'status': ff.status}
if klass is not File:
return {'error': 415}, 415
f = File.from_hash(h)
ret = {'status': f.status}
if f.processor is not None: # When processor is available, ther rest of the information is too, even if the file might not have finished processing yet.
ret[h] = _file_object(f)
ret['hash'] = h
return ret
def status(self, h):
klass = RedisObject.klass(h)
if not klass:
return {'error': 404}, 404
if klass is FailedFile:
ff = klass.from_hash(h)
return {'status': ff.status}
if klass is not File:
return {'error': 415}, 415
f = File.from_hash(h)
ret = {'status': f.status}
if f.processor is not None: # When processor is available, ther rest of the information is too, even if the file might not have finished processing yet.
ret[h] = _file_object(f)
ret['hash'] = h
return ret
def get(self, id, layout):
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
if not f.processor:
# TODO: Better error page
return render_template("error.html", error="Unable to detect file type")
return render_template("view.html", **_template_params(f))
def frame(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['embedded'] = True
v['filename'] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params['embedded'] = True
return render_template("direct.html", **template_params)
def direct(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if (request.user_agent.platform in ["android", "iphone", "ipad"]
or "windows phone" in request.user_agent.string.lower()):
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
return render_template("direct.html", **template_params)
def report(self, id):
f = File.from_hash(id)
f.add_report()
return "ok"
def flags(self, h):
if not File.exists(h):
return {"error": 404}, 404
f = File.from_hash(h)
return {"flags": f.flags.as_dict()}
def flags(self, h):
if not File.exists(h):
return {'error': 404}, 404
f = File.from_hash(h)
return {'flags': f.flags.as_dict()}
def cleanup(results, path, h):
f = File.from_hash(h)
os.unlink(path)
if f.status in ["internal_error", "error", "timeout"]:
delete_file(f)
def _still_image(h):
f = File.from_hash(h)
if f.processor.startswith("image"):
return f.original
return None
def report(self, id):
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
from mediacrush.objects import File
from mediacrush.database import r, _k
from mediacrush.processing.detect import detect
from mediacrush.mimetypes import get_mimetype, extension
from mediacrush.files import file_storage
if __name__ == '__main__':
for h in r.smembers(_k("file")):
f = File.from_hash(h)
ext = extension(f.original)
overrides = {
'jpe': 'image/jpeg',
'ogg': 'audio/ogg',
}
processor, extra = detect(file_storage(f.original))
f.mimetype = overrides.get(ext, get_mimetype(f.original))
f.processor = processor
f.save()
def _still_image(h):
f = File.from_hash(h)
if f.processor.startswith("image"):
return f.original
return None
def report(self, id):
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
def flags(self, h):
if not File.exists(h):
return {'error': 404}, 404
f = File.from_hash(h)
return {'flags': f.flags.as_dict()}
