def frame(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if (
request.user_agent.platform in ["android", "iphone", "ipad"]
or "windows phone" in request.user_agent.string.lower()
):
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v["embedded"] = True
v["filename"] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params["embedded"] = True
return render_template("direct.html", **template_params)
def process_file(path, h, ignore_limit):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def process_file(path, h):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def report(self, id):
rate_limit_update(1, section="report")
if not current_app.debug and rate_limit_exceeded(section="report"):
return {'error': 413}, 413
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
def fragment(self, id):
klass = RedisObject.klass(id)
if klass is not File:
abort(404)
f = File.from_hash(id)
params = _template_params(f)
params['album'] = True
return render_template(params['fragment'], **params)
def cleanup(results, path, h):
f = File.from_hash(h)
os.unlink(path)
if f.status in ["internal_error", "error", "timeout", "unrecognised"]:
failed = FailedFile(hash=h, status=f.status) # Create a "failed file" record
failed.save()
delete_file(f)
def status(self, id):
klass = RedisObject.klass(id)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
if f.status in ['done', 'ready']:
return tor_redirect('/' + f.hash)
return render_template("status.html", **_template_params(f))
def _upload_object(result, status):
if status == 200:
return {'hash': result}
else:
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(result))
resp[result] = f
resp['hash'] = result
return resp, status
def convert_file(self, h, path, p, extra):
f = File.from_hash(h)
if p not in processor_table:
p = 'default'
processor = processor_table[p](path, f, extra)
# Execute the synchronous step.
processor.sync()
# Save compression information
f = File.from_hash(h) # Reload file; user might have changed the config vector while processing
f.compression = compression_rate(path, f)
f.save()
# Notify frontend: sync step is done.
self.update_state(state="READY")
# Execute the asynchronous step.
processor.important = False
processor.async()
def _upload_f(f, filename):
result = upload(f, filename)
if not isinstance(result, tuple):
return {'hash': result}
else:
h, status = result
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(h))
resp[h] = f
resp['hash'] = h
return resp, status
def get(self, id, layout):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
return render_template("view.html", **_template_params(f))
def process_file(path, h):
f = File.from_hash(h)
p, extra = detect(path)
f.processor = p
f.save()
task = convert_file.s(h, path, p, extra)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = task | cleanup.s(path, h)
c.apply_async()
f.taskid = task_result.id
f.save()
def status(self, h):
klass = RedisObject.klass(h)
if not klass:
return {'error': 404}, 404
if klass is not File:
return {'error': 415}, 415
f = File.from_hash(h)
ret = {'status': f.status}
if ret['status'] == 'done':
ret[h] = _file_object(f)
ret['hash'] = h
return ret
def _upload_f(f, filename):
result = upload(f, filename)
if not isinstance(result, tuple):
return {'hash': result}
else:
# It's a tuple which means it was uploaded before or it was rate limited
# jdiez, this is super hacky, please refactor this
h, status = result
resp = {'error': status}
if status == 409:
f = _file_object(File.from_hash(h))
resp[h] = f
resp['hash'] = h
return resp, status
def direct(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
return render_template("direct.html", **template_params)
def process_file(path, h, ignore_limit):
t = time.time() + 2
while True:
f = File.from_hash(h)
if f or time.time() > t:
break
time.sleep(0.05) # Wait for Redis to catch up
try:
result = detect(path)
processor = result['type'] if result else 'default'
except:
processor = 'default'
finally:
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
queue = "priority" if processor.startswith("image") else "celery"
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
args = [h, path, processor, metadata, processor_state, ignore_limit]
task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue})
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def get(self, id, layout):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
return render_template("view.html", **_template_params(f))
def urlinfo(self):
l = request.form['list']
items = l.split(",") if "," in l else [l]
result = {}
for item in items:
key = _k("url.%s" % item)
h = r.get(key)
if h:
f = File.from_hash(h)
if f:
result[item] = _file_object(f)
else:
result[item] = None
r.delete(key)
else:
result[item] = None
return result
def status(self, h):
klass = RedisObject.klass(h)
if not klass:
return {'error': 404}, 404
if klass is FailedFile:
ff = klass.from_hash(h)
return {'status': ff.status}
if klass is not File:
return {'error': 415}, 415
f = File.from_hash(h)
ret = {'status': f.status}
if f.processor is not None: # When processor is available, ther rest of the information is too, even if the file might not have finished processing yet.
ret[h] = _file_object(f)
ret['hash'] = h
return ret
def frame(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['embedded'] = True
v['filename'] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params['embedded'] = True
return render_template("direct.html", **template_params)
def get(self, id, layout):
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
if not f.processor:
# TODO: Better error page
return render_template("error.html", error="Unable to detect file type")
return render_template("view.html", **_template_params(f))
def get(self, id, layout):
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['layout'] = layout
if layout == "random":
random.shuffle(v['items']) # It's in-place
return render_template("albums/%s.html" % layout, **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
if not f.processor:
# TODO: Better error page
return render_template("error.html", error="Unable to detect file type")
return render_template("view.html", **_template_params(f))
def status(self, h):
klass = RedisObject.klass(h)
if not klass:
return {'error': 404}, 404
if klass is FailedFile:
ff = klass.from_hash(h)
return {'status': ff.status}
if klass is not File:
return {'error': 415}, 415
f = File.from_hash(h)
ret = {'status': f.status}
if f.processor is not None: # When processor is available, ther rest of the information is too, even if the file might not have finished processing yet.
ret[h] = _file_object(f)
ret['hash'] = h
return ret
def direct(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if request.user_agent.platform in ['android', 'iphone', 'ipad'] or 'windows phone' in request.user_agent.string.lower():
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
return render_template("direct.html", **template_params)
def frame(self, id):
send = self._send_file(id)
if send:
return send
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
v['embedded'] = True
v['filename'] = id
return render_template("album-embedded.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
template_params['embedded'] = True
return render_template("direct.html", **template_params)
def direct(self, id):
send = self._send_file(id)
if send:
return send
# We only want to maintain one page for mobile, not a dozen
if request.user_agent.platform in ['android', 'iphone', 'ipad'] or 'windows phone' in request.user_agent.string.lower():
return redirect("/" + id, code=302)
klass = RedisObject.klass(id)
if klass is Album:
album = klass.from_hash(id)
v = _album_params(album)
return render_template("album.html", **v)
if klass is not File:
abort(404)
f = File.from_hash(id)
template_params = _template_params(f)
return render_template("direct.html", **template_params)
def report(self, id):
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
def report(self, id):
f = File.from_hash(id)
f.add_report()
return "ok"
newvec |= (1 << self.shifts[name])
object.__setattr__(self, '_vec', newvec)
def as_dict(self):
return dict((flag, getattr(self, flag)) for flag in self.shifts)
def __int__(self):
return self._vec
flags_per_processor = {
'video': ['autoplay', 'loop', 'mute']
}
if __name__ == '__main__':
files = File.get_all()
count = len(files)
print "About to process %d files." % count
done = 0
errors = []
for f in files:
h = f.hash
configvector = 0
try:
path = file_storage(f.original)
result = detect(path)
def flags(self, h):
if not File.exists(h):
return {'error': 404}, 404
f = File.from_hash(h)
return {'flags': f.flags.as_dict()}
def _still_image(h):
f = File.from_hash(h)
if f.processor.startswith("image"):
return f.original
return None
def upload(f, filename):
if not f.content_type:
f.content_type = get_mimetype(filename) or "application/octet-stream"
#if f.content_type.split("/")[0] not in ['video', 'image', 'audio']:
# return "no", 415
if not current_app.debug:
rate_limit_update(file_length(f))
if rate_limit_exceeded():
return "ratelimit", 420
h = get_hash(f)
identifier = to_id(h)
if "." not in filename:
ext = mimetypes.guess_extension(f.content_type)[1:] # This not very scientific, but it works
else:
ext = extension(filename)
filename = "%s.%s" % (identifier, ext)
path = tempfile.NamedTemporaryFile(suffix="." + ext).name # Fix for imagemagick's silliness
if os.path.exists(file_storage(filename)):
if File.exists(identifier):
return identifier, 409
else:
# Delete residual files from storage by creating a dummy File
dummy = File(original=filename)
dummy.delete = lambda: None # nop
delete_file(dummy)
f.seek(0) # Otherwise it'll write a 0-byte file
f.save(path)
file_object = File(hash=identifier)
file_object.compression = os.path.getsize(path)
file_object.original = filename
file_object.mimetype = f.content_type
file_object.ip = secure_ip()
result = process_file.delay(path, identifier)
file_object.taskid = result.id
file_object.save()
return identifier
def exists(self, h):
if not File.exists(h):
return {"exists": False}, 404
return {"exists": True}
def exists(self, h):
if not File.exists(h):
return {'exists': False}, 404
return {'exists': True}
def flags(self, h):
if not File.exists(h):
return {"error": 404}, 404
f = File.from_hash(h)
return {"flags": f.flags.as_dict()}
def flags(self, h):
if not File.exists(h):
return {'error': 404}, 404
f = File.from_hash(h)
return {'flags': f.flags.as_dict()}
def cleanup(results, path, h):
f = File.from_hash(h)
os.unlink(path)
if f.status in ["internal_error", "error", "timeout"]:
delete_file(f)
def _still_image(h):
f = File.from_hash(h)
if f.processor.startswith("image"):
return f.original
return None
def report(self, id):
f = File.from_hash(id)
f.add_report()
return render_template("report.html")
def upload(f, filename):
if not f.content_type:
f.content_type = get_mimetype(filename) or "application/octet-stream"
#if f.content_type.split("/")[0] not in ['video', 'image', 'audio']:
# return "no", 415
ignore_limit = current_app.debug or r.sismember(_k("whitelisted_ips"),
get_ip())
if not ignore_limit:
rate_limit_update(file_length(f))
if rate_limit_exceeded():
return None, 420
h = get_hash(f)
identifier = to_id(h)
if "." not in filename:
ext = mimetypes.guess_extension(
f.content_type)[1:] # This not very scientific, but it works
else:
ext = extension(filename)
filename = "%s.%s" % (identifier, ext)
path = tempfile.NamedTemporaryFile(
suffix="." + ext).name # Fix for imagemagick's silliness
if os.path.exists(file_storage(filename)):
if File.exists(identifier):
return identifier, 409
else:
# Delete residual files from storage by creating a dummy File
dummy = File(original=filename)
dummy.delete = lambda: None # nop
delete_file(dummy)
f.seek(0) # Otherwise it'll write a 0-byte file
f.save(path)
file_object = File(hash=identifier)
file_object.compression = os.path.getsize(path)
file_object.original = filename
file_object.mimetype = f.content_type
file_object.ip = secure_ip()
result = process_file.delay(path, identifier, ignore_limit)
file_object.taskid = result.id
file_object.save()
return identifier, 200
from mediacrush.objects import File, RedisObject
from mediacrush.database import r, _k
from mediacrush.fileutils import file_storage
from mediacrush.processing.invocation import Invocation
from mediacrush.config import _cfg, _cfgi
import sys
import json
if __name__ == '__main__':
files = File.get_all()
count = len(files)
print "About to process %d files." % count
done = 0
errors = []
for f in files:
h = f.hash
k = _k("file.%s" % h)
r.hset(k, "ip", "")
print "\n%d/%d files processed, errors:" % (done, count), errors
def normalise_processor(processor):
if not processor: return None
return processor.split("/")[0] if "/" in processor else processor
def exists(self, h):
if not File.exists(h):
return {'exists': False}, 404
return {'exists': True}
