def frame(self, id): send = self._send_file(id) if send: return send # We only want to maintain one page for mobile, not a dozen if ( request.user_agent.platform in ["android", "iphone", "ipad"] or "windows phone" in request.user_agent.string.lower() ): return redirect("/" + id, code=302) klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v["embedded"] = True v["filename"] = id return render_template("album-embedded.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) template_params["embedded"] = True return render_template("direct.html", **template_params)
def process_file(path, h, ignore_limit): f = File.from_hash(h) result = detect(path) processor = result['type'] if result else 'default' if processor == 'default': # Unrecognised file type failed = FailedFile(hash=h, status="unrecognised") failed.save() delete_file(f) return metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor setattr(f.flags, 'nsfw', False) if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def process_file(path, h): f = File.from_hash(h) result = detect(path) processor = result['type'] if result else 'default' metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() task = convert_file.s(h, path, processor, metadata, processor_state) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def report(self, id): rate_limit_update(1, section="report") if not current_app.debug and rate_limit_exceeded(section="report"): return {'error': 413}, 413 f = File.from_hash(id) f.add_report() return render_template("report.html")
def fragment(self, id): klass = RedisObject.klass(id) if klass is not File: abort(404) f = File.from_hash(id) params = _template_params(f) params['album'] = True return render_template(params['fragment'], **params)
def cleanup(results, path, h): f = File.from_hash(h) os.unlink(path) if f.status in ["internal_error", "error", "timeout", "unrecognised"]: failed = FailedFile(hash=h, status=f.status) # Create a "failed file" record failed.save() delete_file(f)
def status(self, id): klass = RedisObject.klass(id) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) if f.status in ['done', 'ready']: return tor_redirect('/' + f.hash) return render_template("status.html", **_template_params(f))
def _upload_object(result, status): if status == 200: return {'hash': result} else: resp = {'error': status} if status == 409: f = _file_object(File.from_hash(result)) resp[result] = f resp['hash'] = result return resp, status
def convert_file(self, h, path, p, extra): f = File.from_hash(h) if p not in processor_table: p = 'default' processor = processor_table[p](path, f, extra) # Execute the synchronous step. processor.sync() # Save compression information f = File.from_hash(h) # Reload file; user might have changed the config vector while processing f.compression = compression_rate(path, f) f.save() # Notify frontend: sync step is done. self.update_state(state="READY") # Execute the asynchronous step. processor.important = False processor.async()
def _upload_f(f, filename): result = upload(f, filename) if not isinstance(result, tuple): return {'hash': result} else: h, status = result resp = {'error': status} if status == 409: f = _file_object(File.from_hash(h)) resp[h] = f resp['hash'] = h return resp, status
def get(self, id, layout): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("albums/%s.html" % layout, **v) if klass is not File: abort(404) f = File.from_hash(id) return render_template("view.html", **_template_params(f))
def process_file(path, h): f = File.from_hash(h) p, extra = detect(path) f.processor = p f.save() task = convert_file.s(h, path, p, extra) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = task | cleanup.s(path, h) c.apply_async() f.taskid = task_result.id f.save()
def status(self, h): klass = RedisObject.klass(h) if not klass: return {'error': 404}, 404 if klass is not File: return {'error': 415}, 415 f = File.from_hash(h) ret = {'status': f.status} if ret['status'] == 'done': ret[h] = _file_object(f) ret['hash'] = h return ret
def _upload_f(f, filename): result = upload(f, filename) if not isinstance(result, tuple): return {'hash': result} else: # It's a tuple which means it was uploaded before or it was rate limited # jdiez, this is super hacky, please refactor this h, status = result resp = {'error': status} if status == 409: f = _file_object(File.from_hash(h)) resp[h] = f resp['hash'] = h return resp, status
def direct(self, id): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("album.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) return render_template("direct.html", **template_params)
def process_file(path, h, ignore_limit): t = time.time() + 2 while True: f = File.from_hash(h) if f or time.time() > t: break time.sleep(0.05) # Wait for Redis to catch up try: result = detect(path) processor = result['type'] if result else 'default' except: processor = 'default' finally: if processor == 'default': # Unrecognised file type failed = FailedFile(hash=h, status="unrecognised") failed.save() delete_file(f) return metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor queue = "priority" if processor.startswith("image") else "celery" setattr(f.flags, 'nsfw', False) if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() args = [h, path, processor, metadata, processor_state, ignore_limit] task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue}) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def get(self, id, layout): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['layout'] = layout if layout == "random": random.shuffle(v['items']) # It's in-place return render_template("albums/%s.html" % layout, **v) if klass is not File: abort(404) f = File.from_hash(id) return render_template("view.html", **_template_params(f))
def urlinfo(self): l = request.form['list'] items = l.split(",") if "," in l else [l] result = {} for item in items: key = _k("url.%s" % item) h = r.get(key) if h: f = File.from_hash(h) if f: result[item] = _file_object(f) else: result[item] = None r.delete(key) else: result[item] = None return result
def status(self, h): klass = RedisObject.klass(h) if not klass: return {'error': 404}, 404 if klass is FailedFile: ff = klass.from_hash(h) return {'status': ff.status} if klass is not File: return {'error': 415}, 415 f = File.from_hash(h) ret = {'status': f.status} if f.processor is not None: # When processor is available, ther rest of the information is too, even if the file might not have finished processing yet. ret[h] = _file_object(f) ret['hash'] = h return ret
def frame(self, id): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['embedded'] = True v['filename'] = id return render_template("album-embedded.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) template_params['embedded'] = True return render_template("direct.html", **template_params)
def get(self, id, layout): klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['layout'] = layout if layout == "random": random.shuffle(v['items']) # It's in-place return render_template("albums/%s.html" % layout, **v) if klass is not File: abort(404) f = File.from_hash(id) if not f.processor: # TODO: Better error page return render_template("error.html", error="Unable to detect file type") return render_template("view.html", **_template_params(f))
def direct(self, id): send = self._send_file(id) if send: return send # We only want to maintain one page for mobile, not a dozen if request.user_agent.platform in ['android', 'iphone', 'ipad'] or 'windows phone' in request.user_agent.string.lower(): return redirect("/" + id, code=302) klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("album.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) return render_template("direct.html", **template_params)
def report(self, id): f = File.from_hash(id) f.add_report() return render_template("report.html")
def report(self, id): f = File.from_hash(id) f.add_report() return "ok"
newvec |= (1 << self.shifts[name]) object.__setattr__(self, '_vec', newvec) def as_dict(self): return dict((flag, getattr(self, flag)) for flag in self.shifts) def __int__(self): return self._vec flags_per_processor = { 'video': ['autoplay', 'loop', 'mute'] } if __name__ == '__main__': files = File.get_all() count = len(files) print "About to process %d files." % count done = 0 errors = [] for f in files: h = f.hash configvector = 0 try: path = file_storage(f.original) result = detect(path)
def flags(self, h): if not File.exists(h): return {'error': 404}, 404 f = File.from_hash(h) return {'flags': f.flags.as_dict()}
def _still_image(h): f = File.from_hash(h) if f.processor.startswith("image"): return f.original return None
def upload(f, filename): if not f.content_type: f.content_type = get_mimetype(filename) or "application/octet-stream" #if f.content_type.split("/")[0] not in ['video', 'image', 'audio']: # return "no", 415 if not current_app.debug: rate_limit_update(file_length(f)) if rate_limit_exceeded(): return "ratelimit", 420 h = get_hash(f) identifier = to_id(h) if "." not in filename: ext = mimetypes.guess_extension(f.content_type)[1:] # This not very scientific, but it works else: ext = extension(filename) filename = "%s.%s" % (identifier, ext) path = tempfile.NamedTemporaryFile(suffix="." + ext).name # Fix for imagemagick's silliness if os.path.exists(file_storage(filename)): if File.exists(identifier): return identifier, 409 else: # Delete residual files from storage by creating a dummy File dummy = File(original=filename) dummy.delete = lambda: None # nop delete_file(dummy) f.seek(0) # Otherwise it'll write a 0-byte file f.save(path) file_object = File(hash=identifier) file_object.compression = os.path.getsize(path) file_object.original = filename file_object.mimetype = f.content_type file_object.ip = secure_ip() result = process_file.delay(path, identifier) file_object.taskid = result.id file_object.save() return identifier
def exists(self, h): if not File.exists(h): return {"exists": False}, 404 return {"exists": True}
def exists(self, h): if not File.exists(h): return {'exists': False}, 404 return {'exists': True}
def flags(self, h): if not File.exists(h): return {"error": 404}, 404 f = File.from_hash(h) return {"flags": f.flags.as_dict()}
def cleanup(results, path, h): f = File.from_hash(h) os.unlink(path) if f.status in ["internal_error", "error", "timeout"]: delete_file(f)
def upload(f, filename): if not f.content_type: f.content_type = get_mimetype(filename) or "application/octet-stream" #if f.content_type.split("/")[0] not in ['video', 'image', 'audio']: # return "no", 415 ignore_limit = current_app.debug or r.sismember(_k("whitelisted_ips"), get_ip()) if not ignore_limit: rate_limit_update(file_length(f)) if rate_limit_exceeded(): return None, 420 h = get_hash(f) identifier = to_id(h) if "." not in filename: ext = mimetypes.guess_extension( f.content_type)[1:] # This not very scientific, but it works else: ext = extension(filename) filename = "%s.%s" % (identifier, ext) path = tempfile.NamedTemporaryFile( suffix="." + ext).name # Fix for imagemagick's silliness if os.path.exists(file_storage(filename)): if File.exists(identifier): return identifier, 409 else: # Delete residual files from storage by creating a dummy File dummy = File(original=filename) dummy.delete = lambda: None # nop delete_file(dummy) f.seek(0) # Otherwise it'll write a 0-byte file f.save(path) file_object = File(hash=identifier) file_object.compression = os.path.getsize(path) file_object.original = filename file_object.mimetype = f.content_type file_object.ip = secure_ip() result = process_file.delay(path, identifier, ignore_limit) file_object.taskid = result.id file_object.save() return identifier, 200
from mediacrush.objects import File, RedisObject from mediacrush.database import r, _k from mediacrush.fileutils import file_storage from mediacrush.processing.invocation import Invocation from mediacrush.config import _cfg, _cfgi import sys import json if __name__ == '__main__': files = File.get_all() count = len(files) print "About to process %d files." % count done = 0 errors = [] for f in files: h = f.hash k = _k("file.%s" % h) r.hset(k, "ip", "") print "\n%d/%d files processed, errors:" % (done, count), errors def normalise_processor(processor): if not processor: return None return processor.split("/")[0] if "/" in processor else processor