def convert_file(self, h, path, p, metadata, processor_state, ignore_limit): f = File.from_hash(h) if p not in processor_table: p = 'default' processor = processor_table[p](path, f, processor_state, ignore_limit) # Execute the synchronous step. processor.sync() # Save compression information f = File.from_hash( h ) # Reload file; user might have changed the config vector while processing f.compression = compression_rate(path, f) f.metadata = json.dumps(metadata) f.save() # Notify frontend: sync step is done. self.update_state(state="READY") # Execute the asynchronous step. processor.important = False processor. async ()
def process_file(path, h, ignore_limit): f = File.from_hash(h) result = detect(path) processor = result['type'] if result else 'default' if processor == 'default': # Unrecognised file type failed = FailedFile(hash=h, status="unrecognised") failed.save() delete_file(f) return metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor setattr(f.flags, 'nsfw', False) if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def report(self, id): rate_limit_update(1, section="report") if not current_app.debug and rate_limit_exceeded(section="report"): return {'error': 413}, 413 f = File.from_hash(id) f.add_report() return render_template("report.html")
def frame(self, id): send = self._send_file(id) if send: return send # We only want to maintain one page for mobile, not a dozen if ( request.user_agent.platform in ["android", "iphone", "ipad"] or "windows phone" in request.user_agent.string.lower() ): return redirect("/" + id, code=302) klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v["embedded"] = True v["filename"] = id return render_template("album-embedded.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) template_params["embedded"] = True return render_template("direct.html", **template_params)
def process_file(path, h): f = File.from_hash(h) result = detect(path) processor = result['type'] if result else 'default' metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() task = convert_file.s(h, path, processor, metadata, processor_state) task_result = task.freeze( ) # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def frame(self, id): send = self._send_file(id) if send: return send # We only want to maintain one page for mobile, not a dozen if request.user_agent.platform in [ 'android', 'iphone', 'ipad' ] or 'windows phone' in request.user_agent.string.lower(): return redirect("/" + id, code=302) klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['embedded'] = True v['filename'] = id return render_template("album-embedded.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) template_params['embedded'] = True return render_template("direct.html", **template_params)
def process_file(path, h): f = File.from_hash(h) result = detect(path) processor = result['type'] if result else 'default' metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() task = convert_file.s(h, path, processor, metadata, processor_state) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def process_file(path, h, ignore_limit): f = File.from_hash(h) result = detect(path) processor = result['type'] if result else 'default' if processor == 'default': # Unrecognised file type failed = FailedFile(hash=h, status="unrecognised") failed.save() delete_file(f) return metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor setattr(f.flags, 'nsfw', False) if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit) task_result = task.freeze( ) # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def fragment(self, id): klass = RedisObject.klass(id) if klass is not File: abort(404) f = File.from_hash(id) params = _template_params(f) params['album'] = True return render_template(params['fragment'], **params)
def cleanup(results, path, h): f = File.from_hash(h) os.unlink(path) if f.status in ["internal_error", "error", "timeout", "unrecognised"]: failed = FailedFile(hash=h, status=f.status) # Create a "failed file" record failed.save() delete_file(f)
def status(self, id): klass = RedisObject.klass(id) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) if f.status in ['done', 'ready']: return tor_redirect('/' + f.hash) return render_template("status.html", **_template_params(f))
def _upload_object(result, status): if status == 200: return {"hash": result} else: resp = {"error": status} if status == 409: f = _file_object(File.from_hash(result)) resp[result] = f resp["hash"] = result return resp, status
def _upload_object(result, status): if status == 200: return {'hash': result} else: resp = {'error': status} if status == 409: f = _file_object(File.from_hash(result)) resp[result] = f resp['hash'] = result return resp, status
def convert_file(self, h, path, p, extra): f = File.from_hash(h) if p not in processor_table: p = 'default' processor = processor_table[p](path, f, extra) # Execute the synchronous step. processor.sync() # Save compression information f = File.from_hash(h) # Reload file; user might have changed the config vector while processing f.compression = compression_rate(path, f) f.save() # Notify frontend: sync step is done. self.update_state(state="READY") # Execute the asynchronous step. processor.important = False processor.async()
def _upload_f(f, filename): result = upload(f, filename) if not isinstance(result, tuple): return {'hash': result} else: h, status = result resp = {'error': status} if status == 409: f = _file_object(File.from_hash(h)) resp[h] = f resp['hash'] = h return resp, status
def status(self, h): klass = RedisObject.klass(h) if not klass: return {'error': 404}, 404 if klass is not File: return {'error': 415}, 415 f = File.from_hash(h) ret = {'status': f.status} if ret['status'] == 'done': ret[h] = _file_object(f) ret['hash'] = h return ret
def get(self, id): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("album.html", **v) if klass is not File: abort(404) f = File.from_hash(id) return render_template("view.html", **_template_params(f))
def get(self, id, layout): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("albums/%s.html" % layout, **v) if klass is not File: abort(404) f = File.from_hash(id) return render_template("view.html", **_template_params(f))
def process_file(path, h): f = File.from_hash(h) p, extra = detect(path) f.processor = p f.save() task = convert_file.s(h, path, p, extra) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = task | cleanup.s(path, h) c.apply_async() f.taskid = task_result.id f.save()
def process_file(path, h, ignore_limit): t = time.time() + 2 while True: f = File.from_hash(h) if f or time.time() > t: break time.sleep(0.05) # Wait for Redis to catch up try: result = detect(path) processor = result['type'] except Exception as e: print("failed file:", h, e) traceback.print_exc() failed = FailedFile(hash=h, status="unrecognised") failed.save() delete_file(f) return metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor queue = "priority" if processor.startswith("image") else "celery" setattr(f.flags, 'nsfw', False) if result and result['flags']: for flag, value in list(result['flags'].items()): setattr(f.flags, flag, value) f.save() args = [h, path, processor, metadata, processor_state, ignore_limit] task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue}) task_result = task.freeze( ) # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def _upload_f(f, filename): result = upload(f, filename) if not isinstance(result, tuple): return {'hash': result} else: # It's a tuple which means it was uploaded before or it was rate limited # jdiez, this is super hacky, please refactor this h, status = result resp = {'error': status} if status == 409: f = _file_object(File.from_hash(h)) resp[h] = f resp['hash'] = h return resp, status
def direct(self, id): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("album.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) return render_template("direct.html", **template_params)
def process_file(path, h, ignore_limit): t = time.time() + 2 while True: f = File.from_hash(h) if f or time.time() > t: break time.sleep(0.05) # Wait for Redis to catch up try: result = detect(path) processor = result['type'] if result else 'default' except: processor = 'default' finally: if processor == 'default': # Unrecognised file type failed = FailedFile(hash=h, status="unrecognised") failed.save() delete_file(f) return metadata = result['metadata'] if result else {} processor_state = result['processor_state'] if result else {} f.processor = processor queue = "priority" if processor.startswith("image") else "celery" setattr(f.flags, 'nsfw', False) if result and result['flags']: for flag, value in result['flags'].items(): setattr(f.flags, flag, value) f.save() args = [h, path, processor, metadata, processor_state, ignore_limit] task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue}) task_result = task.freeze() # This sets the taskid, so we can pass it to the UI # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished. c = chord(task, cleanup.s(path, h)) c.apply_async() f.taskid = task_result.id f.save()
def urlinfo(self): l = request.form['list'] items = l.split(",") if "," in l else [l] result = {} for item in items: key = _k("url.%s" % item) h = r.get(key) if h: f = File.from_hash(h) if f: result[item] = _file_object(f) else: result[item] = None r.delete(key) else: result[item] = None return result
def get(self, id, layout): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['layout'] = layout if layout == "random": random.shuffle(v['items']) # It's in-place return render_template("albums/%s.html" % layout, **v) if klass is not File: abort(404) f = File.from_hash(id) return render_template("view.html", **_template_params(f))
def frame(self, id): send = self._send_file(id) if send: return send klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['embedded'] = True v['filename'] = id return render_template("album-embedded.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) template_params['embedded'] = True return render_template("direct.html", **template_params)
def direct(self, id): send = self._send_file(id) if send: return send # We only want to maintain one page for mobile, not a dozen if request.user_agent.platform in ['android', 'iphone', 'ipad'] or 'windows phone' in request.user_agent.string.lower(): return redirect("/" + id, code=302) klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("album.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) return render_template("direct.html", **template_params)
def get(self, id, layout): klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) v['layout'] = layout if layout == "random": random.shuffle(v['items']) # It's in-place return render_template("albums/%s.html" % layout, **v) if klass is not File: abort(404) f = File.from_hash(id) if not f.processor: # TODO: Better error page return render_template("error.html", error="Unable to detect file type") return render_template("view.html", **_template_params(f))
def status(self, h): klass = RedisObject.klass(h) if not klass: return {'error': 404}, 404 if klass is FailedFile: ff = klass.from_hash(h) return {'status': ff.status} if klass is not File: return {'error': 415}, 415 f = File.from_hash(h) ret = {'status': f.status} if f.processor is not None: # When processor is available, ther rest of the information is too, even if the file might not have finished processing yet. ret[h] = _file_object(f) ret['hash'] = h return ret
def direct(self, id): send = self._send_file(id) if send: return send # We only want to maintain one page for mobile, not a dozen if (request.user_agent.platform in ["android", "iphone", "ipad"] or "windows phone" in request.user_agent.string.lower()): return redirect("/" + id, code=302) klass = RedisObject.klass(id) if klass is Album: album = klass.from_hash(id) v = _album_params(album) return render_template("album.html", **v) if klass is not File: abort(404) f = File.from_hash(id) template_params = _template_params(f) return render_template("direct.html", **template_params)
def report(self, id): f = File.from_hash(id) f.add_report() return "ok"
def flags(self, h): if not File.exists(h): return {"error": 404}, 404 f = File.from_hash(h) return {"flags": f.flags.as_dict()}
def flags(self, h): if not File.exists(h): return {'error': 404}, 404 f = File.from_hash(h) return {'flags': f.flags.as_dict()}
def cleanup(results, path, h): f = File.from_hash(h) os.unlink(path) if f.status in ["internal_error", "error", "timeout"]: delete_file(f)
def _still_image(h): f = File.from_hash(h) if f.processor.startswith("image"): return f.original return None
def report(self, id): f = File.from_hash(id) f.add_report() return render_template("report.html")
from mediacrush.objects import File from mediacrush.database import r, _k from mediacrush.processing.detect import detect from mediacrush.mimetypes import get_mimetype, extension from mediacrush.files import file_storage if __name__ == '__main__': for h in r.smembers(_k("file")): f = File.from_hash(h) ext = extension(f.original) overrides = { 'jpe': 'image/jpeg', 'ogg': 'audio/ogg', } processor, extra = detect(file_storage(f.original)) f.mimetype = overrides.get(ext, get_mimetype(f.original)) f.processor = processor f.save()