示例#1
0
文件: tasks.py 项目: Jaex/MediaCrush
def process_file(path, h, ignore_limit):
    f = File.from_hash(h)
    result = detect(path)

    processor = result['type'] if result else 'default'
    if processor == 'default': # Unrecognised file type
        failed = FailedFile(hash=h, status="unrecognised")
        failed.save()

        delete_file(f)
        return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit)
    task_result = task.freeze() # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
示例#2
0
def process_file(path, h, ignore_limit):
    f = File.from_hash(h)
    result = detect(path)

    processor = result['type'] if result else 'default'
    if processor == 'default':  # Unrecognised file type
        failed = FailedFile(hash=h, status="unrecognised")
        failed.save()

        delete_file(f)
        return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    task = convert_file.s(h, path, processor, metadata, processor_state,
                          ignore_limit)
    task_result = task.freeze(
    )  # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
示例#3
0
def cleanup(results, path, h):
    f = File.from_hash(h)
    os.unlink(path)

    if f.status in ["internal_error", "error", "timeout", "unrecognised"]:
        failed = FailedFile(hash=h, status=f.status) # Create a "failed file" record
        failed.save()

        delete_file(f)
示例#4
0
def upload(f, filename):
    if not f.content_type:
        f.content_type = get_mimetype(filename) or "application/octet-stream"

    #if f.content_type.split("/")[0] not in ['video', 'image', 'audio']:
    #    return "no", 415

    ignore_limit = current_app.debug or r.sismember(_k("whitelisted_ips"),
                                                    get_ip())
    if not ignore_limit:
        rate_limit_update(file_length(f))
        if rate_limit_exceeded():
            return None, 420

    h = get_hash(f)
    identifier = to_id(h)
    if "." not in filename:
        ext = mimetypes.guess_extension(
            f.content_type)[1:]  # This not very scientific, but it works
    else:
        ext = extension(filename)

    filename = "%s.%s" % (identifier, ext)
    path = tempfile.NamedTemporaryFile(
        suffix="." + ext).name  # Fix for imagemagick's silliness

    if os.path.exists(file_storage(filename)):
        if File.exists(identifier):
            return identifier, 409
        else:
            # Delete residual files from storage by creating a dummy File
            dummy = File(original=filename)
            dummy.delete = lambda: None  # nop
            delete_file(dummy)

    f.seek(0)  # Otherwise it'll write a 0-byte file
    f.save(path)

    file_object = File(hash=identifier)
    file_object.compression = os.path.getsize(path)
    file_object.original = filename
    file_object.mimetype = f.content_type
    file_object.ip = secure_ip()

    result = process_file.delay(path, identifier, ignore_limit)
    file_object.taskid = result.id

    file_object.save()

    return identifier, 200
示例#5
0
def upload(f, filename):
    if not f.content_type:
        f.content_type = get_mimetype(filename) or "application/octet-stream"

    #if f.content_type.split("/")[0] not in ['video', 'image', 'audio']:
    #    return "no", 415

    ignore_limit = current_app.debug or r.sismember(_k("whitelisted_ips"), get_ip())
    if not ignore_limit:
        rate_limit_update(file_length(f))
        if rate_limit_exceeded():
            return None, 420

    h = get_hash(f)
    identifier = to_id(h)
    if "." not in filename:
        ext = mimetypes.guess_extension(f.content_type)[1:] # This not very scientific, but it works
    else:
        ext = extension(filename)

    filename = "%s.%s" % (identifier, ext)
    path = tempfile.NamedTemporaryFile(suffix="." + ext).name # Fix for imagemagick's silliness

    if os.path.exists(file_storage(filename)):
        if File.exists(identifier):
            return identifier, 409
        else:
            # Delete residual files from storage by creating a dummy File
            dummy = File(original=filename)
            dummy.delete = lambda: None # nop
            delete_file(dummy)

    f.seek(0)  # Otherwise it'll write a 0-byte file
    f.save(path)

    file_object = File(hash=identifier)
    file_object.compression = os.path.getsize(path)
    file_object.original = filename
    file_object.mimetype = f.content_type
    file_object.ip = secure_ip()

    result = process_file.delay(path, identifier, ignore_limit)
    file_object.taskid = result.id

    file_object.save()

    return identifier, 200
示例#6
0
def process_file(path, h, ignore_limit):
    t = time.time() + 2
    while True:
        f = File.from_hash(h)
        if f or time.time() > t:
            break
        time.sleep(0.05)  # Wait for Redis to catch up

    try:
        result = detect(path)
        processor = result['type']
    except Exception as e:
        print("failed file:", h, e)
        traceback.print_exc()
        failed = FailedFile(hash=h, status="unrecognised")
        failed.save()

        delete_file(f)
        return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor
    queue = "priority" if processor.startswith("image") else "celery"

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in list(result['flags'].items()):
            setattr(f.flags, flag, value)

    f.save()

    args = [h, path, processor, metadata, processor_state, ignore_limit]
    task = signature("mediacrush.tasks.convert_file",
                     args=args,
                     options={'queue': queue})
    task_result = task.freeze(
    )  # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
示例#7
0
def process_file(path, h, ignore_limit):
    t = time.time() + 2
    while True:
        f = File.from_hash(h)
        if f or time.time() > t:
            break
        time.sleep(0.05) # Wait for Redis to catch up

    try:
        result = detect(path)
        processor = result['type'] if result else 'default'
    except:
        processor = 'default'
    finally:
        if processor == 'default': # Unrecognised file type
            failed = FailedFile(hash=h, status="unrecognised")
            failed.save()

            delete_file(f)
            return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor
    queue = "priority" if processor.startswith("image") else "celery"

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    args = [h, path, processor, metadata, processor_state, ignore_limit]
    task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue})
    task_result = task.freeze() # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
示例#8
0
def cleanup(results, path, h):
    f = File.from_hash(h)
    os.unlink(path)

    if f.status in ["internal_error", "error", "timeout"]:
        delete_file(f)