Example #1
0
def process_file(path, h, ignore_limit):
    f = File.from_hash(h)
    result = detect(path)

    processor = result['type'] if result else 'default'
    if processor == 'default': # Unrecognised file type
        failed = FailedFile(hash=h, status="unrecognised")
        failed.save()

        delete_file(f)
        return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit)
    task_result = task.freeze() # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
Example #2
0
def process_file(path, h, ignore_limit):
    f = File.from_hash(h)
    result = detect(path)

    processor = result['type'] if result else 'default'
    if processor == 'default':  # Unrecognised file type
        failed = FailedFile(hash=h, status="unrecognised")
        failed.save()

        delete_file(f)
        return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    task = convert_file.s(h, path, processor, metadata, processor_state,
                          ignore_limit)
    task_result = task.freeze(
    )  # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
Example #3
0
def process_file(path, h):
    f = File.from_hash(h)
    result = detect(path)

    processor = result['type'] if result else 'default'
    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor

    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    task = convert_file.s(h, path, processor, metadata, processor_state)
    task_result = task.freeze(
    )  # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
Example #4
0
def process_file(path, h):
    f = File.from_hash(h)
    result = detect(path)

    processor = result['type'] if result else 'default'
    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor

    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    task = convert_file.s(h, path, processor, metadata, processor_state)
    task_result = task.freeze() # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
Example #5
0
def process_file(path, h):
    f = File.from_hash(h)
    p, extra = detect(path)

    f.processor = p
    f.save()

    task = convert_file.s(h, path, p, extra)
    task_result = task.freeze() # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = task | cleanup.s(path, h)
    c.apply_async()

    f.taskid = task_result.id
    f.save()
Example #6
0
def process_file(path, h, ignore_limit):
    t = time.time() + 2
    while True:
        f = File.from_hash(h)
        if f or time.time() > t:
            break
        time.sleep(0.05)  # Wait for Redis to catch up

    try:
        result = detect(path)
        processor = result['type']
    except Exception as e:
        print("failed file:", h, e)
        traceback.print_exc()
        failed = FailedFile(hash=h, status="unrecognised")
        failed.save()

        delete_file(f)
        return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor
    queue = "priority" if processor.startswith("image") else "celery"

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in list(result['flags'].items()):
            setattr(f.flags, flag, value)

    f.save()

    args = [h, path, processor, metadata, processor_state, ignore_limit]
    task = signature("mediacrush.tasks.convert_file",
                     args=args,
                     options={'queue': queue})
    task_result = task.freeze(
    )  # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()
Example #7
0
def process_file(path, h, ignore_limit):
    t = time.time() + 2
    while True:
        f = File.from_hash(h)
        if f or time.time() > t:
            break
        time.sleep(0.05) # Wait for Redis to catch up

    try:
        result = detect(path)
        processor = result['type'] if result else 'default'
    except:
        processor = 'default'
    finally:
        if processor == 'default': # Unrecognised file type
            failed = FailedFile(hash=h, status="unrecognised")
            failed.save()

            delete_file(f)
            return

    metadata = result['metadata'] if result else {}
    processor_state = result['processor_state'] if result else {}

    f.processor = processor
    queue = "priority" if processor.startswith("image") else "celery"

    setattr(f.flags, 'nsfw', False)
    if result and result['flags']:
        for flag, value in result['flags'].items():
            setattr(f.flags, flag, value)

    f.save()

    args = [h, path, processor, metadata, processor_state, ignore_limit]
    task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue})
    task_result = task.freeze() # This sets the taskid, so we can pass it to the UI

    # This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
    c = chord(task, cleanup.s(path, h))
    c.apply_async()

    f.taskid = task_result.id
    f.save()