def process_file(path, h, ignore_limit):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state,
ignore_limit)
task_result = task.freeze(
) # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def process_file(path, h, ignore_limit):
f = File.from_hash(h)
result = detect(path)
processor = result['type'] if result else 'default'
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
task = convert_file.s(h, path, processor, metadata, processor_state, ignore_limit)
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def cleanup(results, path, h):
f = File.from_hash(h)
os.unlink(path)
if f.status in ["internal_error", "error", "timeout", "unrecognised"]:
failed = FailedFile(hash=h, status=f.status) # Create a "failed file" record
failed.save()
delete_file(f)
def process_file(path, h, ignore_limit):
t = time.time() + 2
while True:
f = File.from_hash(h)
if f or time.time() > t:
break
time.sleep(0.05) # Wait for Redis to catch up
try:
result = detect(path)
processor = result['type']
except Exception as e:
print("failed file:", h, e)
traceback.print_exc()
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
queue = "priority" if processor.startswith("image") else "celery"
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in list(result['flags'].items()):
setattr(f.flags, flag, value)
f.save()
args = [h, path, processor, metadata, processor_state, ignore_limit]
task = signature("mediacrush.tasks.convert_file",
args=args,
options={'queue': queue})
task_result = task.freeze(
) # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
def process_file(path, h, ignore_limit):
t = time.time() + 2
while True:
f = File.from_hash(h)
if f or time.time() > t:
break
time.sleep(0.05) # Wait for Redis to catch up
try:
result = detect(path)
processor = result['type'] if result else 'default'
except:
processor = 'default'
finally:
if processor == 'default': # Unrecognised file type
failed = FailedFile(hash=h, status="unrecognised")
failed.save()
delete_file(f)
return
metadata = result['metadata'] if result else {}
processor_state = result['processor_state'] if result else {}
f.processor = processor
queue = "priority" if processor.startswith("image") else "celery"
setattr(f.flags, 'nsfw', False)
if result and result['flags']:
for flag, value in result['flags'].items():
setattr(f.flags, flag, value)
f.save()
args = [h, path, processor, metadata, processor_state, ignore_limit]
task = signature("mediacrush.tasks.convert_file", args=args, options={'queue': queue})
task_result = task.freeze() # This sets the taskid, so we can pass it to the UI
# This chord will execute `syncstep` and `asyncstep`, and `cleanup` after both of them have finished.
c = chord(task, cleanup.s(path, h))
c.apply_async()
f.taskid = task_result.id
f.save()
