def export(dst_format, task_id=None, project_id=None, server_url=None, save_images=False):
try:
if task_id is not None:
db_instance = Task.objects.get(pk=task_id)
logger = slogger.task[task_id]
cache_ttl = TASK_CACHE_TTL
export_fn = task.export_task
else:
db_instance = Project.objects.get(pk=project_id)
logger = slogger.project[project_id]
cache_ttl = PROJECT_CACHE_TTL
export_fn = project.export_project
cache_dir = get_export_cache_dir(db_instance)
exporter = EXPORT_FORMATS[dst_format]
output_base = '%s_%s' % ('dataset' if save_images else 'annotations',
make_file_name(to_snake_case(dst_format)))
output_path = '%s.%s' % (output_base, exporter.EXT)
output_path = osp.join(cache_dir, output_path)
instance_time = timezone.localtime(db_instance.updated_date).timestamp()
if isinstance(db_instance, Project):
tasks_update = list(map(lambda db_task: timezone.localtime(db_task.updated_date).timestamp(), db_instance.tasks.all()))
instance_time = max(tasks_update + [instance_time])
if not (osp.exists(output_path) and \
instance_time <= osp.getmtime(output_path)):
os.makedirs(cache_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir:
temp_file = osp.join(temp_dir, 'result')
export_fn(db_instance.id, temp_file, dst_format,
server_url=server_url, save_images=save_images)
os.replace(temp_file, output_path)
archive_ctime = osp.getctime(output_path)
scheduler = django_rq.get_scheduler()
cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl,
func=clear_export_cache,
task_id=task_id,
file_path=output_path, file_ctime=archive_ctime)
logger.info(
"The {} '{}' is exported as '{}' at '{}' "
"and available for downloading for the next {}. "
"Export cache cleaning job is enqueued, id '{}'".format(
"project" if isinstance(db_instance, Project) else 'task',
db_instance.name, dst_format, output_path, cache_ttl,
cleaning_job.id
))
return output_path
except Exception:
log_exception(logger)
raise
def export_task(task_id, dst_format, server_url=None, save_images=False):
try:
db_task = Task.objects.get(pk=task_id)
cache_dir = get_export_cache_dir(db_task)
exporter = EXPORT_FORMATS[dst_format]
output_base = '%s_%s' % ('dataset' if save_images else 'annotations',
make_file_name(to_snake_case(dst_format)))
output_path = '%s.%s' % (output_base, exporter.EXT)
output_path = osp.join(cache_dir, output_path)
task_time = timezone.localtime(db_task.updated_date).timestamp()
if not (osp.exists(output_path) and \
task_time <= osp.getmtime(output_path)):
os.makedirs(cache_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir:
temp_file = osp.join(temp_dir, 'result')
task.export_task(task_id,
temp_file,
dst_format,
server_url=server_url,
save_images=save_images)
os.replace(temp_file, output_path)
archive_ctime = osp.getctime(output_path)
scheduler = django_rq.get_scheduler()
cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL,
func=clear_export_cache,
task_id=task_id,
file_path=output_path,
file_ctime=archive_ctime)
slogger.task[task_id].info(
"The task '{}' is exported as '{}' at '{}' "
"and available for downloading for the next {}. "
"Export cache cleaning job is enqueued, id '{}'".format(
db_task.name, dst_format, output_path, CACHE_TTL,
cleaning_job.id))
return output_path
except Exception:
log_exception(slogger.task[task_id])
raise
def normalize_label(label):
label = make_file_name(label) # basically, convert to ASCII lowercase
label = label.replace('-', '_')
return label