def export_task(task_id, dst_format, server_url=None, save_images=False):
try:
db_task = Task.objects.get(pk=task_id)
cache_dir = get_export_cache_dir(db_task)
exporter = EXPORT_FORMATS[dst_format]
output_base = '%s_%s' % ('dataset' if save_images else 'annotations',
make_file_name(to_snake_case(dst_format)))
output_path = '%s.%s' % (output_base, exporter.EXT)
output_path = osp.join(cache_dir, output_path)
task_time = timezone.localtime(db_task.updated_date).timestamp()
if not (osp.exists(output_path) and \
task_time <= osp.getmtime(output_path)):
os.makedirs(cache_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir:
temp_file = osp.join(temp_dir, 'result')
task.export_task(task_id,
temp_file,
dst_format,
server_url=server_url,
save_images=save_images)
os.replace(temp_file, output_path)
archive_ctime = osp.getctime(output_path)
scheduler = django_rq.get_scheduler()
cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL,
func=clear_export_cache,
task_id=task_id,
file_path=output_path,
file_ctime=archive_ctime)
slogger.task[task_id].info(
"The task '{}' is exported as '{}' at '{}' "
"and available for downloading for the next {}. "
"Export cache cleaning job is enqueued, id '{}'".format(
db_task.name, dst_format, output_path, CACHE_TTL,
cleaning_job.id))
return output_path
except Exception:
log_exception(slogger.task[task_id])
raise
def push(self, user, scheme, host, db_task, last_save):
# Update local repository
self._pull()
os.makedirs(os.path.join(self._cwd,
os.path.dirname(self._annotation_file)),
exist_ok=True)
# Remove old annotation file if it exists
if os.path.exists(self._annotation_file):
os.remove(self._annotation_file)
# Initialize LFS if need
if self._lfs:
updated = False
lfs_settings = [
"*.xml\tfilter=lfs diff=lfs merge=lfs -text\n",
"*.zip\tfilter=lfs diff=lfs merge=lfs -text\n"
]
if not os.path.isfile(os.path.join(self._cwd, ".gitattributes")):
with open(os.path.join(self._cwd, ".gitattributes"),
"w") as gitattributes:
gitattributes.writelines(lfs_settings)
updated = True
else:
with open(os.path.join(self._cwd, ".gitattributes"),
"r+") as gitattributes:
lines = gitattributes.readlines()
for setting in lfs_settings:
if setting not in lines:
updated = True
lines.append(setting)
gitattributes.seek(0)
gitattributes.writelines(lines)
gitattributes.truncate()
if updated:
self._rep.git.add(['.gitattributes'])
# Dump an annotation
timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
dump_name = os.path.join(
db_task.get_task_dirname(),
"git_annotation_{}_{}.zip".format(self._format, timestamp))
export_task(
task_id=self._tid,
dst_file=dump_name,
format_name=self._format,
server_url=scheme + host,
save_images=False,
)
ext = os.path.splitext(self._path)[1]
if ext == '.zip':
shutil.move(dump_name, self._annotation_file)
elif ext == '.xml':
with zipfile.ZipFile(dump_name) as archive:
for f in archive.namelist():
if f.endswith('.xml'):
with open(self._annotation_file, 'wb') as output:
output.write(archive.read(f))
break
os.remove(dump_name)
else:
raise Exception("Got unknown annotation file type")
self._rep.git.add(self._annotation_file)
# Merge diffs
summary_diff = {"update": 0, "create": 0, "delete": 0}
old_diffs_dir = os.path.join(os.path.dirname(self._diffs_dir),
'repos_diffs')
if (os.path.isdir(old_diffs_dir)):
_read_old_diffs(old_diffs_dir, summary_diff)
for diff_name in list(
map(lambda x: os.path.join(self._diffs_dir, x),
os.listdir(self._diffs_dir))):
with open(diff_name, 'r') as f:
diff = json.loads(f.read())
for key in diff:
summary_diff[key] += diff[key]
message = "CVAT Annotation updated by {}. \n".format(
self._user["name"])
message += 'Task URL: {}://{}/dashboard?id={}\n'.format(
scheme, host, db_task.id)
if db_task.bug_tracker:
message += 'Bug Tracker URL: {}\n'.format(db_task.bug_tracker)
message += "Created: {}, updated: {}, deleted: {}\n".format(
summary_diff["create"], summary_diff["update"],
summary_diff["delete"])
message += "Annotation time: {} hours\n".format(
math.ceil((last_save - self._sync_date).total_seconds() / 3600))
message += "Total annotation time: {} hours".format(
math.ceil(
(last_save - db_task.created_date).total_seconds() / 3600))
self._rep.index.commit(message)
self._rep.git.push("origin", self._branch_name, "--force")
shutil.rmtree(old_diffs_dir, True)
shutil.rmtree(self._diffs_dir, True)