def finalize_project(project):
log = task.logger
config = GlobalConfig(task.conf)
paths = PathsConfig(config)
project_id = project["id"]
log.info("--- [{0}] --------------------------------------------".format(project_id))
log.info("Cleaning project ...")
projres = ProjectResults(project)
projres.clean(config)
log.info("Saving project configuration ...")
projres.save_def()
if config.results.use_storage:
log.info("Compressing the database ...")
db_path = os.path.join(projres.path, "project.db")
temp_path = tempfile.mkdtemp(prefix="intogen-mutations-{}-".format(project_id))
compressed_db_path = os.path.join(temp_path, "project.db.gz")
try:
cmd = "gzip -c {} >{}".format(db_path, compressed_db_path)
log.debug("> {}".format(cmd))
retcode = subprocess.call(cmd, shell=True)
if retcode == 1:
raise Exception("Error compressing the project database")
log.info("Uploading project files ...")
exclude = ["sources/*", "project.db", "project.db.gz"]
if not config.results.create_zip:
exclude += ["results.zip"]
object_prefix = "results/projects/{}".format(project_id)
start_callback = lambda path: log.info(" {}".format(path))
if os.path.exists(compressed_db_path):
task.storage.upload(compressed_db_path, object_prefix=object_prefix, overwrite=True,
start_callback=start_callback)
task.storage.upload(projres.path, object_prefix=object_prefix, exclude=exclude, overwrite=True,
start_callback=start_callback)
if config.results.purge_after_upload:
log.info("Purging project files ...")
shutil.rmtree(projres.path)
finally:
if os.path.exists(temp_path):
shutil.rmtree(temp_path)
def init_project(logger, config, paths, storage, project):
project_id = project["id"]
results_path = paths.results_path()
project_path = paths.project_path(project_id)
project_temp_path = paths.project_temp_path(project_path)
if config.results.purge_on_start:
logger.info(" Purging previous results ...")
if os.path.isdir(project_path):
logger.info(" {} ...".format(os.path.relpath(project_path, results_path)))
shutil.rmtree(project_path)
#if os.path.isdir(project_temp_path):
# logger.info(" {} ...".format(os.path.relpath(project_temp_path, results_path)))
# shutil.rmtree(project_temp_path)
for obj_name in storage.list_objects(prefix="results/"):
logger.info(" {} ...".format(obj_name))
storage.delete_object("results/{}".format(obj_name))
ensure_path_exists(project_path)
ensure_path_exists(project_temp_path)
projdb_path = os.path.join(project_path, "project.db")
if "annotations" in project:
annotations = project["annotations"]
if not Data.is_element(annotations):
logger.warn("Overriding project annotations field with an empty dictionary")
project["annotations"] = annotations = Data.element()
else:
project["annotations"] = annotations = Data.element()
# for backward compatibility
for key in project.keys():
if key not in ["id", "assembly", "files", "storage_objects", "annotations", "conf", "oncodriveclust", "oncodrivefm"]:
value = project[key]
del project[key]
annotations[key] = value
project["conf"] = pconf = project.get("conf") or Data.element()
if not Data.is_element(pconf):
logger.warn("Overriding project conf field with an empty dictionary")
project["conf"] = pconf = Data.element()
# for backward compatibility
for key in project.keys():
if key in ["oncodriveclust", "oncodrivefm"]:
value = project[key]
del project[key]
pconf[key] = value
project["path"] = project_path
project["temp_path"] = project_temp_path
project["db"] = projdb_path
if "assembly" not in project:
project["assembly"] = DEFAULT_ASSEMBLY
missing_objects = []
for obj_name in project["storage_objects"]:
if not storage.exists_object(obj_name):
missing_objects += [obj_name]
if len(missing_objects) > 0:
raise InternalError("Project {0} references some missing objects:\n{1}".format(project_id, "\n".join(missing_objects)))
project["files"] = [str(f) for f in project["files"]] #unicode is not json serializable
project["storage_objects"] = [str(f) for f in project["storage_objects"]] #unicode is not json serializable
project = project.to_native()
# save project.conf
projres = ProjectResults(project)
projres.save_def()
return project
