def get_refine_log_files(project: Project, dataset, processing_tool) -> Iterator[Path]:
res_dir = Path(
project.get_dataset_results_dir(dataset), processing_tool, "fspipeline"
)
project_dir = project.project_dir
for path in res_dir.glob("**/*.log"):
if path.is_file():
yield path.relative_to(project_dir)
def scrape_results(project: Project, dataset) -> Iterator[LigfitResult]:
res_dir = project.get_dataset_results_dir(dataset)
for ref_dir in subdirs(res_dir, 2):
score = _scrape_score(ref_dir)
proc_tool = ref_dir.parent.name
refine_tool = ref_dir.name
if score is None:
status = ToolStatus.FAILURE
else:
status = ToolStatus.SUCCESS
yield LigfitResult(proc_tool, refine_tool, "rhofit", status, score)
def _migrate_refine(project: Project, old_paths: OldPaths):
_log("MIGRATE REFINE RESULTS")
for dset in _dbg_cutoff(project.get_datasets()):
for proc_tool in ["xds", "xdsapp", "dials"]:
proc_dir = "xdsxscale" if proc_tool == "xds" else proc_tool
src_dir = Path(old_paths.results_dir, dset.name, proc_dir)
if not src_dir.is_dir():
# no tool results found for this dataset
continue
dest_dir = Path(project.get_dataset_results_dir(dset), proc_tool)
_log(f"{src_dir} -> {dest_dir}")
# note, we can't use ignore_dangling_symlinks=True
# here due to https://bugs.python.org/issue38523
_copytree(src_dir, dest_dir)
for tool in REFINE_TOOLS:
update_refine_tool_status(project, tool, dset)
for tool in LIGFIT_TOOLS:
_maybe_resync_ligfit_tool(project, tool, dset)
def launch_refine_jobs(
project: Project,
filters,
pdb_file,
space_group,
run_aimless,
refine_tool,
refine_tool_commands,
cpus,
):
epoch = round(time.time())
jobs = JobsSet("Refine")
hpc = SITE.get_hpc_runner()
for dset in get_refine_datasets(project, filters, refine_tool):
for tool, input_mtz in _find_input_mtzs(project, dset):
batch = hpc.new_batch_file(
f"refine {tool} {dset.name}",
project_script(project,
f"refine_{tool}_{refine_tool}_{dset.name}.sh"),
project_log_path(
project, f"refine_{tool}_{dset.name}_{epoch}_%j_out.txt"),
project_log_path(
project, f"refine_{tool}_{dset.name}_{epoch}_%j_err.txt"),
cpus,
)
batch.set_options(
time=Duration(hours=12),
nodes=1,
mem_per_cpu=DataSize(gigabyte=5),
)
batch.add_commands(crypt_shell.crypt_cmd(project))
batch.assign_variable("WORK_DIR", "`mktemp -d`")
batch.add_commands(
"cd $WORK_DIR",
crypt_shell.fetch_file(project, pdb_file, "model.pdb"),
crypt_shell.fetch_file(project, input_mtz, "input.mtz"),
)
# TODO: load tool specific modules?
batch.load_modules(HPC_MODULES)
if run_aimless:
batch.add_commands(
_aimless_cmd(space_group.short_name, "input.mtz"))
results_dir = Path(project.get_dataset_results_dir(dset), tool)
batch.add_commands(
*refine_tool_commands,
_upload_result_cmd(project, results_dir),
"cd",
"rm -rf $WORK_DIR",
)
batch.save()
jobs.add_job(batch)
add_update_job(jobs, hpc, project, refine_tool, dset, batch)
jobs.submit()
def scrape_results(project: Project, dataset) -> Iterator[RefineResult]:
res_dir = project.get_dataset_results_dir(dataset)
for sdir in res_dir.glob("*/fspipeline"):
yield _get_results(project, dataset, sdir)
def get_refine_log_files(
project: Project, dataset, processing_tool
) -> Optional[Iterable[Path]]:
res_dir = Path(project.get_dataset_results_dir(dataset), processing_tool, "dimple")
return get_files_by_suffixes(res_dir, ["log"])