def main(demux_fcid_dirs, restrict_to_projects=None, restrict_to_samples=None,
force_update=False, workflow="NGI", already_parsed=False,
config=None, config_file_path=None):
if force_update: force_update = validate_force_update()
if not restrict_to_projects: restrict_to_projects = []
if not restrict_to_samples: restrict_to_samples = []
demux_fcid_dirs_set = set(demux_fcid_dirs)
# Sort/copy each raw demux FC into project/sample/fcid format -- "analysis-ready"
projects_to_analyze = dict()
if already_parsed: # Starting from Project/Sample/Libprep/Seqrun tree format
for demux_fcid_dir in demux_fcid_dirs_set:
p = recreate_project_from_filesystem(demux_fcid_dir)
projects_to_analyze[p.name] = p
else: # Raw illumina flowcell
for demux_fcid_dir in demux_fcid_dirs_set:
# These will be a bunch of Project objects each containing Samples, FCIDs, lists of fastq files
projects_to_analyze = setup_analysis_directory_structure(demux_fcid_dir,
projects_to_analyze,
restrict_to_projects,
restrict_to_samples,
create_files=False,
config=config)
if not projects_to_analyze:
sys.exit("Quitting: no projects found to process in flowcells {}"
"or there was an error gathering required "
"information.".format(",".join(demux_fcid_dirs_set)))
else:
# Don't need the dict functionality anymore; revert to list
projects_to_analyze = projects_to_analyze.values()
for project in projects_to_analyze:
try:
create_charon_entries_from_project(project, workflow=workflow, force_overwrite=force_update)
except Exception as e:
print(e, file=sys.stderr)
def main(demux_fcid_dir, restrict_to_projects=None, restrict_to_samples=None):
demux_fcid_dir = "/proj/a2014205/INBOX/140528_D00415_0049_BC423WACXX" # G.Grigelioniene_14_01
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140702_D00415_0052_AC41A2ANXX" # M.Kaller_14_06 sample P1171_102, P1171_104, P1171_106, P1171_108
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140905_D00415_0057_BC45KVANXX" # M.Kaller_14_06 sample P1171_102, P1171_104, P1171_106 ---- rerun
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140815_SN1025_0222_AC4HA6ACXX" # M.Kaller_14_05 sample P1170_101, P1170_103, P1170_105
process_demultiplexed_flowcell(
demux_fcid_dir, None,
None) # M.Kaller_14_08 sample P1272_101, P1272_104
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140815_SN1025_0223_BC4HAPACXX" # M.Kaller_14_05 sample P1170_101, P1170_103, P1170_105
process_demultiplexed_flowcell(
demux_fcid_dir, None,
None) # M.Kaller_14_08 sample P1272_101, P1272_104
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140919_SN1018_0203_BHA3THADXX" # M.Kaller_14_05 P1170_103, P1170_105 --- rerun
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
###UPPSALA
demux_fcid_dir = "/proj/a2014205/INBOX/140821_D00458_0029_AC45JGANXX" # uppsala run
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140917_D00458_0034_AC4FF3ANXX" # -- rerun
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
#and now a loop to update the DB
time.sleep(3800)
charon_session = CharonSession()
####charon_session.project_delete("ND-0522")
while True:
update_charon_with_local_jobs_status(
) ## this updated local_db and charon accordingly
# grab all projects from Charon
projects_dict = charon_session.projects_get_all()['projects']
for project_charon in projects_dict:
project_name = project_charon["name"]
project_dir = os.path.join(
"/proj/a2014205/nobackup/NGI/analysis_ready/DATA",
project_name)
if os.path.isdir(project_dir):
projectObj = recreate_project_from_filesystem(
project_dir, None)
launch_analysis_for_samples([projectObj])
time.sleep(3800)
def main(demux_fcid_dir, restrict_to_projects=None, restrict_to_samples=None):
demux_fcid_dir = "/proj/a2014205/INBOX/140528_D00415_0049_BC423WACXX" # G.Grigelioniene_14_01
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140702_D00415_0052_AC41A2ANXX" # M.Kaller_14_06 sample P1171_102, P1171_104, P1171_106, P1171_108
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140905_D00415_0057_BC45KVANXX" # M.Kaller_14_06 sample P1171_102, P1171_104, P1171_106 ---- rerun
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140815_SN1025_0222_AC4HA6ACXX" # M.Kaller_14_05 sample P1170_101, P1170_103, P1170_105
process_demultiplexed_flowcell(demux_fcid_dir, None, None) # M.Kaller_14_08 sample P1272_101, P1272_104
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140815_SN1025_0223_BC4HAPACXX" # M.Kaller_14_05 sample P1170_101, P1170_103, P1170_105
process_demultiplexed_flowcell(demux_fcid_dir, None, None) # M.Kaller_14_08 sample P1272_101, P1272_104
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140919_SN1018_0203_BHA3THADXX" # M.Kaller_14_05 P1170_103, P1170_105 --- rerun
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
###UPPSALA
demux_fcid_dir = "/proj/a2014205/INBOX/140821_D00458_0029_AC45JGANXX" # uppsala run
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
demux_fcid_dir = "/proj/a2014205/INBOX/140917_D00458_0034_AC4FF3ANXX" # -- rerun
process_demultiplexed_flowcell(demux_fcid_dir, None, None)
time.sleep(60) #wait for 1 minutes
#and now a loop to update the DB
time.sleep(3800)
charon_session = CharonSession()
####charon_session.project_delete("ND-0522")
while True:
update_charon_with_local_jobs_status() ## this updated local_db and charon accordingly
# grab all projects from Charon
projects_dict = charon_session.projects_get_all()['projects']
for project_charon in projects_dict:
project_name = project_charon["name"]
project_dir = os.path.join("/proj/a2014205/nobackup/NGI/analysis_ready/DATA", project_name)
if os.path.isdir(project_dir):
projectObj = recreate_project_from_filesystem(project_dir, None)
launch_analysis_for_samples([projectObj])
time.sleep(3800)
def main(demux_fcid_dirs,
restrict_to_projects=None,
restrict_to_samples=None,
best_practice_analysis=None,
sequencing_facility=None,
already_parsed=False,
force_update=False,
delete_existing=False,
force_create_project=False,
config=None,
config_file_path=None):
if force_update: force_update = validate_force_update()
if delete_existing: delete_existing = validate_delete_existing()
if not restrict_to_projects: restrict_to_projects = []
if not restrict_to_samples: restrict_to_samples = []
demux_fcid_dirs_set = set(demux_fcid_dirs)
projects_to_analyze = dict()
if already_parsed: # Starting from Project/Sample/Libprep/Seqrun tree format
for demux_fcid_dir in demux_fcid_dirs_set:
p = recreate_project_from_filesystem(
demux_fcid_dir, force_create_project=force_create_project)
projects_to_analyze[p.name] = p
else: # Raw illumina flowcell
for demux_fcid_dir in demux_fcid_dirs_set:
projects_to_analyze = setup_analysis_directory_structure(
demux_fcid_dir,
projects_to_analyze,
restrict_to_projects,
restrict_to_samples,
config=config)
if not projects_to_analyze:
sys.exit("Quitting: no projects found to process in flowcells {}"
"or there was an error gathering required "
"information.".format(",".join(demux_fcid_dirs_set)))
else:
projects_to_analyze = projects_to_analyze.values()
for project in projects_to_analyze:
try:
create_charon_entries_from_project(
project,
best_practice_analysis=best_practice_analysis,
sequencing_facility=sequencing_facility,
force_overwrite=force_update,
delete_existing=delete_existing)
except Exception as e:
print(e, file=sys.stderr)
def main(demux_fcid_dirs, restrict_to_projects=None, restrict_to_samples=None,
best_practice_analysis=None, sequencing_facility=None,
already_parsed=False,
force_update=False, delete_existing=False,
force_create_project=False,
config=None, config_file_path=None):
if force_update: force_update = validate_force_update()
if delete_existing: delete_existing = validate_delete_existing()
if not restrict_to_projects: restrict_to_projects = []
if not restrict_to_samples: restrict_to_samples = []
demux_fcid_dirs_set = set(demux_fcid_dirs)
projects_to_analyze = dict()
if already_parsed: # Starting from Project/Sample/Libprep/Seqrun tree format
for demux_fcid_dir in demux_fcid_dirs_set:
p = recreate_project_from_filesystem(demux_fcid_dir,
force_create_project=force_create_project)
projects_to_analyze[p.name] = p
else: # Raw illumina flowcell
for demux_fcid_dir in demux_fcid_dirs_set:
projects_to_analyze = setup_analysis_directory_structure(demux_fcid_dir,
projects_to_analyze,
restrict_to_projects,
restrict_to_samples,
config=config)
if not projects_to_analyze:
sys.exit("Quitting: no projects found to process in flowcells {}"
"or there was an error gathering required "
"information.".format(",".join(demux_fcid_dirs_set)))
else:
projects_to_analyze = projects_to_analyze.values()
for project in projects_to_analyze:
try:
create_charon_entries_from_project(project, best_practice_analysis=best_practice_analysis,
sequencing_facility=sequencing_facility,
force_overwrite=force_update,
delete_existing=delete_existing)
except Exception as e:
print(e, file=sys.stderr)
if args.__dict__.get("force_update"):
args.force_update = \
validate_dangerous_user_thing("overwrite existing data in Charon")
# Finally execute corresponding functions
## Analyze Project
if 'analyze_project_dirs' in args:
for analyze_project_dir in args.analyze_project_dirs:
try:
project_dir = locate_project(analyze_project_dir)
except ValueError as e:
LOG.error(e)
continue
project_obj = \
recreate_project_from_filesystem(project_dir=project_dir,
restrict_to_samples=args.restrict_to_samples)
launchers.launch_analysis(
[project_obj],
restart_failed_jobs=args.restart_failed_jobs,
restart_finished_jobs=args.restart_finished_jobs,
restart_running_jobs=args.restart_running_jobs,
no_qc=args.no_qc,
quiet=args.quiet,
manual=True)
## QC Project
elif 'qc_project_dirs' in args:
for qc_project_dir in args.qc_project_dirs:
project = recreate_project_from_filesystem(
project_dir=qc_project_dir,
restrict_to_samples=args.restrict_to_samples)
no_qc=args.no_qc,
quiet=args.quiet,
manual=True,
generate_bqsr_bam=args.generate_bqsr_bam,
)
## Analyze Project
elif "analyze_project_dirs" in args:
for analyze_project_dir in args.analyze_project_dirs:
try:
project_dir = locate_project(analyze_project_dir)
except ValueError as e:
LOG.error(e)
continue
project_obj = recreate_project_from_filesystem(
project_dir=project_dir, restrict_to_samples=args.restrict_to_samples
)
launchers.launch_analysis(
[project_obj],
restart_failed_jobs=args.restart_failed_jobs,
restart_finished_jobs=args.restart_finished_jobs,
restart_running_jobs=args.restart_running_jobs,
keep_existing_data=args.keep_existing_data,
no_qc=args.no_qc,
quiet=args.quiet,
manual=True,
generate_bqsr_bam=args.generate_bqsr_bam,
)
elif "delete_proj_analysis" in args:
from ngi_pipeline.conductor.launchers import get_engine_for_bp
def test_recreate_project_from_filesystem(self):
project_obj = self.projects[0]
recreated_project = recreate_project_from_filesystem(
project_dir=os.path.join(self.datadir, project_obj.dirname))
self.assertEqual(project_obj, recreated_project)
launch_analysis_for_samples
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--project", dest="restrict_to_projects", action="append",
help=("Restrict processing to these projects. "
"Use flag multiple times for multiple projects."))
parser.add_argument("-s", "--sample", dest= "restrict_to_samples", action="append",
help=("Restrict processing to these samples. "
"Use flag multiple times for multiple samples."))
parser.add_argument("project_dir", nargs="?", action="store",
help=("The path to the project to be processed."))
parser.add_argument("-f", "--restart-failed", dest="restart_failed_jobs", action="store_true",
help=("Restart jobs marked as FAILED in Charon."))
g = parser.add_mutually_exclusive_group()
g.add_argument("--seqrun_only", action="store_true",
help=("Only process at the seqrun level."))
g.add_argument("--sample_only", action="store_true",
help=("Only process at the sample level."))
args_dict = vars(parser.parse_args())
project = recreate_project_from_filesystem(args_dict['project_dir'],
args_dict['restrict_to_samples'])
if os.path.split(project.base_path)[1] == "DATA":
project.base_path = os.path.split(project.base_path)[0]
if not args_dict['sample_only']:
launch_analysis_for_seqruns([project], args_dict["restart_failed_jobs"])
if not args_dict['seqrun_only']:
launch_analysis_for_samples([project], args_dict["restart_failed_jobs"])
# Finally execute corresponding functions
### TODO change to work with multiple flowcells
if 'analyze_fc_dir' in args:
LOG.info("Starting flowcell analysis in directory {}".format(args.analyze_fc_dir))
flowcell.process_demultiplexed_flowcell(args.analyze_fc_dir,
args.restrict_to_projects,
args.restrict_to_samples,
args.restart_failed_jobs,
args.restart_finished_jobs,
args.restart_running_jobs,
quiet=args.quiet,
manual=True)
### TODO change to work with multiple projects
elif 'analyze_project_dir' in args:
project = recreate_project_from_filesystem(project_dir=args.analyze_project_dir,
restrict_to_samples=args.restrict_to_samples)
if project and os.path.split(project.base_path)[1] == "DATA":
project.base_path = os.path.split(project.base_path)[0]
launchers.launch_analysis([project],
restart_failed_jobs=args.restart_failed_jobs,
restart_finished_jobs=args.restart_finished_jobs,
restart_running_jobs=args.restart_running_jobs,
quiet=args.quiet,
manual=True)
elif 'organize_fc_dirs' in args:
if args.force_update: args.force_update = validate_force_update()
if args.delete_existing: args.delete_existing = validate_delete_existing()
if not args.restrict_to_projects: args.restrict_to_projects = []
if not args.restrict_to_samples: args.restrict_to_samples = []
organize_fc_dirs_set = set(args.organize_fc_dirs)
keep_existing_data=args.keep_existing_data,
no_qc=args.no_qc,
quiet=args.quiet,
manual=True,
generate_bqsr_bam=args.generate_bqsr_bam)
## Analyze Project
elif 'analyze_project_dirs' in args:
for analyze_project_dir in args.analyze_project_dirs:
try:
project_dir = locate_project(analyze_project_dir)
except ValueError as e:
LOG.error(e)
continue
project_obj = \
recreate_project_from_filesystem(project_dir=project_dir,
restrict_to_samples=args.restrict_to_samples)
launchers.launch_analysis([project_obj],
restart_failed_jobs=args.restart_failed_jobs,
restart_finished_jobs=args.restart_finished_jobs,
restart_running_jobs=args.restart_running_jobs,
keep_existing_data=args.keep_existing_data,
no_qc=args.no_qc,
quiet=args.quiet,
manual=True,
generate_bqsr_bam=args.generate_bqsr_bam)
elif 'delete_proj_analysis' in args:
from ngi_pipeline.conductor.launchers import get_engine_for_bp
delete_proj_analysis_list = list(set(args.delete_proj_analysis))
for delete_proj_analysis in delete_proj_analysis_list:
if args.restrict_to_samples:
