Пример #1
0
def check_end_run(run_id, conf):
    """Check the end of a run data transfer.

    Arguments:
        run_id: the run id
        conf: configuration dictionary
    """

    hiseq_data_path = find_hiseq_run_path(run_id, conf)
    reads_number = get_read_count(run_id, conf)

    # TODO
    if hiseq_data_path is False:
        return False

    # if reads_number equals -1, runParameters.xml is missing
    if reads_number == -1:
        return False

    prefix, suffix = ('Basecalling_Netcopy_complete_Read', '.txt') if (
        common.get_rta_major_version(run_id, conf) == 1) else (
        'RTARead', 'Complete.txt')

    # File generate only by HiSeq sequencer
    for i in range(reads_number):
        file_to_test = hiseq_data_path + '/' + run_id + '/' + _build_read_complete_filename(run_id, i, conf)

        if not os.path.exists(file_to_test):
            return False

    if not os.path.exists(hiseq_data_path + '/' + run_id + '/RTAComplete.txt'):
        return False

    return True
Пример #2
0
def get_available_new_run_ids(conf):
    """Get the list of the available runs.

    Arguments:
        conf: configuration dictionary
    """

    result = set()
    hiseq_run_ids_do_not_process = hiseq_run.load_deny_run_ids(conf)

    for hiseq_data_path in hiseq_run.get_hiseq_data_paths(conf):

        files = os.listdir(hiseq_data_path)
        for f in files:

            # Do not process denied runs
            if f in hiseq_run_ids_do_not_process:
                continue

            if not (os.path.isdir(hiseq_data_path + '/' + f) and hiseq_run.check_run_id(f, conf)):
                # No valid entry
                continue

            # NextSeq sequencer create this file after clusterisation step
            if not os.path.exists(hiseq_data_path + '/' + f + '/RunInfo.xml'):
                continue

            if common.get_rta_major_version(f, conf) == 1 and \
                    not os.path.exists(hiseq_data_path + '/' + common.FIRST_BASE_REPORT_FILE):
                continue

            if not hiseq_run.check_end_run(f, conf):
                # TODO
                # os.path.exists(hiseq_data_path + '/' + f + '/First_Base_Report.htm'):
                result.add(f)

    return result
Пример #3
0
def create_run_summary_reports(run_id, conf):
    """ Copy main files and directory from hiseq run directory to save in report run data directory.
        Save data in two distinct directory on hiseq and on report, and tar.bz2 version

    Arguments:
        run_id: the run id
        conf: configuration dictionary
    """

    hiseq_data_path = hiseq_run.find_hiseq_run_path(run_id, conf)
    tmp_base_path = conf[TMP_PATH_KEY]
    reports_data_base_path = conf[REPORTS_DATA_PATH_KEY]

    source_path = hiseq_data_path + '/' + run_id
    reports_data_path = common.get_report_run_data_path(run_id, conf)
    report_prefix = 'report_'
    hiseq_log_prefix = 'hiseq_log_'
    report_archive_file = report_prefix + run_id + '.tar.bz2'
    hiseq_log_archive_file = hiseq_log_prefix + run_id + '.tar.bz2'

    # Save quality control data
    tmp_path = tmp_base_path + '/' + run_id

    # Check if reports_data_path exists
    if not os.path.exists(reports_data_base_path):
        error("Failed to create report archive: Report directory does not exist",
              "Failed to create report archive: Report directory does not exist: " + reports_data_base_path, conf)
        return False

    # Check if temporary directory exists
    if not os.path.exists(tmp_base_path):
        error("Failed to create report archive: Temporary directory does not exist",
              "Failed to create report archive: Temporary directory does not exist: " + tmp_base_path, conf)
        return False

    # Check if reports archive exists
    if os.path.exists(reports_data_path + '/' + report_archive_file):
        error("Failed to create report archive: Report archive already exists for run " + run_id,
              "Failed to create report archive: Report archive already exists for run " + run_id + " : " + report_archive_file, conf)
        return False

    # Check if hiseq log archive exists
    if os.path.exists(reports_data_path + '/' + hiseq_log_archive_file):
        error("Failed to create report archive: Hiseq log archive already exists for run " + run_id,
              "Failed to create report archive: Hiseq log archive already exists for run " + run_id + " : " + hiseq_log_archive_file, conf)
        return False

    # Create if not exist archive directory for the run
    if not os.path.exists(reports_data_path):
        os.mkdir(reports_data_path)

    # Create run tmp  directory
    if os.path.exists(tmp_path):
        error("Failed to create report archive: Temporary run data directory already exists for run " + run_id,
              "Failed to create report archive: Temporary run data directory already exists for run " + run_id + " : " + hiseq_log_archive_file, conf)
    else:
        os.mkdir(tmp_path)

    # Define set file to copy in report archive, check if exists (depend on parameters Illumina)
    files = ['InterOp', 'RunInfo.xml', 'runParameters.xml', 'RunParameters.xml']
    files_to_copy = list_existing_files(source_path, files)

    if len(files_to_copy) == 0:
        common.log("WARNING",
                   "Archive " + hiseq_log_archive_file + " not created: none file exists " + str(files) +
                   ' in ' + source_path, conf)
    else:
        cmd = 'cd ' + quote(source_path) + ' && ' + \
              'cp -rp ' + files_to_copy + ' ' + quote(tmp_path) + ' && ' + \
              'cd ' + quote(tmp_base_path) + ' && ' + \
              'mv ' + quote(run_id) + ' ' + quote(hiseq_log_prefix + run_id) + ' && ' + \
              'tar cjf ' + quote(reports_data_path + '/' + hiseq_log_archive_file) + ' ' + quote(hiseq_log_prefix + run_id) + ' && ' +\
              'rm -rf ' + quote(tmp_path)
        # + ' && rm -rf ' + hiseq_log_prefix + run_id

        common.log("INFO", "exec: " + cmd, conf)
        if os.system(cmd) != 0:
            error("Failed to create report archive: Error while saving Illumina quality control for run " + run_id,
                  "Failed to create report archive: Error saving Illumina quality control.\nCommand line:\n" + cmd, conf)
            return False

    # Save html reports
    if os.path.exists(tmp_path):
        cmd = 'rm -rf ' + quote(tmp_path)

        common.log("INFO", "exec: " + cmd, conf)
        if os.system(cmd) != 0:
            error("Failed to create report archive: Error while removing existing temporary directory",
                  "Failed to create report archive: Error while removing existing temporary directory.\nCommand line:\n" + cmd, conf)
            return False

    os.mkdir(tmp_path)

    # Define set file to copy in report archive, check if exists (depend on parameters Illumina)
    if common.get_rta_major_version(run_id, conf) == 1:
        files = ['./Data/Status_Files', './Data/reports', './Data/Status.htm', './First_Base_Report.htm']
    else:
        files = ['./Config', './Recipe', './RTALogs', './RTAConfiguration.xml', './RunCompletionStatus.xml']

    files_to_copy = list_existing_files(source_path, files)
    if len(files_to_copy) == 0:
        common.log("WARNING", "Archive " + report_archive_file + " not created: none file exists " + str(
            files) + ' in ' + source_path, conf)
    else:
        cmd = 'cd ' + quote(source_path) + ' && ' + \
              'cp -rp ' + files_to_copy + ' ' + quote(tmp_path) + ' && ' + \
              'cd ' + quote(tmp_base_path) + ' && ' + \
              'mv ' + quote(run_id) + ' ' + quote(report_prefix + run_id) + ' && ' + \
              'tar cjf ' + quote(reports_data_path + '/' + report_archive_file) + ' ' + quote(report_prefix + run_id) + ' && ' + \
              'mv ' + quote(report_prefix + run_id) + ' ' + quote(reports_data_path)

        common.log("INFO", "exec: " + cmd, conf)
        if os.system(cmd) != 0:
            error("Failed to create report archive: Error while saving Illumina HTML reports for run " + run_id,
                  "Failed to create report archive: Error saving Illumina HTML reports.\nCommand line:\n" + cmd, conf)
            return False

    # Create index.hml file
    common.create_html_index_file(conf, run_id, [HISEQ_STEP_KEY])

    # Set read only the report directory
    common.chmod_files_in_dir(reports_data_path + '/' + report_prefix, None, conf)

    # Set read only archives files
    common.chmod(reports_data_path + '/' + report_archive_file, conf)
    common.chmod(reports_data_path + '/' + hiseq_log_archive_file, conf)

    return True
Пример #4
0
def _build_read_complete_filename(run_id, i, conf):
    if common.get_rta_major_version(run_id, conf) == 1:
        return 'Basecalling_Netcopy_complete_Read' + str(i + 1) + '.txt'

    else:
        return 'RTARead' + str(i + 1) + 'Complete.txt'