Пример #1
0
def check_output_file(options):  
    '''Check root file for TTree entries
    '''
    # Can only check the outputs for files with output file specified in the command options
    if options.output_file is None:
        print "check_output_file::no output file specified"
        return None

    # Can only check the outputs for files with number of events specified in the command options
    if options.n_events is None:
        print "check_output_file::no number of events specified"
        return None

    # Create command to run python script to check TTree entries
    # Will raise an exception if the checks fail
    ntuple = job_tools.check_processor(options.rat_macro, 'outntuple')
    root = job_tools.check_processor(options.rat_macro, 'outroot')
    soc = job_tools.check_processor(options.rat_macro, 'outsoc')
    if root or soc:
        rtc, out, err = job_tools.execute_ratenv('python', ['check_root_output.py', '-f', '%s.root' % options.output_file,
                                                        '-n', options.n_events, '-v', options.base_version])
        if rtc==0:
            # this must be true (exception raised otherwise)
            print "Outputs file checks successful"
    if ntuple:
        rtc, out, err = job_tools.execute_ratenv('python', ['check_root_output.py', '-f', '%s.ntuple.root' % options.output_file,
                                                        '-n', options.n_events, '-v', options.base_version])
        if rtc==0:
            # this must be true (exception raised otherwise)
            print "Outputs file checks successful"
Пример #2
0
def check_output_file(options):
    '''Check root file for TTree entries
    '''
    # Can only check the outputs for files with output file specified in the command options
    if options.output_file is None:
        print "check_output_file::no output file specified"
        return None

    # Can only check the outputs for files with number of events specified in the command options
    if options.n_events is None:
        print "check_output_file::no number of events specified"
        return None

    # Create command to run python script to check TTree entries
    # Will raise an exception if the checks fail
    ntuple = job_tools.check_processor(options.rat_macro, 'outntuple')
    root = job_tools.check_processor(options.rat_macro, 'outroot')
    soc = job_tools.check_processor(options.rat_macro, 'outsoc')
    if root or soc:
        rtc, out, err = job_tools.execute_ratenv('python', [
            'check_root_output.py', '-f',
            '%s.root' % options.output_file, '-n', options.n_events, '-v',
            options.base_version
        ])
        if rtc == 0:
            # this must be true (exception raised otherwise)
            print "Outputs file checks successful"
    if ntuple:
        rtc, out, err = job_tools.execute_ratenv('python', [
            'check_root_output.py', '-f',
            '%s.ntuple.root' % options.output_file, '-n', options.n_events,
            '-v', options.base_version
        ])
        if rtc == 0:
            # this must be true (exception raised otherwise)
            print "Outputs file checks successful"
Пример #3
0
def check_outputs(options):
    '''Before running RAT, check whether output data already exists!
    '''
    # Can only check the outputs for files with output file specified in the command options
    if options.output_file is None:
        print "check_outputs::no output file specified"
        return None
    if options.grid_mode is None:
        print "check_outputs::no grid mode specified"
        return None
    # For output files, check if there is an output root and output ntuple processor in the macro
    ntuple = job_tools.check_processor(options.rat_macro, 'outntuple')
    root = job_tools.check_processor(options.rat_macro, 'outroot')
    suffixes = []
    exists = []
    if root:
        suffixes.append(".root")
        exists.append(False)
    if ntuple:
        suffixes.append(".ntuple.root")
        exists.append(False)
    all_exist = True
    any_exist = False
    if len(suffixes) == 0:
        print "check_output::no files to check!"
        print suffixes
        all_exist = False
    for i, s in enumerate(suffixes):
        root_file = "%s%s" % (options.output_file, s)
        lfc_path = os.path.join('lfn:/grid/snoplus.snolab.ca',
                                options.output_dir, root_file)
        exists[i] = False
        try:
            job_tools.exists(lfc_path)
            # lfc exists, get the replica
            exists[i] = True
            any_exist = True
            print "check_outputs::LFN already in use %s" % lfc_path
        except job_tools.JobToolsException as e:
            print "check_outputs::LFN unused %s, %s" % (lfc_path, e)
            all_exist = False
    # Now, if all exist, we need to append the return card and show that the job has in fact completed
    # If only some (i.e. any) exist, but not all, then we need to raise this problem by failing the job!
    dump_out = {}
    if all_exist:
        print "check_outputs::All expected outputs already exist on LFN"
        for s in suffixes:
            try:
                root_file = "%s%s" % (options.output_file, s)
                lfc_path = os.path.join('lfn:/grid/snoplus.snolab.ca',
                                        options.output_dir, root_file)
                guid = job_tools.getguid(lfc_path)
                replica = job_tools.listreps(lfc_path)[0]
                checksum = job_tools.checksum(replica)
                size = job_tools.getsize(replica)

                # Note that the actual se and replica information may be different to the local information
                se_name = replica
                if se_name.startswith('srm://'):
                    se_name = se_name[6:]
                se_name = se_name.split('/')[0]  # never any /
                dump_out[root_file] = {}
                dump_out[root_file]['guid'] = guid
                dump_out[root_file]['se'] = replica
                dump_out[root_file]['size'] = size
                dump_out[root_file]['cksum'] = checksum
                dump_out[root_file]['name'] = se_name
                dump_out[root_file]['lfc'] = lfc_path
            except job_tools.JobToolsException as e:
                print "check_outputs::Problem checking output for %s: %s" % (
                    lfc_path, e)
                raise
        return dump_out
    else:
        return None
Пример #4
0
def check_outputs(options):
    '''Before running RAT, check whether output data already exists!
    '''
    # Can only check the outputs for files with output file specified in the command options
    if options.output_file is None:
        print "check_outputs::no output file specified"
        return None
    if options.grid_mode is None:
        print "check_outputs::no grid mode specified"
        return None
    # For output files, check if there is an output root and output ntuple processor in the macro
    ntuple = job_tools.check_processor(options.rat_macro, 'outntuple')
    root = job_tools.check_processor(options.rat_macro, 'outroot')
    suffixes = []
    exists = []
    if root:
        suffixes.append(".root")
        exists.append(False)
    if ntuple:
        suffixes.append(".ntuple.root")
        exists.append(False)
    all_exist = True
    any_exist = False
    if len(suffixes)==0:
        print "check_output::no files to check!"
        print suffixes
        all_exist = False
    for i, s in enumerate(suffixes):
        root_file = "%s%s" % (options.output_file, s)
        lfc_path = os.path.join('lfn:/grid/snoplus.snolab.ca', options.output_dir, root_file)
        exists[i] = False
        try:
            job_tools.exists(lfc_path)
            # lfc exists, get the replica
            exists[i] = True
            any_exist = True
            print "check_outputs::LFN already in use %s" % lfc_path
        except job_tools.JobToolsException as e:
            print "check_outputs::LFN unused %s, %s" % (lfc_path, e)
            all_exist = False
    # Now, if all exist, we need to append the return card and show that the job has in fact completed
    # If only some (i.e. any) exist, but not all, then we need to raise this problem by failing the job!
    dump_out = {}
    if all_exist:
        print "check_outputs::All expected outputs already exist on LFN"
        for s in suffixes:
            try:
                root_file = "%s%s" % (options.output_file, s)
                lfc_path = os.path.join('lfn:/grid/snoplus.snolab.ca', options.output_dir, root_file)
                guid = job_tools.getguid(lfc_path)
                replica = job_tools.listreps(lfc_path)[0]
                checksum = job_tools.checksum(replica)
                size = job_tools.getsize(replica)
                
                # Note that the actual se and replica information may be different to the local information
                se_name = replica
                if se_name.startswith('srm://'):
                    se_name = se_name[6:]
                se_name = se_name.split('/')[0] # never any /
                dump_out[root_file] = {}
                dump_out[root_file]['guid'] = guid
                dump_out[root_file]['se'] = replica
                dump_out[root_file]['size'] = size
                dump_out[root_file]['cksum'] = checksum
                dump_out[root_file]['name'] = se_name
                dump_out[root_file]['lfc'] = lfc_path
            except job_tools.JobToolsException as e:
                print "check_outputs::Problem checking output for %s: %s" % (lfc_path, e)
                raise
        return dump_out
    else:
        return None