Пример #1
0
def datasets(args):
    if args.list:
        _pprint_json(utils.get_datasets())
    elif args.show:
        ds = utils.get_datasets().get(args.show, None)
        if not ds:
            raise ValueError("Dataset not found.")
        _pprint_json(ds)
    elif args.search:
        search_datasets_for_string(args.search)
    elif args.download:
        ds_name, volume = args.download
        try:
            volume = float(volume)
        except ValueError:
            raise ValueError("Volume has to be a valid number, not `{}`".format(volume))
        if not utils.get_datasets().get(ds_name, False):
            raise ValueError("Dataset `{}` not found.".format(ds_name))
        elif args.run_list:
            # is the given run list a subset of the full run list?
            user_run_list = [int(r.strip()) for r in args.run_list.split(",")]
            full_run_list = [int(r.strip()) for r in
                             utils.get_datasets().get(ds_name)['run_list'].split(",")]
            if not set(user_run_list).issubset(full_run_list):
                raise ValueError("Run {} list is not a subset of this periods run list ({})!"
                                 .format(user_run_list, full_run_list))
        utils.download_dataset(ds_name, volume, args.run_list)
Пример #2
0
def datasets(args):
    if args.list:
        utils.pprint_json(utils.get_datasets())
    elif args.show:
        ds = utils.get_datasets().get(args.show, None)
        if not ds:
            raise ValueError("Dataset not found.")
        utils.pprint_json(ds)
    elif args.search:
        search_datasets_for_string(args.search)
    elif args.download:
        ds_name, volume = args.download
        try:
            volume = float(volume)
        except ValueError:
            raise ValueError(
                "Volume has to be a valid number, not `{}`".format(volume))
        if not utils.get_datasets().get(ds_name, False):
            raise ValueError("Dataset `{}` not found.".format(ds_name))
        elif args.run_list:
            # is the given run list a subset of the full run list?
            user_run_list = [int(r.strip()) for r in args.run_list.split(",")]
            full_run_list = [
                int(r.strip()) for r in utils.get_datasets().get(ds_name)
                ['run_list'].split(",")
            ]
            if not set(user_run_list).issubset(full_run_list):
                raise ValueError(
                    "Run {} list is not a subset of this periods run list ({})!"
                    .format(user_run_list, full_run_list))
        utils.download_dataset(ds_name, volume, args.run_list)
Пример #3
0
def run(args):
    utils.is_valid_project_dir()
    output_dir = _prepare_output_dir(args)
    # start the analysis
    os.chdir(output_dir)
    if args.runmode != "grid":
        # generate input file
        ds = utils.get_datasets()[args.dataset]
        # create list of local files
        with open(os.path.join(output_dir, "input_files.dat"),
                  "a") as input_files:
            search_string = os.path.join(settings["local_data_dir"],
                                         ds["datadir"].strip("/"), "*",
                                         ds["data_pattern"].strip("/"))
            search_string = os.path.expanduser(search_string)
            search_results = glob(search_string)
            if len(search_results) == 0:
                raise ValueError(
                    "No local files found at {} matching run list".format(
                        search_string))
            # Filter the found files to match the given run list
            if args.run_list:
                run_list = [r.strip() for r in args.run_list.split(",")]
            else:
                run_list = [r.strip() for r in ds["run_list"].split(",")]
            filtered_results = []
            for path in search_results:
                if any([r for r in run_list if r in path]):
                    filtered_results.append(path)
            if len(filtered_results) == 0:
                run_numbers = []
                for i in search_results:
                    i = i.split(ds["datadir"])[1].split("/")[0]
                    if i not in run_numbers:
                        run_numbers.append(i)
                raise ValueError(
                    "No local files found at {} matching run list good for required analysis. Try run numbers in {}"
                    .format(search_string, ", ".join(run_numbers)))
            input_files.write("\n".join(filtered_results) + "\n")
        # command to start the analysis
        cmd = ["root", "-l", "-q", "run.C"]
    else:
        cmd = ["root", "-l", "-q", "-b", "-x", 'run.C("full")']
    procs = []
    try:
        p = subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        procs.append(p)
        for line in iter(p.stdout.readline, b""):
            # rstrip to remove \n; doesn't like carriage returns
            print((line.rstrip()))
    except KeyboardInterrupt as e:
        for proc in procs:
            print("Killing: ", proc)
            proc.terminate()
        raise e
Пример #4
0
def run(args):
    utils.is_valid_project_dir()
    output_dir = _prepare_output_dir(args)
    # start the analysis
    os.chdir(output_dir)
    if args.runmode != "grid":
        # generate input file
        ds = utils.get_datasets()[args.dataset]
        # create list of local files
        with open(os.path.join(output_dir, "input_files.dat"),
                  'a') as input_files:
            search_string = os.path.join(settings["local_data_dir"],
                                         ds["datadir"].lstrip("/"), "*",
                                         ds["data_pattern"])
            search_string = os.path.expanduser(search_string)
            search_results = glob(search_string)
            # Filter the found files to match the given run list
            if args.run_list:
                run_list = [r.strip() for r in args.run_list.split(",")]
            else:
                run_list = [r.strip() for r in ds["run_list"].split(",")]
            filtered_results = []
            for path in search_results:
                if any([r for r in run_list if r in path]):
                    filtered_results.append(path)
            if len(filtered_results) == 0:
                raise ValueError(
                    "No local files found at {} matching run list".format(
                        search_string))
            input_files.write('\n'.join(filtered_results) + '\n')
        # command to start the analysis
        cmd = ['root', '-l', '-q', 'run.C']
    else:
        cmd = ['root', '-l', '-q', '-b', '-x', 'run.C(\"full\")']
    procs = []
    try:
        p = subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        procs.append(p)
        for line in iter(p.stdout.readline, b''):
            if isinstance(line, bytes):
                line = line.decode()
            print(line.rstrip()
                  )  # rstrip to remove \n; doesn't like carriage returns
    except KeyboardInterrupt as e:
        for proc in procs:
            print("Killing: ", proc)
            proc.terminate()
        raise e
Пример #5
0
def search_datasets_for_string(s):
    def flatten(dictionary):
        for key, value in dictionary.items():
            if isinstance(value, dict):
                # recurse
                for res in flatten(value):
                    yield res
            else:
                yield key, value
    datasets = utils.get_datasets()
    matches = []
    for dset_name, dset in datasets.items():
        for key, value in flatten(dset):
            if s in value:
                matches.append({dset_name: dset})
    _pprint_json(matches)
Пример #6
0
def search_datasets_for_string(s):
    def flatten(dictionary):
        for key, value in dictionary.items():
            if isinstance(value, dict):
                # recurse
                for res in flatten(value):
                    yield res
            else:
                yield key, value

    datasets = utils.get_datasets()
    matches = []
    for dset_name, dset in datasets.items():
        for value in flatten(dset):
            if s in value:
                matches.append({dset_name: dset})
    utils.pprint_json(matches)
Пример #7
0
def run(args):
    utils.is_valid_project_dir()
    output_dir = _prepare_output_dir(args)
    # start the analysis
    os.chdir(output_dir)
    if args.runmode != "grid":
        # generate input file
        ds = utils.get_datasets()[args.dataset]
        # create list of local files
        with open(os.path.join(output_dir, "input_files.dat"), 'a') as input_files:
            search_string = os.path.join(settings["local_data_dir"],
                                         ds["datadir"].lstrip("/"),
                                         "*",
                                         ds["data_pattern"])
            search_string = os.path.expanduser(search_string)
            search_results = glob(search_string)
            # Filter the found files to match the given run list
            if args.run_list:
                run_list = [r.strip() for r in args.run_list.split(",")]
            else:
                run_list = [r.strip() for r in ds["run_list"].split(",")]
            filtered_results = []
            for path in search_results:
                if any([r for r in run_list if r in path]):
                    filtered_results.append(path)
            if len(filtered_results) == 0:
                raise ValueError("No local files found at {} matching run list".format(search_string))
            input_files.write('\n'.join(filtered_results) + '\n')
        # command to start the analysis
        cmd = ['root', '-l', '-q', 'run.C']
    else:
        cmd = ['root', '-l', '-q', '-b', '-x', 'run.C(\"full\")']
    procs = []
    try:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        procs.append(p)
        for line in iter(p.stdout.readline, b''):
            if isinstance(line, bytes):
                line = line.decode()
            print(line.rstrip()) # rstrip to remove \n; doesn't like carriage returns
    except KeyboardInterrupt as e:
        for proc in procs:
            print("Killing: ", proc)
            proc.terminate()
        raise e