def run(args): if (len(args) == 0): args = ["--help"] import libtbx.load_env from libtbx.option_parser import option_parser command_line = (option_parser( usage="%s [options] [var=value] [...] [file_listing_commands] [...]" % libtbx.env.dispatcher_name).option( None, "--dirs", action="store", type="str", help="create a sub-directory for each run.").option( None, "--force_clean_dirs", action="store_true", help="forces removal of existing directories before creation." ).option( None, "--command", action="append", type="str", help="command to be executed, e.g. 'echo $(MULTI:1-4)'" ).option( "-j", "--jobs", action="store", type="int", help="maximum number of parallel jobs (default: all CPUs).") ).process(args=args) co = command_line.options # from libtbx import easy_mp import libtbx.introspection import libtbx.utils from libtbx.utils import Sorry from libtbx.str_utils import show_string # files_listing_commands = [] for arg in command_line.args: earg = op.expandvars(arg) flds = earg.split("=", 1) if (len(flds) != 2): files_listing_commands.append(earg) else: k, v = flds os.environ[k] = v # cmd_infos = [] def cmd_infos_append(line): for l in process_dollar_multi(line): index = len(cmd_infos) cmd_infos.append(libtbx.group_args(index=index, cmd=l, log=None)) if (co.command is not None): for line in co.command: cmd_infos_append(line=line) for file_name in files_listing_commands: file_dir = op.dirname(file_name) for line in open(file_name).read().splitlines(): ll = line.lstrip() if (ll.startswith("#")): continue if (ll.startswith("set ")): continue if (ll.startswith("setenv ")): flds = ll.split(None, 2) if (len(flds) == 2): flds.append("") os.environ[flds[1]] = flds[2] continue cmd_infos_append(line=line.replace("$(DIRNAME)", file_dir)) n_proc = min(len(cmd_infos), libtbx.introspection.number_of_processors()) if (co.jobs is not None): n_proc = max(1, min(co.jobs, n_proc)) print("Number of processors:", n_proc) print("Number of jobs:", len(cmd_infos)) print() sys.stdout.flush() show_times = libtbx.utils.show_times(time_start="now") def show_log(cmd_info): need_log = (cmd_info.index != 0 or co.dirs is not None) if (cmd_info.log is None): if (need_log): print("MISSING: output of command with index %03d:" % cmd_info.index) print(" %s" % cmd_info.cmd) elif (not op.isfile(cmd_info.log)): if (need_log): print("MISSING:", cmd_info.log) else: lines = open(cmd_info.log).read().splitlines() if (len(lines) > 10): print("@BEGIN") if (len(lines) != 0): print("\n".join(lines)) if (len(lines) > 10): print("@END") def show_logs(): for cmd_info in cmd_infos: if (cmd_info.index == 0 and co.dirs is None): continue print("command:", cmd_info.cmd) show_log(cmd_info=cmd_info) if (co.dirs is None): for cmd_info in cmd_infos: cmd_info.log = "log%03d" % cmd_info.index libtbx.utils.remove_files(cmd_info.log) if (n_proc < 2): for cmd_info in cmd_infos: print("command:", cmd_info.cmd) run_one_cmd(cmd_info=cmd_info) show_log(cmd_info=cmd_info) else: easy_mp.pool_map(processes=n_proc, func=run_one_cmd, args=cmd_infos, chunksize=1) show_logs() else: old_dirs = [] for cmd_info in cmd_infos: d = "%s%03d" % (co.dirs, cmd_info.index) if (op.exists(d)): if (not co.force_clean_dirs): print("exists already: %s" % show_string(d), file=sys.stderr) old_dirs.append(d) cmd_info.log = op.join(d, "log") if (len(old_dirs) != 0): if (not co.force_clean_dirs): raise Sorry("Please remove the existing directories or files," " or use a different --dirs assignment.") from libtbx.clear_paths \ import remove_or_rename_files_and_directories_if_possible remaining = remove_or_rename_files_and_directories_if_possible( paths=old_dirs) if (len(remaining) != 0): for d in remaining: print("unable to remove or rename: %s" % show_string(d), file=sys.stderr) raise Sorry("Failure removing existing directories.") easy_mp.pool_map(processes=n_proc, func=run_in_dir, args=cmd_infos, chunksize=1) show_logs() print() show_times() print() sys.stdout.flush()
def run(args): if (len(args) == 0): args = ["--help"] import libtbx.load_env from libtbx.option_parser import option_parser command_line = (option_parser( usage="%s [options] [var=value] [...] [file_listing_commands] [...]" % libtbx.env.dispatcher_name) .option(None, "--dirs", action="store", type="str", help="create a sub-directory for each run.") .option(None, "--force_clean_dirs", action="store_true", help="forces removal of existing directories before creation.") .option(None, "--command", action="append", type="str", help="command to be executed, e.g. 'echo $(MULTI:1-4)'") .option("-j", "--jobs", action="store", type="int", help="maximum number of parallel jobs (default: all CPUs).") ).process(args=args) co = command_line.options # from libtbx import easy_mp import libtbx.introspection import libtbx.utils from libtbx.utils import Sorry from libtbx.str_utils import show_string # files_listing_commands = [] for arg in command_line.args: earg = op.expandvars(arg) flds = earg.split("=", 1) if (len(flds) != 2): files_listing_commands.append(earg) else: k,v = flds os.environ[k] = v # cmd_infos = [] def cmd_infos_append(line): for l in process_dollar_multi(line): index = len(cmd_infos) cmd_infos.append(libtbx.group_args(index=index, cmd=l, log=None)) if (co.command is not None): for line in co.command: cmd_infos_append(line=line) for file_name in files_listing_commands: file_dir = op.dirname(file_name) for line in open(file_name).read().splitlines(): ll = line.lstrip() if (ll.startswith("#")): continue if (ll.startswith("set ")): continue if (ll.startswith("setenv ")): flds = ll.split(None, 2) if (len(flds) == 2): flds.append("") os.environ[flds[1]] = flds[2] continue cmd_infos_append(line=line.replace("$(DIRNAME)", file_dir)) n_proc = min(len(cmd_infos), libtbx.introspection.number_of_processors()) if (co.jobs is not None): n_proc = max(1, min(co.jobs, n_proc)) print "Number of processors:", n_proc print "Number of jobs:", len(cmd_infos) print sys.stdout.flush() show_times = libtbx.utils.show_times(time_start="now") def show_log(cmd_info): need_log = (cmd_info.index != 0 or co.dirs is not None) if (cmd_info.log is None): if (need_log): print "MISSING: output of command with index %03d:" % cmd_info.index print " %s" % cmd_info.cmd elif (not op.isfile(cmd_info.log)): if (need_log): print "MISSING:", cmd_info.log else: lines = open(cmd_info.log).read().splitlines() if (len(lines) > 10): print "@BEGIN" if (len(lines) != 0): print "\n".join(lines) if (len(lines) > 10): print "@END" def show_logs(): for cmd_info in cmd_infos: if (cmd_info.index == 0 and co.dirs is None): continue print "command:", cmd_info.cmd show_log(cmd_info=cmd_info) if (co.dirs is None): for cmd_info in cmd_infos: cmd_info.log = "log%03d" % cmd_info.index libtbx.utils.remove_files(cmd_info.log) if (n_proc < 2): for cmd_info in cmd_infos: print "command:", cmd_info.cmd run_one_cmd(cmd_info=cmd_info) show_log(cmd_info=cmd_info) else: easy_mp.pool_map( processes=n_proc, func=run_one_cmd, args=cmd_infos, chunksize=1) show_logs() else: old_dirs = [] for cmd_info in cmd_infos: d = "%s%03d" % (co.dirs, cmd_info.index) if (op.exists(d)): if (not co.force_clean_dirs): print >> sys.stderr, "exists already: %s" % show_string(d) old_dirs.append(d) cmd_info.log = op.join(d, "log") if (len(old_dirs) != 0): if (not co.force_clean_dirs): raise Sorry( "Please remove the existing directories or files," " or use a different --dirs assignment.") from libtbx.clear_paths \ import remove_or_rename_files_and_directories_if_possible remaining = remove_or_rename_files_and_directories_if_possible( paths=old_dirs) if (len(remaining) != 0): for d in remaining: print >> sys.stderr, \ "unable to remove or rename: %s" % show_string(d) raise Sorry("Failure removing existing directories.") easy_mp.pool_map( processes=n_proc, func=run_in_dir, args=cmd_infos, chunksize=1) show_logs() print show_times() print sys.stdout.flush()
def run(args): remaining = remove_or_rename_files_and_directories_if_possible(paths=args) for path in remaining: "WARNING: unable to remove or rename:", path