def create_run_script(
main_filepath,
argname_lst,
argvalue_lst,
script_filepath,
# entry_folderpath=None,
output_filepath=None,
profile_filepath=None):
sc_lines = ['#!/bin/bash', 'set -e']
# # change into the entry folder if provided.
# if entry_folderpath is not None:
# sc_lines += ['cd %s' % entry_folderpath]
# call the main function.
sc_lines += generate_call_lines(
**tb_ut.subset_dict_via_selection(locals(), [
'main_filepath', 'argname_lst', 'argvalue_lst', 'output_filepath',
'profile_filepath'
]))
# change back to the previous folder if I change to some other folder.
# if entry_folderpath is not None:
# sc_lines += ['cd -']
tb_io.write_textfile(script_filepath, sc_lines, with_newline=True)
# add run permissions.
st = os.stat(script_filepath)
exec_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
os.chmod(script_filepath, st.st_mode | exec_bits)
def register(self,
bash_command,
num_cpus=1,
num_gpus=0,
mem_budget=8.0,
time_budget=60.0,
mem_units='gigabytes',
time_units='minutes',
folderpath=None,
wait_for_output=True,
require_gpu_types=None,
require_nodes=None,
run_on_head_node=False):
# NOTE: this is not implemented for now.
assert not run_on_head_node
# should not specify both.
assert require_gpu_types is None or require_nodes is None
self.jobs.append(
tb_ut.subset_dict_via_selection(locals(), [
'bash_command', 'num_cpus', 'num_gpus', 'mem_budget',
'time_budget', 'mem_units', 'time_units', 'folderpath',
'wait_for_output', 'require_gpu_types', 'require_nodes',
'run_on_head_node'
]))
def list_folders(folderpath,
ignore_hidden_folders=True,
recursive=False,
use_relative_paths=False):
kwargs = tb_ut.subset_dict_via_selection(
locals(), ['recursive', 'ignore_hidden_folders', 'use_relative_paths'])
return list_paths(folderpath, ignore_files=True, **kwargs)
def run_on_matrix(bash_command,
servername,
username,
password=None,
num_cpus=1,
num_gpus=0,
mem_budget=8.0,
time_budget=60.0,
mem_units='gigabytes',
time_units='minutes',
folderpath=None,
wait_for_output=True,
require_gpu_type=None,
run_on_head_node=False,
jobname=None):
assert (not run_on_head_node) or num_gpus == 0
assert require_gpu_type is None ### NOT IMPLEMENTED YET.
# prompts for password if it has not been provided
if password == None:
password = getpass.getpass()
script_cmd = "\n".join(['#!/bin/bash', bash_command])
script_name = "run_%s.sh" % uuid.uuid4()
# either do the call using sbatch, or run directly on the head node.
if not run_on_head_node:
cmd_parts = [
'srun' if wait_for_output else 'sbatch',
'--cpus-per-task=%d' % num_cpus,
'--gres=gpu:%d' % num_gpus,
'--mem=%d' % tb_rs.convert_between_byte_units(
mem_budget, src_units=mem_units, dst_units='megabytes'),
'--time=%d' % tb_lg.convert_between_time_units(
time_budget, time_units, dst_units='minutes')
]
if jobname is not None:
cmd_parts += ['--job-name=%s' % jobname]
cmd_parts += [script_name]
run_script_cmd = ' '.join(cmd_parts)
else:
run_script_cmd = './' + script_name
# actual command to run remotely
remote_cmd = " && ".join([
"echo \'%s\' > %s" % (script_cmd, script_name),
"chmod +x %s" % script_name, run_script_cmd,
"rm %s" % script_name
])
return run_on_server(
remote_cmd,
**tb_ut.subset_dict_via_selection(locals(), [
'servername', 'username', 'password', 'folderpath',
'wait_for_output'
]))
def copy_folder(src_folderpath,
dst_folderpath,
ignore_hidden_files=False,
ignore_hidden_folders=False,
ignore_file_exts=None,
abort_if_dst_exists=True,
create_parent_folders=False):
assert folder_exists(src_folderpath)
assert src_folderpath != dst_folderpath
assert not (abort_if_dst_exists and folder_exists(dst_folderpath))
if (not abort_if_dst_exists) and folder_exists(dst_folderpath):
delete_folder(dst_folderpath, abort_if_nonempty=False)
pref_dst_fo = path_prefix(dst_folderpath)
assert create_parent_folders or folder_exists(pref_dst_fo)
create_folder(dst_folderpath, create_parent_folders=create_parent_folders)
# create all folders in the destination.
fos = list_folders(src_folderpath,
use_relative_paths=True,
recursive=True,
ignore_hidden_folders=ignore_hidden_folders)
for fo in fos:
fo_path = join_paths([dst_folderpath, fo])
create_folder(fo_path, create_parent_folders=True)
# copy all files to the destination.
kwargs = tb_ut.subset_dict_via_selection(
locals(),
['ignore_hidden_folders', 'ignore_hidden_files', 'ignore_file_exts'])
fis = list_files(src_folderpath,
use_relative_paths=True,
recursive=True,
**kwargs)
for fi in fis:
src_fip = join_paths([src_folderpath, fi])
dst_fip = join_paths([dst_folderpath, fi])
copy_file(src_fip, dst_fip)
def run_on_lithium_node(bash_command,
node,
servername,
username,
password=None,
visible_gpu_ids=None,
folderpath=None,
wait_for_output=True,
run_on_head_node=False):
# check that node exists.
assert node in tb_ut.flatten(get_lithium_nodes())
# prompting for password if asked about. (because lithium needs password)
if password == None:
password = getpass.getpass()
# if no visilbe gpu are specified, it creates a list with nothing there.
if visible_gpu_ids is None:
visible_gpu_ids = []
# creating the command to run remotely.
gpu_cmd = 'export CUDA_VISIBLE_DEVICES=%s' % ",".join(
map(str, visible_gpu_ids))
if not run_on_head_node:
cmd = "ssh -T %s \'%s && %s\'" % (node, gpu_cmd, bash_command)
else:
# NOTE: perhaps repetition could be improved here. also, probably head
# node does not have gpus.
cmd = "%s && %s" % (gpu_cmd, bash_command)
return run_on_server(
cmd,
**tb_ut.subset_dict_via_selection(locals(), [
'servername', 'username', 'password', 'folderpath',
'wait_for_output'
]))
def create_experiment_folder(
main_filepath,
argname_lst,
argval_lst_lst,
output_folderpath_argname,
all_experiments_folderpath,
readme,
experiment_name=None,
# entry_folderpath=None,
code_folderpath=None,
# data_folderpath=None,
capture_output=False,
profile_run=False):
assert tb_fs.folder_exists(all_experiments_folderpath)
assert experiment_name is None or (not tb_fs.path_exists(
tb_fs.join_paths([all_experiments_folderpath, experiment_name])))
# assert folder_exists(project_folderpath) and file_exists(tb_fs.join_paths([
# project_folderpath, main_relfilepath]))
# create the main folder where things for the experiment will be.
if experiment_name is None:
experiment_name = get_available_filename(all_experiments_folderpath,
"exp")
experiment_folderpath = tb_fs.join_paths(
[all_experiments_folderpath, experiment_name])
tb_fs.create_folder(experiment_folderpath)
# copy the code to the experiment folder.
if code_folderpath is not None:
code_foldername = tb_fs.path_last_element(code_folderpath)
dst_code_fo = tb_fs.join_paths(
[experiment_folderpath, code_foldername])
tb_fs.copy_folder(code_folderpath,
dst_code_fo,
ignore_hidden_files=True,
ignore_hidden_folders=True,
ignore_file_exts=['.pyc'])
# change main_filepath to use that new code.
main_filepath = tb_fs.join_paths(
[experiment_folderpath, main_filepath])
# NOTE: no data copying for now because it often does not make much sense.
data_folderpath = None ### TODO: remove later.
# # copy the code to the experiment folder.
# if data_folderpath is not None:
# data_foldername = path_last_element(data_folderpath)
# dst_data_fo = join_paths([experiment_folderpath, data_foldername])
# copy_folder(data_folderpath, dst_data_fo,
# ignore_hidden_files=True, ignore_hidden_folders=True)
# write the config for the experiment.
tb_io.write_jsonfile(
tb_ut.subset_dict_via_selection(locals(), [
'main_filepath', 'argname_lst', 'argval_lst_lst',
'output_folderpath_argname', 'all_experiments_folderpath',
'readme', 'experiment_name', 'code_folderpath', 'data_folderpath',
'capture_output', 'profile_run'
]), tb_fs.join_paths([experiment_folderpath, 'config.json']))
# generate the executables for each configuration.
argname_lst = list(argname_lst)
argname_lst.append(output_folderpath_argname)
for (i, vs) in enumerate(argval_lst_lst):
cfg_folderpath = tb_fs.join_paths([experiment_folderpath, "cfg%d" % i])
tb_fs.create_folder(cfg_folderpath)
# create the script
argvalue_lst = list(vs)
argvalue_lst.append(cfg_folderpath)
call_args = tb_ut.subset_dict_via_selection(
locals(), ['argname_lst', 'argvalue_lst', 'main_filepath'])
call_args['script_filepath'] = tb_fs.join_paths(
[cfg_folderpath, 'run.sh'])
if capture_output:
call_args['output_filepath'] = tb_fs.join_paths(
[cfg_folderpath, 'output.txt'])
if profile_run:
call_args['profile_filepath'] = tb_fs.join_paths(
[cfg_folderpath, 'profile.txt'])
create_run_script(**call_args)
# write a config file for each configuration
tb_io.write_jsonfile(tb_ut.create_dict(argname_lst, argvalue_lst),
tb_fs.join_paths([cfg_folderpath, 'config.json']))
# create_runall_script(experiment_folderpath)
create_runall_script_with_parallelization(experiment_folderpath)
return experiment_folderpath
def run(self, run_only_if_enough_resources_for_all=True):
args = tb_ut.subset_dict_via_selection(
vars(self), ['servername', 'username', 'password'])
args['abort_if_any_node_unavailable'] = False
# get the resource availability and filter out unavailable nodes.
d = get_lithium_resource_availability(**args)
d = {k: v for (k, v) in d.iteritems() if v is not None}
g = get_lithium_nodes()
# assignments to each of the registered jobs
run_cfgs = []
for x in self.jobs:
if x['require_nodes'] is not None:
req_nodes = x['require_nodes']
else:
req_nodes = d.keys()
# based on the gpu type restriction.
if x['require_gpu_types'] is not None:
req_gpu_nodes = tb_ut.flatten(
tb_ut.subset_dict_via_selection(g, x['require_gpu_types']))
else:
# NOTE: only consider the nodes that are available anyway.
req_gpu_nodes = d.keys()
# potentially available nodes to place this job.
nodes = list(set(req_nodes).intersection(req_gpu_nodes))
assert len(nodes) > 0
# greedy assigned to a node.
assigned = False
for n in nodes:
r = d[n]
# if there are enough resources on the node, assign it to the
# job.
if ((r['cpus_free'] >= x['num_cpus'])
and (r['gpus_free'] >= x['num_gpus']) and
(r['mem_mbs_free'] >= tb_rs.convert_between_byte_units(
x['mem_budget'],
src_units=x['mem_units'],
dst_units='megabytes'))):
# record information about where to run the job.
run_cfgs.append({
'node':
n,
'visible_gpu_ids':
r['free_gpu_ids'][:x['num_gpus']]
})
# deduct the allocated resources from the available resources
# for that node.
r['cpus_free'] -= x['num_cpus']
r['gpus_free'] -= x['num_gpus']
r['mem_mbs_free'] -= tb_rs.convert_between_byte_units(
x['mem_budget'],
src_units=x['mem_units'],
dst_units='megabytes')
r['free_gpu_ids'] = r['free_gpu_ids'][x['num_gpus']:]
# assigned = True
break
# if not assigned, terminate without doing anything.
if not assigned:
run_cfgs.append(None)
if run_only_if_enough_resources_for_all:
print("Insufficient resources to satisfy"
" (cpus=%d, gpus=%d, mem=%0.3f%s)" %
(x['num_cpus'], x['num_gpus'], x['mem_budget'],
x['mem_units']))
return None
# running the jobs that have a valid config.
remaining_jobs = []
outs = []
for x, c in zip(self.jobs, run_cfgs):
if c is None:
remaining_jobs.append(x)
else:
out = run_on_lithium_node(**tb_ut.merge_dicts([
tb_ut.subset_dict_via_selection(vars(
self), ['servername', 'username', 'password']),
tb_ut.subset_dict_via_selection(x, [
'bash_command', 'folderpath', 'wait_for_output',
'run_on_head_node'
]),
tb_ut.subset_dict_via_selection(
c, ['node', 'visible_gpu_ids'])
]))
outs.append(out)
self.jobs = remaining_jobs
return outs