def download_to_demo(fp):
"""
Args:
fp (str): file path relative to data root.
"""
demo_data_dir = args.demo_data_dir
create_if_not_exists(demo_data_dir)
s3_http_prefix = 'https://s3-us-west-1.amazonaws.com/mousebrainatlas-data/'
url = s3_http_prefix + fp
demo_fp = os.path.join(demo_data_dir, fp)
execute_command('wget -N -P \"%s\" \"%s\"' %
(os.path.dirname(demo_fp), url))
return demo_fp
def delete_file_or_directory(fp):
execute_command("rm -rf %s" % fp)
def run_distributed5(command,
argument_type='single',
kwargs_list=None,
jobs_per_node=1,
node_list=None,
local_only=False,
use_aws=False):
"""
Distributed executing a command.
Args:
local_only: run on local computer instead of AWS cluster
jobs_per_node:
kwargs_list: either dict of lists {kA: [vA1, vA2, ...], kB: [vB1, vB2, ...]} or list of dicts [{kA:vA1, kB:vB1}, {kA:vA2, kB:vB2}, ...].
argument_type: one of list, list2, single. If command takes one input item as argument, use "single". If command takes a list of input items as argument, use "list2". If command takes an argument called "kwargs_str", use "list".
"""
if use_aws:
execute_command(
'rm -f /home/ubuntu/stderr_*; rm -f /home/ubuntu/stdout_*')
else:
execute_command(
'rm -f %s; rm -f %s' %
(os.path.join(DATA_ROOTDIR, 'mousebrainatlas_tmp', 'stderr_*'),
os.path.join(DATA_ROOTDIR, 'mousebrainatlas_tmp', 'stdout_*')))
if local_only:
sys.stderr.write("Run locally.\n")
n_hosts = 1
else:
# Use a fixed node list rather than letting SGE automatically determine the node list.
# This allows for control over which input items go to which node.
if node_list is None:
node_list = get_node_list()
n_hosts = len(node_list)
sys.stderr.write('%d nodes available.\n' % (n_hosts))
if n_hosts == 0:
return
if kwargs_list is None:
kwargs_list = {'dummy': [None] * n_hosts}
if isinstance(kwargs_list, dict):
keys, vals = zip(*kwargs_list.items())
kwargs_list_as_list = [dict(zip(keys, t)) for t in zip(*vals)]
kwargs_list_as_dict = kwargs_list
else:
kwargs_list_as_list = kwargs_list
keys = kwargs_list[0].keys()
vals = [t.values() for t in kwargs_list]
kwargs_list_as_dict = dict(zip(keys, zip(*vals)))
assert argument_type in [
'single', 'list', 'list2'
], 'argument_type must be one of single, list, list2.'
create_if_not_exists(os.path.join(DATA_ROOTDIR, 'mousebrainatlas_tmp'))
for node_i, (fi, li) in enumerate(
first_last_tuples_distribute_over(0,
len(kwargs_list_as_list) - 1,
n_hosts)):
temp_script = os.path.join(DATA_ROOTDIR, 'mousebrainatlas_tmp',
'runall.sh')
temp_f = open(temp_script, 'w')
for j, (fj, lj) in enumerate(
first_last_tuples_distribute_over(fi, li, jobs_per_node)):
if argument_type == 'list':
line = command % {
'kwargs_str': json.dumps(kwargs_list_as_list[fj:lj + 1])
}
elif argument_type == 'list2':
line = command % {
key: json.dumps(vals[fj:lj + 1])
for key, vals in kwargs_list_as_dict.iteritems()
}
elif argument_type == 'single':
# It is important to wrap command_templates and kwargs_list_str in apostrphes.
# That lets bash treat them as single strings.
# Reference: http://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-in-bash-how-do-we-know-it
line = "%(generic_launcher_path)s %(command_template)s %(kwargs_list_str)s" % \
{'generic_launcher_path': os.path.join(os.environ['REPO_DIR'], 'utilities', 'sequential_dispatcher.py'),
'command_template': shell_escape(command),
'kwargs_list_str': shell_escape(json.dumps(kwargs_list_as_list[fj:lj+1]))
}
temp_f.write(line + ' &\n')
temp_f.write('wait')
temp_f.close()
os.chmod(temp_script, 0o777)
# Explicitly specify the node to submit jobs.
# By doing so, we can control which files are available in the local scratch space of which node.
# One can then assign downstream programs to specific nodes so they can read corresponding files from local scratch.
if use_aws:
stdout_template = '/home/ubuntu/stdout_%d.log'
stderr_template = '/home/ubuntu/stderr_%d.log'
else:
stdout_template = os.path.join(DATA_ROOTDIR, 'mousebrainatlas_tmp',
'stdout_%d.log')
stderr_template = os.path.join(DATA_ROOTDIR, 'mousebrainatlas_tmp',
'stderr_%d.log')
if local_only:
stdout_f = open(stdout_template % node_i, "w")
stderr_f = open(stderr_template % node_i, "w")
call(temp_script, shell=True, stdout=stdout_f, stderr=stderr_f)
else:
call('qsub -V -q all.q@%(node)s -o %(stdout_log)s -e %(stderr_log)s %(script)s' % \
dict(node=node_list[node_i], script=temp_script,
stdout_log=stdout_template % node_i, stderr_log=stderr_template % node_i),
shell=True)
sys.stderr.write(
'Jobs submitted. Use wait_qsub_complete() to wait for all execution to finish.\n'
)
def transfer_data(from_fp,
to_fp,
from_hostname,
to_hostname,
is_dir,
include_only=None,
exclude_only=None,
includes=None):
assert from_hostname in [
'localhost', 'workstation', 'oasis', 's3', 'ec2', 's3raw', 'ec2scratch'
], 'from_hostname must be one of localhost, workstation, oasis, s3, s3raw, ec2 or ec2scratch.'
assert to_hostname in [
'localhost', 'workstation', 'oasis', 's3', 'ec2', 's3raw', 'ec2scratch'
], 'to_hostname must be one of localhost, workstation, oasis, s3, s3raw, ec2 or ec2scratch.'
to_parent = os.path.dirname(to_fp)
t = time.time()
if from_hostname in ['localhost', 'ec2', 'workstation', 'ec2scratch']:
# upload
if to_hostname in ['s3', 's3raw']:
if is_dir:
if includes is not None:
execute_command(
'aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --exclude \"*\" %(includes_str)s'
% dict(from_fp=from_fp,
to_fp=to_fp,
includes_str=" ".join(
['--include ' + incl
for incl in includes])))
elif include_only is not None:
execute_command(
'aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --exclude \"*\" --include \"%(include)s\"'
% dict(from_fp=from_fp,
to_fp=to_fp,
include=include_only))
elif exclude_only is not None:
execute_command(
'aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --include \"*\" --exclude \"%(exclude)s\"'
% dict(from_fp=from_fp,
to_fp=to_fp,
exclude=exclude_only))
else:
execute_command('aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\"' % \
dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command('aws s3 cp \"%(from_fp)s\" \"s3://%(to_fp)s\"' % \
dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command("ssh %(to_hostname)s 'rm -rf \"%(to_fp)s\" && mkdir -p \"%(to_parent)s\"' && scp -r \"%(from_fp)s\" %(to_hostname)s:\"%(to_fp)s\"" % \
dict(from_fp=from_fp, to_fp=to_fp, to_hostname=to_hostname, to_parent=to_parent))
elif to_hostname in ['localhost', 'ec2', 'workstation', 'ec2scratch']:
# download
if from_hostname in ['s3', 's3raw']:
# Clear existing folder/file
if not include_only and not includes and not exclude_only:
execute_command(
'rm -rf \"%(to_fp)s\" && mkdir -p \"%(to_parent)s\"' %
dict(to_parent=to_parent, to_fp=to_fp))
# Download from S3 using aws commandline interface.
if is_dir:
if includes is not None:
execute_command(
'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --exclude \"*\" %(includes_str)s'
% dict(from_fp=from_fp,
to_fp=to_fp,
includes_str=" ".join(
['--include ' + incl
for incl in includes])))
elif include_only is not None:
execute_command(
'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --exclude \"*\" --include \"%(include)s\"'
% dict(from_fp=from_fp,
to_fp=to_fp,
include=include_only))
elif exclude_only is not None:
execute_command(
'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --include \"*\" --exclude \"%(exclude)s\"'
% dict(from_fp=from_fp,
to_fp=to_fp,
exclude=exclude_only))
else:
execute_command(
'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\"'
% dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command(
'aws s3 cp \"s3://%(from_fp)s\" \"%(to_fp)s\"' %
dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command(
"scp -r %(from_hostname)s:\"%(from_fp)s\" \"%(to_fp)s\"" %
dict(from_fp=from_fp, to_fp=to_fp,
from_hostname=from_hostname))
else:
# log onto another machine and perform upload from there.
execute_command("ssh %(from_hostname)s \"ssh %(to_hostname)s \'rm -rf \"%(to_fp)s\" && mkdir -p %(to_parent)s && scp -r \"%(from_fp)s\" %(to_hostname)s:\"%(to_fp)s\"\'\"" % \
dict(from_fp=from_fp, to_fp=to_fp, from_hostname=from_hostname, to_hostname=to_hostname, to_parent=to_parent))
sys.stderr.write(output_fn + '\n')
T2 = transform.copy()
T2[:2, 2] = transform[:2, 2] * scale_factor
T = np.linalg.inv(T2)
d = {
'sx': T[0, 0],
'sy': T[1, 1],
'rx': T[1, 0],
'ry': T[0, 1],
'tx': T[0, 2],
'ty': T[1, 2],
'input_fn': os.path.join(input_dir, filename),
'output_fn': os.path.join(output_dir, output_fn),
'x': '+' + str(x * scale_factor) if int(x) >= 0 else str(x * scale_factor),
'y': '+' + str(y * scale_factor) if int(y) >= 0 else str(y * scale_factor),
'w': str(w * scale_factor),
'h': str(h * scale_factor),
}
sys.stderr.write("Background color: %s\n" % background_color)
if background_color == 'black':
execute_command(
"convert \"%(input_fn)s\" -virtual-pixel background -background black +distort AffineProjection '%(sx)f,%(rx)f,%(ry)f,%(sy)f,%(tx)f,%(ty)f' -crop %(w)sx%(h)s%(x)s%(y)s\! -flatten -compress lzw \"%(output_fn)s\""
% d)
elif background_color == 'white':
execute_command(
"convert \"%(input_fn)s\" -virtual-pixel background -background white +distort AffineProjection '%(sx)f,%(rx)f,%(ry)f,%(sy)f,%(tx)f,%(ty)f' -crop %(w)sx%(h)s%(x)s%(y)s\! -flatten -compress lzw \"%(output_fn)s\""
% d)
# elif suffix == 'lossy':
# scale_factor = 16
elif suffix == 'raw':
scale_factor = 32
else:
raise "Unknown resolution: %s" % suffix
T2 = transform.copy()
T2[:2,2] = transform[:2, 2] * scale_factor
T = np.linalg.inv(T2)
create_parent_dir_if_not_exists(output_fp)
execute_command("convert \"%(input_fp)s\" %(init_rotate)s +repage -virtual-pixel background -background %(bg_color)s +distort AffineProjection '%(sx)f,%(rx)f,%(ry)f,%(sy)f,%(tx)f,%(ty)f' -crop %(w)sx%(h)s%(x)s%(y)s\! -flatten -compress lzw \"%(output_fp)s\"" % \
{'init_rotate':init_rotate,
'sx':T[0,0],
'sy':T[1,1],
'rx':T[1,0],
'ry':T[0,1],
'tx':T[0,2],
'ty':T[1,2],
'input_fp': input_fp,
'output_fp': output_fp,
'x': '+' + str(x_tb * scale_factor) if int(x_tb) >= 0 else str(x_tb * scale_factor),
'y': '+' + str(y_tb * scale_factor) if int(y_tb) >= 0 else str(y_tb * scale_factor),
'w': str(w_tb * scale_factor),
'h': str(h_tb * scale_factor),
'bg_color': background_color
})
# output_fn = args.output_fn
#input_fp = os.path.join(input_dir, filename)
input_fp = args.input_fp
output_fp = args.output_fp
#basename = os.path.splitext(os.path.basename(filename))[0]
#blue_fp = os.path.join(output_dir, basename + '_blue.tif')
#output_fp = os.path.join(output_dir, basename + '_blueasgray.tif')
# Cannot chain: final output is not grayscale for some reason.
#execute_command('convert %(input_fp)s -channel B -separate %(blue_fp)s' % \
# dict(input_fp=input_fp, blue_fp=blue_fp))
#execute_command('convert %(blue_fp)s -negate %(output_fp)s' % \
# dict(blue_fp=blue_fp, output_fp=output_fp))
#execute_command('rm %(blue_fp)s' % dict(blue_fp=blue_fp))
from distributed_utilities import *
import tempfile
fd, tmp_fp = tempfile.mkstemp()
try:
# Cannot chain: final output is not grayscale for some reason.
download_from_s3(input_fp)
execute_command('convert %(input_fp)s -channel B -separate %(blue_fp)s' % \
dict(input_fp=input_fp, blue_fp=tmp_fp))
execute_command('convert %(blue_fp)s -negate %(output_fp)s' % \
dict(blue_fp=tmp_fp, output_fp=output_fp))
upload_to_s3(output_fp)
finally:
os.remove(tmp_fp)
scale_factor = 1
# elif suffix == 'lossy':
# scale_factor = 16
elif suffix == 'raw':
scale_factor = 32
else:
raise "Unknown resolution: %s" % suffix
T2 = transform.copy()
T2[:2,2] = transform[:2, 2] * scale_factor
T = np.linalg.inv(T2)
create_parent_dir_if_not_exists(output_fp)
execute_command("convert \"%(input_fp)s\" %(init_rotate)s +repage -virtual-pixel background -background %(bg_color)s +distort AffineProjection '%(sx)f,%(rx)f,%(ry)f,%(sy)f,%(tx)f,%(ty)f' -crop %(w)sx%(h)s%(x)s%(y)s\! -flatten -compress lzw \"%(output_fp)s\"" % \
{'init_rotate':init_rotate,
'sx':T[0,0],
'sy':T[1,1],
'rx':T[1,0],
'ry':T[0,1],
'tx':T[0,2],
'ty':T[1,2],
'input_fp': input_fp,
'output_fp': output_fp,
'x': '+' + str(x_tb * scale_factor) if int(x_tb) >= 0 else str(x_tb * scale_factor),
'y': '+' + str(y_tb * scale_factor) if int(y_tb) >= 0 else str(y_tb * scale_factor),
'w': str(w_tb * scale_factor),
'h': str(h_tb * scale_factor),
'bg_color': background_color
})
# output_fn = args.output_fn
#input_fp = os.path.join(input_dir, filename)
input_fp = args.input_fp
output_fp = args.output_fp
#basename = os.path.splitext(os.path.basename(filename))[0]
#blue_fp = os.path.join(output_dir, basename + '_blue.tif')
#output_fp = os.path.join(output_dir, basename + '_blueasgray.tif')
# Cannot chain: final output is not grayscale for some reason.
#execute_command('convert %(input_fp)s -channel B -separate %(blue_fp)s' % \
# dict(input_fp=input_fp, blue_fp=blue_fp))
#execute_command('convert %(blue_fp)s -negate %(output_fp)s' % \
# dict(blue_fp=blue_fp, output_fp=output_fp))
#execute_command('rm %(blue_fp)s' % dict(blue_fp=blue_fp))
from distributed_utilities import *
import tempfile
fd, tmp_fp = tempfile.mkstemp()
try:
# Cannot chain: final output is not grayscale for some reason.
download_from_s3(input_fp)
execute_command('convert %(input_fp)s -channel B -separate %(blue_fp)s' % \
dict(input_fp=input_fp, blue_fp=tmp_fp))
execute_command('convert %(blue_fp)s -negate %(output_fp)s' % \
dict(blue_fp=tmp_fp, output_fp=output_fp))
upload_to_s3(output_fp)
finally:
os.remove(tmp_fp)
from utilities2015 import execute_command
import argparse
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description='Neurotrace blue to nissl')
parser.add_argument("input_dir", type=str, help="input dir")
parser.add_argument("output_dir", type=str, help="output dir")
parser.add_argument("filename", type=str, help="full filename, with extension")
# parser.add_argument("output_fn", type=str, help="output filename")
args = parser.parse_args()
input_dir = args.input_dir
filename = args.filename
output_dir = args.output_dir
# output_fn = args.output_fn
input_fp = os.path.join(input_dir, filename)
basename = os.path.splitext(os.path.basename(filename))[0]
blue_fp = os.path.join(output_dir, basename + '_blue.tif')
output_fp = os.path.join(output_dir, basename + '_blueAsGrayscale.tif')
# Cannot chain: final output is not grayscale for some reason.
execute_command('convert %(input_fp)s -channel B -separate %(blue_fp)s' % \
dict(input_fp=input_fp, blue_fp=blue_fp))
execute_command('convert %(blue_fp)s -negate -contrast-stretch %%1x%%1 -depth 8 %(output_fp)s' % \
dict(blue_fp=blue_fp, output_fp=output_fp))
execute_command('rm %(blue_fp)s' % dict(blue_fp=blue_fp))
def transfer_data(from_fp, to_fp, from_hostname, to_hostname, is_dir, include_only=None, exclude_only=None, includes=None):
assert from_hostname in ['localhost', 'workstation', 'oasis', 's3', 'ec2', 's3raw', 'ec2scratch'], 'from_hostname must be one of localhost, workstation, oasis, s3, s3raw, ec2 or ec2scratch.'
assert to_hostname in ['localhost', 'workstation', 'oasis', 's3', 'ec2', 's3raw', 'ec2scratch'], 'to_hostname must be one of localhost, workstation, oasis, s3, s3raw, ec2 or ec2scratch.'
to_parent = os.path.dirname(to_fp)
t = time.time()
if from_hostname in ['localhost', 'ec2', 'workstation', 'ec2scratch']:
# upload
if to_hostname in ['s3', 's3raw']:
if is_dir:
if includes is not None:
execute_command('aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --exclude \"*\" %(includes_str)s' % dict(from_fp=from_fp, to_fp=to_fp, includes_str=" ".join(['--include ' + incl for incl in includes])))
elif include_only is not None:
execute_command('aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --exclude \"*\" --include \"%(include)s\"' % dict(from_fp=from_fp, to_fp=to_fp, include=include_only))
elif exclude_only is not None:
execute_command('aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --include \"*\" --exclude \"%(exclude)s\"' % dict(from_fp=from_fp, to_fp=to_fp, exclude=exclude_only))
else:
execute_command('aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\"' % \
dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command('aws s3 cp \"%(from_fp)s\" \"s3://%(to_fp)s\"' % \
dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command("ssh %(to_hostname)s 'rm -rf \"%(to_fp)s\" && mkdir -p \"%(to_parent)s\"' && scp -r \"%(from_fp)s\" %(to_hostname)s:\"%(to_fp)s\"" % \
dict(from_fp=from_fp, to_fp=to_fp, to_hostname=to_hostname, to_parent=to_parent))
elif to_hostname in ['localhost', 'ec2', 'workstation', 'ec2scratch']:
# download
if from_hostname in ['s3', 's3raw']:
# Clear existing folder/file
if not include_only and not includes and not exclude_only:
execute_command('rm -rf \"%(to_fp)s\" && mkdir -p \"%(to_parent)s\"' % dict(to_parent=to_parent, to_fp=to_fp))
# Download from S3 using aws commandline interface.
if is_dir:
if includes is not None:
execute_command('aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --exclude \"*\" %(includes_str)s' % dict(from_fp=from_fp, to_fp=to_fp, includes_str=" ".join(['--include ' + incl for incl in includes])))
elif include_only is not None:
execute_command('aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --exclude \"*\" --include \"%(include)s\"' % dict(from_fp=from_fp, to_fp=to_fp, include=include_only))
elif exclude_only is not None:
execute_command('aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --include \"*\" --exclude \"%(exclude)s\"' % dict(from_fp=from_fp, to_fp=to_fp, exclude=exclude_only))
else:
execute_command('aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\"' % dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command('aws s3 cp \"s3://%(from_fp)s\" \"%(to_fp)s\"' % dict(from_fp=from_fp, to_fp=to_fp))
else:
execute_command("scp -r %(from_hostname)s:\"%(from_fp)s\" \"%(to_fp)s\"" % dict(from_fp=from_fp, to_fp=to_fp, from_hostname=from_hostname))
else:
# log onto another machine and perform upload from there.
execute_command("ssh %(from_hostname)s \"ssh %(to_hostname)s \'rm -rf \"%(to_fp)s\" && mkdir -p %(to_parent)s && scp -r \"%(from_fp)s\" %(to_hostname)s:\"%(to_fp)s\"\'\"" % \
dict(from_fp=from_fp, to_fp=to_fp, from_hostname=from_hostname, to_hostname=to_hostname, to_parent=to_parent))
def delete_file_or_directory(fp):
execute_command("rm -rf %s" % fp)
def run_distributed5(command, argument_type='single', kwargs_list=None, jobs_per_node=1, node_list=None, local_only=False, use_aws=False):
"""
Distributed executing a command.
Args:
local_only: run on local computer instead of AWS cluster
jobs_per_node:
kwargs_list: either dict of lists {kA: [vA1, vA2, ...], kB: [vB1, vB2, ...]} or list of dicts [{kA:vA1, kB:vB1}, {kA:vA2, kB:vB2}, ...].
argument_type: one of list, list2, single. If command takes one input item as argument, use "single". If command takes a list of input items as argument, use "list2". If command takes an argument called "kwargs_str", use "list".
"""
if use_aws:
execute_command('rm -f /home/ubuntu/stderr_*; rm -f /home/ubuntu/stdout_*')
else:
execute_command('rm -f /tmp/stderr_*; rm -f /tmp/stdout_*')
if local_only:
sys.stderr.write("Run locally.\n")
n_hosts = 1
else:
# Use a fixed node list rather than letting SGE automatically determine the node list.
# This allows for control over which input items go to which node.
if node_list is None:
node_list = get_node_list()
n_hosts = len(node_list)
sys.stderr.write('%d nodes available.\n' % (n_hosts))
if n_hosts == 0:
return
if kwargs_list is None:
kwargs_list = {'dummy': [None]*n_hosts}
if isinstance(kwargs_list, dict):
keys, vals = zip(*kwargs_list.items())
kwargs_list_as_list = [dict(zip(keys, t)) for t in zip(*vals)]
kwargs_list_as_dict = kwargs_list
else:
kwargs_list_as_list = kwargs_list
keys = kwargs_list[0].keys()
vals = [t.values() for t in kwargs_list]
kwargs_list_as_dict = dict(zip(keys, zip(*vals)))
assert argument_type in ['single', 'list', 'list2'], 'argument_type must be one of single, list, list2.'
for node_i, (fi, li) in enumerate(first_last_tuples_distribute_over(0, len(kwargs_list_as_list)-1, n_hosts)):
temp_script = '/tmp/runall.sh'
temp_f = open(temp_script, 'w')
for j, (fj, lj) in enumerate(first_last_tuples_distribute_over(fi, li, jobs_per_node)):
if argument_type == 'list':
line = command % {'kwargs_str': json.dumps(kwargs_list_as_list[fj:lj+1])}
elif argument_type == 'list2':
line = command % {key: json.dumps(vals[fj:lj+1]) for key, vals in kwargs_list_as_dict.iteritems()}
elif argument_type == 'single':
# It is important to wrap command_templates and kwargs_list_str in apostrphes.
# That lets bash treat them as single strings.
# Reference: http://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-in-bash-how-do-we-know-it
line = "%(generic_launcher_path)s %(command_template)s %(kwargs_list_str)s" % \
{'generic_launcher_path': os.path.join(os.environ['REPO_DIR'], 'utilities', 'sequential_dispatcher.py'),
'command_template': shell_escape(command),
'kwargs_list_str': shell_escape(json.dumps(kwargs_list_as_list[fj:lj+1]))
}
temp_f.write(line + ' &\n')
temp_f.write('wait')
temp_f.close()
os.chmod(temp_script, 0o777)
# Explicitly specify the node to submit jobs.
# By doing so, we can control which files are available in the local scratch space of which node.
# One can then assign downstream programs to specific nodes so they can read corresponding files from local scratch.
if use_aws:
stdout_template = '/home/ubuntu/stdout_%d.log'
stderr_template = '/home/ubuntu/stderr_%d.log'
else:
stdout_template = '/tmp/stdout_%d.log'
stderr_template = '/tmp/stderr_%d.log'
if local_only:
stdout_f = open(stdout_template % node_i, "w")
stderr_f = open(stderr_template % node_i, "w")
call(temp_script, shell=True, stdout=stdout_f, stderr=stderr_f)
else:
call('qsub -V -q all.q@%(node)s -o %(stdout_log)s -e %(stderr_log)s %(script)s' % \
dict(node=node_list[node_i], script=temp_script,
stdout_log=stdout_template % node_i, stderr_log=stderr_template % node_i),
shell=True)
sys.stderr.write('Jobs submitted. Use wait_qsub_complete() to wait for all execution to finish.\n')
#!/usr/bin/env python
import argparse
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=
"""Align consecutive images. Possible bad alignment pairs are written into a separate file.
Usage 1: align_compose.py in.ini --op from_none_to_aligned
""")
parser.add_argument("input_spec", type=str, help="input specifier. ini")
parser.add_argument("--op", type=str, help="operation id")
args = parser.parse_args()
import os
import sys
sys.path.append(os.environ['REPO_DIR'] + '/utilities')
from utilities2015 import execute_command
execute_command('python align_v3.py %s --op %s' % (args.input_spec, args.op))
execute_command('python compose_v3.py %s --op %s' % (args.input_spec, args.op))
