def dlg_unroll(parser, args):
# Unroll Logical Graph
tool.add_logging_options(parser)
_add_output_options(parser)
apps = _add_unroll_options(parser)
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
dump = _setup_output(opts)
pgt = unroll(opts.lg_path,
opts.oid_prefix,
zerorun=opts.zerorun,
app=apps[opts.app])
dump(init_pgt_unroll_repro_data(pgt))
def dlg_fill(parser, args):
tool.add_logging_options(parser)
_add_output_options(parser)
parser.add_option('-L',
'--logical-graph',
default='-',
help="Path to the Logical Graph (default: stdin)")
parser.add_option(
'-p',
'--parameter',
action='append',
help="Parameter specification (either 'name=value' or a JSON string)",
default=[])
parser.add_option(
'-R',
'--reproducibility',
default='0',
help="Level of reproducibility. Default 0 (NOTHING). Accepts '0'-'5'")
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
dump = _setup_output(opts)
def param_spec_type(s):
if s.startswith('{'):
return 'json'
elif '=' in s:
return 'kv'
else:
return None
# putting all parameters together in a single dictionary
for p in opts.parameter:
if param_spec_type(p) is None:
parser.error(
'Parameter %s is neither JSON nor has it key=value form' % p)
params = [
p.split('=') for p in opts.parameter if param_spec_type(p) == 'kv'
]
params = dict(params)
for json_param in (json.loads(p) for p in opts.parameter
if param_spec_type(p) == 'json'):
params.update(json_param)
from ..dropmake.pg_generator import fill
graph = fill(_open_i(opts.logical_graph), params)
dump(init_lg_repro_data(init_lgt_repro_data(graph, opts.reproducibility)))
def dlg_unroll_and_partition(parser, args):
tool.add_logging_options(parser)
_add_output_options(parser)
apps = _add_unroll_options(parser)
_add_partition_options(parser)
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
dump = _setup_output(opts)
pgt = unroll(opts.lg_path,
opts.oid_prefix,
zerorun=opts.zerorun,
app=apps[opts.app])
init_pgt_unroll_repro_data(pgt)
repro = pgt.pop() # TODO: Re-integrate
pgt = partition(pgt, opts)
pgt.append(repro)
dump(init_pgt_partition_repro_data(pgt))
def dlg_partition(parser, args):
tool.add_logging_options(parser)
_add_output_options(parser)
_add_partition_options(parser)
parser.add_option(
'-P',
'--physical-graph-template',
action='store',
dest='pgt_path',
type='string',
help='Path to the Physical Graph Template (default: stdin)',
default='-')
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
dump = _setup_output(opts)
with _open_i(opts.pgt_path) as fi:
pgt = json.load(fi)
repro = pgt.pop() # TODO: Re-integrate
pgt = partition(pgt, opts)
pgt.append(repro)
dump(init_pgt_partition_repro_data(pgt))
def cwl(parser, args):
tool.add_logging_options(parser)
_add_output_options(parser)
parser.add_option(
'-P',
'--physical-graph-template',
action='store',
dest='pgt_path',
type='string',
help='Path to the Physical Graph Template (default: stdin)',
default='-')
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
# load the pgt
with _open_i(opts.pgt_path) as fi:
pgt = json.load(fi)
# create the CWL workflow
from ..dropmake.cwl import create_workflow
# write to file
with _open_o(opts.output, "wb") as f:
create_workflow(pgt, "workflow.cwl", f)
def dlg_submit(parser, args):
import dlg.constants as con
# Submit Physical Graph
_add_output_options(parser)
tool.add_logging_options(parser)
parser.add_option('-H',
'--host',
action='store',
dest='host',
help='The host we connect to to deploy the graph',
default='localhost')
parser.add_option("-p",
"--port",
action="store",
type="int",
dest='port',
help='The port we connect to to deploy the graph',
default=con.ISLAND_DEFAULT_REST_PORT)
parser.add_option(
'-P',
'--physical-graph',
action='store',
dest='pg_path',
type='string',
help='Path to the Physical Graph to submit (default: stdin)',
default='-')
parser.add_option('-s',
'--session-id',
action='store',
dest='session_id',
type='string',
help='Session ID (default: <pg_name>-<current-time>)',
default=None)
parser.add_option('-S',
'--skip-deploy',
action='store_true',
dest='skip_deploy',
help='Skip the deployment step (default: False)',
default=False)
parser.add_option(
'-w',
'--wait',
action='store_true',
help='Wait for the graph execution to finish (default: False)',
default=False)
parser.add_option(
'-i',
'--poll-interval',
type='float',
help='Polling interval used for monitoring the execution (default: 10)',
default=10)
parser.add_option(
'-R',
'--reproducibility',
action='store_true',
dest='reproducibility',
help=
'Fetch (and output) reproducibility data for the final execution graph (default: False)'
)
(opts, args) = parser.parse_args(args)
with _open_i(opts.pg_path) as f:
pg = json.load(f)
repro = pg[-1]
submit(pg, opts)
pg.append(repro)
def dlg_map(parser, args):
import dlg.constants as con
tool.add_logging_options(parser)
_add_output_options(parser)
parser.add_option('-H',
'--host',
action='store',
dest='host',
help='The host we connect to to deploy the graph',
default='localhost')
parser.add_option("-p",
"--port",
action="store",
type="int",
dest='port',
help='The port we connect to to deploy the graph',
default=con.ISLAND_DEFAULT_REST_PORT)
parser.add_option(
'-P',
'--physical-graph-template',
action='store',
dest='pgt_path',
type='string',
help='Path to the Physical Graph to submit (default: stdin)',
default='-')
parser.add_option(
"-N",
"--nodes",
action="store",
dest="nodes",
help=
"The nodes where the Physical Graph will be distributed, comma-separated",
default=None)
parser.add_option("-i",
"--islands",
action="store",
type="int",
dest="islands",
help="Number of islands to use during the partitioning",
default=1)
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
dump = _setup_output(opts)
from ..dropmake import pg_generator
from dlg.clients import CompositeManagerClient
if opts.nodes:
nodes = [n for n in opts.nodes.split(',') if n]
else:
client = CompositeManagerClient(opts.host, opts.port, timeout=10)
nodes = [opts.host] + client.nodes()
n_nodes = len(nodes)
if n_nodes <= opts.islands:
raise Exception(
"#nodes (%d) should be bigger than number of islands (%d)" %
(n_nodes, opts.islands))
with _open_i(opts.pgt_path) as f:
pgt = json.load(f)
repro = pgt.pop() # TODO: Re-include
pg = pg_generator.resource_map(pgt, nodes, opts.islands)
pg.append(repro)
dump(init_pg_repro_data(pg))