def make(job_list, context, cq,
echo=DefaultsToConfig('echo'),
new_process=DefaultsToConfig('new_process'),
recurse=DefaultsToConfig('recurse')):
"""
Makes selected targets; or all targets if none specified.
Options:
make recurse=1 Recursive make: put generated jobs in the
queue.
make new_process=1 Run the jobs in a new Python process.
make echo=1 Displays the stdout/stderr for the job on
the console.
make new_process=1 echo=1 Not supported yet.
"""
db = context.get_compmake_db()
if not job_list:
job_list = list(top_targets(db=db))
manager = ManagerLocal(context=context, cq=cq,
recurse=recurse, new_process=new_process, echo=echo)
manager.add_targets(job_list)
manager.process()
return raise_error_if_manager_failed(manager)
def make(job_list,
context,
cq,
echo=DefaultsToConfig('echo'),
new_process=DefaultsToConfig('new_process'),
recurse=DefaultsToConfig('recurse')):
"""
Makes selected targets; or all targets if none specified.
Options:
make recurse=1 Recursive make: put generated jobs in the
queue.
make new_process=1 Run the jobs in a new Python process.
make echo=1 Displays the stdout/stderr for the job on
the console.
make new_process=1 echo=1 Not supported yet.
"""
db = context.get_compmake_db()
if not job_list:
job_list = list(top_targets(db=db))
manager = ManagerLocal(context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
echo=echo)
manager.add_targets(job_list)
manager.process()
return raise_error_if_manager_failed(manager)
def cloudmake(job_list, context, cq,
n=DefaultsToConfig('multyvac_max_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo'),
skipsync=False,
rdb=True):
"""
Multyvac backend
"""
# TODO: check it exists
# noinspection PyUnresolvedReferences
import multyvac
disable_logging_if_config(context)
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
volumes = sync_data_up(context, skipsync)
if rdb:
rdb_vol, rdb_db = synchronize_db_up(context, job_list)
else:
rdb_vol, rdb_db = None, None
publish(context, 'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = MVacManager(num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo,
volumes=volumes,
rdb=rdb,
rdb_vol=rdb_vol,
rdb_db=rdb_db,
)
publish(context, 'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
if not skipsync:
sync_data_down(context)
return raise_error_if_manager_failed(manager)
def parmake(job_list,
context,
cq,
n=DefaultsToConfig('max_parallel_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo')):
"""
Parallel equivalent of make.
Uses multiprocessing.Process as a backend and a Python queue to
communicate with the workers.
Options:
parmake n=10 Uses 10 workers
parmake recurse=1 Recursive make: put generated jobs in the
queue.
parmake new_process=1 Run the jobs in a new Python process.
parmake echo=1 Shows the output of the jobs. This might
slow down everything.
parmake new_process=1 echo=1 Not supported yet.
"""
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
publish(context,
'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = PmakeManager(num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo)
publish(context,
'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
return raise_error_if_manager_failed(manager)
def parmake(job_list, context, cq,
n=DefaultsToConfig('max_parallel_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo')):
"""
Parallel equivalent of make.
Uses multiprocessing.Process as a backend and a Python queue to
communicate with the workers.
Options:
parmake n=10 Uses 10 workers
parmake recurse=1 Recursive make: put generated jobs in the
queue.
parmake new_process=1 Run the jobs in a new Python process.
parmake echo=1 Shows the output of the jobs. This might
slow down everything.
parmake new_process=1 echo=1 Not supported yet.
"""
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
publish(context, 'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = PmakeManager(num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo)
publish(context, 'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
return raise_error_if_manager_failed(manager)
def make(job_list):
'''Makes selected targets; or all targets if none specified. '''
job_list = list(job_list)
if not job_list:
job_list = list(top_targets())
#print "Making %d jobs" % len(job_list)
manager = ManagerLocal()
manager.add_targets(job_list)
manager.process()
if manager.failed:
return RET_CODE_JOB_FAILED
else:
return 0
def parmake(job_list, n=None):
'''Parallel equivalent of "make".
Usage:
parmake [n=<num>] [joblist]
'''
job_list = list(job_list)
if not job_list:
job_list = list(top_targets())
manager = MultiprocessingManager(n)
manager.add_targets(job_list, more=False)
manager.process()
if manager.failed:
return RET_CODE_JOB_FAILED
else:
return 0
def clustmake(job_list):
'''Cluster equivalent of "make".
Note: you should use the Redis backend to use multiprocessing.
'''
job_list = list(job_list)
if not job_list:
job_list = list(top_targets())
cluster_conf = compmake_config.cluster_conf #@UndefinedVariable
if not os.path.exists(cluster_conf):
raise UserError('Configuration file "%s" does not exist.' % cluster_conf)
hosts = parse_yaml_configuration(open(cluster_conf))
manager = ClusterManager(hosts)
manager.add_targets(job_list)
manager.process()
if manager.failed:
return RET_CODE_JOB_FAILED
else:
return 0
def graph(job_list, filename='compmake', compact=0,
filter='dot', format='png'):
'''Creates a graph of the given targets and dependencies
graph filename=filename compact=0,1 format=png,...
Params:
filename: name of generated filename in the dot format
compact=0: whether to include the job names in the nodes
filter=[dot,circo,twopi,...] which algorithm to use to arrange
the nodes. This depends on the topology of your
computation. The default is 'dot'
(hierarchy top-bottom).
format=[png,...] The output file format.
'''
if not job_list:
job_list = top_targets()
job_list = tree(job_list)
try:
import gvgen #@UnresolvedImport
except:
gvgen_url = 'http://software.inl.fr/trac/wiki/GvGen'
raise UserError('To use the "graph" command' +
' you have to install the "gvgen" package from %s' %
gvgen_url)
graph = gvgen.GvGen()
state2color = {
Cache.NOT_STARTED: 'grey',
Cache.IN_PROGRESS: 'yellow',
Cache.MORE_REQUESTED: 'blue',
Cache.FAILED: 'red',
Cache.DONE: 'green'
}
job2node = {}
for job_id in job_list:
if int(compact):
job2node[job_id] = graph.newItem("")
else:
job2node[job_id] = graph.newItem(job_id)
cache = get_job_cache(job_id)
graph.styleAppend(job_id, "style", "filled")
graph.styleAppend(job_id, "fillcolor", state2color[cache.state])
graph.styleApply(job_id, job2node[job_id])
for job_id in job_list:
#c = get_computation(job_id)
#children_id = [x.job_id for x in c.depends]
for child in direct_children(job_id):
graph.newLink(job2node[job_id], job2node[child])
# TODO: add check?
with open(filename, 'w') as f:
graph.dot(f)
output = filename + '.' + format
cmd_line = '%s %s -T%s -o%s' % (filter, filename, format, output)
try:
os.system(cmd_line)
except:
raise UserError("Could not run dot (cmdline='%s')\
Make sure graphviz is installed" % cmd_line) # XXX maybe not UserError
info("Written output on files %s, %s." % (filename, output))
def graph(job_list, context, filename='compmake-graph',
filter='dot', format='png', # @ReservedAssignment
label='id', color=True,
cluster=False, processing=set()):
"""
Creates a graph of the given targets and dependencies.
Usage:
@: graph filename=filename label=[id,function,none] color=[0|1] format=png filter=[dot|circo|...]
Options:
filename: name of generated filename in the dot format
label='none','id','function'
color=[0|1]: whether to color the nodes
filter=[dot,circo,twopi,...] which algorithm to use to arrange
the nodes. The best choice depends on
the topology of your
computation. The default is 'dot'
(hierarchy top-bottom).
format=[png,...] The output file format.
"""
possible = ['none', 'id', 'function']
if not label in possible:
msg = 'Invalid label method %r not in %r.' % (label, possible)
raise ValueError(msg)
db = context.get_compmake_db()
if not job_list:
job_list = list(top_targets(db))
print('jobs: %s' % job_list)
print('processing: %s' % processing)
print('Importing gvgen')
try:
# import gvgen
pass
except:
gvgen_url = 'https://github.com/stricaud/gvgen'
msg = ('To use the "graph" command you have to install the "gvgen" '
'package from %s') % gvgen_url
raise UserError(msg)
print('Getting all jobs in tree')
cq = CacheQueryDB(db)
job_list = set(job_list)
# all the dependencies
job_list.update(cq.tree(job_list))
# plus all the jobs that were defined by them
job_list.update(definition_closure(job_list, db))
job_list = set(job_list)
# print('closure: %s' % sorted(job_list))
if cluster:
ggraph = create_graph2_clusters(cq, job_list, label=label, color=color,
processing=processing)
else:
ggraph = create_graph1(cq, job_list, label=label, color=color,
processing=processing)
print('Writing graph on %r.' % filename)
# TODO: add check?
with open(filename, 'w') as f:
ggraph.dot(f)
print('Running rendering')
output = filename + '.' + format
cmd_line = '%s %s -T%s -o%s' % (filter, filename, format, output)
print(' %s' % cmd_line)
try:
os.system(cmd_line)
except:
msg = "Could not run dot (cmdline='%s') Make sure graphviz is " \
"installed" % cmd_line
raise UserError(msg) # XXX maybe not UserError
info("Written output on files %s, %s." % (filename, output))
def cloudmake(job_list,
context,
cq,
n=DefaultsToConfig('multyvac_max_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo'),
skipsync=False,
rdb=True):
"""
Multyvac backend
"""
# TODO: check it exists
import multyvac # @UnusedImport
disable_logging_if_config(context)
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
volumes = sync_data_up(context, skipsync)
if rdb:
rdb_vol, rdb_db = synchronize_db_up(context, job_list)
else:
rdb_vol, rdb_db = None, None
publish(context,
'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = MVacManager(
num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo,
volumes=volumes,
rdb=rdb,
rdb_vol=rdb_vol,
rdb_db=rdb_db,
)
publish(context,
'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
if not skipsync:
sync_data_down(context)
return raise_error_if_manager_failed(manager)
