def create_network(self, nodes=[]):
cmds = 0
session = Session()
for res in session.query(Overlay):
msg = json.dumps({'oid': res.id, 'time': time.time()})
# If nodes are passed, process only those overlaps containing
# the provided node(s)
if nodes:
for r in res.overlaps:
if r in nodes:
self.redis_queue.rpush(
config['redis']['processing_queue'], msg)
cmds += 1
break
else:
self.redis_queue.rpush(config['redis']['processing_queue'],
msg)
cmds += 1
script = 'acn_create_network'
spawn_jobarr(script,
cmds,
mem=config['cluster']['processing_memory'],
queue=config['cluster']['queue'],
env=config['python']['env_name'])
session.close()
def apply(self, function, on='edge',out=None, args=(), walltime='01:00:00', **kwargs):
options = {
'edge' : self.edges,
'edges' : self.edges,
'e' : self.edges,
0 : self.edges,
'node' : self.nodes,
'nodes' : self.nodes,
'n' : self.nodes,
1 : self.nodes
}
# Determine which obj will be called
onobj = options[on]
res = []
key = 1
if isinstance(on, EdgeView):
key = 2
for job_counter, elem in enumerate(onobj.data('data')):
# Determine if we are working with an edge or a node
if len(elem) > 2:
id = (elem[0], elem[1])
image_path = (elem[2].source['image_path'],
elem[2].destination['image_path'])
else:
id = (elem[0])
image_path = elem[1]['image_path']
msg = {'id':id,
'func':function,
'args':args,
'kwargs':kwargs,
'walltime':walltime,
'image_path':image_path,
'param_step':1}
self.redis_queue.rpush(self.processing_queue, json.dumps(msg))
# SLURM is 1 based, while enumerate is 0 based
job_counter += 1
# Submit the jobs
spawn_jobarr('acn_submit', job_counter,
mem=config['cluster']['processing_memory'],
time=walltime,
queue=config['cluster']['queue'],
outdir=config['cluster']['cluster_log_dir']+'/slurm-%A_%a.out',
env=config['python']['env_name'])
return job_counter