def instance_job(self, job_id):
publish(self.context,
'worker-status',
job_id=job_id,
status='apply_async')
assert len(self.sub_available) > 0
name = sorted(self.sub_available)[0]
self.sub_available.remove(name)
assert not name in self.sub_processing
self.sub_processing.add(name)
sub = self.subs[name]
self.job2subname[job_id] = name
if self.new_process:
f = parmake_job2_new_process
args = (job_id, self.context)
else:
f = parmake_job2
args = (job_id, self.context, self.event_queue_name,
self.show_output)
async_result = sub.apply_async(f, args)
return async_result
def parmake_pool(job_list, context, cq,
n=DefaultsToConfig('max_parallel_jobs'), recurse=False):
"""
Parallel equivalent of "make", using multiprocessing.Pool. (buggy)
Usage:
parmake [n=<num>] [joblist]
"""
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
job_list = list(top_targets(db=db))
publish(context, 'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = MultiprocessingManager(num_processes=n,
cq=cq,
context=context,
recurse=recurse)
publish(context, 'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
return raise_error_if_manager_failed(manager)
def cloudmake(job_list, context, cq,
n=DefaultsToConfig('multyvac_max_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo'),
skipsync=False,
rdb=True):
"""
Multyvac backend
"""
# TODO: check it exists
# noinspection PyUnresolvedReferences
import multyvac
disable_logging_if_config(context)
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
volumes = sync_data_up(context, skipsync)
if rdb:
rdb_vol, rdb_db = synchronize_db_up(context, job_list)
else:
rdb_vol, rdb_db = None, None
publish(context, 'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = MVacManager(num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo,
volumes=volumes,
rdb=rdb,
rdb_vol=rdb_vol,
rdb_db=rdb_db,
)
publish(context, 'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
if not skipsync:
sync_data_down(context)
return raise_error_if_manager_failed(manager)
def instance_job(self, job_id):
publish(self.context,
'worker-status',
job_id=job_id,
status='apply_async')
assert len(self.sub_available) > 0
name = sorted(self.sub_available)[0]
self.sub_available.remove(name)
assert not name in self.sub_processing
self.sub_processing.add(name)
sub = self.subs[name]
self.job2subname[job_id] = name
self.subname2job[name] = job_id
job = get_job(job_id, self.db)
if self.rdb:
f = mvac_job_rdb
args = (job_id, self.context, self.event_queue_name,
self.show_output, self.volumes, self.rdb_vol.name,
self.rdb_db, os.getcwd())
else:
if job.needs_context:
# if self.new_process:
# f = parmake_job2_new_process
# args = (job_id, self.context)
#
# else:
f = parmake_job2
args = (job_id, self.context, self.event_queue_name,
self.show_output)
else:
f = mvac_job
args = (job_id, self.context, self.event_queue_name,
self.show_output, self.volumes, os.getcwd())
if True:
async_result = sub.apply_async(f, args)
else:
warnings.warn('Debugging synchronously')
async_result = f(args)
return async_result
def parmake(job_list,
context,
cq,
n=DefaultsToConfig('max_parallel_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo')):
"""
Parallel equivalent of make.
Uses multiprocessing.Process as a backend and a Python queue to
communicate with the workers.
Options:
parmake n=10 Uses 10 workers
parmake recurse=1 Recursive make: put generated jobs in the
queue.
parmake new_process=1 Run the jobs in a new Python process.
parmake echo=1 Shows the output of the jobs. This might
slow down everything.
parmake new_process=1 echo=1 Not supported yet.
"""
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
publish(context,
'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = PmakeManager(num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo)
publish(context,
'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
return raise_error_if_manager_failed(manager)
def instance_job(self, job_id):
publish(self.context, 'worker-status', job_id=job_id,
status='apply_async')
assert len(self.sub_available) > 0
name = sorted(self.sub_available)[0]
self.sub_available.remove(name)
assert not name in self.sub_processing
self.sub_processing.add(name)
sub = self.subs[name]
self.job2subname[job_id] = name
self.subname2job[name] = job_id
job = get_job(job_id, self.db)
if self.rdb:
f = mvac_job_rdb
args = (job_id, self.context,
self.event_queue_name, self.show_output,
self.volumes, self.rdb_vol.name, self.rdb_db, os.getcwd())
else:
if job.needs_context:
# if self.new_process:
# f = parmake_job2_new_process
# args = (job_id, self.context)
#
# else:
f = parmake_job2
args = (job_id, self.context,
self.event_queue_name, self.show_output)
else:
f = mvac_job
args = (job_id, self.context,
self.event_queue_name, self.show_output,
self.volumes, os.getcwd())
if True:
async_result = sub.apply_async(f, args)
else:
warnings.warn('Debugging synchronously')
async_result = f(args)
return async_result
def parmake(job_list, context, cq,
n=DefaultsToConfig('max_parallel_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo')):
"""
Parallel equivalent of make.
Uses multiprocessing.Process as a backend and a Python queue to
communicate with the workers.
Options:
parmake n=10 Uses 10 workers
parmake recurse=1 Recursive make: put generated jobs in the
queue.
parmake new_process=1 Run the jobs in a new Python process.
parmake echo=1 Shows the output of the jobs. This might
slow down everything.
parmake new_process=1 echo=1 Not supported yet.
"""
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
publish(context, 'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = PmakeManager(num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo)
publish(context, 'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
return raise_error_if_manager_failed(manager)
def instance_job(self, job_id):
publish(self.context, 'worker-status', job_id=job_id,
status='apply_async')
assert len(self.sub_available) > 0
name = sorted(self.sub_available)[0]
self.sub_available.remove(name)
assert not name in self.sub_processing
self.sub_processing.add(name)
sub = self.subs[name]
self.job2subname[job_id] = name
if self.new_process:
f = parmake_job2_new_process
args = (job_id, self.context)
else:
f = parmake_job2
args = (job_id, self.context,
self.event_queue_name, self.show_output)
async_result = sub.apply_async(f, args)
return async_result
def parmake_pool(job_list,
context,
cq,
n=DefaultsToConfig('max_parallel_jobs'),
recurse=False):
"""
Parallel equivalent of "make", using multiprocessing.Pool. (buggy)
Usage:
parmake [n=<num>] [joblist]
"""
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
job_list = list(top_targets(db=db))
publish(context,
'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = MultiprocessingManager(num_processes=n,
cq=cq,
context=context,
recurse=recurse)
publish(context,
'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
return raise_error_if_manager_failed(manager)
def progress_callback(stack):
publish(context, 'job-progress-plus', job_id=job_id, host=host,
stack=stack)
def progress_callback(stack):
publish(context,
'job-progress-plus',
job_id=job_id,
host=host,
stack=stack)
def parmake_job2(args):
"""
args = tuple job_id, context, queue_name, show_events
Returns a dictionary with fields "user_object", "new_jobs", 'delete_jobs'.
"user_object" is set to None because we do not want to
load in our thread if not necessary. Sometimes it is necessary
because it might contain a Promise.
"""
job_id, context, event_queue_name, show_output = args # @UnusedVariable
check_isinstance(job_id, str)
check_isinstance(event_queue_name, str)
from .pmake_manager import PmakeManager
event_queue = PmakeManager.queues[event_queue_name]
db = context.get_compmake_db()
setproctitle('compmake:%s' % job_id)
class G():
nlostmessages = 0
try:
# We register a handler for the events to be passed back
# to the main process
def handler(event):
try:
if not CompmakeConstants.disable_interproc_queue:
event_queue.put(event, block=False)
except Full:
G.nlostmessages += 1
# Do not write messages here, it might create a recursive
# problem.
# sys.stderr.write('job %s: Queue is full, message is lost.\n'
# % job_id)
remove_all_handlers()
if show_output:
register_handler("*", handler)
def proctitle(event):
stat = '[%s/%s %s] (compmake)' % (event.progress, event.goal,
event.job_id)
setproctitle(stat)
register_handler("job-progress", proctitle)
publish(context, 'worker-status', job_id=job_id, status='started')
# Note that this function is called after the fork.
# All data is conserved, but resources need to be reopened
try:
db.reopen_after_fork() # @UndefinedVariable
except:
pass
publish(context, 'worker-status', job_id=job_id, status='connected')
res = make(job_id, context=context)
publish(context, 'worker-status', job_id=job_id, status='ended')
res['user_object'] = None
result_dict_check(res)
return res
except KeyboardInterrupt:
assert False, 'KeyboardInterrupt should be captured by make() (' \
'inside Job.compute())'
except JobInterrupted:
publish(context, 'worker-status', job_id=job_id, status='interrupted')
raise
except JobFailed:
raise
except BaseException:
# XXX
raise
except:
raise
finally:
publish(context, 'worker-status', job_id=job_id, status='cleanup')
setproctitle('compmake-worker-finished %s' % job_id)
def parmake_job2(args):
"""
args = tuple job_id, context, queue_name, show_events
Returns a dictionary with fields "user_object", "new_jobs", 'delete_jobs'.
"user_object" is set to None because we do not want to
load in our thread if not necessary. Sometimes it is necessary
because it might contain a Promise.
"""
job_id, context, event_queue_name, show_output = args # @UnusedVariable
check_isinstance(job_id, str)
check_isinstance(event_queue_name, str)
from .pmake_manager import PmakeManager
event_queue = PmakeManager.queues[event_queue_name]
db = context.get_compmake_db()
setproctitle('compmake:%s' % job_id)
class G():
nlostmessages = 0
try:
# We register a handler for the events to be passed back
# to the main process
def handler( event):
try:
if not CompmakeConstants.disable_interproc_queue:
event_queue.put(event, block=False)
except Full:
G.nlostmessages += 1
# Do not write messages here, it might create a recursive
# problem.
# sys.stderr.write('job %s: Queue is full, message is lost.\n'
# % job_id)
remove_all_handlers()
if show_output:
register_handler("*", handler)
def proctitle(event):
stat = '[%s/%s %s] (compmake)' % (event.progress,
event.goal, event.job_id)
setproctitle(stat)
register_handler("job-progress", proctitle)
publish(context, 'worker-status', job_id=job_id, status='started')
# Note that this function is called after the fork.
# All data is conserved, but resources need to be reopened
try:
db.reopen_after_fork() # @UndefinedVariable
except:
pass
publish(context, 'worker-status', job_id=job_id, status='connected')
res = make(job_id, context=context)
publish(context, 'worker-status', job_id=job_id, status='ended')
res['user_object'] = None
result_dict_check(res)
return res
except KeyboardInterrupt:
assert False, 'KeyboardInterrupt should be captured by make() (' \
'inside Job.compute())'
except JobInterrupted:
publish(context, 'worker-status', job_id=job_id, status='interrupted')
raise
except JobFailed:
raise
except BaseException:
# XXX
raise
except:
raise
finally:
publish(context, 'worker-status', job_id=job_id, status='cleanup')
setproctitle('compmake-worker-finished')
def cloudmake(job_list,
context,
cq,
n=DefaultsToConfig('multyvac_max_jobs'),
recurse=DefaultsToConfig('recurse'),
new_process=DefaultsToConfig('new_process'),
echo=DefaultsToConfig('echo'),
skipsync=False,
rdb=True):
"""
Multyvac backend
"""
# TODO: check it exists
import multyvac # @UnusedImport
disable_logging_if_config(context)
publish(context, 'parmake-status', status='Obtaining job list')
job_list = list(job_list)
db = context.get_compmake_db()
if not job_list:
# XXX
job_list = list(top_targets(db=db))
volumes = sync_data_up(context, skipsync)
if rdb:
rdb_vol, rdb_db = synchronize_db_up(context, job_list)
else:
rdb_vol, rdb_db = None, None
publish(context,
'parmake-status',
status='Starting multiprocessing manager (forking)')
manager = MVacManager(
num_processes=n,
context=context,
cq=cq,
recurse=recurse,
new_process=new_process,
show_output=echo,
volumes=volumes,
rdb=rdb,
rdb_vol=rdb_vol,
rdb_db=rdb_db,
)
publish(context,
'parmake-status',
status='Adding %d targets.' % len(job_list))
manager.add_targets(job_list)
publish(context, 'parmake-status', status='Processing')
manager.process()
if not skipsync:
sync_data_down(context)
return raise_error_if_manager_failed(manager)