def __init__(self, template, debug=False, dry_run=False):
self.config = Configuration()
self.debug = debug
self.dry_run = dry_run
# The work queue will figure out a valid combination of MongoDB access
# parameters, e.g., host/port, URI, or replica set discovery via DNS
self.wq = WorkQueue(host=self.config.mongodb_host,
port=self.config.mongodb_port,
uri=self.config.mongodb_uri,
srv_name=self.config.mongodb_rs_srv,
database=self.config.mongodb_queue_db,
replicaset=self.config.mongodb_rs,
collection=self.config.mongodb_queue_col)
if not os.path.exists(template):
raise Exception("Template file does not exist")
self.template_dir = os.path.dirname(template)
self.template_file = os.path.basename(template)
if self.template_file == "":
raise Exception("Template must be a file, not a directory")
self.jinja = jinja2.Environment(loader=jinja2.FileSystemLoader(
self.template_dir),
autoescape=False)
def init_generator(self,
module,
source,
node_type,
instance_id,
collectors,
client_context,
http_port,
sandesh_req_uve_pkg_list=None,
discovery_client=None,
connect_to_collector=True,
logger_class=None,
logger_config_file=None,
host_ip='127.0.0.1',
alarm_ack_callback=None):
self._role = self.SandeshRole.GENERATOR
self._module = module
self._source = source
self._node_type = node_type
self._instance_id = instance_id
self._host_ip = host_ip
self._client_context = client_context
self._collectors = collectors
self._connect_to_collector = connect_to_collector
self._rcv_queue = WorkQueue(self._process_rx_sandesh)
self._init_logger(module,
logger_class=logger_class,
logger_config_file=logger_config_file)
self._logger.info('SANDESH: CONNECT TO COLLECTOR: %s',
connect_to_collector)
self._stats = SandeshStats()
self._trace = trace.Trace()
self._sandesh_request_dict = {}
self._alarm_ack_callback = alarm_ack_callback
self._uve_type_maps = SandeshUVETypeMaps(self._logger)
if sandesh_req_uve_pkg_list is None:
sandesh_req_uve_pkg_list = []
# Initialize the request handling
# Import here to break the cyclic import dependency
import sandesh_req_impl
sandesh_req_impl = sandesh_req_impl.SandeshReqImpl(self)
sandesh_req_uve_pkg_list.append('pysandesh.gen_py')
for pkg_name in sandesh_req_uve_pkg_list:
self._create_sandesh_request_and_uve_lists(pkg_name)
self._gev_httpd = None
if http_port != -1:
self._http_server = SandeshHttp(self, module, http_port,
sandesh_req_uve_pkg_list)
self._gev_httpd = gevent.spawn(self._http_server.start_http_server)
primary_collector = None
secondary_collector = None
if self._collectors is not None:
if len(self._collectors) > 0:
primary_collector = self._collectors[0]
if len(self._collectors) > 1:
secondary_collector = self._collectors[1]
if self._connect_to_collector:
self._client = SandeshClient(self, primary_collector,
secondary_collector, discovery_client)
self._client.initiate()
def run(self, args):
'''Crea los canales de eventos y el proxy que sera mostrado por teminal
para que el cliente pueda conectarse a el'''
adapter = self.communicator().createObjectAdapter("FactoryAdapter")
Downloader.SyncEventPrx.uncheckedCast(
self.get_topic(SYNC_TOPIC).getPublisher())
progress = Downloader.ProgressEventPrx.uncheckedCast(
self.get_topic(PROGRESS_TOPIC).getPublisher())
queue = WorkQueue(progress)
proxy = adapter.addWithUUID(SchedulerFactoryI(queue))
print(proxy, flush=True)
topic_mgr = self.get_topic(SYNC_TOPIC)
servant = SyncEventI()
subscriber = adapter.addWithUUID(servant)
qos = {}
topic_mgr.subscribeAndGetPublisher(qos, subscriber)
while self.communicator().isShutdown():
subscriber.requestSync()
time.sleep(10)
adapter.activate()
queue.start()
self.shutdownOnInterrupt()
self.communicator().waitForShutdown()
queue.destroy()
return 0
def run(self, args):
broker = self.communicator()
adapter = broker.createObjectAdapter('FactoryAdapter')
publisher = self.get_topic(STATUS_TOPIC).getPublisher()
progress_publisher = Downloader.ProgressEventPrx.uncheckedCast(
publisher)
print("ProgressTopic created")
queue = WorkQueue(progress_publisher)
servant = SchedulerFactoryI(queue)
proxy = adapter.addWithUUID(servant)
sync_topic = self.get_topic(SYNC_TOPIC)
pub = sync_topic.subscribeAndGetPublisher({}, proxy)
servant.sync_publisher = Downloader.SyncEventPrx.uncheckedCast(pub)
print("SynTopic created")
print(proxy, flush=True)
adapter.activate()
queue.start()
self.shutdownOnInterrupt()
broker.waitForShutdown()
queue.destroy()
return 0
def run(self, argv):
topic_mgr = self.get_topic_manager() #proxy to topic
if not topic_mgr:
print(': invalid proxy')
return 2
topic_name = "ProgressTopic"
try:
topic = topic_mgr.retrieve(topic_name)
except IceStorm.NoSuchTopic:
topic = topic_mgr.create(topic_name)
publisher = topic.getPublisher();
progress_topic = Example.ProgressSubscriberPrx.unCheckedCast(publisher);
work_queue = WorkQueue(progress_top)
servant = DownloaderI(work_queue)
adapter = broker.createObjectAdapter("DownloaderAdapter")
printer(adapter.add(servant, broker.stringToIdentity("dowloader1")))
work_queue.start();
self.shutdownOnInterrupt()
broker.waitForShutdown()
work_queue.destroy()
return 0
def run(self, args):
topic_mgr = self.get_topic_manager()
if not topic_mgr:
print('invalid proxy')
return 2
topic_name = "ProgressTopic"
try:
topic = topic_mgr.retrieve(topic_name)
except IceStorm.NoSuchTopic:
topic = topic_mgr.create(topic_name)
publisher = topic.getPublisher()
progress = Downloader.ProgressEventPrx.uncheckedCast(publisher)
if progress is None:
print("Progress topic not found")
work_queue = WorkQueue(progress)
ic = self.communicator()
adapter = ic.createObjectAdapter("FactoryAdapter")
servant = SchedulerFactoryI(work_queue, ic)
proxy = adapter.add(servant, ic.stringToIdentity("Factory1"))
work_queue.start()
adapter.activate()
print(proxy, flush=True)
self.shutdownOnInterrupt()
ic.waitForShutdown()
return 0
def init_generator(self,
module,
source,
node_type,
instance_id,
collectors,
client_context,
http_port,
sandesh_req_uve_pkg_list=None,
connect_to_collector=True,
logger_class=None,
logger_config_file=None,
host_ip='127.0.0.1',
alarm_ack_callback=None,
config=None):
self._role = self.SandeshRole.GENERATOR
self._module = module
self._source = source
self._node_type = node_type
self._instance_id = instance_id
self._sandesh_req_uve_pkg_list = sandesh_req_uve_pkg_list or []
self._host_ip = host_ip
self._client_context = client_context
self._connect_to_collector = connect_to_collector
self._rcv_queue = WorkQueue(self._process_rx_sandesh)
self._send_level = SandeshLevel.INVALID
self._init_logger(self._module,
logger_class=logger_class,
logger_config_file=logger_config_file)
self._logger.info('SANDESH: CONNECT TO COLLECTOR: %s',
connect_to_collector)
from sandesh_stats import SandeshMessageStatistics
self._msg_stats = SandeshMessageStatistics()
self._trace = trace.Trace()
self._sandesh_request_map = {}
self._alarm_ack_callback = alarm_ack_callback
self._config = config or SandeshConfig.from_parser_arguments()
self._uve_type_maps = SandeshUVETypeMaps(self._logger)
# Initialize the request handling
# Import here to break the cyclic import dependency
import sandesh_req_impl
sandesh_req_impl = sandesh_req_impl.SandeshReqImpl(self)
self._sandesh_req_uve_pkg_list.append('pysandesh.gen_py')
for pkg_name in self._sandesh_req_uve_pkg_list:
self._create_sandesh_request_and_uve_lists(pkg_name)
if self._config.disable_object_logs is not None:
self.disable_sending_object_logs(self._config.disable_object_logs)
if self._config.system_logs_rate_limit is not None:
SandeshSystem.set_sandesh_send_rate_limit(
self._config.system_logs_rate_limit)
self._gev_httpd = None
if http_port != -1:
self.run_introspect_server(http_port)
if self._connect_to_collector:
self._client = SandeshClient(self)
self._client.initiate(collectors)
def __init__(self, sandesh_instance, server, event_handler,
sandesh_msg_handler):
self._sandesh_instance = sandesh_instance
self._logger = sandesh_instance._logger
self._event_handler = event_handler
self._reader = SandeshReader(self, sandesh_msg_handler)
self._writer = SandeshWriter(self)
self._send_queue = WorkQueue(self._send_sandesh,
self._is_ready_to_send_sandesh)
TcpSession.__init__(self, server)
def main():
args = parse_args()
# Create a temporary file store for task results.
tmpdir = tempfile.mkdtemp()
# Load the task input data here
task_inputs = load_inputs()
print('Creating tasks')
tasks = generate_tasks(args.command, task_inputs,
args.infiles, args.outfile,
tmpdir, args.max_retries)
# Create the Work Queue master that manages task distribution.
work_queue.cctools_debug_flags_set("all")
work_queue.cctools_debug_config_file(f'{args.name}.debug')
work_queue.cctools_debug_config_file_size(0)
wq = WorkQueue(port=args.port, name=args.name, shutdown=True)
wq.specify_log(f'{args.name}.log')
# Submit all tasks to the queue.
print('Submitting tasks')
for t in tasks.values():
wq.submit(t)
# The main loop waits for a task to get done then handles success or
# failure accordingly
print('Entering main loop')
while not all([done_check(t) for t in tasks.values()]):
t = wq.wait(10) # This blocks for 10s or until a task is done.
if t is not None:
tasks[t.tag] = t # Update the task map with the correct status
# On success, post-process the task. If the maximum number of
# submissions for a task has been reached, make a note. Otherwise,
# report the failure and resubmit.
if t.return_status == 0 and t.result == WORK_QUEUE_RESULT_SUCCESS:
print(f'Task {t.tag} completed successfully.')
input_idx = int(t.tag.split('_')[1])
handle_success(t, tmpdir, args.outfile)
elif t.result == WORK_QUEUE_RESULT_MAX_RETRIES:
print(f'Task {t.tag} resubmitted too many times.')
else:
wq.submit(t)
print(f'Task {t.tag} failed with result {t.result}')
print(t.output)
print('All tasks completed or hit max retries.')
print('Cleaning up...')
shutil.rmtree(tmpdir)
print('Done')
def init_generator(self,
module,
source,
node_type,
instance_id,
collectors,
client_context,
http_port,
sandesh_req_uve_pkg_list=None,
discovery_client=None):
self._role = self.SandeshRole.GENERATOR
self._module = module
self._source = source
self._node_type = node_type
self._instance_id = instance_id
self._client_context = client_context
self._collectors = collectors
self._rcv_queue = WorkQueue(self._process_rx_sandesh)
self._init_logger(source + ':' + module + ':' + node_type + ':' \
+ instance_id)
self._stats = SandeshStats()
self._trace = trace.Trace()
self._sandesh_request_dict = {}
self._uve_type_maps = SandeshUVETypeMaps()
if sandesh_req_uve_pkg_list is None:
sandesh_req_uve_pkg_list = []
# Initialize the request handling
# Import here to break the cyclic import dependency
import sandesh_req_impl
sandesh_req_impl = sandesh_req_impl.SandeshReqImpl(self)
sandesh_req_uve_pkg_list.append('pysandesh.gen_py')
for pkg_name in sandesh_req_uve_pkg_list:
self._create_sandesh_request_and_uve_lists(pkg_name)
if http_port != -1:
self._http_server = SandeshHttp(self, module, http_port,
sandesh_req_uve_pkg_list)
gevent.spawn(self._http_server.start_http_server)
primary_collector = None
secondary_collector = None
if self._collectors is not None:
if len(self._collectors) > 0:
primary_collector = self._collectors[0]
if len(self._collectors) > 1:
secondary_collector = self._collectors[1]
self._client = SandeshClient(self, primary_collector,
secondary_collector, discovery_client)
self._client.initiate()
def run(self, argv):
work_queue = WorkQueue()
servant = MathI(work_queue)
broker = self.communicator()
adapter = broker.createObjectAdapter("MathAdapter")
print adapter.add(servant, broker.stringToIdentity("math1"))
adapter.activate()
work_queue.start()
self.shutdownOnInterrupt()
broker.waitForShutdown()
work_queue.destroy()
return 0
async def main():
r = redis.Redis()
wq = WorkQueue(r, tidy_interval=5, stale_time=3)
prod = Producer(work_queue=wq, delay=0.75)
worker = Worker("Good Worker", work_queue=wq, delay=1, fail_rate=0.2)
bad_worker = Worker("Bad Worker", work_queue=wq, delay=2.5, fail_rate=0.7)
try:
print('Start')
await wq.schedule_tidy_task()
await prod.start()
await worker.start()
await bad_worker.start()
await asyncio.sleep(61)
finally:
await wq.stop_tidy_task()
await prod.stop()
await worker.stop()
await bad_worker.stop()
def __init__(self, connection, logger, primary_collector,
secondary_collector):
def _update_connection_state(e, status):
from connection_info import ConnectionState
from gen_py.process_info.ttypes import ConnectionType
collector_addr = e.sm._active_collector
if collector_addr is None:
collector_addr = ''
ConnectionState.update(conn_type=ConnectionType.COLLECTOR,
name='',
status=status,
server_addrs=[collector_addr],
message='%s to %s on %s' %
(e.src, e.dst, e.event))
#end _update_connection_state
def _connection_state_up(e):
from gen_py.process_info.ttypes import ConnectionStatus
_update_connection_state(e, ConnectionStatus.UP)
#end _connection_state_up
def _connection_state_down(e):
from gen_py.process_info.ttypes import ConnectionStatus
_update_connection_state(e, ConnectionStatus.DOWN)
#end _connection_state_down
def _connection_state_init(e):
from gen_py.process_info.ttypes import ConnectionStatus
_update_connection_state(e, ConnectionStatus.INIT)
#end _connection_state_init
def _on_idle(e):
if e.sm._connect_timer is not None:
e.sm._cancel_connect_timer()
# Reset active and backup collector
self._active_collector = self._connection.primary_collector()
self._backup_collector = self._connection.secondary_collector()
# clean up existing connection
e.sm._delete_session()
if e.sm._disable != True:
e.sm._start_idle_hold_timer()
# update connection state
_connection_state_down(e)
#end _on_idle
def _on_disconnect(e):
# update connection state
_connection_state_down(e)
#end _on_disconnect
def _on_connect(e):
if e.sm._idle_hold_timer is not None:
e.sm._cancel_idle_hold_timer()
e.sm._connection.reset_collector()
# clean up existing connection
e.sm._delete_session()
if e.sm._active_collector is not None:
# update connection state
_connection_state_init(e)
e.sm._create_session()
e.sm._start_connect_timer()
e.sm._session.connect()
else:
e.sm.enqueue_event(Event(event=Event._EV_COLLECTOR_UNKNOWN))
#end _on_connect
def _on_connect_to_backup(e):
if e.sm._connect_timer is not None:
e.sm._cancel_connect_timer()
# clean up existing connection
e.sm._delete_session()
# try to connect to the backup collector, if known
if e.sm._backup_collector is not None:
e.sm._active_collector, e.sm._backup_collector = \
e.sm._backup_collector, e.sm._active_collector
# update connection state
_connection_state_init(e)
e.sm._create_session()
e.sm._start_connect_timer()
e.sm._session.connect()
else:
e.sm.enqueue_event(
Event(event=Event._EV_BACKUP_COLLECTOR_UNKNOWN))
#end _on_connect_to_backup
def _on_client_init(e):
e.sm._connects += 1
gevent.spawn(e.sm._session.read)
e.sm._connection.handle_initialized(e.sm._connects)
e.sm._connection.sandesh_instance().send_generator_info()
# update connection state
_connection_state_init(e)
#end _on_client_init
def _on_established(e):
e.sm._cancel_connect_timer()
e.sm._connection.set_collector(e.sm_event.source)
e.sm._connection.handle_sandesh_ctrl_msg(e.sm_event.msg)
e.sm._connection.sandesh_instance().send_generator_info()
# update connection state
_connection_state_up(e)
#end _on_established
# FSM - Fysom
self._fsm = Fysom({
'initial': {
'state': State._IDLE,
'event': Event._EV_START,
'defer': True
},
'events': [
# _IDLE
{
'name': Event._EV_IDLE_HOLD_TIMER_EXPIRED,
'src': State._IDLE,
'dst': State._CONNECT
},
{
'name': Event._EV_COLLECTOR_CHANGE,
'src': State._IDLE,
'dst': State._CONNECT
},
{
'name': Event._EV_START,
'src': State._IDLE,
'dst': State._CONNECT
},
# _DISCONNECT
{
'name': Event._EV_COLLECTOR_CHANGE,
'src': State._DISCONNECT,
'dst': State._CONNECT
},
# _CONNECT
{
'name': Event._EV_COLLECTOR_UNKNOWN,
'src': State._CONNECT,
'dst': State._DISCONNECT
},
{
'name': Event._EV_TCP_CONNECT_FAIL,
'src': State._CONNECT,
'dst': State._CONNECT_TO_BACKUP
},
{
'name': Event._EV_CONNECT_TIMER_EXPIRED,
'src': State._CONNECT,
'dst': State._CONNECT_TO_BACKUP
},
{
'name': Event._EV_COLLECTOR_CHANGE,
'src': State._CONNECT,
'dst': State._IDLE
},
{
'name': Event._EV_TCP_CONNECTED,
'src': State._CONNECT,
'dst': State._CLIENT_INIT
},
# _CONNECT_TO_BACKUP
{
'name': Event._EV_BACKUP_COLLECTOR_UNKNOWN,
'src': State._CONNECT_TO_BACKUP,
'dst': State._IDLE
},
{
'name': Event._EV_TCP_CONNECT_FAIL,
'src': State._CONNECT_TO_BACKUP,
'dst': State._IDLE
},
{
'name': Event._EV_CONNECT_TIMER_EXPIRED,
'src': State._CONNECT_TO_BACKUP,
'dst': State._IDLE
},
{
'name': Event._EV_COLLECTOR_CHANGE,
'src': State._CONNECT_TO_BACKUP,
'dst': State._IDLE
},
{
'name': Event._EV_TCP_CONNECTED,
'src': State._CONNECT_TO_BACKUP,
'dst': State._CLIENT_INIT
},
# _CLIENT_INIT
{
'name': Event._EV_CONNECT_TIMER_EXPIRED,
'src': State._CLIENT_INIT,
'dst': State._IDLE
},
{
'name': Event._EV_TCP_CLOSE,
'src': State._CLIENT_INIT,
'dst': State._IDLE
},
{
'name': Event._EV_COLLECTOR_CHANGE,
'src': State._CLIENT_INIT,
'dst': State._IDLE
},
{
'name': Event._EV_SANDESH_CTRL_MESSAGE_RECV,
'src': State._CLIENT_INIT,
'dst': State._ESTABLISHED
},
# _ESTABLISHED
{
'name': Event._EV_TCP_CLOSE,
'src': State._ESTABLISHED,
'dst': State._CONNECT_TO_BACKUP
},
{
'name': Event._EV_STOP,
'src': State._ESTABLISHED,
'dst': State._IDLE
},
{
'name': Event._EV_COLLECTOR_CHANGE,
'src': State._ESTABLISHED,
'dst': State._CONNECT
}
],
'callbacks': {
'on' + State._IDLE: _on_idle,
'on' + State._CONNECT: _on_connect,
'on' + State._CONNECT_TO_BACKUP: _on_connect_to_backup,
'on' + State._CLIENT_INIT: _on_client_init,
'on' + State._ESTABLISHED: _on_established,
}
})
self._connection = connection
self._session = None
self._connects = 0
self._disable = False
self._idle_hold_timer = None
self._connect_timer = None
self._active_collector = primary_collector
self._backup_collector = secondary_collector
self._logger = logger
self._event_queue = WorkQueue(self._dequeue_event,
self._is_ready_to_dequeue_event)
from work_queue import Task, WorkQueue, set_debug_flag
from work_queue import WORK_QUEUE_SCHEDULE_FCFS, WORK_QUEUE_SCHEDULE_FILES
from work_queue import WORK_QUEUE_RANDOM_PORT
from work_queue import WORK_QUEUE_OUTPUT
#from workqueue import WORK_QUEUE_MASTER_MODE_STANDALONE, WORK_QUEUE_WORKER_MODE_SHARED
from work_queue import WORK_QUEUE_TASK_ORDER_LIFO
import os
import sys
import time
set_debug_flag('debug')
set_debug_flag('wq')
wq = WorkQueue(WORK_QUEUE_RANDOM_PORT,
name='workqueue_example',
catalog=True,
exclusive=False)
os.environ['PATH'] = '../../../work_queue/src:' + os.environ['PATH']
os.system('work_queue_worker -d all localhost %d &' % wq.port)
print wq.name
wq.specify_algorithm(WORK_QUEUE_SCHEDULE_FCFS)
#wq.specify_name('workqueue_example')
#wq.specify_master_mode(WORK_QUEUE_MASTER_MODE_STANDALONE)
#wq.specify_worker_mode(WORK_QUEUE_WORKER_MODE_SHARED)
wq.specify_task_order(WORK_QUEUE_TASK_ORDER_LIFO)
if wq.empty():
print 'work queue is empty'
def work_queue_main(items, function, accumulator, **kwargs):
"""Execute using Work Queue
For valid parameters, see :py:func:`work_queue_executor` in :py:mod:`executor`.
For more information, see :ref:`intro-coffea-wq`
"""
global _wq_queue
_check_dynamic_chunksize_targets(kwargs["dynamic_chunksize"])
clevel = kwargs["compression"]
if clevel is not None:
function = _compression_wrapper(clevel, function)
accumulate_fn = _compression_wrapper(clevel, accumulate_result_files)
else:
accumulate_fn = accumulate_result_files
_vprint.verbose_mode = kwargs["verbose"] or kwargs["print_stdout"]
_vprint.status_mode = kwargs["status"]
if not kwargs["port"]:
kwargs["port"] = 0 if kwargs["master_name"] else 9123
if kwargs["environment_file"] and not kwargs["wrapper"]:
raise ValueError(
"Location of python_package_run could not be determined automatically.\nUse 'wrapper' argument to the work_queue_executor."
)
if _wq_queue is None:
_wq_queue = WorkQueue(
port=kwargs["port"],
name=kwargs["master_name"],
debug_log=kwargs["debug_log"],
stats_log=kwargs["stats_log"],
transactions_log=kwargs["transactions_log"],
)
# Make use of the stored password file, if enabled.
if kwargs["password_file"] is not None:
_wq_queue.specify_password_file(kwargs["password_file"])
print("Listening for work queue workers on port {}...".format(_wq_queue.port))
# perform a wait to print any warnings before progress bars
_wq_queue.wait(0)
_declare_resources(kwargs)
# Working within a custom temporary directory:
with tempfile.TemporaryDirectory(
prefix="wq-executor-tmp-", dir=kwargs["filepath"]
) as tmpdir:
fn_wrapper = _create_fn_wrapper(kwargs["x509_proxy"], tmpdir=tmpdir)
infile_function = _function_to_file(
function, prefix_name=kwargs["function_name"], tmpdir=tmpdir
)
infile_accum_fn = _function_to_file(
accumulate_fn, prefix_name="accum", tmpdir=tmpdir
)
if kwargs["custom_init"]:
kwargs["custom_init"](_wq_queue)
if kwargs["desc"] == "Preprocessing":
return _work_queue_preprocessing(
items, accumulator, fn_wrapper, infile_function, tmpdir, kwargs
)
else:
return _work_queue_processing(
items,
accumulator,
fn_wrapper,
infile_function,
infile_accum_fn,
tmpdir,
kwargs,
)
def WorkQueueSubmitThread(task_queue=multiprocessing.Queue(),
queue_lock=threading.Lock(),
launch_cmd=None,
env=None,
collector_queue=multiprocessing.Queue(),
see_worker_output=False,
data_dir=".",
full=False,
cancel_value=multiprocessing.Value('i', 1),
port=WORK_QUEUE_DEFAULT_PORT,
wq_log_dir=None,
project_password=None,
project_password_file=None,
project_name=None):
"""Thread to handle Parsl app submissions to the Work Queue objects.
Takes in Parsl functions submitted using submit(), and creates a
Work Queue task with the appropriate specifications, which is then
submitted to Work Queue. After tasks are completed, processes the
exit status and exit code of the task, and sends results to the
Work Queue collector thread.
"""
logger.debug("Starting WorkQueue Submit/Wait Process")
# Enable debugging flags and create logging file
if wq_log_dir is not None:
logger.debug("Setting debugging flags and creating logging file")
wq_debug_log = os.path.join(wq_log_dir, "debug_log")
cctools_debug_flags_set("all")
cctools_debug_config_file(wq_debug_log)
# Create WorkQueue queue object
logger.debug("Creating WorkQueue Object")
try:
logger.debug("Listening on port {}".format(port))
q = WorkQueue(port)
except Exception as e:
logger.error("Unable to create WorkQueue object: {}".format(e))
raise e
# Specify WorkQueue queue attributes
if project_name:
q.specify_name(project_name)
if project_password:
q.specify_password(project_password)
elif project_password_file:
q.specify_password_file(project_password_file)
# Only write logs when the wq_log_dir is specified, which it most likely will be
if wq_log_dir is not None:
wq_master_log = os.path.join(wq_log_dir, "master_log")
wq_trans_log = os.path.join(wq_log_dir, "transaction_log")
if full:
wq_resource_log = os.path.join(wq_log_dir, "resource_logs")
q.enable_monitoring_full(dirname=wq_resource_log)
q.specify_log(wq_master_log)
q.specify_transactions_log(wq_trans_log)
wq_tasks = set()
orig_ppid = os.getppid()
continue_running = True
while (continue_running):
# Monitor the task queue
ppid = os.getppid()
if ppid != orig_ppid:
logger.debug("new Process")
continue_running = False
continue
# Submit tasks
while task_queue.qsize() > 0:
if cancel_value.value == 0:
logger.debug("cancel value set to cancel")
continue_running = False
break
# Obtain task from task_queue
try:
item = task_queue.get(timeout=1)
logger.debug("Removing task from queue")
except queue.Empty:
continue
parsl_id = item["task_id"]
# Extract information about the task
function_data_loc = item["data_loc"]
function_data_loc_remote = function_data_loc.split("/")[-1]
function_result_loc = item["result_loc"]
function_result_loc_remote = function_result_loc.split("/")[-1]
input_files = item["input_files"]
output_files = item["output_files"]
std_files = item["std_files"]
full_script_name = workqueue_worker.__file__
script_name = full_script_name.split("/")[-1]
remapping_string = ""
std_string = ""
# Parse input file information
logger.debug("Looking at input")
for item in input_files:
if item[3] == "std":
std_string += "mv " + item[1] + " " + item[0] + "; "
else:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
# Parse output file information
logger.debug("Looking at output")
for item in output_files:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
if len(input_files) + len(output_files) > 0:
remapping_string = "-r " + remapping_string
remapping_string = remapping_string[:-1]
# Create command string
logger.debug(launch_cmd)
command_str = launch_cmd.format(
input_file=function_data_loc_remote,
output_file=function_result_loc_remote,
remapping_string=remapping_string)
command_str = std_string + command_str
logger.debug(command_str)
# Create WorkQueue task for the command
logger.debug("Sending task {} with command: {}".format(
parsl_id, command_str))
try:
t = Task(command_str)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
continue
# Specify environment variables for the task
if env is not None:
for var in env:
t.specify_environment_variable(var, env[var])
# Specify script, and data/result files for task
t.specify_file(full_script_name,
script_name,
WORK_QUEUE_INPUT,
cache=True)
t.specify_file(function_data_loc,
function_data_loc_remote,
WORK_QUEUE_INPUT,
cache=False)
t.specify_file(function_result_loc,
function_result_loc_remote,
WORK_QUEUE_OUTPUT,
cache=False)
t.specify_tag(str(parsl_id))
logger.debug("Parsl ID: {}".format(t.id))
# Specify all input/output files for task
for item in input_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_INPUT,
cache=item[2])
for item in output_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
for item in std_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
# Submit the task to the WorkQueue object
logger.debug("Submitting task {} to WorkQueue".format(parsl_id))
try:
wq_id = q.submit(t)
wq_tasks.add(wq_id)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
msg = {
"tid": parsl_id,
"result_received": False,
"reason": "Workqueue Task Start Failure",
"status": 1
}
collector_queue.put_nowait(msg)
continue
logger.debug("Task {} submitted to WorkQueue with id {}".format(
parsl_id, wq_id))
if cancel_value.value == 0:
continue_running = False
# If the queue is not empty wait on the WorkQueue queue for a task
task_found = True
if not q.empty() and continue_running:
while task_found is True:
if cancel_value.value == 0:
continue_running = False
task_found = False
continue
# Obtain the task from the queue
t = q.wait(1)
if t is None:
task_found = False
continue
else:
parsl_tid = t.tag
logger.debug(
"Completed WorkQueue task {}, parsl task {}".format(
t.id, parsl_tid))
status = t.return_status
task_result = t.result
msg = None
# Task failure
if status != 0 or (task_result != WORK_QUEUE_RESULT_SUCCESS
and task_result !=
WORK_QUEUE_RESULT_OUTPUT_MISSING):
logger.debug(
"Wrapper Script status: {}\nWorkQueue Status: {}".
format(status, task_result))
# Wrapper script failure
if status != 0:
logger.debug(
"WorkQueue task {} failed with status {}".
format(t.id, status))
reason = "Wrapper Script Failure: "
if status == 1:
reason += "problem parsing command line options"
elif status == 2:
reason += "problem loading function data"
elif status == 3:
reason += "problem remapping file names"
elif status == 4:
reason += "problem writing out function result"
else:
reason += "unable to process wrapper script failure with status = {}".format(
status)
reason += "\nTrace:\n" + str(t.output)
logger.debug(
"WorkQueue runner script failed for task {} because {}\n"
.format(parsl_tid, reason))
# WorkQueue system failure
else:
reason = "WorkQueue System Failure: "
if task_result == 1:
reason += "missing input file"
elif task_result == 2:
reason += "unable to generate output file"
elif task_result == 4:
reason += "stdout has been truncated"
elif task_result == 1 << 3:
reason += "task terminated with a signal"
elif task_result == 2 << 3:
reason += "task used more resources than requested"
elif task_result == 3 << 3:
reason += "task ran past the specified end time"
elif task_result == 4 << 3:
reason += "result could not be classified"
elif task_result == 5 << 3:
reason += "task failed, but not a task error"
elif task_result == 6 << 3:
reason += "unable to complete after specified number of retries"
elif task_result == 7 << 3:
reason += "task ran for more than the specified time"
elif task_result == 8 << 3:
reason += "task needed more space to complete task"
else:
reason += "unable to process Work Queue system failure"
msg = {
"tid": parsl_tid,
"result_received": False,
"reason": reason,
"status": status
}
collector_queue.put_nowait(msg)
# Task Success
else:
# Print the output from the task
if see_worker_output:
print(t.output)
# Load result into result file
result_loc = os.path.join(
data_dir,
"task_" + str(parsl_tid) + "_function_result")
logger.debug(
"Looking for result in {}".format(result_loc))
f = open(result_loc, "rb")
result = pickle.load(f)
f.close()
msg = {
"tid": parsl_tid,
"result_received": True,
"result": result
}
wq_tasks.remove(t.id)
collector_queue.put_nowait(msg)
if continue_running is False:
logger.debug("Exiting WorkQueue Master Thread event loop")
break
# Remove all WorkQueue tasks that remain in the queue object
for wq_task in wq_tasks:
logger.debug("Cancelling WorkQueue Task {}".format(wq_task))
q.cancel_by_taskid(wq_task)
logger.debug("Exiting WorkQueue Monitoring Process")
return 0
def _work_queue_submit_wait(task_queue=multiprocessing.Queue(),
launch_cmd=None,
env=None,
collector_queue=multiprocessing.Queue(),
data_dir=".",
full=False,
shared_fs=False,
autolabel=False,
autolabel_window=None,
autocategory=False,
should_stop=None,
port=WORK_QUEUE_DEFAULT_PORT,
wq_log_dir=None,
project_password_file=None,
project_name=None):
"""Thread to handle Parsl app submissions to the Work Queue objects.
Takes in Parsl functions submitted using submit(), and creates a
Work Queue task with the appropriate specifications, which is then
submitted to Work Queue. After tasks are completed, processes the
exit status and exit code of the task, and sends results to the
Work Queue collector thread.
To avoid python's global interpreter lock with work queue's wait, this
function should be launched as a process, not as a lightweight thread. This
means that any communication should be done using the multiprocessing
module capabilities, rather than shared memory.
"""
logger.debug("Starting WorkQueue Submit/Wait Process")
# Enable debugging flags and create logging file
wq_debug_log = None
if wq_log_dir is not None:
logger.debug("Setting debugging flags and creating logging file")
wq_debug_log = os.path.join(wq_log_dir, "debug_log")
# Create WorkQueue queue object
logger.debug("Creating WorkQueue Object")
try:
logger.debug("Listening on port {}".format(port))
q = WorkQueue(port, debug_log=wq_debug_log)
except Exception as e:
logger.error("Unable to create WorkQueue object: {}".format(e))
raise e
# Specify WorkQueue queue attributes
if project_name:
q.specify_name(project_name)
if project_password_file:
q.specify_password_file(project_password_file)
if autolabel:
q.enable_monitoring()
if autolabel_window is not None:
q.tune('category-steady-n-tasks', autolabel_window)
# Only write logs when the wq_log_dir is specified, which it most likely will be
if wq_log_dir is not None:
wq_master_log = os.path.join(wq_log_dir, "master_log")
wq_trans_log = os.path.join(wq_log_dir, "transaction_log")
if full:
wq_resource_log = os.path.join(wq_log_dir, "resource_logs")
q.enable_monitoring_full(dirname=wq_resource_log)
q.specify_log(wq_master_log)
q.specify_transactions_log(wq_trans_log)
orig_ppid = os.getppid()
result_file_of_task_id = {
} # Mapping taskid -> result file for active tasks.
while not should_stop.value:
# Monitor the task queue
ppid = os.getppid()
if ppid != orig_ppid:
logger.debug("new Process")
break
# Submit tasks
while task_queue.qsize() > 0 and not should_stop.value:
# Obtain task from task_queue
try:
task = task_queue.get(timeout=1)
logger.debug("Removing task from queue")
except queue.Empty:
continue
pkg_pfx = ""
if task.env_pkg is not None:
pkg_pfx = "./{} -e {} ".format(
os.path.basename(package_run_script),
os.path.basename(task.env_pkg))
# Create command string
logger.debug(launch_cmd)
command_str = launch_cmd.format(
package_prefix=pkg_pfx,
mapping=os.path.basename(task.map_file),
function=os.path.basename(task.function_file),
result=os.path.basename(task.result_file))
logger.debug(command_str)
# Create WorkQueue task for the command
logger.debug("Sending task {} with command: {}".format(
task.id, command_str))
try:
t = Task(command_str)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
collector_queue.put_nowait(
WqTaskToParsl(
id=task.id,
result_received=False,
result=None,
reason="task could not be created by work queue",
status=-1))
continue
t.specify_category(task.category)
if autolabel:
q.specify_category_mode(
task.category, WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT)
# Specify environment variables for the task
if env is not None:
for var in env:
t.specify_environment_variable(var, env[var])
if task.env_pkg is not None:
t.specify_input_file(package_run_script, cache=True)
t.specify_input_file(task.env_pkg, cache=True)
# Specify script, and data/result files for task
t.specify_input_file(exec_parsl_function.__file__, cache=True)
t.specify_input_file(task.function_file, cache=False)
t.specify_input_file(task.map_file, cache=False)
t.specify_output_file(task.result_file, cache=False)
t.specify_tag(str(task.id))
result_file_of_task_id[str(task.id)] = task.result_file
logger.debug("Parsl ID: {}".format(task.id))
# Specify input/output files that need to be staged.
# Absolute paths are assumed to be in shared filesystem, and thus
# not staged by work queue.
if not shared_fs:
for spec in task.input_files:
if spec.stage:
t.specify_input_file(spec.parsl_name,
spec.parsl_name,
cache=spec.cache)
for spec in task.output_files:
if spec.stage:
t.specify_output_file(spec.parsl_name,
spec.parsl_name,
cache=spec.cache)
# Submit the task to the WorkQueue object
logger.debug("Submitting task {} to WorkQueue".format(task.id))
try:
wq_id = q.submit(t)
except Exception as e:
logger.error(
"Unable to submit task to work queue: {}".format(e))
collector_queue.put_nowait(
WqTaskToParsl(
id=task.id,
result_received=False,
result=None,
reason="task could not be submited to work queue",
status=-1))
continue
logger.debug("Task {} submitted to WorkQueue with id {}".format(
task.id, wq_id))
# If the queue is not empty wait on the WorkQueue queue for a task
task_found = True
if not q.empty():
while task_found and not should_stop.value:
# Obtain the task from the queue
t = q.wait(1)
if t is None:
task_found = False
continue
# When a task is found:
parsl_id = t.tag
logger.debug(
"Completed WorkQueue task {}, parsl task {}".format(
t.id, t.tag))
result_file = result_file_of_task_id.pop(t.tag)
# A tasks completes 'succesfully' if it has result file,
# and it can be loaded. This may mean that the 'success' is
# an exception.
logger.debug("Looking for result in {}".format(result_file))
try:
with open(result_file, "rb") as f_in:
result = pickle.load(f_in)
logger.debug("Found result in {}".format(result_file))
collector_queue.put_nowait(
WqTaskToParsl(id=parsl_id,
result_received=True,
result=result,
reason=None,
status=t.return_status))
# If a result file could not be generated, explain the
# failure according to work queue error codes. We generate
# an exception and wrap it with RemoteExceptionWrapper, to
# match the positive case.
except Exception as e:
reason = _explain_work_queue_result(t)
logger.debug(
"Did not find result in {}".format(result_file))
logger.debug(
"Wrapper Script status: {}\nWorkQueue Status: {}".
format(t.return_status, t.result))
logger.debug(
"Task with id parsl {} / wq {} failed because:\n{}".
format(parsl_id, t.id, reason))
collector_queue.put_nowait(
WqTaskToParsl(id=parsl_id,
result_received=False,
result=e,
reason=reason,
status=t.return_status))
logger.debug("Exiting WorkQueue Monitoring Process")
return 0
def WorkQueueSubmitThread(task_queue=multiprocessing.Queue(),
queue_lock=threading.Lock(),
launch_cmd=None,
env=None,
collector_queue=multiprocessing.Queue(),
see_worker_output=False,
data_dir=".",
full=False,
cancel_value=multiprocessing.Value('i', 1),
port=WORK_QUEUE_DEFAULT_PORT,
wq_log_dir=None,
project_password=None,
project_password_file=None,
project_name=None):
logger.debug("Starting WorkQueue Submit/Wait Process")
orig_ppid = os.getppid()
wq_tasks = set()
continue_running = True
if wq_log_dir is not None:
wq_debug_log = os.path.join(wq_log_dir, "debug")
cctools_debug_flags_set("all")
cctools_debug_config_file(wq_debug_log)
logger.debug("Creating Workqueue Object")
try:
q = WorkQueue(port)
except Exception as e:
logger.error("Unable to create Workqueue object: {}", format(e))
raise e
if project_name:
q.specify_name(project_name)
if project_password:
q.specify_password(project_password)
elif project_password_file:
q.specify_password_file(project_password_file)
# Only write Logs when the log_dir is specified, which is most likely always will be
if wq_log_dir is not None:
wq_master_log = os.path.join(wq_log_dir, "master_log")
wq_trans_log = os.path.join(wq_log_dir, "transaction_log")
if full:
wq_resource_log = os.path.join(wq_log_dir, "resource_logs")
q.enable_monitoring_full(dirname=wq_resource_log)
q.specify_log(wq_master_log)
q.specify_transactions_log(wq_trans_log)
while (continue_running):
# Monitor the Task Queue
ppid = os.getppid()
if ppid != orig_ppid:
continue_running = False
continue
# Submit Tasks
while task_queue.qsize() > 0:
if cancel_value.value == 0:
continue_running = False
break
try:
# item = task_queue.get_nowait()
item = task_queue.get(timeout=1)
logger.debug("Removing task from queue")
except queue.Empty:
continue
parsl_id = item["task_id"]
function_data_loc = item["data_loc"]
function_result_loc = item["result_loc"]
function_result_loc_remote = function_result_loc.split("/")[-1]
function_data_loc_remote = function_data_loc.split("/")[-1]
input_files = item["input_files"]
output_files = item["output_files"]
std_files = item["std_files"]
full_script_name = workqueue_worker.__file__
script_name = full_script_name.split("/")[-1]
remapping_string = ""
std_string = ""
logger.debug("looking at input")
for item in input_files:
if item[3] == "std":
std_string += "mv " + item[1] + " " + item[0] + "; "
else:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
logger.debug("looking at output")
for item in output_files:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
if len(input_files) + len(output_files) > 0:
remapping_string = "-r " + remapping_string
remapping_string = remapping_string[:-1]
logger.debug(launch_cmd)
command_str = launch_cmd.format(
input_file=function_data_loc_remote,
output_file=function_result_loc_remote,
remapping_string=remapping_string)
logger.debug(command_str)
command_str = std_string + command_str
logger.debug(command_str)
logger.debug("Sending task {} with command: {}".format(
parsl_id, command_str))
try:
t = Task(command_str)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
continue
if env is not None:
for var in env:
t.specify_environment_variable(var, env[var])
t.specify_file(full_script_name,
script_name,
WORK_QUEUE_INPUT,
cache=True)
t.specify_file(function_result_loc,
function_result_loc_remote,
WORK_QUEUE_OUTPUT,
cache=False)
t.specify_file(function_data_loc,
function_data_loc_remote,
WORK_QUEUE_INPUT,
cache=False)
t.specify_tag(str(parsl_id))
for item in input_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_INPUT,
cache=item[2])
for item in output_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
for item in std_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
logger.debug("Submitting task {} to workqueue".format(parsl_id))
try:
wq_id = q.submit(t)
wq_tasks.add(wq_id)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
msg = {
"tid": parsl_id,
"result_received": False,
"reason": "Workqueue Task Start Failure",
"status": 1
}
collector_queue.put_nowait(msg)
continue
logger.debug("Task {} submitted workqueue with id {}".format(
parsl_id, wq_id))
if cancel_value.value == 0:
continue_running = False
# Wait for Tasks
task_found = True
# If the queue is not empty wait on the workqueue queue for a task
if not q.empty() and continue_running:
while task_found is True:
if cancel_value.value == 0:
continue_running = False
task_found = False
continue
t = q.wait(1)
if t is None:
task_found = False
continue
else:
parsl_tid = t.tag
logger.debug(
"Completed workqueue task {}, parsl task {}".format(
t.id, parsl_tid))
status = t.return_status
task_result = t.result
msg = None
if status != 0 or (task_result != WORK_QUEUE_RESULT_SUCCESS
and task_result !=
WORK_QUEUE_RESULT_OUTPUT_MISSING):
if task_result == WORK_QUEUE_RESULT_SUCCESS:
logger.debug(
"Workqueue task {} failed with status {}".
format(t.id, status))
reason = "Wrapper Script Failure: "
if status == 1:
reason += "command line parsing"
if status == 2:
reason += "problem loading function data"
if status == 3:
reason += "problem remapping file names"
if status == 4:
reason += "problem writing out function result"
reason += "\nTrace:\n" + t.output
logger.debug(
"Workqueue runner script failed for task {} because {}\n"
.format(parsl_tid, reason))
else:
reason = "Workqueue system failure\n"
msg = {
"tid": parsl_tid,
"result_received": False,
"reason": reason,
"status": status
}
collector_queue.put_nowait(msg)
else:
if see_worker_output:
print(t.output)
result_loc = os.path.join(
data_dir,
"task_" + str(parsl_tid) + "_function_result")
logger.debug(
"Looking for result in {}".format(result_loc))
f = open(result_loc, "rb")
result = pickle.load(f)
f.close()
msg = {
"tid": parsl_tid,
"result_received": True,
"result": result
}
wq_tasks.remove(t.id)
collector_queue.put_nowait(msg)
if continue_running is False:
logger.debug("Exiting WorkQueue Master Thread event loop")
break
for wq_task in wq_tasks:
logger.debug("Cancelling Workqueue Task {}".format(wq_task))
q.cancel_by_taskid(wq_task)
logger.debug("Exiting WorkQueue Monitoring Process")
return 0
