def __init__(self, name, publisher_progress=None):
''' Constructor '''
self.name = name
self.song_list = set([])
self.publisher_progress = publisher_progress
self.work_queue = work_queue.WorkQueue(self)
self.work_queue.start()
def create_queue(port=wq.WORK_QUEUE_DEFAULT_PORT):
"""Create Queue
Create a work_queue WorkQueue on a given port and print a useful help message on success. Exit (1) on fail.
:param port: OPTIONAL, port to listen for workers. Default is work_queue's default port.
:return: WorkQueue object.
"""
# Spawn queue.
try:
q = wq.WorkQueue(port)
ip = get_ip()
# Print instructional message.
if ip != -1:
print("Listening for workers @ %s on port %s" % (ip, q.port))
print("(this is a best guess IP, depending on your computing environment you may need to adjust.)")
print("\nHINT: To start a worker, you can probably use this command:")
print("work_queue_worker -d all --cores 0 %s %s\n" % (ip, q.port))
else:
print("Listening for workers on port %s" % q.port)
except Exception as e:
print("WorkQueue Launch Failed.")
print(e)
sys.exit(1)
# Return queue.
return q
def run_quantum():
ESP = create_esp_surfaces(M)
work_queue.set_debug_flag('all')
wq = work_queue.WorkQueue(wq_port, exclusive=False, shutdown=False)
wq.specify_name('forcebalance')
os.makedirs('calcs')
os.chdir('calcs')
for i in range(M.ns):
dnm = eval(formstr % i)
os.makedirs(dnm)
os.chdir(dnm)
M.edit_qcrems({'igdesp': len(ESP[i])})
M.write("qchem.in", select=i)
ESPBohr = np.array(ESP[i]) / bohrang
np.savetxt('ESPGrid', ESPBohr)
print "Queueing up job", dnm
queue_up(wq,
command='qchem40 qchem.in qchem.out',
input_files=["qchem.in", "ESPGrid"],
output_files=["qchem.out", "plot.esp", "efield.dat"],
verbose=False)
os.chdir('..')
for i in range(M.ns):
wq_wait(wq)
os.chdir('..')
def __init__(self, port):
work_queue.set_debug_flag('all')
wq = work_queue.WorkQueue(port=port, exclusive=False, shutdown=False)
wq.tasks_failed = 0 # Counter for tasks that fail at the application level
wq.specify_keepalive_interval(8640000)
wq.specify_name('dihedral')
print('Work Queue listening on %d' % (wq.port))
self.wq = wq
def createWorkQueue(wq_port, debug=True):
global WORK_QUEUE
if debug:
work_queue.set_debug_flag('all')
WORK_QUEUE = work_queue.WorkQueue(port=wq_port, catalog=True, exclusive=False, shutdown=False)
WORK_QUEUE.specify_name('forcebalance')
#WORK_QUEUE.specify_keepalive_timeout(8640000)
WORK_QUEUE.specify_keepalive_interval(8640000)
def __init__(self, port, name='dihedral'):
# work_queue.set_debug_flag('all')
wq = work_queue.WorkQueue(port=port)
wq.specify_keepalive_interval(8640000)
wq.specify_name(name)
self.wq = wq
self.tasks_failed = 0 # Our own counter for tasks that failed
self.queue_status = None
self.last_print_time = 0
print('Work Queue listening on %d' % wq.port, file=sys.stderr)
def __init__(self, *args, **kwargs):
self._queue = work_queue.WorkQueue(*args, **kwargs)
self._thread = threading.Thread(target = self._sync_loop)
self._stop_queue_event = threading.Event()
self._tasks_to_return = ThreadQueue.Queue()
self._tasks_to_submit = ThreadQueue.Queue()
# calls to synchronous WorkQueue are coordinated with _queue_lock
self._queue_lock = threading.Lock()
self._thread.daemon = True
self._thread.start()
def _mk_wq(self):
global _AWE_WORK_QUEUE
if _AWE_WORK_QUEUE is not None:
### warn
awe.log('WARNING: using previously created WorkQueue instance')
else:
if self.debug:
WQ.set_debug_flag(self.debug)
if self.wq_logfile:
awe.util.makedirs_parent(self.wq_logfile)
WQ.cctools_debug_config_file(self.wq_logfile)
WQ.cctools_debug_config_file_size(0)
if self.name:
self.catalog = True
wq = WQ.WorkQueue(name=self.name,
port=self.port,
shutdown=self.shutdown,
catalog=self.catalog,
exclusive=self.exclusive)
wq.specify_algorithm(self.schedule)
if self.monitor:
wq.enable_monitoring(self.summaryfile)
if self.capacity:
wq.estimate_capacity()
awe.log('Running on port %d...' % wq.port)
if wq.name:
awe.log('Using project name %s' % wq.name)
if self.debug and self.wq_logfile:
awe.log('Logging WorkQueue to %s' % self.wq_logfile)
typ = type(self.fastabort)
if typ is float or typ is int:
wq.activate_fast_abort(self.fastabort)
_AWE_WORK_QUEUE = wq
awe.util.makedirs_parent(self.wqstats_logfile)
_AWE_WORK_QUEUE.specify_log(self.wqstats_logfile)
return _AWE_WORK_QUEUE
def __init__(self, *args, **kwargs):
local_worker_args = kwargs.get('local_worker', None)
if local_worker_args:
del kwargs['local_worker']
if local_worker_args is True:
# local_worker_args can be a dictionary of worker options, or
# simply 'True' to get the defaults (1 core, 512MB memory,
# 1000MB of disk)
local_worker_args = {}
# calls to synchronous WorkQueueFutures are coordinated with _queue_lock
self._queue_lock = threading.Lock()
self._stop_queue_event = threading.Event()
# set when queue is empty
self._join_event = threading.Event()
self._tasks_to_submit = ThreadQueue.Queue()
self._tasks_before_callbacks = ThreadQueue.Queue()
self._sync_loop = threading.Thread(target = self._sync_loop)
self._sync_loop.daemon = True
self._callback_loop = threading.Thread(target = self._callback_loop)
self._callback_loop.daemon = True
self._local_worker = None
self._queue = work_queue.WorkQueue(*args, **kwargs)
if local_worker_args:
self._local_worker = Worker(self.port, **local_worker_args)
self._sync_loop.start()
self._callback_loop.start()
atexit.register(self._terminate)
#!/usr/bin/env python
import work_queue
import os
work_queue.set_debug_flag('all')
wq = work_queue.WorkQueue(port=work_queue.WORK_QUEUE_RANDOM_PORT, exclusive=False, shutdown=True)
wq.specify_name('test')
for i in range(5):
task = work_queue.Task('date')
task.specify_algorithm(work_queue.WORK_QUEUE_SCHEDULE_FCFS)
task.specify_tag('current date/time [%d]' % i)
task.specify_input_file('/bin/date')
print task.id
print task.algorithm
print task.command
print task.tag
wq.submit(task)
os.environ['PATH'] = '../../../dttools/src:' + os.environ['PATH']
os.system('work_queue_worker -d all -t 5 localhost %d &' % wq.port)
while not wq.empty():
print '** wait for task'
task = wq.wait(1)
if task:
print 'task'
def work_queue_executor(items, function, accumulator, **kwargs):
"""Execute using Work Queue
Parameters
----------
items : list
List of input arguments
function : callable
A function to be called on each input, which returns an accumulator instance
accumulator : AccumulatorABC
An accumulator to collect the output of the function
status : bool
If true (default), enable progress bar
unit : str
Label of progress bar unit
desc : str
Label of progress bar description
compression : int, optional
Compress accumulator outputs in flight with LZ4, at level specified (default 1)
Set to ``None`` for no compression.
# work queue specific options:
environment-file : str
Python environment to use. Required.
cores : int
Number of cores for work queue task. If unset, use a whole worker.
memory : int
Amount of memory (in MB) for work queue task. If unset, use a whole worker.
disk : int
Amount of disk space (in MB) for work queue task. If unset, use a whole worker.
resources-mode : one of 'fixed', or 'auto'. Default is 'fixed'.
'fixed' - allocate cores, memory, and disk specified for each task.
'auto' - use cores, memory, and disk as maximum values to allocate.
Useful when the resources used by a task are not known, as
it lets work queue find an efficient value for maximum
throughput.
debug-log : str
Filename for debug output
stats-log : str
Filename for tasks statistics output
transactions-log : str
Filename for tasks lifetime reports output
master-name : str
Name to refer to this work queue master.
Sets port to 0 (any available port) if port not given.
port : int
Port number for work queue master program. Defaults to 9123 if
master-name not given.
wrapper : str
Wrapper script to run/open python environment tarball. Defaults to python_package_run found in PATH.
print-stdout : bool
If true (default), print the standard output of work queue task on completion.
queue-mode : one of 'persistent' or 'one-per-stage'. Default is 'persistent'.
'persistent' - One queue is used for all stages of processing.
'one-per-stage' - A new queue is used for each of the stages of processing.
resource-monitor : bool
If true, (false is the default) turns on resource monitoring for Work Queue.
"""
try:
import work_queue as wq
import tempfile
import dill
import os
from os.path import basename
except ImportError as e:
print('You must have Work Queue and dill installed to use work_queue_executor!')
raise e
global _wq_queue
debug_log = kwargs.pop('debug-log', None)
stats_log = kwargs.pop('stats-log', None)
trans_log = kwargs.pop('transactions-log', None)
master_name = kwargs.pop('master-name', None)
port = kwargs.pop('port', None)
if port is None:
if master_name:
port = 0
else:
port = 9123
queue_mode = kwargs.pop('queue-mode', 'persistent')
if _wq_queue is None or queue_mode == 'one-per-stage':
_wq_queue = wq.WorkQueue(port, name=master_name, debug_log=debug_log, stats_log=stats_log, transactions_log=trans_log)
print('Listening for work queue workers on port {}...'.format(_wq_queue.port))
unit = kwargs.pop('unit', 'items')
status = kwargs.pop('status', True)
desc = kwargs.pop('desc', 'Processing')
clevel = kwargs.pop('compression', 1)
filepath = kwargs.pop('filepath', '.')
output = kwargs.pop('print-stdout', False)
if clevel is not None:
function = _compression_wrapper(clevel, function)
# work queue specific options:
env_file = kwargs.pop('environment-file', None)
wrapper = kwargs.pop('wrapper', shutil.which('python_package_run'))
if not env_file:
raise TypeError("environment-file argument missing. It should name a conda environment as a tar file.")
elif not os.path.exists(env_file):
raise ValueError("environment-file does not name an existing conda environment as a tar file.")
if not wrapper:
raise ValueError("Location of python_package_run could not be determined automatically.\nUse 'wrapper' argument to the work_queue_executor.")
# fixed, or auto
resources_mode = kwargs.pop('resources-mode', 'fixed')
cores = kwargs.pop('cores', None)
memory = kwargs.pop('memory', None)
disk = kwargs.pop('disk', None)
resource_monitor = kwargs.pop('resource-monitor', False)
default_resources = {}
if cores:
default_resources['cores'] = cores
if memory:
default_resources['memory'] = memory
if disk:
default_resources['disk'] = disk
with tempfile.TemporaryDirectory(prefix="wq-executor-tmp-", dir=filepath) as tmpdir:
# Pickle function
with open(os.path.join(tmpdir, 'function.p'), 'wb') as wf:
dill.dump(function, wf)
# Set up Work Queue
command_path = _coffea_fn_as_file_wrapper(tmpdir)
if resource_monitor:
_wq_queue.enable_monitoring()
_wq_queue.specify_category_max_resources('default', default_resources)
if resources_mode == 'auto':
_wq_queue.tune('category-steady-n-tasks', 3)
_wq_queue.specify_category_max_resources('default', {})
_wq_queue.specify_category_mode('default', wq.WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT)
# Define function input here
infile_function = os.path.join(tmpdir, 'function.p')
# Dictionary to keep track of output file corresponding to task id
id_output = {}
# Iterative Executor Specifications
if len(items) == 0:
return accumulator
add_fn = _iadd
for i, item in tqdm(enumerate(items), disable=not status, unit=unit, total=len(items), desc=desc):
with open(os.path.join(tmpdir, 'item_{}.p'.format(i)), 'wb') as wf:
dill.dump(item, wf)
infile_item = os.path.join(tmpdir, 'item_{}.p'.format(i))
outfile = os.path.join(tmpdir, 'output_{}.p'.format(i))
coffea_command = 'python {} {} {} {}'.format(basename(command_path), basename(infile_function), basename(infile_item), basename(outfile))
wrapped_command = './{}'.format(basename(wrapper))
wrapped_command += ' --environment {}'.format(basename(env_file))
wrapped_command += ' --unpack-to "$WORK_QUEUE_SANDBOX"/{}-env {}'.format(env_file, coffea_command)
t = wq.Task(wrapped_command)
t.specify_category('default')
t.specify_input_file(command_path, cache=True)
t.specify_input_file(infile_function, cache=False)
t.specify_input_file(infile_item, cache=False)
# conda environment files
t.specify_input_file(env_file, cache=True)
t.specify_input_file(wrapper, cache=True)
if re.search('://', item.filename):
# This looks like an URL. Not transfering file.
pass
else:
t.specify_input_file(item.filename, remote_name=item.filename, cache=True)
t.specify_output_file(outfile, cache=False)
task_id = _wq_queue.submit(t)
# Add pair to dict
id_output['{}'.format(task_id)] = outfile
print('Submitted task (id #{}): {}'.format(task_id, wrapped_command))
print('Waiting for tasks to complete...')
while not _wq_queue.empty():
t = _wq_queue.wait(5)
if t:
print('Task (id #{}) complete: {} (return code {})'.format(t.id, t.command, t.return_status))
if output:
print('Output:\n{}'.format(t.output))
print('allocated cores: {}, memory: {} MB, disk: {} MB'.format(
t.resources_allocated.cores,
t.resources_allocated.memory,
t.resources_allocated.disk))
if resource_monitor:
print('measured cores: {}, memory: {} MB, disk {} MB, runtime {}'.format(
t.resources_measured.cores,
t.resources_measured.memory,
t.resources_measured.disk,
t.resources_measured.wall_time / 1000000))
if t.result != 0:
print('Task id #{} failed with code: {}'.format(t.id, t.result))
print('Stopping execution')
break
# Unpickle output, add to accumulator
with open(id_output['{}'.format(t.id)], 'rb') as rf:
unpickle_output = dill.load(rf)
add_fn(accumulator, unpickle_output)
if os.path.exists(command_path):
os.remove(command_path)
return accumulator
return t
port_file = None
try:
port_file = sys.argv[1]
except IndexError:
sys.stderr.write("Usage: {} PORTFILE\n".format(sys.argv[0]))
with open(path.join(test_dir, input_file), 'w') as f:
f.write('hello world\n')
shutil.copyfile('/bin/cat', path.join(test_dir, exec_file))
os.chmod(path.join(test_dir, exec_file), stat.S_IRWXU)
q = wq.WorkQueue(0)
with open(port_file, 'w') as f:
print('Writing port {port} to file {file}'.format(port=q.port,
file=port_file))
f.write(str(q.port))
# simple task
# define a task, sending stderr to console, and stdout to output
output = output_file()
t = wq.Task("./{exe} {input} 2>&1 > {output}".format(exe=exec_file,
input=input_file,
output=output))
t.specify_input_file(path.join(test_dir, exec_file), exec_file)
t.specify_input_file(path.join(test_dir, input_file), input_file)
t.specify_output_file(path.join(test_dir, output), output)
# Folder filter.
dirs = [i for i in dirs if i.startswith("PAH/phenbenz")]
dirs = ["ecig-manual/ecigrhfr6"]
# If we are doing a certain set of calculations, then we automatically read them.
if args.qg: args.rg = True
if args.qt: args.rt = True
if args.rt: args.rg = True
if args.qf: args.rf = True
RUN = False
if args.qa or args.qg or args.qt or args.qf:
work_queue.set_debug_flag('all')
wq = work_queue.WorkQueue(port=args.p, exclusive=False, shutdown=False)
wq.specify_keepalive_interval(8640000)
wq.specify_name('nanoreactor')
print('Work Queue named %s listening on %d' % (wq.name, wq.port))
RUN = True
def getN(xyz, frm):
# Return a nanoreactor object corresponding to the selected frame of this xyz.
cwd = os.getcwd()
dnm, xyz = os.path.split(xyz)
os.chdir(dnm)
M = Molecule(xyz)
M[frm].write('.tmp.xyz')
NR = Nanoreactor('.tmp.xyz', enhance=1.2, printlvl=-1, boring=[])
os.chdir(cwd)
return NR
import sys
import work_queue
# Constants
LENGTH = int(sys.argv[1])
ATTEMPTS = int(sys.argv[2])
HASHES = sys.argv[3]
TASKS = int(sys.argv[4])
SOURCES = ('mini-hulk.py', HASHES)
PORT = 8451
# Main Execution
if __name__ == '__main__':
queue = work_queue.WorkQueue(PORT, name='fury-luke', catalog=True)
queue.specify_log('mini-fury.log')
for _ in range(TASKS):
command = './mini-hulk.py {} {} {}'.format(LENGTH, ATTEMPTS, HASHES)
task = work_que.TASK(command)
for source in SOURCES:
task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT)
queue.submit(task)
while not queue.empty():
task = queue.wait()
if task and task.return_status == 0:
sys.stdout.write(task.output)
def __init__(self):
self._work_queue = work_queue.WorkQueue()
def sprint(self):
with util.PartiallyMutable.unlock():
self.source = TaskProvider(self.config)
action = actions.Actions(self.config, self.source)
logger.info("using wq from {0}".format(wq.__file__))
logger.info("running Lobster version {0}".format(util.get_version()))
logger.info("current PID is {0}".format(os.getpid()))
wq.cctools_debug_flags_set("all")
wq.cctools_debug_config_file(
os.path.join(self.config.workdir, "work_queue_debug.log"))
wq.cctools_debug_config_file_size(1 << 29)
self.queue = wq.WorkQueue(self.config.advanced.wq_port)
self.queue.specify_min_taskid(self.source.max_taskid() + 1)
self.queue.specify_log(
os.path.join(self.config.workdir, "work_queue.log"))
self.queue.specify_transactions_log(
os.path.join(self.config.workdir, "transactions.log"))
self.queue.specify_name("lobster_" + self.config.label)
self.queue.specify_keepalive_timeout(300)
# self.queue.tune("short-timeout", 600)
self.queue.tune("transfer-outlier-factor", 4)
self.queue.specify_algorithm(wq.WORK_QUEUE_SCHEDULE_RAND)
if self.config.advanced.full_monitoring:
self.queue.enable_monitoring_full(None)
else:
self.queue.enable_monitoring(None)
logger.info("starting queue as {0}".format(self.queue.name))
abort_active = False
abort_threshold = self.config.advanced.abort_threshold
abort_multiplier = self.config.advanced.abort_multiplier
wq_max_retries = self.config.advanced.wq_max_retries
if util.checkpoint(self.config.workdir, 'KILLED') == 'PENDING':
util.register_checkpoint(self.config.workdir, 'KILLED', 'RESTART')
# time in seconds to wait for WQ to return tasks, with minimum wait
# time in case no more tasks are waiting
interval = 120
interval_minimum = 30
tasks_left = 0
units_left = 0
successful_tasks = 0
categories = []
self.setup_logging('all')
# Workflows can be assigned categories, with each category having
# different cpu/memory/walltime requirements that WQ will automatically
# fine-tune
for category in self.config.categories:
constraints = category.wq()
if category.name != 'merge':
categories.append(category.name)
self.setup_logging(category.name)
self.queue.specify_category_mode(category.name, category.mode)
if category.mode == wq.WORK_QUEUE_ALLOCATION_MODE_FIXED:
self.queue.specify_category_max_resources(
category.name, constraints)
else:
self.queue.specify_category_first_allocation_guess(
category.name, constraints)
logger.debug('Category {0}: {1}'.format(category.name,
constraints))
if 'wall_time' not in constraints:
self.queue.activate_fast_abort_category(
category.name, abort_multiplier)
proxy_email_sent = False
while not self.source.done():
with self.measure('status'):
tasks_left = self.source.tasks_left()
units_left = self.source.work_left()
logger.debug("expecting {0} tasks, still".format(tasks_left))
self.queue.specify_num_tasks_left(tasks_left)
for c in categories + ['all']:
self.log(c, units_left)
if util.checkpoint(self.config.workdir, 'KILLED') == 'PENDING':
util.register_checkpoint(self.config.workdir, 'KILLED',
str(datetime.datetime.utcnow()))
# let the task source shut down gracefully
logger.info("terminating task source")
self.source.terminate()
logger.info("terminating gracefully")
break
with self.measure('create'):
have = {}
for c in categories:
cstats = self.queue.stats_category(c)
have[c] = {
'running': cstats.tasks_running,
'queued': cstats.tasks_waiting
}
stats = self.queue.stats_hierarchy
tasks = self.source.obtain(stats.total_cores, have)
expiry = None
if self.config.advanced.proxy:
expiry = self.config.advanced.proxy.expires()
proxy_time_left = self.config.advanced.proxy.time_left()
if proxy_time_left >= 24 * 3600:
proxy_email_sent = False
if proxy_time_left < 24 * 3600 and not proxy_email_sent:
util.sendemail(
"Your proxy is about to expire.\n" + "Timeleft: " +
str(datetime.timedelta(seconds=proxy_time_left)),
self.config)
proxy_email_sent = True
for category, cmd, id, inputs, outputs, env, dir in tasks:
task = wq.Task(cmd)
task.specify_category(category)
task.specify_tag(id)
task.specify_max_retries(wq_max_retries)
task.specify_monitor_output(
os.path.join(dir, 'resource_monitor'))
for k, v in env.items():
task.specify_environment_variable(k, v)
for (local, remote, cache) in inputs:
cache_opt = wq.WORK_QUEUE_CACHE if cache else wq.WORK_QUEUE_NOCACHE
if os.path.isfile(local) or os.path.isdir(local):
task.specify_input_file(str(local), str(remote),
cache_opt)
else:
logger.critical(
"cannot send file to worker: {0}".format(
local))
raise NotImplementedError
for (local, remote) in outputs:
task.specify_output_file(str(local), str(remote))
if expiry:
task.specify_end_time(expiry * 10**6)
self.queue.submit(task)
with self.measure('status'):
stats = self.queue.stats_hierarchy
logger.info(
"{0} out of {1} workers busy; {2} tasks running, {3} waiting; {4} units left"
.format(stats.workers_busy,
stats.workers_busy + stats.workers_ready,
stats.tasks_running, stats.tasks_waiting,
units_left))
with self.measure('update'):
self.source.update(self.queue)
# recurring actions are triggered here; plotting etc should run
# while we have WQ hand us back tasks w/o any database
# interaction
with self.measure('action'):
if action:
action.take()
with self.measure('fetch'):
starttime = time.time()
task = self.queue.wait(interval)
tasks = []
while task:
if task.return_status == 0:
successful_tasks += 1
elif task.return_status in self.config.advanced.bad_exit_codes:
logger.warning(
"blacklisting host {0} due to bad exit code from task {1}"
.format(task.hostname, task.tag))
self.queue.blacklist(task.hostname)
tasks.append(task)
remaining = int(starttime + interval - time.time())
if (interval - remaining < interval_minimum
or self.queue.stats.tasks_waiting > 0
) and remaining > 0:
task = self.queue.wait(remaining)
else:
task = None
# TODO do we really need this? We have everything based on
# categories by now, so this should not be needed.
if abort_threshold > 0 and successful_tasks >= abort_threshold and not abort_active:
logger.info(
"activating fast abort with multiplier: {0}".format(
abort_multiplier))
abort_active = True
self.queue.activate_fast_abort(abort_multiplier)
if len(tasks) > 0:
try:
with self.measure('return'):
self.source.release(tasks)
except Exception:
tb = traceback.format_exc()
logger.critical(
"cannot recover from the following exception:\n" + tb)
util.sendemail(
"Your Lobster project has crashed from the following exception:\n"
+ tb, self.config)
for task in tasks:
logger.critical(
"tried to return task {0} from {1}".format(
task.tag, task.hostname))
raise
if units_left == 0:
logger.info("no more work left to do")
util.sendemail("Your Lobster project is done!", self.config)
if self.config.elk:
self.config.elk.end()
if action:
action.take(True)
import json
import os
import itertools
#Constants
SOURCES = ('hulk.py', 'hashes.txt')
PORT = 9123
ALPHABET = string.ascii_lowercase + string.digits
# Main Execution
if __name__ == '__main__':
# Create a master
# 1. Random port between 9000 - 9999
# 2. Project name of fury-samuel
# 3. Catalog mode enabled
queue = work_queue.WorkQueue(PORT, name='fury-mchen6apaek1', catalog=True)
queue.specify_log('fury.log') #Specify Work Queue log location
# Create task list
# Check passwords for length of <6
for num in range(5):
command = './hulk.py -l {}'.format(num)
task = work_queue.Task(command)
# Add source files (transfer files to workers)
for source in SOURCES:
task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT)
# Submit tasks
queue.submit(task)
import itertools
import string
import json
#Constants
ALPHABET = string.ascii_lowercase + string.digits
HASHES = "hashes.txt"
SOURCES = ('hulk.py', HASHES)
PORT = 9284
JOURNAL = {}
#Main Execution
if __name__ == '__main__':
#Start work queue
queue = work_queue.WorkQueue(PORT, name='hulk-alam3', catalog=True)
queue.specify_log('fury.log')
#Combinations of length 1-5
for length in range(1, 6):
command = './hulk.py -l {}'.format(length)
#Need to make sure machines being used have hulk.py in them
if command not in JOURNAL:
task = work_queue.Task(command)
#Add source files
for source in SOURCES:
task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT)
queue.submit(task)
else:
print >> sys.stderr, 'Already did', command
import sys
import os
import work_queue
import itertools
import string
import json
HASHES = "hashes.txt"
PORT = 9199
TASKS = 100
ALPHABET = string.ascii_lowercase + string.digits
if __name__ == '__main__':
JOURNAL = json.load(open('journal.json'))
queue = work_queue.WorkQueue(PORT, name='hulk-amunch', catalog=True)
queue.specify_log('fury.log')
for num in range(1, 6):
command = './hulk.py -l {}'.format(num)
if command in JOURNAL:
print >> sys.stderr, 'Already did', command
else:
task = work_queue.Task(command)
for source in ('hulk.py', HASHES):
task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT)
queue.submit(task)
for num in range(1, 4):
for prefix in itertools.product(ALPHABET, repeat=int(num)):
prefix = ''.join(prefix)
return y / x
class NoResult(Exception):
""" Exception used as the result of a function evaluation of a task that failed. """
def __repr__(self):
return 'NoResult()'
def __str__(self):
return 'NoResult'
if __name__ == '__main__':
with tempfile.TemporaryDirectory() as tmpdir:
q = wq.WorkQueue(port=9123,
debug_log='debug.log',
transactions_log='tr.log')
task_counter = 0
total_tasks = 5
for i in range(total_tasks):
task = create_work_queue_task(
task_counter,
tmpdir,
application_function,
input_args=[i, 100]
) # all tasks compute application_function(i, 100) which is 100/i
q.submit(task)
task_counter += 1
# serialized and sent back to the Work Queue manager, where it is read and can
# be used as a regular python value. If an exception occurs, it is set as the
# result of the function, where it can be handled as appropiate.
#
import work_queue as wq
def application_function(x, y):
""" Example function to be executed remotely. """
return y / x
if __name__ == '__main__':
q = wq.WorkQueue(9123)
total_tasks = 20
for i in range(total_tasks):
task = wq.PythonTask(
application_function, i + 1, 100
) # all tasks compute application_function(i, 100) which is 100/i
task.specify_environment('venv.tar.gz')
q.submit(task)
while not q.empty():
t = q.wait(5)
if t:
x = t.output
print(x)
# lidx = opt.index('2>')+1
# cDict['sqCMD']['LOG'] = opt[lidx]
# cDict['wqIO']['2>'] = opt[lidx].split('/')[-1]
cDict['CMD'] = './'+cDict['BLAST']+' '+ \
' '.join([k+' '+cDict['wqCMD'][k] \
for k in cDict['wqCMD'].keys()])+' '+ \
c1Opts+c2Opts+' '+' '.join([k+' '+cDict['wqIO'][k] \
for k in cDict['wqIO'].keys()])
return cDict
##
q = wq.WorkQueue(port)
while True:
ts = q.wait(1)
# ts = q.wait(wq.WORK_QUEUE_WAITFORTASK)
time.sleep(2)
seq = os.path.isdir('/proc/' + str(pid))
if not seq:
sys.exit()
nfiles = glob.glob(wdir + '/*.cmd')
for file in nfiles:
command = readCMD(file)
def _mk_wq(self):
"""
Only one instance of WorkQueue should be run per process. This grants
access to the WorkQueue singleton or else creates a new WorkQueue
instance. This also ensures that the cctools WorkQueue object can
handle more workers.
Parameters:
None
Returns:
The cctools WorkQueue singleton object
"""
global _AWE_WORK_QUEUE
if _AWE_WORK_QUEUE is not None:
### warn
awe.log('WARNING: using previously created WorkQueue instance')
else:
if self.debug:
# Set up debugging parameters for the cctools WorkQueue object.
# It has inbuilt debugging capabilities.
WQ.set_debug_flag(self.debug)
if self.wq_logfile:
awe.util.makedirs_parent(self.wq_logfile)
WQ.cctools_debug_config_file(self.wq_logfile)
WQ.cctools_debug_config_file_size(0)
if self.name:
self.catalog = True
# Create the cctools WorkQueue object
wq = WQ.WorkQueue(name = self.name,
port = self.port,
shutdown = self.shutdown,
catalog = self.catalog,
exclusive = self.exclusive)
# Specify the task scheduling algorithm
wq.specify_algorithm(self.schedule)
# Turn cctools WorkQueue object status monitoring on or off
if self.monitor:
wq.enable_monitoring(self.summaryfile)
if self.capacity:
# Determine the number of workers the WorkQueue object can handle
wq.estimate_capacity()
# Display information about this run of AWE-WQ
awe.log('Running on port %d...' % wq.port)
if wq.name:
awe.log('Using project name %s' % wq.name)
if self.debug and self.wq_logfile:
awe.log('Logging WorkQueue to %s' % self.wq_logfile)
# Set up fast abort procedures
typ = type(self.fastabort)
if typ is float or typ is int:
wq.activate_fast_abort(self.fastabort)
# Ensure that the singleton is set to the new instance
_AWE_WORK_QUEUE = wq
# Ensure that the singleton is logging to the correct files
awe.util.makedirs_parent(self.wqstats_logfile)
_AWE_WORK_QUEUE.specify_log(self.wqstats_logfile)
# Return a reference to teh singleton
return _AWE_WORK_QUEUE
import work_queue as WQ
# in case we want ${USER} for master name
from os import environ
import time
#### Customize your master name
master_name = environ['USER'] + '-master'
# run at some port at random
q = WQ.WorkQueue(name=master_name, port=0)
# enable the transactions log
q.specify_transactions_log('my_transactions.log')
print 'WorkQueue on port: {}'.format(q.port)
# enable the measuring of resources
q.enable_monitoring()
# create a category for all tasks
q.specify_category_max_resources('my-tasks', {'cores': 1, 'disk': 500})
q.specify_category_mode('my-tasks',
WQ.WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT)
# create 30 tasks. Each task simply creates a 200MB file, using 10MB of memory
# buffer.
for i in range(0, 30):
t = WQ.Task('python task.py')
t.specify_input_file('task.py', cache=True)
t.specify_category('my-tasks')
