def __init__(self, name, publisher_progress=None): ''' Constructor ''' self.name = name self.song_list = set([]) self.publisher_progress = publisher_progress self.work_queue = work_queue.WorkQueue(self) self.work_queue.start()
def create_queue(port=wq.WORK_QUEUE_DEFAULT_PORT): """Create Queue Create a work_queue WorkQueue on a given port and print a useful help message on success. Exit (1) on fail. :param port: OPTIONAL, port to listen for workers. Default is work_queue's default port. :return: WorkQueue object. """ # Spawn queue. try: q = wq.WorkQueue(port) ip = get_ip() # Print instructional message. if ip != -1: print("Listening for workers @ %s on port %s" % (ip, q.port)) print("(this is a best guess IP, depending on your computing environment you may need to adjust.)") print("\nHINT: To start a worker, you can probably use this command:") print("work_queue_worker -d all --cores 0 %s %s\n" % (ip, q.port)) else: print("Listening for workers on port %s" % q.port) except Exception as e: print("WorkQueue Launch Failed.") print(e) sys.exit(1) # Return queue. return q
def run_quantum(): ESP = create_esp_surfaces(M) work_queue.set_debug_flag('all') wq = work_queue.WorkQueue(wq_port, exclusive=False, shutdown=False) wq.specify_name('forcebalance') os.makedirs('calcs') os.chdir('calcs') for i in range(M.ns): dnm = eval(formstr % i) os.makedirs(dnm) os.chdir(dnm) M.edit_qcrems({'igdesp': len(ESP[i])}) M.write("qchem.in", select=i) ESPBohr = np.array(ESP[i]) / bohrang np.savetxt('ESPGrid', ESPBohr) print "Queueing up job", dnm queue_up(wq, command='qchem40 qchem.in qchem.out', input_files=["qchem.in", "ESPGrid"], output_files=["qchem.out", "plot.esp", "efield.dat"], verbose=False) os.chdir('..') for i in range(M.ns): wq_wait(wq) os.chdir('..')
def __init__(self, port): work_queue.set_debug_flag('all') wq = work_queue.WorkQueue(port=port, exclusive=False, shutdown=False) wq.tasks_failed = 0 # Counter for tasks that fail at the application level wq.specify_keepalive_interval(8640000) wq.specify_name('dihedral') print('Work Queue listening on %d' % (wq.port)) self.wq = wq
def createWorkQueue(wq_port, debug=True): global WORK_QUEUE if debug: work_queue.set_debug_flag('all') WORK_QUEUE = work_queue.WorkQueue(port=wq_port, catalog=True, exclusive=False, shutdown=False) WORK_QUEUE.specify_name('forcebalance') #WORK_QUEUE.specify_keepalive_timeout(8640000) WORK_QUEUE.specify_keepalive_interval(8640000)
def __init__(self, port, name='dihedral'): # work_queue.set_debug_flag('all') wq = work_queue.WorkQueue(port=port) wq.specify_keepalive_interval(8640000) wq.specify_name(name) self.wq = wq self.tasks_failed = 0 # Our own counter for tasks that failed self.queue_status = None self.last_print_time = 0 print('Work Queue listening on %d' % wq.port, file=sys.stderr)
def __init__(self, *args, **kwargs): self._queue = work_queue.WorkQueue(*args, **kwargs) self._thread = threading.Thread(target = self._sync_loop) self._stop_queue_event = threading.Event() self._tasks_to_return = ThreadQueue.Queue() self._tasks_to_submit = ThreadQueue.Queue() # calls to synchronous WorkQueue are coordinated with _queue_lock self._queue_lock = threading.Lock() self._thread.daemon = True self._thread.start()
def _mk_wq(self): global _AWE_WORK_QUEUE if _AWE_WORK_QUEUE is not None: ### warn awe.log('WARNING: using previously created WorkQueue instance') else: if self.debug: WQ.set_debug_flag(self.debug) if self.wq_logfile: awe.util.makedirs_parent(self.wq_logfile) WQ.cctools_debug_config_file(self.wq_logfile) WQ.cctools_debug_config_file_size(0) if self.name: self.catalog = True wq = WQ.WorkQueue(name=self.name, port=self.port, shutdown=self.shutdown, catalog=self.catalog, exclusive=self.exclusive) wq.specify_algorithm(self.schedule) if self.monitor: wq.enable_monitoring(self.summaryfile) if self.capacity: wq.estimate_capacity() awe.log('Running on port %d...' % wq.port) if wq.name: awe.log('Using project name %s' % wq.name) if self.debug and self.wq_logfile: awe.log('Logging WorkQueue to %s' % self.wq_logfile) typ = type(self.fastabort) if typ is float or typ is int: wq.activate_fast_abort(self.fastabort) _AWE_WORK_QUEUE = wq awe.util.makedirs_parent(self.wqstats_logfile) _AWE_WORK_QUEUE.specify_log(self.wqstats_logfile) return _AWE_WORK_QUEUE
def __init__(self, *args, **kwargs): local_worker_args = kwargs.get('local_worker', None) if local_worker_args: del kwargs['local_worker'] if local_worker_args is True: # local_worker_args can be a dictionary of worker options, or # simply 'True' to get the defaults (1 core, 512MB memory, # 1000MB of disk) local_worker_args = {} # calls to synchronous WorkQueueFutures are coordinated with _queue_lock self._queue_lock = threading.Lock() self._stop_queue_event = threading.Event() # set when queue is empty self._join_event = threading.Event() self._tasks_to_submit = ThreadQueue.Queue() self._tasks_before_callbacks = ThreadQueue.Queue() self._sync_loop = threading.Thread(target = self._sync_loop) self._sync_loop.daemon = True self._callback_loop = threading.Thread(target = self._callback_loop) self._callback_loop.daemon = True self._local_worker = None self._queue = work_queue.WorkQueue(*args, **kwargs) if local_worker_args: self._local_worker = Worker(self.port, **local_worker_args) self._sync_loop.start() self._callback_loop.start() atexit.register(self._terminate)
#!/usr/bin/env python import work_queue import os work_queue.set_debug_flag('all') wq = work_queue.WorkQueue(port=work_queue.WORK_QUEUE_RANDOM_PORT, exclusive=False, shutdown=True) wq.specify_name('test') for i in range(5): task = work_queue.Task('date') task.specify_algorithm(work_queue.WORK_QUEUE_SCHEDULE_FCFS) task.specify_tag('current date/time [%d]' % i) task.specify_input_file('/bin/date') print task.id print task.algorithm print task.command print task.tag wq.submit(task) os.environ['PATH'] = '../../../dttools/src:' + os.environ['PATH'] os.system('work_queue_worker -d all -t 5 localhost %d &' % wq.port) while not wq.empty(): print '** wait for task' task = wq.wait(1) if task: print 'task'
def work_queue_executor(items, function, accumulator, **kwargs): """Execute using Work Queue Parameters ---------- items : list List of input arguments function : callable A function to be called on each input, which returns an accumulator instance accumulator : AccumulatorABC An accumulator to collect the output of the function status : bool If true (default), enable progress bar unit : str Label of progress bar unit desc : str Label of progress bar description compression : int, optional Compress accumulator outputs in flight with LZ4, at level specified (default 1) Set to ``None`` for no compression. # work queue specific options: environment-file : str Python environment to use. Required. cores : int Number of cores for work queue task. If unset, use a whole worker. memory : int Amount of memory (in MB) for work queue task. If unset, use a whole worker. disk : int Amount of disk space (in MB) for work queue task. If unset, use a whole worker. resources-mode : one of 'fixed', or 'auto'. Default is 'fixed'. 'fixed' - allocate cores, memory, and disk specified for each task. 'auto' - use cores, memory, and disk as maximum values to allocate. Useful when the resources used by a task are not known, as it lets work queue find an efficient value for maximum throughput. debug-log : str Filename for debug output stats-log : str Filename for tasks statistics output transactions-log : str Filename for tasks lifetime reports output master-name : str Name to refer to this work queue master. Sets port to 0 (any available port) if port not given. port : int Port number for work queue master program. Defaults to 9123 if master-name not given. wrapper : str Wrapper script to run/open python environment tarball. Defaults to python_package_run found in PATH. print-stdout : bool If true (default), print the standard output of work queue task on completion. queue-mode : one of 'persistent' or 'one-per-stage'. Default is 'persistent'. 'persistent' - One queue is used for all stages of processing. 'one-per-stage' - A new queue is used for each of the stages of processing. resource-monitor : bool If true, (false is the default) turns on resource monitoring for Work Queue. """ try: import work_queue as wq import tempfile import dill import os from os.path import basename except ImportError as e: print('You must have Work Queue and dill installed to use work_queue_executor!') raise e global _wq_queue debug_log = kwargs.pop('debug-log', None) stats_log = kwargs.pop('stats-log', None) trans_log = kwargs.pop('transactions-log', None) master_name = kwargs.pop('master-name', None) port = kwargs.pop('port', None) if port is None: if master_name: port = 0 else: port = 9123 queue_mode = kwargs.pop('queue-mode', 'persistent') if _wq_queue is None or queue_mode == 'one-per-stage': _wq_queue = wq.WorkQueue(port, name=master_name, debug_log=debug_log, stats_log=stats_log, transactions_log=trans_log) print('Listening for work queue workers on port {}...'.format(_wq_queue.port)) unit = kwargs.pop('unit', 'items') status = kwargs.pop('status', True) desc = kwargs.pop('desc', 'Processing') clevel = kwargs.pop('compression', 1) filepath = kwargs.pop('filepath', '.') output = kwargs.pop('print-stdout', False) if clevel is not None: function = _compression_wrapper(clevel, function) # work queue specific options: env_file = kwargs.pop('environment-file', None) wrapper = kwargs.pop('wrapper', shutil.which('python_package_run')) if not env_file: raise TypeError("environment-file argument missing. It should name a conda environment as a tar file.") elif not os.path.exists(env_file): raise ValueError("environment-file does not name an existing conda environment as a tar file.") if not wrapper: raise ValueError("Location of python_package_run could not be determined automatically.\nUse 'wrapper' argument to the work_queue_executor.") # fixed, or auto resources_mode = kwargs.pop('resources-mode', 'fixed') cores = kwargs.pop('cores', None) memory = kwargs.pop('memory', None) disk = kwargs.pop('disk', None) resource_monitor = kwargs.pop('resource-monitor', False) default_resources = {} if cores: default_resources['cores'] = cores if memory: default_resources['memory'] = memory if disk: default_resources['disk'] = disk with tempfile.TemporaryDirectory(prefix="wq-executor-tmp-", dir=filepath) as tmpdir: # Pickle function with open(os.path.join(tmpdir, 'function.p'), 'wb') as wf: dill.dump(function, wf) # Set up Work Queue command_path = _coffea_fn_as_file_wrapper(tmpdir) if resource_monitor: _wq_queue.enable_monitoring() _wq_queue.specify_category_max_resources('default', default_resources) if resources_mode == 'auto': _wq_queue.tune('category-steady-n-tasks', 3) _wq_queue.specify_category_max_resources('default', {}) _wq_queue.specify_category_mode('default', wq.WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT) # Define function input here infile_function = os.path.join(tmpdir, 'function.p') # Dictionary to keep track of output file corresponding to task id id_output = {} # Iterative Executor Specifications if len(items) == 0: return accumulator add_fn = _iadd for i, item in tqdm(enumerate(items), disable=not status, unit=unit, total=len(items), desc=desc): with open(os.path.join(tmpdir, 'item_{}.p'.format(i)), 'wb') as wf: dill.dump(item, wf) infile_item = os.path.join(tmpdir, 'item_{}.p'.format(i)) outfile = os.path.join(tmpdir, 'output_{}.p'.format(i)) coffea_command = 'python {} {} {} {}'.format(basename(command_path), basename(infile_function), basename(infile_item), basename(outfile)) wrapped_command = './{}'.format(basename(wrapper)) wrapped_command += ' --environment {}'.format(basename(env_file)) wrapped_command += ' --unpack-to "$WORK_QUEUE_SANDBOX"/{}-env {}'.format(env_file, coffea_command) t = wq.Task(wrapped_command) t.specify_category('default') t.specify_input_file(command_path, cache=True) t.specify_input_file(infile_function, cache=False) t.specify_input_file(infile_item, cache=False) # conda environment files t.specify_input_file(env_file, cache=True) t.specify_input_file(wrapper, cache=True) if re.search('://', item.filename): # This looks like an URL. Not transfering file. pass else: t.specify_input_file(item.filename, remote_name=item.filename, cache=True) t.specify_output_file(outfile, cache=False) task_id = _wq_queue.submit(t) # Add pair to dict id_output['{}'.format(task_id)] = outfile print('Submitted task (id #{}): {}'.format(task_id, wrapped_command)) print('Waiting for tasks to complete...') while not _wq_queue.empty(): t = _wq_queue.wait(5) if t: print('Task (id #{}) complete: {} (return code {})'.format(t.id, t.command, t.return_status)) if output: print('Output:\n{}'.format(t.output)) print('allocated cores: {}, memory: {} MB, disk: {} MB'.format( t.resources_allocated.cores, t.resources_allocated.memory, t.resources_allocated.disk)) if resource_monitor: print('measured cores: {}, memory: {} MB, disk {} MB, runtime {}'.format( t.resources_measured.cores, t.resources_measured.memory, t.resources_measured.disk, t.resources_measured.wall_time / 1000000)) if t.result != 0: print('Task id #{} failed with code: {}'.format(t.id, t.result)) print('Stopping execution') break # Unpickle output, add to accumulator with open(id_output['{}'.format(t.id)], 'rb') as rf: unpickle_output = dill.load(rf) add_fn(accumulator, unpickle_output) if os.path.exists(command_path): os.remove(command_path) return accumulator
return t port_file = None try: port_file = sys.argv[1] except IndexError: sys.stderr.write("Usage: {} PORTFILE\n".format(sys.argv[0])) with open(path.join(test_dir, input_file), 'w') as f: f.write('hello world\n') shutil.copyfile('/bin/cat', path.join(test_dir, exec_file)) os.chmod(path.join(test_dir, exec_file), stat.S_IRWXU) q = wq.WorkQueue(0) with open(port_file, 'w') as f: print('Writing port {port} to file {file}'.format(port=q.port, file=port_file)) f.write(str(q.port)) # simple task # define a task, sending stderr to console, and stdout to output output = output_file() t = wq.Task("./{exe} {input} 2>&1 > {output}".format(exe=exec_file, input=input_file, output=output)) t.specify_input_file(path.join(test_dir, exec_file), exec_file) t.specify_input_file(path.join(test_dir, input_file), input_file) t.specify_output_file(path.join(test_dir, output), output)
# Folder filter. dirs = [i for i in dirs if i.startswith("PAH/phenbenz")] dirs = ["ecig-manual/ecigrhfr6"] # If we are doing a certain set of calculations, then we automatically read them. if args.qg: args.rg = True if args.qt: args.rt = True if args.rt: args.rg = True if args.qf: args.rf = True RUN = False if args.qa or args.qg or args.qt or args.qf: work_queue.set_debug_flag('all') wq = work_queue.WorkQueue(port=args.p, exclusive=False, shutdown=False) wq.specify_keepalive_interval(8640000) wq.specify_name('nanoreactor') print('Work Queue named %s listening on %d' % (wq.name, wq.port)) RUN = True def getN(xyz, frm): # Return a nanoreactor object corresponding to the selected frame of this xyz. cwd = os.getcwd() dnm, xyz = os.path.split(xyz) os.chdir(dnm) M = Molecule(xyz) M[frm].write('.tmp.xyz') NR = Nanoreactor('.tmp.xyz', enhance=1.2, printlvl=-1, boring=[]) os.chdir(cwd) return NR
import sys import work_queue # Constants LENGTH = int(sys.argv[1]) ATTEMPTS = int(sys.argv[2]) HASHES = sys.argv[3] TASKS = int(sys.argv[4]) SOURCES = ('mini-hulk.py', HASHES) PORT = 8451 # Main Execution if __name__ == '__main__': queue = work_queue.WorkQueue(PORT, name='fury-luke', catalog=True) queue.specify_log('mini-fury.log') for _ in range(TASKS): command = './mini-hulk.py {} {} {}'.format(LENGTH, ATTEMPTS, HASHES) task = work_que.TASK(command) for source in SOURCES: task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT) queue.submit(task) while not queue.empty(): task = queue.wait() if task and task.return_status == 0: sys.stdout.write(task.output)
def __init__(self): self._work_queue = work_queue.WorkQueue()
def sprint(self): with util.PartiallyMutable.unlock(): self.source = TaskProvider(self.config) action = actions.Actions(self.config, self.source) logger.info("using wq from {0}".format(wq.__file__)) logger.info("running Lobster version {0}".format(util.get_version())) logger.info("current PID is {0}".format(os.getpid())) wq.cctools_debug_flags_set("all") wq.cctools_debug_config_file( os.path.join(self.config.workdir, "work_queue_debug.log")) wq.cctools_debug_config_file_size(1 << 29) self.queue = wq.WorkQueue(self.config.advanced.wq_port) self.queue.specify_min_taskid(self.source.max_taskid() + 1) self.queue.specify_log( os.path.join(self.config.workdir, "work_queue.log")) self.queue.specify_transactions_log( os.path.join(self.config.workdir, "transactions.log")) self.queue.specify_name("lobster_" + self.config.label) self.queue.specify_keepalive_timeout(300) # self.queue.tune("short-timeout", 600) self.queue.tune("transfer-outlier-factor", 4) self.queue.specify_algorithm(wq.WORK_QUEUE_SCHEDULE_RAND) if self.config.advanced.full_monitoring: self.queue.enable_monitoring_full(None) else: self.queue.enable_monitoring(None) logger.info("starting queue as {0}".format(self.queue.name)) abort_active = False abort_threshold = self.config.advanced.abort_threshold abort_multiplier = self.config.advanced.abort_multiplier wq_max_retries = self.config.advanced.wq_max_retries if util.checkpoint(self.config.workdir, 'KILLED') == 'PENDING': util.register_checkpoint(self.config.workdir, 'KILLED', 'RESTART') # time in seconds to wait for WQ to return tasks, with minimum wait # time in case no more tasks are waiting interval = 120 interval_minimum = 30 tasks_left = 0 units_left = 0 successful_tasks = 0 categories = [] self.setup_logging('all') # Workflows can be assigned categories, with each category having # different cpu/memory/walltime requirements that WQ will automatically # fine-tune for category in self.config.categories: constraints = category.wq() if category.name != 'merge': categories.append(category.name) self.setup_logging(category.name) self.queue.specify_category_mode(category.name, category.mode) if category.mode == wq.WORK_QUEUE_ALLOCATION_MODE_FIXED: self.queue.specify_category_max_resources( category.name, constraints) else: self.queue.specify_category_first_allocation_guess( category.name, constraints) logger.debug('Category {0}: {1}'.format(category.name, constraints)) if 'wall_time' not in constraints: self.queue.activate_fast_abort_category( category.name, abort_multiplier) proxy_email_sent = False while not self.source.done(): with self.measure('status'): tasks_left = self.source.tasks_left() units_left = self.source.work_left() logger.debug("expecting {0} tasks, still".format(tasks_left)) self.queue.specify_num_tasks_left(tasks_left) for c in categories + ['all']: self.log(c, units_left) if util.checkpoint(self.config.workdir, 'KILLED') == 'PENDING': util.register_checkpoint(self.config.workdir, 'KILLED', str(datetime.datetime.utcnow())) # let the task source shut down gracefully logger.info("terminating task source") self.source.terminate() logger.info("terminating gracefully") break with self.measure('create'): have = {} for c in categories: cstats = self.queue.stats_category(c) have[c] = { 'running': cstats.tasks_running, 'queued': cstats.tasks_waiting } stats = self.queue.stats_hierarchy tasks = self.source.obtain(stats.total_cores, have) expiry = None if self.config.advanced.proxy: expiry = self.config.advanced.proxy.expires() proxy_time_left = self.config.advanced.proxy.time_left() if proxy_time_left >= 24 * 3600: proxy_email_sent = False if proxy_time_left < 24 * 3600 and not proxy_email_sent: util.sendemail( "Your proxy is about to expire.\n" + "Timeleft: " + str(datetime.timedelta(seconds=proxy_time_left)), self.config) proxy_email_sent = True for category, cmd, id, inputs, outputs, env, dir in tasks: task = wq.Task(cmd) task.specify_category(category) task.specify_tag(id) task.specify_max_retries(wq_max_retries) task.specify_monitor_output( os.path.join(dir, 'resource_monitor')) for k, v in env.items(): task.specify_environment_variable(k, v) for (local, remote, cache) in inputs: cache_opt = wq.WORK_QUEUE_CACHE if cache else wq.WORK_QUEUE_NOCACHE if os.path.isfile(local) or os.path.isdir(local): task.specify_input_file(str(local), str(remote), cache_opt) else: logger.critical( "cannot send file to worker: {0}".format( local)) raise NotImplementedError for (local, remote) in outputs: task.specify_output_file(str(local), str(remote)) if expiry: task.specify_end_time(expiry * 10**6) self.queue.submit(task) with self.measure('status'): stats = self.queue.stats_hierarchy logger.info( "{0} out of {1} workers busy; {2} tasks running, {3} waiting; {4} units left" .format(stats.workers_busy, stats.workers_busy + stats.workers_ready, stats.tasks_running, stats.tasks_waiting, units_left)) with self.measure('update'): self.source.update(self.queue) # recurring actions are triggered here; plotting etc should run # while we have WQ hand us back tasks w/o any database # interaction with self.measure('action'): if action: action.take() with self.measure('fetch'): starttime = time.time() task = self.queue.wait(interval) tasks = [] while task: if task.return_status == 0: successful_tasks += 1 elif task.return_status in self.config.advanced.bad_exit_codes: logger.warning( "blacklisting host {0} due to bad exit code from task {1}" .format(task.hostname, task.tag)) self.queue.blacklist(task.hostname) tasks.append(task) remaining = int(starttime + interval - time.time()) if (interval - remaining < interval_minimum or self.queue.stats.tasks_waiting > 0 ) and remaining > 0: task = self.queue.wait(remaining) else: task = None # TODO do we really need this? We have everything based on # categories by now, so this should not be needed. if abort_threshold > 0 and successful_tasks >= abort_threshold and not abort_active: logger.info( "activating fast abort with multiplier: {0}".format( abort_multiplier)) abort_active = True self.queue.activate_fast_abort(abort_multiplier) if len(tasks) > 0: try: with self.measure('return'): self.source.release(tasks) except Exception: tb = traceback.format_exc() logger.critical( "cannot recover from the following exception:\n" + tb) util.sendemail( "Your Lobster project has crashed from the following exception:\n" + tb, self.config) for task in tasks: logger.critical( "tried to return task {0} from {1}".format( task.tag, task.hostname)) raise if units_left == 0: logger.info("no more work left to do") util.sendemail("Your Lobster project is done!", self.config) if self.config.elk: self.config.elk.end() if action: action.take(True)
import json import os import itertools #Constants SOURCES = ('hulk.py', 'hashes.txt') PORT = 9123 ALPHABET = string.ascii_lowercase + string.digits # Main Execution if __name__ == '__main__': # Create a master # 1. Random port between 9000 - 9999 # 2. Project name of fury-samuel # 3. Catalog mode enabled queue = work_queue.WorkQueue(PORT, name='fury-mchen6apaek1', catalog=True) queue.specify_log('fury.log') #Specify Work Queue log location # Create task list # Check passwords for length of <6 for num in range(5): command = './hulk.py -l {}'.format(num) task = work_queue.Task(command) # Add source files (transfer files to workers) for source in SOURCES: task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT) # Submit tasks queue.submit(task)
import itertools import string import json #Constants ALPHABET = string.ascii_lowercase + string.digits HASHES = "hashes.txt" SOURCES = ('hulk.py', HASHES) PORT = 9284 JOURNAL = {} #Main Execution if __name__ == '__main__': #Start work queue queue = work_queue.WorkQueue(PORT, name='hulk-alam3', catalog=True) queue.specify_log('fury.log') #Combinations of length 1-5 for length in range(1, 6): command = './hulk.py -l {}'.format(length) #Need to make sure machines being used have hulk.py in them if command not in JOURNAL: task = work_queue.Task(command) #Add source files for source in SOURCES: task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT) queue.submit(task) else: print >> sys.stderr, 'Already did', command
import sys import os import work_queue import itertools import string import json HASHES = "hashes.txt" PORT = 9199 TASKS = 100 ALPHABET = string.ascii_lowercase + string.digits if __name__ == '__main__': JOURNAL = json.load(open('journal.json')) queue = work_queue.WorkQueue(PORT, name='hulk-amunch', catalog=True) queue.specify_log('fury.log') for num in range(1, 6): command = './hulk.py -l {}'.format(num) if command in JOURNAL: print >> sys.stderr, 'Already did', command else: task = work_queue.Task(command) for source in ('hulk.py', HASHES): task.specify_file(source, source, work_queue.WORK_QUEUE_INPUT) queue.submit(task) for num in range(1, 4): for prefix in itertools.product(ALPHABET, repeat=int(num)): prefix = ''.join(prefix)
return y / x class NoResult(Exception): """ Exception used as the result of a function evaluation of a task that failed. """ def __repr__(self): return 'NoResult()' def __str__(self): return 'NoResult' if __name__ == '__main__': with tempfile.TemporaryDirectory() as tmpdir: q = wq.WorkQueue(port=9123, debug_log='debug.log', transactions_log='tr.log') task_counter = 0 total_tasks = 5 for i in range(total_tasks): task = create_work_queue_task( task_counter, tmpdir, application_function, input_args=[i, 100] ) # all tasks compute application_function(i, 100) which is 100/i q.submit(task) task_counter += 1
# serialized and sent back to the Work Queue manager, where it is read and can # be used as a regular python value. If an exception occurs, it is set as the # result of the function, where it can be handled as appropiate. # import work_queue as wq def application_function(x, y): """ Example function to be executed remotely. """ return y / x if __name__ == '__main__': q = wq.WorkQueue(9123) total_tasks = 20 for i in range(total_tasks): task = wq.PythonTask( application_function, i + 1, 100 ) # all tasks compute application_function(i, 100) which is 100/i task.specify_environment('venv.tar.gz') q.submit(task) while not q.empty(): t = q.wait(5) if t: x = t.output print(x)
# lidx = opt.index('2>')+1 # cDict['sqCMD']['LOG'] = opt[lidx] # cDict['wqIO']['2>'] = opt[lidx].split('/')[-1] cDict['CMD'] = './'+cDict['BLAST']+' '+ \ ' '.join([k+' '+cDict['wqCMD'][k] \ for k in cDict['wqCMD'].keys()])+' '+ \ c1Opts+c2Opts+' '+' '.join([k+' '+cDict['wqIO'][k] \ for k in cDict['wqIO'].keys()]) return cDict ## q = wq.WorkQueue(port) while True: ts = q.wait(1) # ts = q.wait(wq.WORK_QUEUE_WAITFORTASK) time.sleep(2) seq = os.path.isdir('/proc/' + str(pid)) if not seq: sys.exit() nfiles = glob.glob(wdir + '/*.cmd') for file in nfiles: command = readCMD(file)
def _mk_wq(self): """ Only one instance of WorkQueue should be run per process. This grants access to the WorkQueue singleton or else creates a new WorkQueue instance. This also ensures that the cctools WorkQueue object can handle more workers. Parameters: None Returns: The cctools WorkQueue singleton object """ global _AWE_WORK_QUEUE if _AWE_WORK_QUEUE is not None: ### warn awe.log('WARNING: using previously created WorkQueue instance') else: if self.debug: # Set up debugging parameters for the cctools WorkQueue object. # It has inbuilt debugging capabilities. WQ.set_debug_flag(self.debug) if self.wq_logfile: awe.util.makedirs_parent(self.wq_logfile) WQ.cctools_debug_config_file(self.wq_logfile) WQ.cctools_debug_config_file_size(0) if self.name: self.catalog = True # Create the cctools WorkQueue object wq = WQ.WorkQueue(name = self.name, port = self.port, shutdown = self.shutdown, catalog = self.catalog, exclusive = self.exclusive) # Specify the task scheduling algorithm wq.specify_algorithm(self.schedule) # Turn cctools WorkQueue object status monitoring on or off if self.monitor: wq.enable_monitoring(self.summaryfile) if self.capacity: # Determine the number of workers the WorkQueue object can handle wq.estimate_capacity() # Display information about this run of AWE-WQ awe.log('Running on port %d...' % wq.port) if wq.name: awe.log('Using project name %s' % wq.name) if self.debug and self.wq_logfile: awe.log('Logging WorkQueue to %s' % self.wq_logfile) # Set up fast abort procedures typ = type(self.fastabort) if typ is float or typ is int: wq.activate_fast_abort(self.fastabort) # Ensure that the singleton is set to the new instance _AWE_WORK_QUEUE = wq # Ensure that the singleton is logging to the correct files awe.util.makedirs_parent(self.wqstats_logfile) _AWE_WORK_QUEUE.specify_log(self.wqstats_logfile) # Return a reference to teh singleton return _AWE_WORK_QUEUE
import work_queue as WQ # in case we want ${USER} for master name from os import environ import time #### Customize your master name master_name = environ['USER'] + '-master' # run at some port at random q = WQ.WorkQueue(name=master_name, port=0) # enable the transactions log q.specify_transactions_log('my_transactions.log') print 'WorkQueue on port: {}'.format(q.port) # enable the measuring of resources q.enable_monitoring() # create a category for all tasks q.specify_category_max_resources('my-tasks', {'cores': 1, 'disk': 500}) q.specify_category_mode('my-tasks', WQ.WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT) # create 30 tasks. Each task simply creates a 200MB file, using 10MB of memory # buffer. for i in range(0, 30): t = WQ.Task('python task.py') t.specify_input_file('task.py', cache=True) t.specify_category('my-tasks')