def run(self): args = list(islice(self.reqs, self.requests)) if self.shuffle: random.shuffle(args) print("Total requests: %d" % len(args)) print("Concurrency : %d" % self.concurrency) starttime = time.time() q, p = Queue(), Queue() for _ in six.moves.range(self.concurrency): t = Thread(target=worker, args=(self.host, q, p, self.verbose)) t.daemon = True t.start() for a in args: q.put(a) q.join() outputs = [] for _ in six.moves.range(self.requests): outputs.append(p.get()) elapsed = time.time() - starttime print() print("Total requests: %d" % len(args)) print("Concurrency : %d" % self.concurrency) print("Elapsed time : %.3fs" % elapsed) print("Avg time p/req: %.3fs" % (elapsed / len(args))) print("Received (per status code or error):") for c, n in Counter(outputs).items(): print(" %s: %d" % (c, n))
def lines(self, fuseki_process): """ Provides an iterator generating the encoded string representation of each member of this metarelate mapping translation. Returns: An iterator of string. """ msg = '\tGenerating phenomenon translation {!r}.' print(msg.format(self.mapping_name)) lines = ['\n%s = {\n' % self.mapping_name] # Retrieve encodings for the collection of mapping instances. # Retrieval is threaded as it is heavily bound by resource resolution # over http. # Queue for metarelate mapping instances mapenc_queue = Queue() for mapping in self.mappings: mapenc_queue.put(mapping) # deque to contain the results of the jobs processed from the queue mapencs = deque() # run worker threads for i in range(MAXTHREADS): MappingEncodeWorker(mapenc_queue, mapencs, fuseki_process).start() # block progress until the queue is empty mapenc_queue.join() # end of threaded retrieval process. # now sort the payload payload = [mapenc.encoding for mapenc in mapencs] payload.sort(key=self._key) lines.extend(payload) lines.append(' }\n') return iter(lines)
class _BatchWriter(object): #: Truncate overly big items to that many bytes for the error message. ERRMSG_DATA_TRUNCATION_LEN = 1024 def __init__(self, url, start, auth, size, interval, qsize, maxitemsize, content_encoding, uploader, callback=None): self.url = url self.offset = start self._nextid = count(start) self.auth = auth self.size = size self.interval = interval self.maxitemsize = maxitemsize self.content_encoding = content_encoding self.checkpoint = time.time() self.itemsq = Queue(size * 2 if qsize is None else qsize) self.closed = False self.flushme = False self.uploader = uploader self.callback = callback def write(self, item): assert not self.closed, 'attempting writes to a closed writer' data = jsonencode(item) if len(data) > self.maxitemsize: truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..." raise ValueTooLarge( 'Value exceeds max encoded size of {}: {!r}'.format( sizeof_fmt(self.maxitemsize), truncated_data)) self.itemsq.put(data) if self.itemsq.full(): self.uploader.interrupt() return next(self._nextid) def flush(self): self.flushme = True self._waitforq() self.flushme = False def close(self, block=True): self.closed = True if block: self._waitforq() def _waitforq(self): self.uploader.interrupt() self.itemsq.join() def __str__(self): return self.url
class Pool(object): class Error(Exception): pass def __init__(self, threads, host, port, ssl, user, password): self._threads = [] self._queue = Queue(maxsize=1000) count = 0 while len(self._threads) < threads and count < 3 * threads: try: count += 1 w = Downloader(self._queue, host, port, ssl, user, password) w.start() self._threads.append(w) except SOFT_ERRORS as e: log.warn('Cannot create downloder thread: %s', e) if len(self._threads) != threads: log.error('Cannot create enough workers') raise Pool.Error('Cannot create enough workers') def wait_finish(self): self._queue.join() def stop(self): for t in self._threads: t.stop() def download(self, **kwargs): kwargs['retry'] = 0 self._queue.put(kwargs)
class SimpleThreadPool: def __init__(self, num_threads=5): self._num_threads = num_threads self._queue = Queue(2000) self._lock = Lock() self._active = False self._workers = list() self._finished = False def add_task(self, func, *args, **kwargs): if not self._active: with self._lock: if not self._active: self._active = True for i in range(self._num_threads): w = WorkerThread(self._queue) self._workers.append(w) w.start() self._queue.put((func, args, kwargs)) def release(self): while self._queue.empty() is False: time.sleep(1) def wait_completion(self): self._queue.join() self._finished = True def get_result(self): assert self._finished detail = [worker.get_result() for worker in self._workers] succ_all = all([tp[1] == 0 for tp in detail]) return {'success_all': succ_all, 'detail': detail}
class Pool(object): class Error(Exception): pass def __init__(self, threads, host, port, ssl, user, password): self._threads=[] self._queue = Queue(maxsize=1000) count=0 while len(self._threads) < threads and count < 3* threads: try: count+=1 w=Downloader(self._queue, host, port, ssl, user, password) w.start() self._threads.append(w) except SOFT_ERRORS as e: log.warn('Cannot create downloder thread: %s', e) if len(self._threads) != threads: log.error('Cannot create enough workers') raise Pool.Error('Cannot create enough workers') def wait_finish(self): self._queue.join() def stop(self): for t in self._threads: t.stop() def download(self, **kwargs): kwargs['retry']=0 self._queue.put(kwargs)
def daily_metadata(year, month, day, dst_folder, writers=[file_writer], geometry_check=None, num_worker_threads=1): """ Extra metadata for all products in a specific date """ threaded = False counter = { 'products': 0, 'saved_tiles': 0, 'skipped_tiles': 0, 'skipped_tiles_paths': [] } if num_worker_threads > 1: threaded = True queue = Queue() # create folders year_dir = os.path.join(dst_folder, str(year)) month_dir = os.path.join(year_dir, str(month)) day_dir = os.path.join(month_dir, str(day)) product_list = get_products_metadata_path(year, month, day) logger.info('There are %s products in %s-%s-%s' % (len(list(iterkeys(product_list))), year, month, day)) for name, product in iteritems(product_list): product_dir = os.path.join(day_dir, name) if threaded: queue.put([product, product_dir, counter, writers, geometry_check]) else: counter = product_metadata(product, product_dir, counter, writers, geometry_check) if threaded: def worker(): while not queue.empty(): args = queue.get() try: product_metadata(*args) except Exception: exc = sys.exc_info() logger.error('%s tile skipped due to error: %s' % (threading.current_thread().name, exc[1].__str__())) args[2]['skipped_tiles'] += 1 queue.task_done() threads = [] for i in range(num_worker_threads): t = threading.Thread(target=worker) t.start() threads.append(t) queue.join() return counter
class ThreadPool: def __init__(self, context={}, size=2): self.tasks = Queue(size) for num in range(size): Worker(self.tasks, context) def add(self, func, *args, **kargs): self.tasks.put((func, args, kargs)) def wait(self): self.tasks.join()
class Copier(object): def __init__(self, num_workers=4): self.num_workers = num_workers self.queue = None self.workers = [] self.errors = [] self.running = False def start(self): if self.num_workers: self.queue = Queue(maxsize=self.num_workers) self.workers = [_Worker(self) for _ in range(self.num_workers)] for worker in self.workers: worker.start() self.running = True def stop(self): if self.running and self.num_workers: for worker in self.workers: self.queue.put(None) for worker in self.workers: worker.join() self.queue.join() self.running = False def add_error(self, error): self.errors.append(error) def __enter__(self): self.start() return self def __exit__( self, exc_type, # type: Optional[Type[BaseException]] exc_value, # type: Optional[BaseException] traceback # type: Optional[TracebackType] ): self.stop() def copy(self, src_fs, src_path, dst_fs, dst_path): """Copy a file from on fs to another.""" src_file = src_fs.openbin(src_path, 'r') try: dst_file = dst_fs.openbin(dst_path, 'w') except Exception: # If dst file fails to open, explicitly close src_file src_file.close() raise task = _CopyTask(src_file, dst_file) if self.num_workers: self.queue.put(task) else: task()
class _BatchWriter(object): #: Truncate overly big items to that many bytes for the error message. ERRMSG_DATA_TRUNCATION_LEN = 1024 def __init__(self, url, start, auth, size, interval, qsize, maxitemsize, content_encoding, uploader, callback=None): self.url = url self.offset = start self._nextid = count(start) self.auth = auth self.size = size self.interval = interval self.maxitemsize = maxitemsize self.content_encoding = content_encoding self.checkpoint = time.time() self.itemsq = Queue(size * 2 if qsize is None else qsize) self.closed = False self.flushme = False self.uploader = uploader self.callback = callback def write(self, item): assert not self.closed, 'attempting writes to a closed writer' data = jsonencode(item) if len(data) > self.maxitemsize: truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..." raise ValueTooLarge( 'Value exceeds max encoded size of {} bytes: {!r}' .format(self.maxitemsize, truncated_data)) self.itemsq.put(data) if self.itemsq.full(): self.uploader.interrupt() return next(self._nextid) def flush(self): self.flushme = True self._waitforq() self.flushme = False def close(self, block=True): self.closed = True if block: self._waitforq() def _waitforq(self): self.uploader.interrupt() self.itemsq.join() def __str__(self): return self.url
class ThreadPool: """Pool of threads consuming tasks from a queue.""" def __init__(self, num_threads): self.tasks = Queue(num_threads) for _ in range(num_threads): Worker(self.tasks) def add_task(self, func, *args, **kargs): """Add a task to the queue.""" self.tasks.put((func, args, kargs)) def wait_completion(self): """Wait for completion of all the tasks in the queue.""" self.tasks.join()
class WorkerPool(object): def __init__(self, num_threads=None): if num_threads is None: num_threads = multiprocessing.cpu_count() self.tasks = Queue(num_threads) for _ in range(num_threads): Worker(self.tasks) def add_task(self, func, *args, **kargs): self.tasks.put((func, args, kargs)) def wait_for_completion(self): self.tasks.join()
def fetch_adblock_list(): queue = Queue() for url in FILTER_URLS: queue.put(url) worker_count = min(len(FILTER_URLS), multiprocessing.cpu_count()) workers = [] for _ in range(worker_count): worker = Worker(queue) worker.start() workers.append(worker) queue.join() hosts_str = '\n' for worker in workers: hosts_str += worker.result return hosts_str
def main(): volumes = 900 num_worker_threads = 25 task = Queue() poems = Queue() for i in range(num_worker_threads): t = Thread(target=worker, args=(task, poems)) t.daemon = True t.start() write_thread = Thread(target=write_poems, args=('./data/poems.txt', poems)) write_thread.start() for js in range(1, volumes + 1): task.put(js) task.join() poems.join() poems.put(None) write_thread.join()
class SimpleThreadPool: def __init__(self, num_threads=3): self._num_threads = num_threads self._queue = Queue(2000) self._lock = Lock() self._active = False self._workers = [] self._finished = False def add_task(self, func, *args, **kwargs): if not self._active: with self._lock: if not self._active: self._workers = [] self._active = True for i in range(self._num_threads): w = WorkerThread(self._queue) self._workers.append(w) w.start() self._queue.put((func, args, kwargs)) def release(self): while self._queue.empty() is False: time.sleep(1) def wait_completion(self): self._queue.join() self._finished = True # 已经结束的任务, 需要将线程都退出, 防止卡死 for i in range(self._num_threads): self._queue.put((None, None, None)) self._active = False def complete(self): self._finished = True def get_result(self): assert self._finished detail = [worker.get_result() for worker in self._workers] succ_num = sum([tp[0] for tp in detail]) fail_num = sum([tp[1] for tp in detail]) return {'success_num': succ_num, 'fail_num': fail_num}
def test_resolve(self): repo_fname = os.path.join(os.path.dirname(__file__), "results", "imagerepo.json") with open(repo_fname, "rb") as fi: repo = json.load(codecs.getreader("utf-8")(fi)) uris = list(chain.from_iterable(six.itervalues(repo))) uri_list = deque() exceptions = deque() uri_queue = Queue() for uri in uris: if uri.startswith("https://scitools.github.io"): uri_queue.put(uri) else: msg = "{} is not a valid resource.".format(uri) exceptions.append(ValueError(msg)) for i in range(MAXTHREADS): _ResolveWorkerThread(uri_queue, uri_list, exceptions).start() uri_queue.join() self.assertEqual(deque(), exceptions)
class SimpleThreadPool: def __init__(self, num_threads=5, num_queue=0): self._num_threads = num_threads self._queue = Queue(num_queue) self._lock = Lock() self._active = False self._workers = list() self._finished = False def add_task(self, func, *args, **kwargs): if not self._active: with self._lock: if not self._active: self._workers = [] self._active = True for i in range(self._num_threads): w = WorkerThread(self._queue) self._workers.append(w) w.start() self._queue.put((func, args, kwargs)) def wait_completion(self): self._queue.join() self._finished = True # 已经结束的任务, 需要将线程都退出, 防止卡死 for i in range(self._num_threads): self._queue.put((None, None, None)) self._active = False def get_result(self): assert self._finished detail = [worker.get_result() for worker in self._workers] succ_all = all([tp[1] == 0 for tp in detail]) return {'success_all': succ_all, 'detail': detail}
class WebRunner: resp_queue = None def __init__(self): self.resp_queue = Queue() def request(self, index, url, **kwargs): kwargs = flatten_kwargs(index, **kwargs) try: method = kwargs.pop('method', 'GET') self.resp_queue.put( (index, requests.request(method=method, url=url, **kwargs))) except Exception as e: self.resp_queue.put((index, None)) print('Failed to download %s because %s.' % (url, e)) def runner(self, **kwargs): while True: index, url = self.work_queue.get() if 'http://' not in url and 'https://' not in url: url = 'http://' + url self.request(index, url, **kwargs) self.work_queue.task_done() def run(self, urls, concurrency=4, **kwargs): self.work_queue = Queue() for url in enumerate(urls): self.work_queue.put(url) for i in range(concurrency): t = Thread(target=self.runner, kwargs=kwargs) t.daemon = True t.start() self.work_queue.join() responses = list(self.resp_queue.queue) responses = sorted(responses, key=lambda x: x[0]) return [r[1] for r in responses]
class GraphiteReporter(threading.Thread): """A graphite reporter thread.""" def __init__(self, host, port, maxQueueSize=10000): """Connect to a Graphite server on host:port.""" threading.Thread.__init__(self) self.host, self.port = host, port self.sock = None self.queue = Queue() self.maxQueueSize = maxQueueSize self.daemon = True def run(self): """Run the thread.""" while True: try: try: name, value, valueType, stamp = self.queue.get() except TypeError: break self.log(name, value, valueType, stamp) finally: self.queue.task_done() def connect(self): """Connects to the Graphite server if not already connected.""" if self.sock is not None: return backoff = 0.01 while True: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5) sock.connect((self.host, self.port)) self.sock = sock return except socket.error: time.sleep(random.uniform(0, 2.0*backoff)) backoff = min(backoff*2.0, 5.0) def disconnect(self): """Disconnect from the Graphite server if connected.""" if self.sock is not None: try: self.sock.close() except socket.error: pass finally: self.sock = None def _sendMsg(self, msg): """Send a line to graphite. Retry with exponential backoff.""" if not self.sock: self.connect() if not isinstance(msg, binary_type): msg = msg.encode("UTF-8") backoff = 0.001 while True: try: self.sock.sendall(msg) break except socket.error: log.warning('Graphite connection error', exc_info = True) self.disconnect() time.sleep(random.uniform(0, 2.0*backoff)) backoff = min(backoff*2.0, 5.0) self.connect() def _sanitizeName(self, name): """Sanitize a metric name.""" return name.replace(' ', '-') def log(self, name, value, valueType=None, stamp=None): """Log a named numeric value. The value type may be 'value', 'count', or None.""" if type(value) == float: form = "%s%s %2.2f %d\n" else: form = "%s%s %s %d\n" if valueType is not None and len(valueType) > 0 and valueType[0] != '.': valueType = '.' + valueType if not stamp: stamp = time.time() self._sendMsg(form % (self._sanitizeName(name), valueType or '', value, stamp)) def enqueue(self, name, value, valueType=None, stamp=None): """Enqueue a call to log.""" # If queue is too large, refuse to log. if self.maxQueueSize and self.queue.qsize() > self.maxQueueSize: return # Stick arguments into the queue self.queue.put((name, value, valueType, stamp)) def flush(self): """Block until all stats have been sent to Graphite.""" self.queue.join() def shutdown(self): """Shut down the background thread.""" self.queue.put(None) self.flush()
def execute( trans, tool, param_combinations, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None ): """ Execute a tool and return object containing summary (output data, number of failures, etc...). """ all_jobs_timer = ExecutionTimer() execution_tracker = ToolExecutionTracker( tool, param_combinations, collection_info ) app = trans.app execution_cache = ToolExecutionCache(trans) def execute_single_job(params): job_timer = ExecutionTimer() if workflow_invocation_uuid: params[ '__workflow_invocation_uuid__' ] = workflow_invocation_uuid elif '__workflow_invocation_uuid__' in params: # Only workflow invocation code gets to set this, ignore user supplied # values or rerun parameters. del params[ '__workflow_invocation_uuid__' ] job, result = tool.handle_single_execution( trans, rerun_remap_job_id, params, history, collection_info, execution_cache ) if job: message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer) log.debug(message) execution_tracker.record_success( job, result ) else: execution_tracker.record_error( result ) config = app.config burst_at = getattr( config, 'tool_submission_burst_at', 10 ) burst_threads = getattr( config, 'tool_submission_burst_threads', 1 ) tool_action = tool.tool_action if hasattr( tool_action, "check_inputs_ready" ): for params in execution_tracker.param_combinations: # This will throw an exception if the tool is not ready. tool_action.check_inputs_ready( tool, trans, params, history ) job_count = len(execution_tracker.param_combinations) if job_count < burst_at or burst_threads < 2: for params in execution_tracker.param_combinations: execute_single_job(params) else: q = Queue() def worker(): while True: params = q.get() execute_single_job(params) q.task_done() for i in range(burst_threads): t = Thread(target=worker) t.daemon = True t.start() for params in execution_tracker.param_combinations: q.put(params) q.join() log.debug("Executed %d job(s) for tool %s request: %s" % (job_count, tool.id, all_jobs_timer)) if collection_info: history = history or tool.get_default_history_by_trans( trans ) if len(param_combinations) == 0: template = "Attempting to map over an empty collection, this is not yet implemented. colleciton_info is [%s]" message = template % collection_info log.warn(message) raise Exception(message) params = param_combinations[0] execution_tracker.create_output_collections( trans, history, params ) return execution_tracker
def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None, completed_jobs=None, workflow_resource_parameters=None): """ Execute a tool and return object containing summary (output data, number of failures, etc...). """ if max_num_jobs: assert invocation_step is not None if rerun_remap_job_id: assert invocation_step is None all_jobs_timer = ExecutionTimer() if invocation_step is None: execution_tracker = ToolExecutionTracker(trans, tool, mapping_params, collection_info) else: execution_tracker = WorkflowStepExecutionTracker(trans, tool, mapping_params, collection_info, invocation_step, job_callback=job_callback) app = trans.app execution_cache = ToolExecutionCache(trans) def execute_single_job(execution_slice, completed_job): job_timer = ExecutionTimer() params = execution_slice.param_combination if workflow_invocation_uuid: params['__workflow_invocation_uuid__'] = workflow_invocation_uuid elif '__workflow_invocation_uuid__' in params: # Only workflow invocation code gets to set this, ignore user supplied # values or rerun parameters. del params['__workflow_invocation_uuid__'] if workflow_resource_parameters: params['__workflow_resource_params__'] = workflow_resource_parameters elif '__workflow_resource_params__' in params: # Only workflow invocation code gets to set this, ignore user supplied # values or rerun parameters. del params['__workflow_resource_params__'] job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache, completed_job) if job: message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer) log.debug(message) execution_tracker.record_success(execution_slice, job, result) else: execution_tracker.record_error(result) tool_action = tool.tool_action if hasattr(tool_action, "check_inputs_ready"): for params in execution_tracker.param_combinations: # This will throw an exception if the tool is not ready. tool_action.check_inputs_ready( tool, trans, params, history ) execution_tracker.ensure_implicit_collections_populated(history, mapping_params.param_template) config = app.config burst_at = getattr(config, 'tool_submission_burst_at', 10) burst_threads = getattr(config, 'tool_submission_burst_threads', 1) job_count = len(execution_tracker.param_combinations) jobs_executed = 0 has_remaining_jobs = False if (job_count < burst_at or burst_threads < 2): for i, execution_slice in enumerate(execution_tracker.new_execution_slices()): if max_num_jobs and jobs_executed >= max_num_jobs: has_remaining_jobs = True break else: execute_single_job(execution_slice, completed_jobs[i]) else: # TODO: re-record success... q = Queue() def worker(): while True: params = q.get() execute_single_job(params) q.task_done() for i in range(burst_threads): t = Thread(target=worker) t.daemon = True t.start() for i, execution_slice in enumerate(execution_tracker.new_execution_slices()): if max_num_jobs and jobs_executed >= max_num_jobs: has_remaining_jobs = True break else: q.put(execution_slice, completed_jobs[i]) jobs_executed += 1 q.join() if has_remaining_jobs: raise PartialJobExecution(execution_tracker) else: execution_tracker.finalize_dataset_collections(trans) log.debug("Executed %d job(s) for tool %s request: %s" % (job_count, tool.id, all_jobs_timer)) return execution_tracker
class ThreadPool(object): def __init__(self, thread_manager, thread_count=10): """Initialization method :param thread_manager: the thread manager to use :param thread_count: the number of workers to instantiate """ self.logger = logging.getLogger( 'storj.downstream_farmer.utils.ThreadPool') self.tasks = Queue() self.thread_manager = thread_manager self.workers = list() self.workers_lock = threading.Lock() for i in range(0, thread_count): self._add_thread() # managed monitor thread self.monitor_thread = self.thread_manager.create_thread( name='MonitorThread', target=self._monitor) self.load_minimum = 0.01 self.load_maximum = 0.25 def thread_count(self): with self.workers_lock: return len(self.workers) def _add_thread(self): # unmanaged worker threads self.logger.debug( '{0} : adding worker'.format(threading.current_thread())) worker = WorkerThread(self) with self.workers_lock: self.workers.append(worker) return worker def _remove_thread(self): with self.workers_lock: if (len(self.workers) > 1): self.logger.debug( '{0} : removing worker'.format(threading.current_thread())) # make sure to retain one worker thread = self.workers.popleft() thread.stop() def calculate_loading(self): total_time = 0 work_time = 0 with self.workers_lock: for w in self.workers: total_time += w.load_tracker.total_time() work_time += w.load_tracker.work_time() if (total_time > 0): load = float(work_time) / float(total_time) else: load = 0 return load def max_load(self): max = 0 with self.workers_lock: for w in self.workers: load = w.load_tracker.load() if (load > max): max = load return max def check_loading(self): self.monitor_thread.wake() def _monitor(self): """This runs until the thread manager wakes it up during shutdown, at which time it will wait for any unfinished work in the queue, and then finish, allowing the program to exit """ # wait until shutdown is called while (self.thread_manager.running): # check loading every second to see if we should add another # thread. load = self.calculate_loading() if (load > self.load_maximum): worker = self._add_thread() worker.start() elif (load < self.load_minimum): self._remove_thread() self.thread_manager.sleep(10) # wait for any existing work to finish self.logger.debug('MonitorThread waiting for tasks to finish') self.tasks.join() self.logger.debug('MonitorThread finishing') # now, managed thread can exit so program can close cleanly def put_work(self, target, args=[], kwargs={}): """Puts work in the work queue. :param work: callable work object """ self.tasks.put(WorkItem(target, args, kwargs)) def start(self): """Starts the thread pool and all its workers and the monitor thread """ with self.workers_lock: for worker in self.workers: worker.start() self.monitor_thread.start()
def test_cancel_fn(executor, caplog): task_id_queue = Queue() tasks = {} poll_fn = partial(poll_tasks, tasks) def cancel_fn(task): if task.startswith("cancel-true-"): return True if task.startswith("cancel-false-"): return False raise RuntimeError("simulated cancel error") poll_executor = PollExecutor(executor, poll_fn, cancel_fn, default_interval=0.01) def make_task(x): got = task_id_queue.get(True) task_id_queue.task_done() return "%s-%s" % (x, got) inputs = ["cancel-true", "cancel-false", "cancel-error"] futures = [poll_executor.submit(make_task, x) for x in inputs] # The futures should not currently be able to progress. assert_that(not any([f.done() for f in futures])) # Allow tasks to be created. task_id_queue.put("x") task_id_queue.put("y") task_id_queue.put("z") # Wait until all tasks were created and futures moved # into poll mode task_id_queue.join() # Wait until the make_task function definitely completed in each thread, # which can be determined by running==False assert_soon(lambda: assert_that(all([not f.running() for f in futures]))) # Should be able to cancel soon. # Why "soon" instead of "now" - because even though the futures above are # not running, the delegate may not have been cleared yet. Cancel needs # to wait until the future's delegate is cleared and the future has # transitioned fully into "poll mode". assert_soon(lambda: assert_that(futures[0].cancel())) # The other two futures don't need assert_soon since the cancel result is negative. # Cancel behavior should be consistent (calling multiple times same # as calling once) for _ in 1, 2: # Cancelling the cancel-true task should be allowed. assert_that(futures[0].cancel()) # Cancelling the cancel-false task should not be allowed. assert_that(not futures[1].cancel()) # Cancelling the cancel-error task should not be allowed. assert_that(not futures[2].cancel()) # An error should have been logged due to the cancel function raising. if caplog: assert_that( caplog.record_tuples, has_item( contains( "PollExecutor", logging.ERROR, matches_regexp(r"Exception during cancel .*/cancel-error"), )), )
class GraphiteReporter(threading.Thread): """A graphite reporter thread.""" def __init__(self, host, port, maxQueueSize=10000): """Connect to a Graphite server on host:port.""" threading.Thread.__init__(self) self.host, self.port = host, port self.sock = None self.queue = Queue() self.maxQueueSize = maxQueueSize self.daemon = True def run(self): """Run the thread.""" while True: try: try: name, value, valueType, stamp = self.queue.get() except TypeError: break self.log(name, value, valueType, stamp) finally: self.queue.task_done() def connect(self): """Connects to the Graphite server if not already connected.""" if self.sock is not None: return backoff = 0.01 while True: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5) sock.connect((self.host, self.port)) self.sock = sock return except socket.error: time.sleep(random.uniform(0, 2.0 * backoff)) backoff = min(backoff * 2.0, 5.0) def disconnect(self): """Disconnect from the Graphite server if connected.""" if self.sock is not None: try: self.sock.close() except socket.error: pass finally: self.sock = None def _sendMsg(self, msg): """Send a line to graphite. Retry with exponential backoff.""" if not self.sock: self.connect() backoff = 0.001 while True: try: self.sock.sendall(msg) break except socket.error: logging.warning('Graphite connection error', exc_info=True) self.disconnect() time.sleep(random.uniform(0, 2.0 * backoff)) backoff = min(backoff * 2.0, 5.0) self.connect() def _sanitizeName(self, name): """Sanitize a metric name.""" return name.replace(' ', '-') def log(self, name, value, valueType=None, stamp=None): """Log a named numeric value. The value type may be 'value', 'count', or None.""" if type(value) == float: form = "%s%s %2.2f %d\n" else: form = "%s%s %s %d\n" if valueType is not None and len( valueType) > 0 and valueType[0] != '.': valueType = '.' + valueType if not stamp: stamp = time.time() self._sendMsg( form % (self._sanitizeName(name), valueType or '', value, stamp)) def enqueue(self, name, value, valueType=None, stamp=None): """Enqueue a call to log.""" # If queue is too large, refuse to log. if self.maxQueueSize and self.queue.qsize() > self.maxQueueSize: return # Stick arguments into the queue self.queue.put((name, value, valueType, stamp)) def flush(self): """Block until all stats have been sent to Graphite.""" self.queue.join() def shutdown(self): """Shut down the background thread.""" self.queue.put(None) self.flush()
def read_images_in_memory(job_model, dataset, node, trainer): """ Reads all images into memory and applies augmentation if enabled """ concurrent = psutil.cpu_count() dataset_config = dataset['config'] controller = {'running': True} q = Queue(concurrent) result = { 'X_train': [], 'Y_train': [], 'X_test': [], 'Y_test': [] } images = [] max = 0 path = job_model.get_dataset_downloads_dir(dataset) if 'path' in dataset['config']: path = dataset['config']['path'] classes_count = 0 category_map = {} classes = [] trainer.set_status('LOAD IMAGES INTO MEMORY') try: for i in range(concurrent): t = ImageReadWorker(q, job_model, node, path, images, controller) t.daemon = True t.start() for validation_or_training in ['validation', 'training']: if os.path.isdir(os.path.normpath(path + '/' + validation_or_training)): for category_name in os.listdir(os.path.normpath(path + '/' + validation_or_training)): if os.path.isdir(os.path.normpath(path + '/' + validation_or_training + '/' + category_name)): if category_name not in category_map: category_map[category_name] = classes_count if 'classes' in dataset_config and 'category_' in category_name: category_idx = int(category_name.replace('category_', '')) category_map[category_name] = category_idx target_category = dataset_config['classes'][category_idx] classes.append(target_category['title'] or 'Class %s' % (category_idx, )) else: classes.append(category_name) classes_count += 1 for id in os.listdir(os.path.normpath(path + '/' + validation_or_training + '/' + category_name)): file_path = os.path.join(path, validation_or_training, category_name, id) q.put([file_path, validation_or_training == 'validation', category_name]) max += 1 q.join() controller['running'] = False train_images = [] test_images = [] for v in images: image, validation, category_dir = v if validation is True: test_images.append([image, category_map[category_dir]]) else: train_images.append([image, category_map[category_dir]]) train_datagen = None augmentation = bool(get_option(dataset_config, 'augmentation', False)) if augmentation: train_datagen = get_image_data_augmentor_from_dataset(dataset) train = InMemoryDataGenerator(train_datagen, train_images, classes_count, job_model.job['config']['batchSize']) test = InMemoryDataGenerator(None, test_images, classes_count, job_model.job['config']['batchSize']) nb_sample = len(train_images) trainer.set_info('Dataset size', {'training': nb_sample, 'validation': len(test_images)}) trainer.set_generator_training_nb(nb_sample) trainer.set_generator_validation_nb(len(test_images)) trainer.logger.info(("Found %d classes, %d images (%d in training [%saugmented], %d in validation). Read all images into memory from %s" % (classes_count, max, len(train_images), 'not ' if augmentation is False else '', len(test_images), path))) if classes_count == 0: trainer.logger.warning("Could not find any classes. Does the directory contains images?") sys.exit(1) trainer.output_size = classes_count trainer.set_info('classes', classes) trainer.classes = classes result['X_train'] = train result['Y_train'] = train result['X_test'] = test result['Y_test'] = test return result except KeyboardInterrupt: controller['running'] = False sys.exit(1)
def get_images(job_model, dataset, node, trainer): concurrent = 15 from PIL import ImageFile if hasattr(ImageFile, 'LOAD_TRUNCATED_IMAGES'): ImageFile.LOAD_TRUNCATED_IMAGES = True q = Queue(concurrent) config = dataset['config'] dir = trainer.job_model.get_dataset_downloads_dir(dataset) ensure_dir(dir) if 'classes' not in config or not config['classes']: print("Dataset %s does not contain any classes." % (dataset['id'], )) return { 'X_train': np.array([]), 'Y_train': np.array([]), 'X_test': np.array([]), 'Y_test': np.array([]) } classes = config['classes'] trainer.set_status('PREPARE_IMAGES') max = 0 images = {} dataset_path = trainer.job_model.get_dataset_downloads_dir(dataset) meta_information_file = dataset_path + '/meta.json' classes_changed = False config_changed = False had_previous = False classes_md5 = hashlib.md5( json.dumps(classes, default=invalid_json_values).encode('utf-8')).hexdigest() validationFactor = 0.2 if os.path.isdir(dataset_path): if os.path.isfile(meta_information_file): with open(meta_information_file) as f: meta = json.load(f) if meta: had_previous = True if 'classes_md5' in meta and meta[ 'classes_md5'] != classes_md5: classes_changed = True trigger_changed = [ 'resize', 'resizeWidth', 'resizeHeight', 'resizeCompression' ] for i in trigger_changed: if i in meta['config'] and i in config and meta[ 'config'][i] != config[i]: config_changed = True else: config_changed = True else: config_changed = True need_download = classes_changed or config_changed if need_download: if had_previous: print("Reset dataset and re-download images to " + dir) if classes_changed: print(" .. because classes changed") if config_changed: print(" .. because settings changed") else: print("Download images to " + dir) resize = bool(get_option(config, 'resize', True)) if resize: resizeSize = (int(get_option(config, 'resizeWidth', 64)), int(get_option(config, 'resizeHeight', 64))) print(" .. with resizing to %dx%d " % resizeSize) # we need to donwload all images shutil.rmtree(dataset_path) controller = {'running': True} try: for category in classes: max += len(category['images']) for i in range(concurrent): t = ImageDownloaderWorker(q, trainer, dataset, max, images, controller) t.daemon = True t.start() for category_idx, category in enumerate(classes): for image in category['images']: q.put([image, category_idx]) q.join() controller['running'] = False def move_image(image, category='training'): if image['id'] in images and os.path.isfile( images[image['id']]): target_path = dataset_path + \ '/%s/category_%s/%s' % (category, category_idx, os.path.basename(images[image['id']])) ensure_dir(os.path.dirname(target_path)) os.rename(images[image['id']], target_path) for category_idx, category in enumerate(classes): random.shuffle(category['images']) position = int( math.ceil(len(category['images']) * validationFactor)) ensure_dir(dataset_path + '/training') ensure_dir(dataset_path + '/validation') for image in category['images'][position:]: # test data if image['id'] in images and os.path.isfile( images[image['id']]): move_image(image, 'training') for image in category['images'][:position]: # validation data if image['id'] in images and os.path.isfile( images[image['id']]): move_image(image, 'validation') with open(meta_information_file, 'w') as f: meta = { 'loaded_at': classes_md5, 'classes_md5': classes_md5, 'config': config } json.dump(meta, f, default=invalid_json_values) except KeyboardInterrupt: controller['running'] = False sys.exit(1) else: print("Downloaded images up2date in " + dir) print( " - Remove this directory if you want to re-download all images of your dataset and re-shuffle training/validation images." ) trainer.output_size = len(classes) trainer.set_status('LOAD IMAGE DONE') # change to type local_images dataset_transformed = dataset.copy() dataset_transformed['config']['path'] = dir all_memory = get_option(dataset['config'], 'allMemory', False, 'bool') if all_memory: return read_images_in_memory(job_model, dataset_transformed, node, trainer) else: return read_images_keras_generator(job_model, dataset_transformed, node, trainer)
def read_images_in_memory(job_model, dataset, node, trainer): """ Reads all images into memory and applies augmentation if enabled """ concurrent = psutil.cpu_count() dataset_config = dataset['config'] controller = {'running': True} config = dataset['config'] # TODO: config not used q = Queue(concurrent) result = {'X_train': [], 'Y_train': [], 'X_test': [], 'Y_test': []} images = [] max = 0 path = trainer.job_model.get_dataset_downloads_dir(dataset) if 'path' in dataset['config']: path = dataset['config']['path'] classes_count = 0 category_map = {} classes = [] try: for i in range(concurrent): t = ImageReadWorker(q, job_model, node, path, images, controller) t.daemon = True t.start() for validation_or_training in ['validation', 'training']: if os.path.isdir(path + '/' + validation_or_training): for category_name in os.listdir(path + '/' + validation_or_training): if os.path.isdir(path + '/' + validation_or_training + '/' + category_name): if category_name not in category_map: category_map[category_name] = classes_count if 'classes' in dataset_config and 'category_' in category_name: category_idx = int( category_name.replace('category_', '')) category_map[category_name] = category_idx target_category = dataset_config['classes'][ category_idx] classes.append(target_category['title'] or 'Class %s' % (category_idx, )) else: classes.append(category_name) classes_count += 1 for id in os.listdir(path + '/' + validation_or_training + '/' + category_name): file_path = os.path.join(path, validation_or_training, category_name, id) q.put([ file_path, validation_or_training == 'validation', category_name ]) max += 1 q.join() controller['running'] = False train_images = [] test_images = [] for v in images: image, validation, category_dir = v if validation is True: test_images.append([image, category_map[category_dir]]) else: train_images.append([image, category_map[category_dir]]) train_datagen = None augmentation = bool(get_option(dataset_config, 'augmentation', False)) if augmentation: train_datagen = get_image_data_augmentor_from_dataset(dataset) train = InMemoryDataGenerator( train_datagen, train_images, classes_count, job_model.job['config']['settings']['batchSize']) test = InMemoryDataGenerator( None, test_images, classes_count, job_model.job['config']['settings']['batchSize']) nb_sample = len(train_images) trainer.set_generator_training_nb(nb_sample) trainer.set_generator_validation_nb(len(test_images)) print(( "Found %d classes, %d images (%d in training [%saugmented], %d in validation). Read all images into memory from %s" % (classes_count, max, len(train_images), 'not ' if augmentation is False else '', len(test_images), path))) if classes_count == 0: print( "Could not find any classes. Does the directory contains images?" ) sys.exit(1) trainer.output_size = classes_count trainer.set_job_system_info('classes', classes) trainer.classes = classes result['X_train'] = train result['Y_train'] = train result['X_test'] = test result['Y_test'] = test return result except KeyboardInterrupt: controller['running'] = False sys.exit(1)
class TaskIO(object): """Object used to stream I/O between a running Mesos task and the local terminal. :param task: task ID :type task: str :param cmd: a command to launch inside the task's container :type cmd: str :param args: Additional arguments for the command :type args: str :param interactive: whether to attach STDIN of the current terminal to the new command being launched :type interactive: bool :param tty: whether to allocate a tty for this command and attach the local terminal to it :type tty: bool """ # The interval to send heartbeat messages to # keep persistent connections alive. HEARTBEAT_INTERVAL = 30 HEARTBEAT_INTERVAL_NANOSECONDS = HEARTBEAT_INTERVAL * 1000000000 def __init__(self, task_id, cmd=None, args=None, interactive=False, tty=False): # Store relevant parameters of the call for later. self.cmd = cmd self.interactive = interactive self.tty = tty self.args = args # Create a client and grab a reference to the DC/OS master. client = DCOSClient() master = get_master(client) # Get the task and make sure its container was launched by the UCR. # Since task's containers are launched by the UCR by default, we want # to allow most tasks to pass through unchecked. The only exception is # when a task has an explicit container specified and it is not of type # "MESOS". Having a type of "MESOS" implies that it was launched by the # UCR -- all other types imply it was not. task_obj = master.task(task_id) if "container" in task_obj.dict(): if "type" in task_obj.dict()["container"]: if task_obj.dict()["container"]["type"] != "MESOS": raise DCOSException( "This command is only supported for tasks" " launched by the Universal Container Runtime (UCR).") # Get the URL to the agent running the task. if client._mesos_master_url: self.agent_url = client.slave_url( slave_id="", private_url=task_obj.slave().http_url(), path="api/v1") else: self.agent_url = client.slave_url(slave_id=task_obj.slave()['id'], private_url="", path="api/v1") # Grab a reference to the container ID for the task. self.parent_id = master.get_container_id(task_obj) # Generate a new UUID for the nested container # used to run commands passed to `task exec`. self.container_id = str(uuid.uuid4()) # Set up a recordio encoder and decoder # for any incoming and outgoing messages. self.encoder = recordio.Encoder( lambda s: bytes(json.dumps(s, ensure_ascii=False), "UTF-8")) self.decoder = recordio.Decoder( lambda s: json.loads(s.decode("UTF-8"))) # Set up queues to send messages between threads used for # reading/writing to STDIN/STDOUT/STDERR and threads # sending/receiving data over the network. self.input_queue = Queue() self.output_queue = Queue() # Set up an event to block attaching # input until attaching output is complete. self.attach_input_event = threading.Event() self.attach_input_event.clear() # Set up an event to block printing the output # until an attach input event has successfully # been established. self.print_output_event = threading.Event() self.print_output_event.clear() # Set up an event to block the main thread # from exiting until signaled to do so. self.exit_event = threading.Event() self.exit_event.clear() # Use a class variable to store exceptions thrown on # other threads and raise them on the main thread before # exiting. self.exception = None def run(self): """Run the helper threads in this class which enable streaming of STDIN/STDOUT/STDERR between the CLI and the Mesos Agent API. If a tty is requested, we take over the current terminal and put it into raw mode. We make sure to reset the terminal back to its original settings before exiting. """ # Without a TTY. if not self.tty: try: self._start_threads() self.exit_event.wait() except Exception as e: self.exception = e if self.exception: raise self.exception return # With a TTY. if util.is_windows_platform(): raise DCOSException( "Running with the '--tty' flag is not supported on windows.") if not sys.stdin.isatty(): raise DCOSException( "Must be running in a tty to pass the '--tty flag'.") fd = sys.stdin.fileno() oldtermios = termios.tcgetattr(fd) try: if self.interactive: tty.setraw(fd, when=termios.TCSANOW) self._window_resize(signal.SIGWINCH, None) signal.signal(signal.SIGWINCH, self._window_resize) self._start_threads() self.exit_event.wait() except Exception as e: self.exception = e termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, oldtermios) if self.exception: raise self.exception def _thread_wrapper(self, func): """A wrapper around all threads used in this class If a thread throws an exception, it will unblock the main thread and save the exception in a class variable. The main thread will then rethrow the exception before exiting. :param func: The start function for the thread :type func: function """ try: func() except Exception as e: self.exception = e self.exit_event.set() def _start_threads(self): """Start all threads associated with this class """ if self.interactive: # Collects input from STDIN and puts # it in the input_queue as data messages. thread = threading.Thread(target=self._thread_wrapper, args=(self._input_thread, )) thread.daemon = True thread.start() # Prepares heartbeat control messages and # puts them in the input queueaat a specific # heartbeat interval. thread = threading.Thread(target=self._thread_wrapper, args=(self._heartbeat_thread, )) thread.daemon = True thread.start() # Opens a persistent connection with the mesos agent and # feeds it both control and data messages from the input # queue via ATTACH_CONTAINER_INPUT messages. thread = threading.Thread(target=self._thread_wrapper, args=(self._attach_container_input, )) thread.daemon = True thread.start() # Opens a persistent connection with a mesos agent, reads # data messages from it and feeds them to an output_queue. thread = threading.Thread( target=self._thread_wrapper, args=(self._launch_nested_container_session, )) thread.daemon = True thread.start() # Collects data messages from the output queue and writes # their content to STDOUT and STDERR. thread = threading.Thread(target=self._thread_wrapper, args=(self._output_thread, )) thread.daemon = True thread.start() def _launch_nested_container_session(self): """Sends a request to the Mesos Agent to launch a new nested container and attach to its output stream. The output stream is then sent back in the response. """ message = { 'type': "LAUNCH_NESTED_CONTAINER_SESSION", 'launch_nested_container_session': { 'container_id': { 'parent': self.parent_id, 'value': self.container_id }, 'command': { 'value': self.cmd, 'arguments': [self.cmd] + self.args, 'shell': False } } } if self.tty: message['launch_nested_container_session']['container'] = { 'type': 'MESOS', 'tty_info': {} } req_extra_args = { 'stream': True, 'headers': { 'Content-Type': 'application/json', 'Accept': 'application/recordio', 'Message-Accept': 'application/json' } } response = http.post(self.agent_url, data=json.dumps(message), timeout=None, **req_extra_args) self._process_output_stream(response) def _process_output_stream(self, response): """Gets data streamed over the given response and places the returned messages into our output_queue. Only expects to receive data messages. :param response: Response from an http post :type response: requests.models.Response """ # Now that we are ready to process the output stream (meaning # our output connection has been established), allow the input # stream to be attached by setting an event. self.attach_input_event.set() # If we are running in interactive mode, wait to make sure that # our input connection succeeds before pushing any output to the # output queue. if self.interactive: self.print_output_event.wait() try: for chunk in response.iter_content(chunk_size=None): records = self.decoder.decode(chunk) for r in records: if r.get('type') and r['type'] == 'DATA': self.output_queue.put(r['data']) except Exception as e: raise DCOSException( "Error parsing output stream: {error}".format(error=e)) self.output_queue.join() self.exit_event.set() def _attach_container_input(self): """Streams all input data (e.g. STDIN) from the client to the agent """ def _initial_input_streamer(): """Generator function yielding the initial ATTACH_CONTAINER_INPUT message for streaming. We have a separate generator for this so that we can attempt the connection once before committing to a persistent connection where we stream the rest of the input. :returns: A RecordIO encoded message """ message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'CONTAINER_ID', 'container_id': { 'parent': self.parent_id, 'value': self.container_id } } } yield self.encoder.encode(message) def _input_streamer(): """Generator function yielding ATTACH_CONTAINER_INPUT messages for streaming. It yields the _intitial_input_streamer() message, followed by messages from the input_queue on each subsequent call. :returns: A RecordIO encoded message """ yield next(_initial_input_streamer()) while True: record = self.input_queue.get() if not record: break yield record req_extra_args = { 'headers': { 'Content-Type': 'application/recordio', 'Message-Content-Type': 'application/json', 'Accept': 'application/json', 'Connection': 'close', 'Transfer-Encoding': 'chunked' } } # Ensure we don't try to attach our input to a container that isn't # fully up and running by waiting until the # `_process_output_stream` function signals us that it's ready. self.attach_input_event.wait() # Send an intial "Test" message to ensure that we are able to # establish a connection with the agent. If we aren't we will throw # an exception and break out of this thread. However, in cases where # we receive a 500 response from the agent, we actually want to # continue without throwing an exception. A 500 error indicates that # we can't connect to the container because it has already finished # running. In that case we continue running to allow the output queue # to be flushed. try: http.post(self.agent_url, data=_initial_input_streamer(), **req_extra_args) except DCOSHTTPException as e: if not e.response.status_code == 500: raise e # If we succeeded with that connection, unblock process_output_stream() # from sending output data to the output thread. self.print_output_event.set() # Begin streaming the input. http.post(self.agent_url, data=_input_streamer(), timeout=None, **req_extra_args) def _input_thread(self): """Reads from STDIN and places a message with that data onto the input_queue. """ message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'PROCESS_IO', 'process_io': { 'type': 'DATA', 'data': { 'type': 'STDIN', 'data': '' } } } } for chunk in iter(partial(os.read, sys.stdin.fileno(), 1024), b''): message['attach_container_input']['process_io']['data'][ 'data'] = base64.b64encode(chunk).decode('utf-8') self.input_queue.put(self.encoder.encode(message)) # Push an empty string to indicate EOF to the server and push # 'None' to signal that we are done processing input. message['attach_container_input']['process_io']['data']['data'] = '' self.input_queue.put(self.encoder.encode(message)) self.input_queue.put(None) def _output_thread(self): """Reads from the output_queue and writes the data to the appropriate STDOUT or STDERR. """ while True: # Get a message from the output queue and decode it. # Then write the data to the appropriate stdout or stderr. output = self.output_queue.get() if not output.get('data'): raise DCOSException("Error no 'data' field in output message") data = output['data'] data = base64.b64decode(data.encode('utf-8')) if output.get('type') and output['type'] == 'STDOUT': sys.stdout.buffer.write(data) sys.stdout.flush() elif output.get('type') and output['type'] == 'STDERR': sys.stderr.buffer.write(data) sys.stderr.flush() else: raise DCOSException("Unsupported data type in output stream") self.output_queue.task_done() def _heartbeat_thread(self): """Generates a heartbeat message to send over the ATTACH_CONTAINER_INPUT stream every `interval` seconds and inserts it in the input queue. """ interval = self.HEARTBEAT_INTERVAL nanoseconds = self.HEARTBEAT_INTERVAL_NANOSECONDS message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'PROCESS_IO', 'process_io': { 'type': 'CONTROL', 'control': { 'type': 'HEARTBEAT', 'heartbeat': { 'interval': { 'nanoseconds': nanoseconds } } } } } } while True: self.input_queue.put(self.encoder.encode(message)) time.sleep(interval) def _window_resize(self, signum, frame): """Signal handler for SIGWINCH. Generates a message with the current demensions of the terminal and puts it in the input_queue. :param signum: the signal number being handled :type signum: int :param frame: current stack frame :type frame: frame """ # Determine the size of our terminal, and create the message to be sent rows, columns = os.popen('stty size', 'r').read().split() message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'PROCESS_IO', 'process_io': { 'type': 'CONTROL', 'control': { 'type': 'TTY_INFO', 'tty_info': { 'window_size': { 'rows': int(rows), 'columns': int(columns) } } } } } } self.input_queue.put(self.encoder.encode(message))
def execute(trans, tool, param_combinations, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None): """ Execute a tool and return object containing summary (output data, number of failures, etc...). """ all_jobs_timer = ExecutionTimer() execution_tracker = ToolExecutionTracker(tool, param_combinations, collection_info) app = trans.app execution_cache = ToolExecutionCache(trans) def execute_single_job(params): job_timer = ExecutionTimer() if workflow_invocation_uuid: params['__workflow_invocation_uuid__'] = workflow_invocation_uuid elif '__workflow_invocation_uuid__' in params: # Only workflow invocation code gets to set this, ignore user supplied # values or rerun parameters. del params['__workflow_invocation_uuid__'] job, result = tool.handle_single_execution(trans, rerun_remap_job_id, params, history, collection_info, execution_cache) if job: message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer) log.debug(message) execution_tracker.record_success(job, result) else: execution_tracker.record_error(result) config = app.config burst_at = getattr(config, 'tool_submission_burst_at', 10) burst_threads = getattr(config, 'tool_submission_burst_threads', 1) tool_action = tool.action if hasattr(tool_action, "check_inputs_ready"): for params in execution_tracker.param_combinations: # This will throw an exception if the tool is not ready. tool_action.check_inputs_ready(tool, trans, params, history) job_count = len(execution_tracker.param_combinations) if job_count < burst_at or burst_threads < 2: for params in execution_tracker.param_combinations: execute_single_job(params) else: q = Queue() def worker(): while True: params = q.get() execute_single_job(params) q.task_done() for i in range(burst_threads): t = Thread(target=worker) t.daemon = True t.start() for params in execution_tracker.param_combinations: q.put(params) q.join() log.debug("Executed %d job(s) for tool %s request: %s" % (job_count, tool.id, all_jobs_timer)) if collection_info: history = history or tool.get_default_history_by_trans(trans) if len(param_combinations) == 0: template = "Attempting to map over an empty collection, this is not yet implemented. colleciton_info is [%s]" message = template % collection_info log.warn(message) raise Exception(message) params = param_combinations[0] execution_tracker.create_output_collections(trans, history, params) return execution_tracker
class RisMonolith_v1_0_0(Dictable): """Monolithic cache of RIS data""" def __init__(self, client): """Initialize RisMonolith :param client: client to utilize :type client: RmcClient object """ self._client = client self.name = "Monolithic output of RIS Service" self.types = OrderedDict() self._visited_urls = list() self._current_location = '/' # "root" self.queue = Queue() self._type = None self._name = None self.progress = 0 self.reload = False self._typestring = '@odata.type' self._hrefstring = '@odata.id' def _get_type(self): """Return monolith version type""" return "Monolith.1.0.0" type = property(_get_type, None) def update_progress(self): """Simple function to increment the dot progress""" if self.progress % 6 == 0: sys.stdout.write('.') def get_visited_urls(self): """Return the visited URLS""" return self._visited_urls def set_visited_urls(self, visited_urls): """Set visited URLS to given list.""" self._visited_urls = visited_urls def load(self, path=None, includelogs=False, skipinit=False, \ skipcrawl=False, loadtype='href', loadcomplete=False): """Walk entire RIS model and cache all responses in self. :param path: path to start load from. :type path: str. :param includelogs: flag to determine if logs should be downloaded also. :type includelogs: boolean. :param skipinit: flag to determine if first run of load. :type skipinit: boolean. :param skipcrawl: flag to determine if load should traverse found links. :type skipcrawl: boolean. :param loadtype: flag to determine if load is meant for only href items. :type loadtype: str. :param loadcomplete: flag to download the entire monolith :type loadcomplete: boolean """ if not skipinit: if LOGGER.getEffectiveLevel() == 40: sys.stdout.write("Discovering data...") else: LOGGER.info("Discovering data...") self.name = self.name + ' at %s' % self._client.base_url if not self.types: self.types = OrderedDict() if not threading.active_count() >= 6: for _ in range(5): workhand = SuperDuperWorker(self.queue) workhand.setDaemon(True) workhand.start() selectivepath = path if not selectivepath: selectivepath = self._client._rest_client.default_prefix self._load(selectivepath, skipcrawl=skipcrawl, includelogs=includelogs,\ skipinit=skipinit, loadtype=loadtype, loadcomplete=loadcomplete) self.queue.join() if not skipinit: if LOGGER.getEffectiveLevel() == 40: sys.stdout.write("Done\n") else: LOGGER.info("Done\n") def _load(self, path, skipcrawl=False, originaluri=None, includelogs=False,\ skipinit=False, loadtype='href', loadcomplete=False): """Helper function to main load function. :param path: path to start load from. :type path: str. :param skipcrawl: flag to determine if load should traverse found links. :type skipcrawl: boolean. :param originaluri: variable to assist in determining originating path. :type originaluri: str. :param includelogs: flag to determine if logs should be downloaded also. :type includelogs: boolean. :param skipinit: flag to determine if first run of load. :type skipinit: boolean. :param loadtype: flag to determine if load is meant for only href items. :type loadtype: str. :param loadcomplete: flag to download the entire monolith :type loadcomplete: boolean """ if path.endswith("?page=1"): return elif not includelogs: if "/Logs/" in path: return #TODO: need to find a better way to support non ascii characters path = path.replace("|", "%7C") #remove fragments newpath = urlparse(path) newpath = list(newpath[:]) newpath[-1] = '' path = urlunparse(tuple(newpath)) LOGGER.debug('_loading %s', path) if not self.reload: if path.lower() in self._visited_urls: return resp = self._client.get(path) if resp.status != 200: path = path + '/' resp = self._client.get(path) if resp.status == 401: raise SessionExpiredRis("Invalid session. Please logout and "\ "log back in or include credentials.") elif resp.status != 200: return self.queue.put((resp, path, skipinit, self)) if loadtype == 'href': #follow all the href attributes jsonpath_expr = jsonpath_rw.parse("$..'@odata.id'") matches = jsonpath_expr.find(resp.dict) if 'links' in resp.dict and 'NextPage' in resp.dict['links']: if originaluri: next_link_uri = originaluri + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=originaluri, \ includelogs=includelogs, skipcrawl=skipcrawl, \ skipinit=skipinit) else: next_link_uri = path + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=path, includelogs=includelogs,\ skipcrawl=skipcrawl, skipinit=skipinit) if not skipcrawl: for match in matches: if str(match.full_path) == "*****@*****.**" or \ str(match.full_path) == "*****@*****.**": continue if match.value == path: continue href = '%s' % match.value self._load(href, skipcrawl=skipcrawl, \ originaluri=originaluri, includelogs=includelogs, \ skipinit=skipinit) if loadcomplete: for match in matches: self._load(match.value, skipcrawl=skipcrawl, originaluri=\ originaluri, includelogs=includelogs, skipinit=skipinit) def branch_worker(self, resp, path, skipinit): """Helper for load function, creates threaded worker :param resp: response received. :type resp: str. :param path: path correlating to the response. :type path: str. :param skipinit: flag to determine if progress bar should be updated. :type skipinit: boolean. """ self._visited_urls.append(path.lower()) member = RisMonolithMember_v1_0_0(resp) if not member.type: return self.update_member(member) if not skipinit: self.progress += 1 if LOGGER.getEffectiveLevel() == 40: self.update_progress() def update_member(self, member): """Adds member to this monolith. If the member already exists the""" """ data is updated in place. :param member: Ris monolith member object made by branch worker. :type member: RisMonolithMember_v1_0_0. """ if member.maj_type not in self.types: self.types[member.maj_type] = OrderedDict() self.types[member.maj_type]['Instances'] = list() found = False for indices in range(len(self.types[member.maj_type]['Instances'])): inst = self.types[member.maj_type]['Instances'][indices] if inst.resp.request.path == member.resp.request.path: self.types[member.maj_type]['Instances'][indices] = member self.types[member.maj_type]['Instances'][indices].patches.\ extend([patch for patch in inst.patches]) found = True break if not found: self.types[member.maj_type]['Instances'].append(member) def load_from_dict(self, src): """Load data to monolith from dict :param src: data receive from rest operation. :type src: str. """ self._type = src['Type'] self._name = src['Name'] self.types = OrderedDict() for typ in src['Types']: for inst in typ['Instances']: member = RisMonolithMember_v1_0_0(None) member.load_from_dict(inst) self.update_member(member) return def to_dict(self): """Convert data to monolith from dict""" result = OrderedDict() result['Type'] = self.type result['Name'] = self.name types_list = list() for typ in list(self.types.keys()): type_entry = OrderedDict() type_entry['Type'] = typ type_entry['Instances'] = list() for inst in self.types[typ]['Instances']: type_entry['Instances'].append(inst.to_dict()) types_list.append(type_entry) result['Types'] = types_list return result def reduce(self): """Reduce monolith data""" result = OrderedDict() result['Type'] = self.type result['Name'] = self.name types_list = list() for typ in list(self.types.keys()): type_entry = OrderedDict() type_entry['Type'] = typ for inst in self.types[typ]['Instances']: type_entry['Instances'] = inst.reduce() types_list.append(type_entry) result['Types'] = types_list return result def _jsonpath2jsonpointer(self, instr): """Convert json path to json pointer :param instr: input path to be converted to pointer. :type instr: str. """ outstr = instr.replace('.[', '[') outstr = outstr.replace('[', '/') outstr = outstr.replace(']', '/') if outstr.endswith('/'): outstr = outstr[:-1] return outstr def _get_current_location(self): """Return current location""" return self._current_location def _set_current_location(self, newval): """Set current location""" self._current_location = newval location = property(_get_current_location, _set_current_location) def list(self, lspath=None): """Function for list command :param lspath: path list. :type lspath: list. """ results = list() path_parts = ['Types'] # Types is always assumed if isinstance(lspath, list) and len(lspath) > 0: lspath = lspath[0] path_parts.extend(lspath.split('/')) elif not lspath: lspath = '/' else: path_parts.extend(lspath.split('/')) currpos = self.to_dict() for path_part in path_parts: if not path_part: continue if isinstance(currpos, RisMonolithMember_v1_0_0): break elif isinstance(currpos, dict) and path_part in currpos: currpos = currpos[path_part] elif isinstance(currpos, list): for positem in currpos: if 'Type' in positem and path_part == positem['Type']: currpos = positem break results.append(currpos) return results def killthreads(self): """Function to kill threads on logout""" threads = [] for thread in threading.enumerate(): if isinstance(thread, SuperDuperWorker): self.queue.put(('KILL', 'KILL', 'KILL', 'KILL')) threads.append(thread) for thread in threads: thread.join()
def get_images(job_model, dataset, node, trainer): concurrent = 15 from PIL import ImageFile if hasattr(ImageFile, 'LOAD_TRUNCATED_IMAGES'): ImageFile.LOAD_TRUNCATED_IMAGES = True q = Queue(concurrent) config = dataset['config'] dir = job_model.get_dataset_downloads_dir(dataset) ensure_dir(dir) if 'classes' not in config or not config['classes']: trainer.logger.warning("Dataset %s does not contain any classes." % (dataset['id'],)) return { 'X_train': np.array([]), 'Y_train': np.array([]), 'X_test': np.array([]), 'Y_test': np.array([]) } classes = config['classes'] trainer.set_status('LOAD IMAGES') max = 0 images = {} dataset_path = job_model.get_dataset_downloads_dir(dataset) meta_information_file = dataset_path + '/meta.json' classes_changed = False config_changed = False had_previous = False classes_md5 = hashlib.md5(simplejson.dumps(classes, default=invalid_json_values, sort_keys=True).encode('utf-8')).hexdigest() validationFactor = 0.2 meta = {} if os.path.isdir(dataset_path): if os.path.isfile(meta_information_file): with open(meta_information_file) as f: meta = simplejson.load(f) if meta: had_previous = True if 'classes_md5' in meta and meta['classes_md5'] != classes_md5: classes_changed = True trigger_changed = ['resize', 'resizeWidth', 'resizeHeight', 'resizeCompression'] for i in trigger_changed: if i in meta['config'] and i in config and meta['config'][i] != config[i]: config_changed = True else: config_changed = True else: config_changed = True need_download = classes_changed or config_changed if need_download: if had_previous: trainer.logger.info("Reset dataset and re-download images to " + dir) if classes_changed: trainer.logger.info(" .. because classes changed in", meta['classes_md5'], classes_md5, meta_information_file) if config_changed: trainer.logger.info(" .. because settings changed in", meta_information_file) else: trainer.logger.info("Download images to " + dir) resize = bool(get_option(config, 'resize', True)) if resize: resizeSize = (int(get_option(config, 'resizeWidth', 64)), int(get_option(config, 'resizeHeight', 64))) trainer.logger.info(" .. with resizing to %dx%d " % resizeSize) # # we need to donwload all images shutil.rmtree(dataset_path) controller = {'running': True} try: for category in classes: max += len(category['images']) progress = trainer.job_backend.create_progress('dataset-download-images', max) progress.label('Download dataset images') for i in range(concurrent): t = ImageDownloaderWorker(q, progress, dataset, max, images, controller) t.daemon = True t.start() for category_idx, category in enumerate(classes): for image in category['images']: local_name = image['id'] local_path = '%s/%s' % (trainer.job_model.get_dataset_downloads_dir(dataset), local_name) q.put([image, category_idx, local_path]) q.join() controller['running'] = False def move_image(image, category='training'): if image['id'] in images and os.path.isfile(images[image['id']]): target_path = dataset_path + \ '/%s/category_%s/%s' % (category, category_idx, os.path.basename(images[image['id']])) ensure_dir(os.path.dirname(target_path)) os.rename(images[image['id']], target_path) for category_idx, category in enumerate(classes): random.shuffle(category['images']) position = int(math.ceil(len(category['images']) * validationFactor)) ensure_dir(dataset_path + '/training') ensure_dir(dataset_path + '/validation') for image in category['images'][position:]: # test data if image['id'] in images and os.path.isfile(images[image['id']]): move_image(image, 'training') for image in category['images'][:position]: # validation data if image['id'] in images and os.path.isfile(images[image['id']]): move_image(image, 'validation') with open(meta_information_file, 'w') as f: meta = { 'loaded_at': classes_md5, 'classes_md5': classes_md5, 'config': config } simplejson.dump(meta, f, default=invalid_json_values) except KeyboardInterrupt: controller['running'] = False sys.exit(1) else: trainer.logger.info("Downloaded images up2date in " + dir) trainer.logger.info(" - Remove this directory if you want to re-download all images of your dataset and re-shuffle training/validation images.") trainer.output_size = len(classes) # change to type local_images dataset_transformed = dataset.copy() dataset_transformed['config']['path'] = dir all_memory = get_option(dataset['config'], 'allMemory', False, 'bool') if all_memory: return read_images_in_memory(job_model, dataset_transformed, node, trainer) else: return read_images_keras_generator(job_model, dataset_transformed, node, trainer)
class Copier(object): """Copy files in worker threads.""" def __init__(self, num_workers=4): # type: (int) -> None if num_workers < 0: raise ValueError("num_workers must be >= 0") self.num_workers = num_workers self.queue = None # type: Optional[Queue[_Task]] self.workers = [] # type: List[_Worker] self.errors = [] # type: List[Exception] self.running = False def start(self): """Start the workers.""" if self.num_workers: self.queue = Queue(maxsize=self.num_workers) self.workers = [_Worker(self) for _ in range(self.num_workers)] for worker in self.workers: worker.start() self.running = True def stop(self): """Stop the workers (will block until they are finished).""" if self.running and self.num_workers: for worker in self.workers: self.queue.put(None) for worker in self.workers: worker.join() # Free up references help by workers del self.workers[:] self.queue.join() self.running = False def add_error(self, error): """Add an exception raised by a task.""" self.errors.append(error) def __enter__(self): self.start() return self def __exit__( self, exc_type, # type: Optional[Type[BaseException]] exc_value, # type: Optional[BaseException] traceback, # type: Optional[TracebackType] ): self.stop() if traceback is None and self.errors: raise BulkCopyFailed(self.errors) def copy(self, src_fs, src_path, dst_fs, dst_path): # type: (FS, Text, FS, Text) -> None """Copy a file from one fs to another.""" if self.queue is None: # This should be the most performant for a single-thread copy_file_internal(src_fs, src_path, dst_fs, dst_path) else: src_file = src_fs.openbin(src_path, "r") try: dst_file = dst_fs.openbin(dst_path, "w") except Exception: src_file.close() raise task = _CopyTask(src_file, dst_file) self.queue.put(task)
class Rigger(object): """ A Rigger event framework instance. The Rigger object holds all configuration and instances of plugins. By default Rigger accepts a configuration file name to parse, though it is perfectly acceptable to pass the configuration into the ``self.config`` attribute. Args: config_file: A configuration file holding all of Riggers base and plugin configuration. """ def __init__(self, config_file): self.gdl = threading.Lock() self.pre_callbacks = defaultdict(dict) self.post_callbacks = defaultdict(dict) self.plugins = {} self.config_file = config_file self.squash_exceptions = False self.initialized = False self._task_list = {} self._queue_lock = threading.Lock() self._global_queue = Queue() self._background_queue = Queue() self._server_shutdown = False self._zmq_event_handler_shutdown = False self._global_queue_shutdown = False self._background_queue_shutdown = False globt = threading.Thread(target=self.process_queue, name="global_queue_processor") globt.start() bgt = threading.Thread(target=self.process_background_queue, name="background_queue_processor") bgt.start() def process_queue(self): """ The ``process_queue`` thread manages taking events on and off of the global queue. Both TCP and in-object fire_hooks place events onto the global_queue and these are both handled by the same handler called ``process_hook``. If there is an exception during processing, the exception is printed and execution continues. """ while not self._global_queue_shutdown: while not self._global_queue.empty(): with self._queue_lock: tid = self._global_queue.get() obj = self._task_list[tid].json_dict self._task_list[tid].status = Task.RUNNING try: loc, glo = self.process_hook(obj['hook_name'], **obj['data']) combined_dict = {} combined_dict.update(glo) combined_dict.update(loc) self._task_list[tid].output = combined_dict except Exception as e: self.log_message(e) with self._queue_lock: self._global_queue.task_done() self._task_list[tid].status = Task.FINISHED if not self._task_list[tid].json_dict.get('grab_result', None): del self._task_list[tid] time.sleep(0.1) def process_background_queue(self): """ The ``process_background_queue`` manages the hooks which have been backgrounded. In this respect the tasks that are completed are not required to continue with the test and as such can be forgotten about. An example of this would be some that sends an email, or tars up files, it has all the information it needs and the main process doesn't need to wait for it to complete. """ while not self._background_queue_shutdown: while not self._background_queue.empty(): obj = self._background_queue.get() try: local, globals_updates = self.process_callbacks( obj['cb'], obj['kwargs']) with self.gdl: self.global_data = recursive_update( self.global_data, globals_updates) except Exception as e: self.log_message(e) self._background_queue.task_done() time.sleep(0.1) def zmq_event_handler(self, zmq_socket_address): """ The ``zmq_event_handler`` thread receives (and responds to) updates from the zmq socket, which is normally embedded in the web server running alongside this riggerlib instance, in its own process. """ ctx = zmq.Context() zmq_socket = ctx.socket(zmq.REP) zmq_socket.set(zmq.RCVTIMEO, 300) zmq_socket.bind(zmq_socket_address) def zmq_reply(message, **extra): payload = {'message': message} payload.update(extra) zmq_socket.send_json(payload) bad_request = partial(zmq_reply, 'BAD REQUEST') while not self._zmq_event_handler_shutdown: try: json_dict = zmq_socket.recv_json() except zmq.Again: continue try: event_name = json_dict['event_name'] except KeyError: bad_request() if event_name == 'fire_hook': tid = self._fire_internal_hook(json_dict) if tid: zmq_reply('OK', tid=tid) else: bad_request() elif event_name == 'task_check': try: tid = json_dict['tid'] extra = { "tid": tid, "status": self._task_list[tid].status, } if json_dict['grab_result']: extra["output"] = self._task_list[tid].output zmq_reply('OK', **extra) except KeyError: zmq_reply('NOT FOUND') elif event_name == 'task_delete': try: tid = json_dict['tid'] del self._task_list[tid] zmq_reply('OK', tid=tid) except KeyError: zmq_reply('OK', tid=tid) elif event_name == 'shutdown': zmq_reply('OK') # We gotta initiate server stop from here and stop this thread self._server_shutdown = True break elif event_name == 'ping': zmq_reply('PONG') else: bad_request() zmq_socket.close() def read_config(self, config_file): """ Reads in the config file and parses the yaml data. Args: config_file: A configuration file holding all of Riggers base and plugin configuration. Raises: IOError: If the file can not be read. Exception: If there is any error parsing the configuration file. """ try: with open(config_file, "r") as stream: data = yaml.load(stream) except IOError: print("!!! Configuration file could not be loaded...exiting") sys.exit(127) except Exception as e: print(e) print("!!! Error parsing Configuration file") sys.exit(127) self.config = data def parse_config(self): """ Takes the configuration data from ``self.config`` and sets up the plugin instances. """ self.read_config(self.config_file) self.setup_plugin_instances() self.start_server() def setup_plugin_instances(self): """ Sets up instances into a dict called ``self.instances`` and instantiates each instance of the plugin. It also sets the ``self._threaded`` option to determine if plugins will be processed synchronously or asynchronously. """ self.instances = {} self._threaded = self.config.get("threaded", False) plugins = self.config.get("plugins", {}) for ident, config in plugins.items(): self.setup_instance(ident, config) def setup_instance(self, ident, config): """ Sets up a single instance into the ``self.instances`` dict. If the instance does not exist, a warning is printed out. Args: ident: A plugin instance identifier. config: Configuration dict from the yaml. """ plugin_name = config.get('plugin', {}) if plugin_name in self.plugins: obj = self.plugins[plugin_name] if obj: obj_instance = obj(ident, config, self) self.instances[ident] = RiggerPluginInstance( ident, obj_instance, config) else: msg = "Plugin [{}] was not found, "\ "disabling instance [{}]".format(plugin_name, ident) self.log_message(msg) def start_server(self): """ Starts the ZMQ server if the ``server_enabled`` is True in the config. """ self._server_hostname = self.config.get('server_address', '127.0.0.1') self._server_port = self.config.get('server_port', 21212) self._server_enable = self.config.get('server_enabled', False) if self._server_enable: zmq_socket_address = 'tcp://{}:{}'.format(self._server_hostname, self._server_port) # set up reciever thread for zmq event handling zeh = threading.Thread(target=self.zmq_event_handler, args=(zmq_socket_address, ), name="zmq_event_handler") zeh.start() exect = threading.Thread(target=self.await_shutdown, name="executioner") exect.start() def await_shutdown(self): while not self._server_shutdown: time.sleep(0.3) self.stop_server() def stop_server(self): """ Responsible for the following: - stopping the zmq event handler (unless already stopped through 'terminate') - stopping the global queue - stopping the background queue """ self.log_message("Shutdown initiated : {}".format( self._server_hostname)) # The order here is important self._zmq_event_handler_shutdown = True self._global_queue.join() self._global_queue_shutdown = True self._background_queue.join() self._background_queue_shutdown = True raise SystemExit def fire_hook(self, hook_name, **kwargs): """ Parses the hook information into a dict for passing to process_hook. This is used to enable both the TCP and in-object fire_hook methods to use the same process_hook method call. Args: hook_name: The name of the hook to fire. kwargs: The kwargs to pass to the hooks. """ json_dict = {'hook_name': hook_name, 'data': kwargs} self._fire_internal_hook(json_dict) def _fire_internal_hook(self, json_dict): task = Task(json_dict) tid = task.tid.hexdigest() self._task_list[tid] = task if self._global_queue: with self._queue_lock: self._global_queue.put(tid) return tid else: return None def process_hook(self, hook_name, **kwargs): """ Takes a hook_name and a selection of kwargs and fires off the appropriate callbacks. This function is the guts of Rigger and is responsible for running the callback and hook functions. It first loads some blank dicts to collect the updates for the local and global namespaces. After this, it loads the pre_callback functions along with the kwargs into the callback collector processor. The return values are then classifed into local and global dicts and updates proceed. After this, the plugin hooks themselves are then run using the same methodology. Their return values are merged with the existing dicts and then the same process happens for the post_callbacks. Note: If the instance of the plugin has been marked as a background instance, and hooks which are called in that instance will be backgrounded. The hook will also not be able to return any data to the post-hook callback, although updates to globals will be processed as and when the backgrounded task is completed. Args: hook_name: The name of the hook to fire. kwargs: The kwargs to pass to the hooks. """ if not self.initialized: return kwargs_updates = {} globals_updates = {} kwargs.update({'config': self.config}) # First fire off any pre-hook callbacks if self.pre_callbacks.get(hook_name): # print "Running pre hook callback for {}".format(hook_name) kwargs_updates, globals_updates = self.process_callbacks( self.pre_callbacks[hook_name].values(), kwargs) # Now we can update the kwargs passed to the real hook with the updates with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) # Now fire off each plugin hook event_hooks = [] for instance_name, instance in self.instances.items(): callbacks = instance.obj.callbacks enabled = instance.data.get('enabled', None) if callbacks.get(hook_name) and enabled: cb = callbacks[hook_name] if instance.data.get('background', False): self._background_queue.put({'cb': [cb], 'kwargs': kwargs}) elif cb['bg']: self._background_queue.put({'cb': [cb], 'kwargs': kwargs}) else: event_hooks.append(cb) kwargs_updates, globals_updates = self.process_callbacks( event_hooks, kwargs) # One more update for the post_hook callback with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) # Finally any post-hook callbacks if self.post_callbacks.get(hook_name): # print "Running post hook callback for {}".format(hook_name) kwargs_updates, globals_updates = self.process_callbacks( self.post_callbacks[hook_name].values(), kwargs) with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) return kwargs, self.global_data def process_callbacks(self, callback_collection, kwargs): """ Processes a collection of callbacks or hooks for a particular event, namely pre, hook or post. The functions are passed in as an array to ``callback_collection`` and process callbacks first iterates each function and ensures that each one has the correct arguments available to it. If not, an Exception is raised. Then, depending on whether Threading is enabled or not, the functions are either run sequentially, or loaded into a ThreadPool and executed asynchronously. The returned local and global updates are either collected and processed sequentially, as in the case of the non-threaded behaviour, or collected at the end of the callback_collection processing and handled there. Note: It is impossible to predict the order of the functions being run. If the order is important, it is advised to create a second event hook that will be fired before the other. Rigger has no concept of hook or callback order and is unlikely to ever have. Args: callback_collection: A list of functions to call. kwargs: A set of kwargs to pass to the functions. Returns: A tuple of local and global namespace updates. """ loc_collect = {} glo_collect = {} if self._threaded: results_list = [] pool = ThreadPool(10) for cb in callback_collection: required_args = [ sig for sig in cb['args'] if isinstance(cb['args'][sig].default, type) ] missing = list( set(required_args).difference(set( self.global_data.keys())).difference(set(kwargs.keys()))) if not missing: new_kwargs = self.build_kwargs(cb['args'], kwargs) if self._threaded: results_list.append( pool.apply_async(cb['func'], [], new_kwargs)) else: obtain_result = self.handle_results( cb['func'], [], new_kwargs) loc_collect, glo_collect = self.handle_collects( obtain_result, loc_collect, glo_collect) else: raise Exception('Function {} is missing kwargs {}'.format( cb['func'].__name__, missing)) if self._threaded: pool.close() pool.join() for result in results_list: obtain_result = self.handle_results(result.get, [], {}) loc_collect, glo_collect = self.handle_collects( obtain_result, loc_collect, glo_collect) return loc_collect, glo_collect def handle_results(self, call, args, kwargs): """ Handles results and depending on configuration, squashes exceptions and logs or returns the obtained result. Args: call: The function call. args: The positional arguments. kwargs: The keyword arguments. Returns: The obtained result of the callback or hook. """ try: obtain_result = call(*args, **kwargs) except: if self.squash_exceptions: obtain_result = None self.handle_failure(sys.exc_info()) else: raise return obtain_result def handle_collects(self, result, loc_collect, glo_collect): """ Handles extracting the information from the hook/callback result. If the hook/callback returns None, then the dicts are returned unaltered, else they are updated with local, global namespace updates. Args: result: The result to process. loc_collect: The local namespace updates collection. glo_collect: The global namespace updates collection. Returns: A tuple containing the local and global updates. """ if result: if result[0]: loc_collect = recursive_update(loc_collect, result[0]) if result[1]: glo_collect = recursive_update(glo_collect, result[1]) return loc_collect, glo_collect def build_kwargs(self, args, kwargs): """ Builds a new kwargs from a list of allowed args. Functions only receive a single set of kwargs, and so the global and local namespaces have to be collapsed. In this way, the local overrides the global namespace, hence if a key exists in both local and global, the local value will be passed to the function under the the key name and the global value will be forgotten. The args parameter ensures that only the expected arguments are supplied. Args: args: A list of allowed argument names kwargs: A dict of kwargs from the local namespace. Returns: A consolidated global/local namespace with local overrides. """ returned_args = {} returned_args.update({ name: self.global_data[name] for name in args if name in self.global_data }) returned_args.update( {name: kwargs[name] for name in args if name in kwargs}) return returned_args def register_hook_callback(self, hook_name=None, ctype="pre", callback=None, name=None): """ Registers pre and post callbacks. Takes a callback function and assigns it to the hook_name with an optional identifier. The optional identifier makes it possible to hot bind functions into hooks and to remove them at a later date with ``unregister_hook_callback``. Args: hook_name: The name of the event hook to respond to. ctype: The call back type, either ``pre`` or ``post``. callback: The callback function. name: An optional name for the callback instance binding. """ if hook_name and callback: callback_instance = self.create_callback(callback) if not name: name = hashlib.sha1( str(time.time()) + hook_name + str(callback_instance['args'])).hexdigest() if ctype == "pre": self.pre_callbacks[hook_name][name] = callback_instance elif ctype == "post": self.post_callbacks[hook_name][name] = callback_instance def unregister_hook_callback(self, hook_name, ctype, name): """ Unregisters a pre or post callback. If the binding has a known name, this function allows the removal of a binding. Args: hook_name: The event hook name. ctype: The callback type, either ``pre`` or ``post``. name: An optional name for the callback instance binding. """ if ctype == "pre": del self.pre_callbacks[hook_name][name] elif ctype == "post": del self.post_callbacks[hook_name][name] def register_plugin(self, cls, plugin_name=None): """ Registers a plugin class to a name. Multiple instances of the same plugin can be used in Rigger, ``self.plugins`` stores un-initialized class defintions to be used by ``setup_instances``. Args: cls: The class. plugin_name: The name of the plugin. """ if plugin_name in self.plugins: print("Plugin name already taken [{}]".format(plugin_name)) elif plugin_name is None: print("Plugin name cannot be None") else: # print "Registering plugin {}".format(plugin_name) self.plugins[plugin_name] = cls def get_instance_obj(self, name): """ Gets the instance object for a given ident name. Args: name: The ident name of the instance. Returns: The object of the instance. """ if name in self.instances: return self.instances[name].obj else: return None def get_instance_data(self, name): """ Gets the instance data(config) for a given ident name. Args: name: The ident name of the instance. Returns: The data(config) of the instance. """ if name in self.instances: return self.instances[name].data else: return None def configure_plugin(self, name, *args, **kwargs): """ Attempts to configure an instance, passing it the args and kwargs. Args: name: The ident name of the instance. args: The positional args. kwargs: The keyword arguments. """ obj = self.get_instance_obj(name) if obj: obj.configure(*args, **kwargs) @staticmethod def create_callback(callback, bg=False): """ Simple function to inspect a function and return it along with it param names wrapped up in a nice dict. This forms a callback object. Args: callback: The callback function. Returns: A dict of function and param names. """ params = signature(callback).parameters return {'func': callback, 'args': params, 'bg': bg} def handle_failure(self, exc): """ Handles an exception. It is expected that this be overidden. """ self.log_message(exc) def log_message(self, message): """ "Logs" a message. It is expected that this be overidden. """ print(message)
class RisMonolith(Dictable): """Monolithic cache of RIS data. This takes a :class:`redfish.rest.v1.RestClient` and uses it to gather data from a server and saves it in a modifiable database called monolith. :param client: client to use for data retrieval. Client is saved as a weakref, using it requires parenthesis and will not survive if the client used in init is removed. :type client: :class:`redfish.rest.v1.RestClient` :param typepath: The compatibility class to use for differentiating between Redfish/LegacyRest. :type typepath: :class:`redfish.rest.ris.Typesandpathdefines` :param directory_load: The flag to quick load using resource directory if possible. When set to True this will load paths, etags, and types, but not create responses for every monolith member. When responses are needed, they will need to be loaded separately. :type directory_load: bool """ def __init__(self, client, typepath, directory_load=True): self._client = weakref.ref(client) self.name = "Monolithic output of RIS Service" self._visited_urls = list() self._type = None self._name = None self.progress = 0 self._directory_load = directory_load self.is_redfish = self._client().is_redfish self.typesadded = defaultdict(set) self.paths = dict() self.ctree = defaultdict(set) self.colltypes = defaultdict(set) self.typepath = typepath self.collstr = self.typepath.defs.collectionstring self.etagstr = 'ETag' if self.is_redfish: self._resourcedir = '/redfish/v1/ResourceDirectory/' else: self._resourcedir = '/rest/v1/ResourceDirectory' #MultiThreading self.get_queue = Queue() self.threads = [] @property def directory_load(self): """The flag to gather information about a tree without downloading every path. Only usable on HPE systems with a ResourceDirectory. type""" return self._directory_load @directory_load.setter def directory_load(self, dir_load): """Set the directory_load flag""" self._directory_load = dir_load @property def type(self): """Return monolith version type""" return "Monolith.1.0.0" @property def visited_urls(self): """The urls visited by the monolith""" return list(set(self._visited_urls) | set(self.paths.keys())) @visited_urls.setter def visited_urls(self, visited_urls): """Set visited URLS.""" self._visited_urls = visited_urls @property def types(self): """Returns list of types for members in the monolith :rtype: list """ return list(self.typesadded.keys()) @types.setter def types(self, member): """Adds a member to monolith :param member: Member created based on response. :type member: RisMonolithMemberv100 """ self.typesadded[member.maj_type].add(member.path) patches = [] if member.path in list(self.paths.keys()): patches = self.paths[member.path].patches self.paths[member.path] = member self.paths[member.path].patches.extend([patch for patch in patches]) def path(self, path): """Provides the member corresponding to the path specified. Case sensitive. :param path: path of the monolith member to return :type path: str :rtype: RisMonolithMemberv100 """ try: return self.paths[path] except: return None def iter(self, typeval=None): """An iterator that yields each member of monolith associated with a specific type. In the case that no type is included this will yield all members in the monolith. :rtype: RisMonolithMemberv100 """ if not typeval: for _, val in self.paths.items(): yield val else: for typename in self.gettypename(typeval): for item in self.typesadded[typename]: yield self.paths[item] # types = next(self.gettypename(typeval), None) # if types in self.typesadded: # for item in self.typesadded[types]: # yield self.paths[item] # else: # raise RisInstanceNotFoundError("Unable to locate instance for" \ # " '%s'" % typeval) def itertype(self, typeval): """Iterator that yields member(s) of given type in the monolith and raises an error if no member of that type is found. :param typeval: type name of the requested member. :type typeval: str :rtype: RisMonolithMemberv100 """ typeiter = self.gettypename(typeval) types = next(typeiter, None) if types: while types: for item in self.typesadded[types]: yield self.paths[item] types = next(typeiter, None) else: raise RisInstanceNotFoundError( "Unable to locate instance for '%s'" % typeval) def typecheck(self, types): """Check if a member of given type exists in the monolith :param types: type to check. :type types: str :rtype: bool """ if any(types in val for val in self.types): return True return False def gettypename(self, types): """Takes a full type response and returns all major types associated. Example: #Type.v1_0_0.Type will return iter(Type.1) :param types: The type of the requested response. :type types: str :rtype: iter of major types """ types = types[1:] if types[0] in ("#", u"#") else types return iter( (xt for xt in self.types if xt and types.lower() in xt.lower())) def update_member(self, member=None, resp=None, path=None, init=True): """Adds member to the monolith. If the member already exists the data is updated in place. Takes either a RisMonolithMemberv100 instance or a :class:`redfish.rest.containers.RestResponse` along with that responses path. :param member: The monolith member to add to the monolith. :type member: RisMonolithMemberv100 :param resp: The rest response to add to the monolith. :type resp: :class:`redfish.rest.containers.RestResponse` :param path: The path correlating to the response. :type path: str :param init: Flag if addition is part of the initial load. Set this to false if you are calling this by itself. :type init: bool """ if not member and resp and path: self._visited_urls.append(path.lower()) member = RisMonolithMemberv100(resp, self.is_redfish) if not member: #Assuming for lack of member and not member.type return if not member.type: member.deftype = 'object' #Hack for general schema with no type self.types = member if init: self.progress += 1 if LOGGER.getEffectiveLevel() == 40: self._update_progress() def load(self, path=None, includelogs=False, init=False, \ crawl=True, loadtype='href', loadcomplete=False, path_refresh=False): """Walks the entire data model and caches all responses or loads an individual path into the monolith. Supports both threaded and sequential crawling. :param path: The path to start the crawl from the provided path if crawling or loads the path into monolith. If path is not included, crawl will start with the default. The default is */redfish/v1/* or */rest/v1* depending on if the system is Redfish or LegacyRest. :type path: str. :param includelogs: Flag to determine if logs should be downloaded as well in the crawl. :type includelogs: bool :param init: Flag to determine if this is the initial load. :type init: bool :param crawl: Flag to determine if load should crawl through found links. :type crawl: bool :param loadtype: Flag to determine if loading standard links: *href* or schema links: *ref*. :type loadtype: str. :param loadcomplete: Flag to download the entire data model including registries and schemas. :type loadcomplete: bool :param path_refresh: Flag to reload the path specified, clearing any patches and overwriting the current data in the monolith. :type path_refresh: bool """ if init: if LOGGER.getEffectiveLevel() == 40: sys.stdout.write("Discovering data...") else: LOGGER.info("Discovering data...") self.name = self.name + ' at %s' % self._client().base_url selectivepath = path if not selectivepath: selectivepath = self._client().default_prefix if loadtype == 'href' and not self._client().base_url.startswith( "blobstore://."): if not self.threads: for _ in range(6): workhand = LoadWorker(self.get_queue) workhand.setDaemon(True) workhand.start() self.threads.append(workhand) self.get_queue.put((selectivepath, includelogs, loadcomplete, crawl, \ path_refresh, init, None, None, self)) self.get_queue.join() #Raise any errors from threads, and set them back to None after excp = None for thread in self.threads: if excp == None: excp = thread.get_exception() thread.exception = None if excp: raise excp #self.member_queue.join() else: #We can't load ref or local client in a threaded manner self._load(selectivepath, originaluri=None, crawl=crawl, \ includelogs=includelogs, init=init, loadtype=loadtype, \ loadcomplete=loadcomplete, path_refresh=path_refresh, prevpath=None) if init: if LOGGER.getEffectiveLevel() == 40: sys.stdout.write("Done\n") else: LOGGER.info("Done\n") if self.directory_load and init: self._populatecollections() def _load(self, path, crawl=True, originaluri=None, includelogs=False,\ init=True, loadtype='href', loadcomplete=False, \ path_refresh=False, prevpath=None): """Sequential version of loading monolith and parsing schemas. :param path: path to start load from. :type path: str :param crawl: flag to determine if load should traverse found links. :type crawl: bool :param originaluri: variable to assist in determining originating path. :type originaluri: str :param includelogs: flag to determine if logs should be downloaded also. :type includelogs: bool :param init: flag to determine if first run of load. :type init: bool :param loadtype: flag to determine if load is meant for only href items. :type loadtype: str. :param loadcomplete: flag to download the entire monolith :type loadcomplete: bool :param path_refresh: flag to reload the members in the monolith instead of skip if they exist. :type path_refresh: bool """ if path.endswith("?page=1") and not loadcomplete: #Don't download schemas in crawl unless we are loading absolutely everything return elif not includelogs and crawl: #Only include logs when asked as there can be an extreme amount of entries if "/log" in path.lower(): return #TODO: need to find a better way to support non ascii characters path = path.replace("|", "%7C") #remove fragments newpath = urlparse(path) newpath = list(newpath[:]) newpath[-1] = '' path = urlunparse(tuple(newpath)) if prevpath and prevpath != path: self.ctree[prevpath].update([path]) if not path_refresh: if path.lower() in self.visited_urls: return LOGGER.debug('_loading %s', path) resp = self._client().get(path) if resp.status != 200 and path.lower() == self.typepath.defs.biospath: raise BiosUnregisteredError() elif resp.status == 401: raise SessionExpired("Invalid session. Please logout and "\ "log back in or include credentials.") elif resp.status not in (201, 200): self.removepath(path) return if loadtype == "ref": try: if resp.status in (201, 200): self.update_member(resp=resp, path=path, init=init) self._parse_schema(resp) except jsonpointer.JsonPointerException: raise SchemaValidationError() self.update_member(resp=resp, path=path, init=init) fpath = lambda pa, path: path if pa.endswith(self.typepath.defs.hrefstring) and \ pa.startswith((self.collstr, 'Entries')) else None if loadtype == 'href': #follow all the href attributes if self.is_redfish: jsonpath_expr = jsonpath_rw.parse("$..'@odata.id'") else: jsonpath_expr = jsonpath_rw.parse('$..href') matches = jsonpath_expr.find(resp.dict) if 'links' in resp.dict and 'NextPage' in resp.dict['links']: if originaluri: next_link_uri = originaluri + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=originaluri, \ includelogs=includelogs, crawl=crawl, \ init=init, prevpath=None, loadcomplete=loadcomplete) else: next_link_uri = path + '?page=' + str( resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=path, includelogs=includelogs,\ crawl=crawl, init=init, prevpath=None, loadcomplete=loadcomplete) #Only use monolith if we are set to matchrdirpath = next((match for match in matches if match.value == \ self._resourcedir), None) if self.directory_load else None if not matchrdirpath and crawl: for match in matches: if path == "/rest/v1" and not loadcomplete: if str(match.full_path) == "links.Schemas.href" or \ str(match.full_path) == "links.Registries.href": continue elif not loadcomplete: if str(match.full_path) == "*****@*****.**" or \ str(match.full_path) == "*****@*****.**": continue if match.value == path: continue elif not isinstance(match.value, six.string_types): continue href = '%s' % match.value self._load(href, crawl=crawl, \ originaluri=originaluri, includelogs=includelogs, \ init=init, prevpath=fpath(str(match.full_path), path), \ loadcomplete=loadcomplete) elif crawl: href = '%s' % matchrdirpath.value self._load(href, crawl=crawl, originaluri=originaluri, \ includelogs=includelogs, init=init, prevpath=path, loadcomplete=loadcomplete) if loadcomplete: if path == '/rest/v1': schemamatch = jsonpath_rw.parse('$..extref') else: schemamatch = jsonpath_rw.parse('$..Uri') smatches = schemamatch.find(resp.dict) matches = matches + smatches for match in matches: if isinstance(match.value, six.string_types): self._load(match.value, crawl=crawl, originaluri=originaluri,\ includelogs=includelogs, init=init, loadcomplete=loadcomplete,\ prevpath=fpath(str(match.full_path), path)) def _parse_schema(self, resp): """Function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str """ #pylint: disable=maybe-no-member if not self.typepath.gencompany: return self._parse_schema_gen(resp) jsonpath_expr = jsonpath_rw.parse('$.."$ref"') matches = jsonpath_expr.find(resp.dict) respcopy = resp.dict typeregex = '([#,@].*?\.)' if matches: for match in matches: fullpath = str(match.full_path) jsonfile = match.value.split('#')[0] jsonpath = match.value.split('#')[1] listmatch = None found = None if 'redfish.dmtf.org' in jsonfile: if 'odata' in jsonfile: jsonpath = jsonpath.replace(jsonpath.split('/')[-1], \ 'odata' + jsonpath.split('/')[-1]) jsonfile = 'Resource.json' found = re.search(typeregex, fullpath) if found: repitem = fullpath[found.regs[0][0]:found.regs[0][1]] schemapath = '/' + fullpath.replace(repitem, '~').\ replace('.', '/').replace('~', repitem) else: schemapath = '/' + fullpath.replace('.', '/') if '.json' in jsonfile: itempath = schemapath if self.is_redfish: if resp.request.path[-1] == '/': newpath = '/'.join(resp.request.path.split('/')\ [:-2]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')\ [:-1]) + '/' + jsonfile + '/' else: newpath = '/'.join( resp.request.path.split('/')[:-1]) + '/' + jsonfile if 'href.json' in newpath: continue if newpath.lower() not in self.visited_urls: self.load(newpath, crawl=False, includelogs=False, \ init=False, loadtype='ref') instance = list() #deprecated type "string" for Type.json if 'string' in self.types: for item in self.iter('string'): instance.append(item) if 'object' in self.types: for item in self.iter('object'): instance.append(item) for item in instance: if jsonfile in item.path: if 'anyOf' in fullpath: break dictcopy = item.dict listmatch = re.search('[[][0-9]+[]]', itempath) if listmatch: start = listmatch.regs[0][0] end = listmatch.regs[0][1] newitempath = [ itempath[:start], itempath[end:] ] start = jsonpointer.JsonPointer(newitempath[0]) end = jsonpointer.JsonPointer(newitempath[1]) del start.parts[-1], end.parts[-1] vals = start.resolve(respcopy) count = 0 for val in vals: try: if '$ref' in six.iterkeys( end.resolve(val)): end.resolve(val).pop('$ref') end.resolve(val).update(dictcopy) replace_pointer = jsonpointer.\ JsonPointer(end.path + jsonpath) data = replace_pointer.resolve(val) set_pointer(val, end.path, data) start.resolve( respcopy)[count].update(val) break except: count += 1 else: itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in six.iterkeys( itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(dictcopy) break if jsonpath: if 'anyOf' in fullpath: continue if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = schemapath.replace('/$ref', '') if re.search('\[\d]', schemapath): schemapath = schemapath.translate(None, '[]') schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: if not listmatch: schemapath = schemapath.replace('/$ref', '') replacepath = schemapath + jsonpath replace_pointer = jsonpointer.JsonPointer( replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.loaddict(respcopy) else: resp.loaddict(respcopy) def _parse_schema_gen(self, resp): """Redfish general function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str """ #pylint: disable=maybe-no-member getval = lambda inmat: getval(inmat.left) + '/' + str(inmat.right) \ if hasattr(inmat, 'left') else str(inmat) respcopy = resp.dict jsonpath_expr = jsonpath_rw.parse('$.."anyOf"') while True: matches = jsonpath_expr.find(respcopy) if not matches: break match = matches[0] newval = None schlist = match.value schlist = [ele for ele in list(schlist) if ele != {"type": "null"}] norefsch = [ele for ele in list(schlist) if isinstance(ele, dict) and \ len(ele.keys()) > 1] if norefsch: newval = norefsch[0] else: newsc = [ ele for ele in list(schlist) if not ele["$ref"].split('#')[0] ] newval = newsc[0] if newsc else None if not newval: schlist = [ele["$ref"] for ele in list(schlist) if "$ref" in ele.keys() and \ (ele["$ref"].split('#')[0].endswith('.json') and 'odata' not in \ ele["$ref"].split('#')[0])] maxsch = max(schlist) newval = {"$ref": maxsch} itempath = '/' + getval(match.full_path) if re.search('\[\d+]', itempath): itempath = itempath.translate(None, '[]') itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if 'anyOf' in six.iterkeys(itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('anyOf') itempath.resolve(respcopy).update(newval) jsonpath_expr = jsonpath_rw.parse('$.."$ref"') matches = jsonpath_expr.find(respcopy) if matches: for _, match in enumerate(matches): jsonfile = match.value.split('#')[0] jsonfile = '' if jsonfile.lower() == resp.request.path.lower( ) else jsonfile jsonpath = match.value.split('#')[1] schemapath = '/' + getval(match.full_path) if jsonfile: itempath = schemapath if '/' not in jsonfile: inds = -2 if resp.request.path[-1] == '/' else -1 jsonfile = '/'.join(resp.request.path.split('/')[:inds]) \ + '/' + jsonfile + '/' if jsonfile not in self.paths: self.load(jsonfile, crawl=False, includelogs=False, \ init=False, loadtype='ref') item = self.paths[ jsonfile] if jsonfile in self.paths else None if not item: if not 'anyOf' in schemapath: raise SchemaValidationError() continue if re.search('\[\d+]', itempath): itempath = itempath.translate(None, '[]') itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in six.iterkeys(itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(item.dict) if jsonpath: schemapath = schemapath.replace('/$ref', '') if re.search('\[\d+]', schemapath): schemapath = schemapath.translate(None, '[]') if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: replacepath = schemapath + jsonpath replace_pointer = jsonpointer.\ JsonPointer(replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.loaddict(respcopy) else: resp.loaddict(respcopy) def load_from_dict(self, src): """Load data to monolith from a dict. This is the reverse of :func:`to_dict`. :param src: data receive from rest operation. :type src: str """ self._type = src['Type'] self._name = src['Name'] self.typesadded = defaultdict( set, {ki: set(val) for ki, val in src['typepath'].iteritems()}) self.ctree = defaultdict( set, {ki: set(val) for ki, val in src['ctree'].iteritems()}) self.colltypes = defaultdict( set, {ki: set(val) for ki, val in src['colls'].iteritems()}) for _, resp in list(src['resps'].items()): member = RisMonolithMemberv100(None, self.is_redfish) member.load_from_dict(resp) self.update_member(member=member, init=False) def to_dict(self): """Convert data to a dict from monolith. This is the reverse of :func:`load_from_dict`.""" result = OrderedDict() result['Type'] = self.type result['Name'] = self.name result["typepath"] = self.typesadded result['ctree'] = self.ctree result['colls'] = self.colltypes result["resps"] = {x: v.to_dict() for x, v in list(self.paths.items())} return result def markmodified(self, opath, path=None, modpaths=None): """Mark the paths to be modifed which are connected to current path. When calling this function you only need to include `opath`. :param opath: original path which has been modified :type opath: str """ modpaths = set() if modpaths is None else modpaths path = path if path else opath if not path: return modpaths.update(self.ctree[path] if path in self.ctree else set()) self.paths[path].modified = True for npath in [unmodpath for unmodpath in modpaths if unmodpath \ in self.paths and not self.paths[unmodpath].modified]: self.markmodified(opath, path=npath, modpaths=modpaths) return modpaths def checkmodified(self, opath, path=None, modpaths=None): """Check if the path or its children are modified. When calling this function you only need to include `opath`. :param opath: original path which has been modified :type opath: str """ #return [paths for paths in self.ctree[path] if self.paths[paths].modified] modpaths = set() if modpaths is None else modpaths path = path if path else opath newpaths = set() if not path: return if path in self.paths and self.paths[path].modified: newpaths = set([conn for conn in self.ctree[path] if conn in \ self.paths and self.paths[path].modified]) - modpaths modpaths.update(newpaths | set([path])) for npath in [unmodpath for unmodpath in newpaths]: self.checkmodified(opath, path=npath, modpaths=modpaths) return modpaths def removepath(self, path): """Remove a given path from the cache :param path: path which is to be checked if modified :type path: str """ if path in self._visited_urls: self._visited_urls.remove(path) if not path in self.paths: return if path in self.typesadded[self.paths[path].maj_type]: self.typesadded[self.paths[path].maj_type].remove(path) if not self.typesadded[self.paths[path].maj_type]: del self.typesadded[self.paths[path].maj_type] del self.paths[path] if path in self.ctree: del self.ctree[path] _ = [ self.ctree[paths].remove(path) for paths in self.ctree if path in self.ctree[paths] ] def _populatecollections(self): """Populate the collections type and types depending on resourcedirectory""" if not self._resourcedir in self.paths: return self.colltypes = defaultdict(set) alltypes = [] colls = [] for item in self.paths[self._resourcedir].dict["Instances"]: #Fix for incorrect RDir instances. if not self.typepath.defs.typestring in item or item[self.typepath.defs.hrefstring] \ in self.paths: continue typename = ".".join(item[self.typepath.defs.typestring].split(".", 2)[:2])\ .split('#')[-1] _ = [ alltypes.append(typename) if not 'Collection' in typename else None ] _ = [colls.append(typename) if 'Collection' in typename else None] member = RisMonolithMemberv100(None, self.is_redfish) member.popdefs(typename, item[self.typepath.defs.hrefstring], item[self.etagstr]) self.update_member(member=member, init=False) for coll in colls: collname = coll.split('Collection')[0].split('#')[-1] typename = next( (name for name in alltypes if name.startswith(collname)), None) colltype = ".".join(coll.split(".", 2)[:2]).split('#')[-1] self.colltypes[typename].add(colltype) def capture(self, redmono=False): """Crawls the server specified by the client and returns the entire monolith. :param redmono: Flag to return only the headers and responses instead of the entire monolith member data. :type redmono: bool :rtype: dict """ self.load(includelogs=True, crawl=True, loadcomplete=True, path_refresh=True, init=True) return self.to_dict() if not redmono else {x:{"Headers":v.resp.getheaders(), \ "Response":v.resp.dict} for x, v in list(self.paths.items()) if v} def killthreads(self): """Function to kill threads on logout""" threads = [] for thread in threading.enumerate(): if isinstance(thread, LoadWorker): self.get_queue.put(('KILL', 'KILL', 'KILL', 'KILL',\ 'KILL', 'KILL', 'KILL', 'KILL', 'KILL', 'KILL')) threads.append(thread) for thread in threads: thread.join() def _update_progress(self): """Simple function to increment the dot progress""" if self.progress % 6 == 0: sys.stdout.write('.')
if choice != '': job_queue.put({ 'q': entity_id, 'action': 'set_description', 'lang': 'nb', 'value': choice, 'summary': '#no_to_nb cleanup drive' }) if labels['no'].get('description') is not None: job_queue.put({ 'q': entity_id, 'action': 'remove_description', 'lang': 'no', 'summary': '#no_to_nb cleanup drive' }) completed += 1 t1 = time.time() - t0 if not bg_thread.isAlive(): print('Thread exited. Starting a new') bg_thread = start_thread() while job_queue.qsize() > 100: print('Job queue length: %d, sleeping a while' % job_queue.qsize()) time.sleep(10) print('Status: Fixed %s items. %s items left to go, time: %.2f sec/item, job queue length: %d' % (completed, len(rows), t1 / completed, job_queue.qsize())) print('*** Main thread waiting') job_queue.join() print('*** Done')
class Rigger(object): """ A Rigger event framework instance. The Rigger object holds all configuration and instances of plugins. By default Rigger accepts a configuration file name to parse, though it is perfectly acceptable to pass the configuration into the ``self.config`` attribute. Args: config_file: A configuration file holding all of Riggers base and plugin configuration. """ def __init__(self, config_file): self.gdl = threading.Lock() self.pre_callbacks = defaultdict(dict) self.post_callbacks = defaultdict(dict) self.plugins = {} self.config_file = config_file self.squash_exceptions = False self.initialized = False self._task_list = {} self._queue_lock = threading.Lock() self._global_queue = Queue() self._background_queue = Queue() self._server_shutdown = False self._zmq_event_handler_shutdown = False self._global_queue_shutdown = False self._background_queue_shutdown = False globt = threading.Thread(target=self.process_queue, name="global_queue_processor") globt.start() bgt = threading.Thread( target=self.process_background_queue, name="background_queue_processor") bgt.start() def process_queue(self): """ The ``process_queue`` thread manages taking events on and off of the global queue. Both TCP and in-object fire_hooks place events onto the global_queue and these are both handled by the same handler called ``process_hook``. If there is an exception during processing, the exception is printed and execution continues. """ while not self._global_queue_shutdown: while not self._global_queue.empty(): with self._queue_lock: tid = self._global_queue.get() obj = self._task_list[tid].json_dict self._task_list[tid].status = Task.RUNNING try: loc, glo = self.process_hook(obj['hook_name'], **obj['data']) combined_dict = {} combined_dict.update(glo) combined_dict.update(loc) self._task_list[tid].output = combined_dict except Exception as e: self.log_message(e) with self._queue_lock: self._global_queue.task_done() self._task_list[tid].status = Task.FINISHED if not self._task_list[tid].json_dict.get('grab_result', None): del self._task_list[tid] time.sleep(0.1) def process_background_queue(self): """ The ``process_background_queue`` manages the hooks which have been backgrounded. In this respect the tasks that are completed are not required to continue with the test and as such can be forgotten about. An example of this would be some that sends an email, or tars up files, it has all the information it needs and the main process doesn't need to wait for it to complete. """ while not self._background_queue_shutdown: while not self._background_queue.empty(): obj = self._background_queue.get() try: local, globals_updates = self.process_callbacks(obj['cb'], obj['kwargs']) with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) except Exception as e: self.log_message(e) self._background_queue.task_done() time.sleep(0.1) def zmq_event_handler(self, zmq_socket_address): """ The ``zmq_event_handler`` thread receives (and responds to) updates from the zmq socket, which is normally embedded in the web server running alongside this riggerlib instance, in its own process. """ ctx = zmq.Context() zmq_socket = ctx.socket(zmq.REP) zmq_socket.set(zmq.RCVTIMEO, 300) zmq_socket.bind(zmq_socket_address) def zmq_reply(message, **extra): payload = {'message': message} payload.update(extra) zmq_socket.send_json(payload) bad_request = partial(zmq_reply, 'BAD REQUEST') while not self._zmq_event_handler_shutdown: try: json_dict = zmq_socket.recv_json() except zmq.Again: continue try: event_name = json_dict['event_name'] except KeyError: bad_request() if event_name == 'fire_hook': tid = self._fire_internal_hook(json_dict) if tid: zmq_reply('OK', tid=tid) else: bad_request() elif event_name == 'task_check': try: tid = json_dict['tid'] extra = { "tid": tid, "status": self._task_list[tid].status, } if json_dict['grab_result']: extra["output"] = self._task_list[tid].output zmq_reply('OK', **extra) except KeyError: zmq_reply('NOT FOUND') elif event_name == 'task_delete': try: tid = json_dict['tid'] del self._task_list[tid] zmq_reply('OK', tid=tid) except KeyError: zmq_reply('OK', tid=tid) elif event_name == 'shutdown': zmq_reply('OK') # We gotta initiate server stop from here and stop this thread self._server_shutdown = True break elif event_name == 'ping': zmq_reply('PONG') else: bad_request() zmq_socket.close() def read_config(self, config_file): """ Reads in the config file and parses the yaml data. Args: config_file: A configuration file holding all of Riggers base and plugin configuration. Raises: IOError: If the file can not be read. Exception: If there is any error parsing the configuration file. """ try: with open(config_file, "r") as stream: data = yaml.load(stream) except IOError: print("!!! Configuration file could not be loaded...exiting") sys.exit(127) except Exception as e: print(e) print("!!! Error parsing Configuration file") sys.exit(127) self.config = data def parse_config(self): """ Takes the configuration data from ``self.config`` and sets up the plugin instances. """ self.read_config(self.config_file) self.setup_plugin_instances() self.start_server() def setup_plugin_instances(self): """ Sets up instances into a dict called ``self.instances`` and instantiates each instance of the plugin. It also sets the ``self._threaded`` option to determine if plugins will be processed synchronously or asynchronously. """ self.instances = {} self._threaded = self.config.get("threaded", False) plugins = self.config.get("plugins", {}) for ident, config in plugins.items(): self.setup_instance(ident, config) def setup_instance(self, ident, config): """ Sets up a single instance into the ``self.instances`` dict. If the instance does not exist, a warning is printed out. Args: ident: A plugin instance identifier. config: Configuration dict from the yaml. """ plugin_name = config.get('plugin', {}) if plugin_name in self.plugins: obj = self.plugins[plugin_name] if obj: obj_instance = obj(ident, config, self) self.instances[ident] = RiggerPluginInstance(ident, obj_instance, config) else: msg = "Plugin [{}] was not found, "\ "disabling instance [{}]".format(plugin_name, ident) self.log_message(msg) def start_server(self): """ Starts the ZMQ server if the ``server_enabled`` is True in the config. """ self._server_hostname = self.config.get('server_address', '127.0.0.1') self._server_port = self.config.get('server_port', 21212) self._server_enable = self.config.get('server_enabled', False) if self._server_enable: zmq_socket_address = 'tcp://{}:{}'.format(self._server_hostname, self._server_port) # set up reciever thread for zmq event handling zeh = threading.Thread( target=self.zmq_event_handler, args=(zmq_socket_address,), name="zmq_event_handler") zeh.start() exect = threading.Thread(target=self.await_shutdown, name="executioner") exect.start() def await_shutdown(self): while not self._server_shutdown: time.sleep(0.3) self.stop_server() def stop_server(self): """ Responsible for the following: - stopping the zmq event handler (unless already stopped through 'terminate') - stopping the global queue - stopping the background queue """ self.log_message("Shutdown initiated : {}".format(self._server_hostname)) # The order here is important self._zmq_event_handler_shutdown = True self._global_queue.join() self._global_queue_shutdown = True self._background_queue.join() self._background_queue_shutdown = True raise SystemExit def fire_hook(self, hook_name, **kwargs): """ Parses the hook information into a dict for passing to process_hook. This is used to enable both the TCP and in-object fire_hook methods to use the same process_hook method call. Args: hook_name: The name of the hook to fire. kwargs: The kwargs to pass to the hooks. """ json_dict = {'hook_name': hook_name, 'data': kwargs} self._fire_internal_hook(json_dict) def _fire_internal_hook(self, json_dict): task = Task(json_dict) tid = task.tid.hexdigest() self._task_list[tid] = task if self._global_queue: with self._queue_lock: self._global_queue.put(tid) return tid else: return None def process_hook(self, hook_name, **kwargs): """ Takes a hook_name and a selection of kwargs and fires off the appropriate callbacks. This function is the guts of Rigger and is responsible for running the callback and hook functions. It first loads some blank dicts to collect the updates for the local and global namespaces. After this, it loads the pre_callback functions along with the kwargs into the callback collector processor. The return values are then classifed into local and global dicts and updates proceed. After this, the plugin hooks themselves are then run using the same methodology. Their return values are merged with the existing dicts and then the same process happens for the post_callbacks. Note: If the instance of the plugin has been marked as a background instance, and hooks which are called in that instance will be backgrounded. The hook will also not be able to return any data to the post-hook callback, although updates to globals will be processed as and when the backgrounded task is completed. Args: hook_name: The name of the hook to fire. kwargs: The kwargs to pass to the hooks. """ if not self.initialized: return kwargs_updates = {} globals_updates = {} kwargs.update({'config': self.config}) # First fire off any pre-hook callbacks if self.pre_callbacks.get(hook_name): # print "Running pre hook callback for {}".format(hook_name) kwargs_updates, globals_updates = self.process_callbacks( self.pre_callbacks[hook_name].values(), kwargs) # Now we can update the kwargs passed to the real hook with the updates with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) # Now fire off each plugin hook event_hooks = [] for instance_name, instance in self.instances.items(): callbacks = instance.obj.callbacks enabled = instance.data.get('enabled', None) if callbacks.get(hook_name) and enabled: cb = callbacks[hook_name] if instance.data.get('background', False): self._background_queue.put({'cb': [cb], 'kwargs': kwargs}) elif cb['bg']: self._background_queue.put({'cb': [cb], 'kwargs': kwargs}) else: event_hooks.append(cb) kwargs_updates, globals_updates = self.process_callbacks(event_hooks, kwargs) # One more update for the post_hook callback with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) # Finally any post-hook callbacks if self.post_callbacks.get(hook_name): # print "Running post hook callback for {}".format(hook_name) kwargs_updates, globals_updates = self.process_callbacks( self.post_callbacks[hook_name].values(), kwargs) with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) return kwargs, self.global_data def process_callbacks(self, callback_collection, kwargs): """ Processes a collection of callbacks or hooks for a particular event, namely pre, hook or post. The functions are passed in as an array to ``callback_collection`` and process callbacks first iterates each function and ensures that each one has the correct arguments available to it. If not, an Exception is raised. Then, depending on whether Threading is enabled or not, the functions are either run sequentially, or loaded into a ThreadPool and executed asynchronously. The returned local and global updates are either collected and processed sequentially, as in the case of the non-threaded behaviour, or collected at the end of the callback_collection processing and handled there. Note: It is impossible to predict the order of the functions being run. If the order is important, it is advised to create a second event hook that will be fired before the other. Rigger has no concept of hook or callback order and is unlikely to ever have. Args: callback_collection: A list of functions to call. kwargs: A set of kwargs to pass to the functions. Returns: A tuple of local and global namespace updates. """ loc_collect = {} glo_collect = {} if self._threaded: results_list = [] pool = ThreadPool(10) for cb in callback_collection: required_args = [sig for sig in cb['args'] if isinstance(cb['args'][sig].default, type)] missing = list(set(required_args).difference(set(self.global_data.keys())) .difference(set(kwargs.keys()))) if not missing: new_kwargs = self.build_kwargs(cb['args'], kwargs) if self._threaded: results_list.append(pool.apply_async(cb['func'], [], new_kwargs)) else: obtain_result = self.handle_results(cb['func'], [], new_kwargs) loc_collect, glo_collect = self.handle_collects( obtain_result, loc_collect, glo_collect) else: raise Exception('Function {} is missing kwargs {}' .format(cb['func'].__name__, missing)) if self._threaded: pool.close() pool.join() for result in results_list: obtain_result = self.handle_results(result.get, [], {}) loc_collect, glo_collect = self.handle_collects( obtain_result, loc_collect, glo_collect) return loc_collect, glo_collect def handle_results(self, call, args, kwargs): """ Handles results and depending on configuration, squashes exceptions and logs or returns the obtained result. Args: call: The function call. args: The positional arguments. kwargs: The keyword arguments. Returns: The obtained result of the callback or hook. """ try: obtain_result = call(*args, **kwargs) except: if self.squash_exceptions: obtain_result = None self.handle_failure(sys.exc_info()) else: raise return obtain_result def handle_collects(self, result, loc_collect, glo_collect): """ Handles extracting the information from the hook/callback result. If the hook/callback returns None, then the dicts are returned unaltered, else they are updated with local, global namespace updates. Args: result: The result to process. loc_collect: The local namespace updates collection. glo_collect: The global namespace updates collection. Returns: A tuple containing the local and global updates. """ if result: if result[0]: loc_collect = recursive_update(loc_collect, result[0]) if result[1]: glo_collect = recursive_update(glo_collect, result[1]) return loc_collect, glo_collect def build_kwargs(self, args, kwargs): """ Builds a new kwargs from a list of allowed args. Functions only receive a single set of kwargs, and so the global and local namespaces have to be collapsed. In this way, the local overrides the global namespace, hence if a key exists in both local and global, the local value will be passed to the function under the the key name and the global value will be forgotten. The args parameter ensures that only the expected arguments are supplied. Args: args: A list of allowed argument names kwargs: A dict of kwargs from the local namespace. Returns: A consolidated global/local namespace with local overrides. """ returned_args = {} returned_args.update({ name: self.global_data[name] for name in args if name in self.global_data}) returned_args.update({ name: kwargs[name] for name in args if name in kwargs}) return returned_args def register_hook_callback(self, hook_name=None, ctype="pre", callback=None, name=None): """ Registers pre and post callbacks. Takes a callback function and assigns it to the hook_name with an optional identifier. The optional identifier makes it possible to hot bind functions into hooks and to remove them at a later date with ``unregister_hook_callback``. Args: hook_name: The name of the event hook to respond to. ctype: The call back type, either ``pre`` or ``post``. callback: The callback function. name: An optional name for the callback instance binding. """ if hook_name and callback: callback_instance = self.create_callback(callback) if not name: name = hashlib.sha1( str(time.time()) + hook_name + str(callback_instance['args'])).hexdigest() if ctype == "pre": self.pre_callbacks[hook_name][name] = callback_instance elif ctype == "post": self.post_callbacks[hook_name][name] = callback_instance def unregister_hook_callback(self, hook_name, ctype, name): """ Unregisters a pre or post callback. If the binding has a known name, this function allows the removal of a binding. Args: hook_name: The event hook name. ctype: The callback type, either ``pre`` or ``post``. name: An optional name for the callback instance binding. """ if ctype == "pre": del self.pre_callbacks[hook_name][name] elif ctype == "post": del self.post_callbacks[hook_name][name] def register_plugin(self, cls, plugin_name=None): """ Registers a plugin class to a name. Multiple instances of the same plugin can be used in Rigger, ``self.plugins`` stores un-initialized class defintions to be used by ``setup_instances``. Args: cls: The class. plugin_name: The name of the plugin. """ if plugin_name in self.plugins: print("Plugin name already taken [{}]".format(plugin_name)) elif plugin_name is None: print("Plugin name cannot be None") else: # print "Registering plugin {}".format(plugin_name) self.plugins[plugin_name] = cls def get_instance_obj(self, name): """ Gets the instance object for a given ident name. Args: name: The ident name of the instance. Returns: The object of the instance. """ if name in self.instances: return self.instances[name].obj else: return None def get_instance_data(self, name): """ Gets the instance data(config) for a given ident name. Args: name: The ident name of the instance. Returns: The data(config) of the instance. """ if name in self.instances: return self.instances[name].data else: return None def configure_plugin(self, name, *args, **kwargs): """ Attempts to configure an instance, passing it the args and kwargs. Args: name: The ident name of the instance. args: The positional args. kwargs: The keyword arguments. """ obj = self.get_instance_obj(name) if obj: obj.configure(*args, **kwargs) @staticmethod def create_callback(callback, bg=False): """ Simple function to inspect a function and return it along with it param names wrapped up in a nice dict. This forms a callback object. Args: callback: The callback function. Returns: A dict of function and param names. """ params = signature(callback).parameters return { 'func': callback, 'args': params, 'bg': bg } def handle_failure(self, exc): """ Handles an exception. It is expected that this be overidden. """ self.log_message(exc) def log_message(self, message): """ "Logs" a message. It is expected that this be overidden. """ print(message)
def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None, completed_jobs=None): """ Execute a tool and return object containing summary (output data, number of failures, etc...). """ if max_num_jobs: assert invocation_step is not None if rerun_remap_job_id: assert invocation_step is None all_jobs_timer = ExecutionTimer() if invocation_step is None: execution_tracker = ToolExecutionTracker(trans, tool, mapping_params, collection_info) else: execution_tracker = WorkflowStepExecutionTracker( trans, tool, mapping_params, collection_info, invocation_step, job_callback=job_callback) app = trans.app execution_cache = ToolExecutionCache(trans) def execute_single_job(execution_slice, completed_job): job_timer = ExecutionTimer() params = execution_slice.param_combination if workflow_invocation_uuid: params['__workflow_invocation_uuid__'] = workflow_invocation_uuid elif '__workflow_invocation_uuid__' in params: # Only workflow invocation code gets to set this, ignore user supplied # values or rerun parameters. del params['__workflow_invocation_uuid__'] job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache, completed_job) if job: message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer) log.debug(message) execution_tracker.record_success(execution_slice, job, result) else: execution_tracker.record_error(result) tool_action = tool.tool_action if hasattr(tool_action, "check_inputs_ready"): for params in execution_tracker.param_combinations: # This will throw an exception if the tool is not ready. tool_action.check_inputs_ready(tool, trans, params, history) execution_tracker.ensure_implicit_collections_populated( history, mapping_params.param_template) config = app.config burst_at = getattr(config, 'tool_submission_burst_at', 10) burst_threads = getattr(config, 'tool_submission_burst_threads', 1) job_count = len(execution_tracker.param_combinations) jobs_executed = 0 has_remaining_jobs = False if (job_count < burst_at or burst_threads < 2): for i, execution_slice in enumerate( execution_tracker.new_execution_slices()): if max_num_jobs and jobs_executed >= max_num_jobs: has_remaining_jobs = True break else: execute_single_job(execution_slice, completed_jobs[i]) else: # TODO: re-record success... q = Queue() def worker(): while True: params = q.get() execute_single_job(params) q.task_done() for i in range(burst_threads): t = Thread(target=worker) t.daemon = True t.start() for i, execution_slice in enumerate( execution_tracker.new_execution_slices()): if max_num_jobs and jobs_executed >= max_num_jobs: has_remaining_jobs = True break else: q.put(execution_slice, completed_jobs[i]) jobs_executed += 1 q.join() if has_remaining_jobs: raise PartialJobExecution(execution_tracker) else: execution_tracker.finalize_dataset_collections(trans) log.debug("Executed %d job(s) for tool %s request: %s" % (job_count, tool.id, all_jobs_timer)) return execution_tracker
class RisMonolithv100(Dictable): """Monolithic cache of RIS data""" def __init__(self, client): """Initialize RisMonolith :param client: client to utilize :type client: RmcClient object """ self._client = client self.name = "Monolithic output of RIS Service" self.types = OrderedDict() self._visited_urls = list() self._current_location = '/' # "root" self.queue = Queue() self._type = None self._name = None self.progress = 0 self.reload = False self.is_redfish = client._rest_client.is_redfish if self.is_redfish: self._resourcedir = '/redfish/v1/ResourceDirectory/' self._typestring = '@odata.type' self._hrefstring = '@odata.id' else: self._resourcedir = '/rest/v1/ResourceDirectory' self._typestring = 'Type' self._hrefstring = 'href' def _get_type(self): """Return monolith version type""" return "Monolith.1.0.0" type = property(_get_type, None) def update_progress(self): """Simple function to increment the dot progress""" if self.progress % 6 == 0: sys.stdout.write('.') def get_visited_urls(self): """Return the visited URLS""" return self._visited_urls def set_visited_urls(self, visited_urls): """Set visited URLS to given list.""" self._visited_urls = visited_urls def load(self, path=None, includelogs=False, skipinit=False, \ skipcrawl=False, loadtype='href', loadcomplete=False): """Walk entire RIS model and cache all responses in self. :param path: path to start load from. :type path: str. :param includelogs: flag to determine if logs should be downloaded also. :type includelogs: boolean. :param skipinit: flag to determine if first run of load. :type skipinit: boolean. :param skipcrawl: flag to determine if load should traverse found links. :type skipcrawl: boolean. :param loadtype: flag to determine if load is meant for only href items. :type loadtype: str. :param loadcomplete: flag to download the entire monolith :type loadcomplete: boolean """ if not skipinit: if LOGGER.getEffectiveLevel() == 40: sys.stdout.write("Discovering data...") else: LOGGER.info("Discovering data...") self.name = self.name + ' at %s' % self._client.base_url if not self.types: self.types = OrderedDict() if not threading.active_count() >= 6: for _ in range(5): workhand = SuperDuperWorker(self.queue) workhand.setDaemon(True) workhand.start() selectivepath = path if not selectivepath: selectivepath = self._client._rest_client.default_prefix self._load(selectivepath, skipcrawl=skipcrawl, includelogs=includelogs,\ skipinit=skipinit, loadtype=loadtype, loadcomplete=loadcomplete) self.queue.join() if not skipinit: if LOGGER.getEffectiveLevel() == 40: sys.stdout.write("Done\n") else: LOGGER.info("Done\n") def _load(self, path, skipcrawl=False, originaluri=None, includelogs=False,\ skipinit=False, loadtype='href', loadcomplete=False): """Helper function to main load function. :param path: path to start load from. :type path: str. :param skipcrawl: flag to determine if load should traverse found links. :type skipcrawl: boolean. :param originaluri: variable to assist in determining originating path. :type originaluri: str. :param includelogs: flag to determine if logs should be downloaded also. :type includelogs: boolean. :param skipinit: flag to determine if first run of load. :type skipinit: boolean. :param loadtype: flag to determine if load is meant for only href items. :type loadtype: str. :param loadcomplete: flag to download the entire monolith :type loadcomplete: boolean """ if path.endswith("?page=1"): return elif not includelogs: if "/Logs/" in path: return #TODO: need to find a better way to support non ascii characters path = path.replace("|", "%7C") #remove fragments newpath = urlparse(path) newpath = list(newpath[:]) newpath[-1] = '' path = urlunparse(tuple(newpath)) LOGGER.debug('_loading %s', path) if not self.reload: if path.lower() in self._visited_urls: return resp = self._client.get(path) if resp.status != 200 and path.lower() == self._client.typepath.defs.\ biospath: raise BiosUnregisteredError() elif resp.status != 200: path = path + '/' resp = self._client.get(path) if resp.status == 401: raise SessionExpiredRis("Invalid session. Please logout and "\ "log back in or include credentials.") elif resp.status != 200: return if loadtype == "ref": self.parse_schema(resp) self.queue.put((resp, path, skipinit, self)) if loadtype == 'href': #follow all the href attributes if self.is_redfish: jsonpath_expr = jsonpath_rw.parse("$..'@odata.id'") else: jsonpath_expr = jsonpath_rw.parse('$..href') matches = jsonpath_expr.find(resp.dict) if 'links' in resp.dict and 'NextPage' in resp.dict['links']: if originaluri: next_link_uri = originaluri + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=originaluri, \ includelogs=includelogs, skipcrawl=skipcrawl, \ skipinit=skipinit) else: next_link_uri = path + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=path, includelogs=includelogs,\ skipcrawl=skipcrawl, skipinit=skipinit) (newversion, dirmatch) = self.check_for_directory(matches) if not newversion and not skipcrawl: for match in matches: if path == "/rest/v1": if str(match.full_path) == "links.Schemas.href" or \ str(match.full_path) == "links.Registries.href": continue else: if str(match.full_path) == "*****@*****.**" or \ str(match.full_path) == "*****@*****.**": continue if match.value == path: continue href = '%s' % match.value self._load(href, skipcrawl=skipcrawl, \ originaluri=originaluri, includelogs=includelogs, \ skipinit=skipinit) elif not skipcrawl: href = '%s' % dirmatch.value self._load(href, skipcrawl=skipcrawl, originaluri=originaluri, \ includelogs=includelogs, skipinit=skipinit) if loadcomplete: for match in matches: self._load(match.value, skipcrawl=skipcrawl, originaluri=\ originaluri, includelogs=includelogs, skipinit=skipinit) def parse_schema(self, resp): """Function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str. """ #pylint: disable=maybe-no-member jsonpath_expr = jsonpath_rw.parse('$.."$ref"') matches = jsonpath_expr.find(resp.dict) respcopy = resp.dict typeregex = '([#,@].*?\.)' if matches: for match in matches: fullpath = str(match.full_path) jsonfile = match.value.split('#')[0] jsonpath = match.value.split('#')[1] listmatch = None found = None if 'redfish.dmtf.org' in jsonfile: if 'odata' in jsonfile: jsonpath = jsonpath.replace(jsonpath.split('/')[-1], \ 'odata' + jsonpath.split('/')[-1]) jsonfile = 'Resource.json' found = re.search(typeregex, fullpath) if found: repitem = fullpath[found.regs[0][0]:found.regs[0][1]] schemapath = '/' + fullpath.replace(repitem, '~').\ replace('.', '/').replace('~', repitem) else: schemapath = '/' + fullpath.replace('.', '/') if '.json' in jsonfile: itempath = schemapath if self.is_redfish: if resp.request.path[-1] == '/': newpath = '/'.join(resp.request.path.split('/')\ [:-2]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')\ [:-1]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')[:-1]) \ + '/' + jsonfile if 'href.json' in newpath: continue if not newpath.lower() in self._visited_urls: self.load(newpath, skipcrawl=True, includelogs=False, \ skipinit=True, loadtype='ref') instance = list() if 'st' in self.types: for stitem in self.types['st']['Instances']: instance.append(stitem) if 'ob' in self.types: for obitem in self.types['ob']['Instances']: instance.append(obitem) for item in instance: if jsonfile in item.resp._rest_request._path: if 'anyOf' in fullpath: break dictcopy = item.resp.dict listmatch = re.search('[[][0-9]+[]]', itempath) if listmatch: start = listmatch.regs[0][0] end = listmatch.regs[0][1] newitempath = [ itempath[:start], itempath[end:] ] start = jsonpointer.JsonPointer(newitempath[0]) end = jsonpointer.JsonPointer(newitempath[1]) del start.parts[-1], end.parts[-1] vals = start.resolve(respcopy) count = 0 for val in vals: try: if '$ref' in six.iterkeys( end.resolve(val)): end.resolve(val).pop('$ref') end.resolve(val).update(dictcopy) replace_pointer = jsonpointer.\ JsonPointer(end.path + jsonpath) data = replace_pointer.resolve(val) set_pointer(val, end.path, data) start.resolve(respcopy)[count].\ update(val) break except: count += 1 else: itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in six.iterkeys( itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(dictcopy) if jsonpath: if 'anyOf' in fullpath: continue if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = schemapath.replace('/$ref', '') if re.search('\[\d]', schemapath): schemapath = schemapath.translate(None, '[]') schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: if not listmatch: schemapath = schemapath.replace('/$ref', '') replacepath = schemapath + jsonpath replace_pointer = jsonpointer.\ JsonPointer(replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.json(respcopy) else: resp.json(respcopy) def check_for_directory(self, matches): """Function to allow checking for new directory :param matches: current found matches. :type matches: dict. """ for match in matches: if match.value == self._resourcedir: return (True, match) return (False, None) def branch_worker(self, resp, path, skipinit): """Helper for load function, creates threaded worker :param resp: response received. :type resp: str. :param path: path correlating to the response. :type path: str. :param skipinit: flag to determine if progress bar should be updated. :type skipinit: boolean. """ self._visited_urls.append(path.lower()) member = RisMonolithMemberv100(resp, self.is_redfish) if not member.type: return self.update_member(member) if not skipinit: self.progress += 1 if LOGGER.getEffectiveLevel() == 40: self.update_progress() def update_member(self, member): """Adds member to this monolith. If the member already exists the data is updated in place. :param member: Ris monolith member object made by branch worker. :type member: RisMonolithMemberv100. """ if member.maj_type not in self.types: self.types[member.maj_type] = OrderedDict() self.types[member.maj_type]['Instances'] = list() found = False for indices in range(len(self.types[member.maj_type]['Instances'])): inst = self.types[member.maj_type]['Instances'][indices] if inst.resp.request.path == member.resp.request.path: self.types[member.maj_type]['Instances'][indices] = member self.types[member.maj_type]['Instances'][indices].patches.\ extend([patch for patch in inst.patches]) found = True break if not found: self.types[member.maj_type]['Instances'].append(member) def load_from_dict(self, src): """Load data to monolith from dict :param src: data receive from rest operation. :type src: str. """ self._type = src['Type'] self._name = src['Name'] self.types = OrderedDict() for typ in src['Types']: for inst in typ['Instances']: member = RisMonolithMemberv100(None, self.is_redfish) member.load_from_dict(inst) self.update_member(member) return def to_dict(self): """Convert data to monolith from dict""" result = OrderedDict() result['Type'] = self.type result['Name'] = self.name types_list = list() for typ in list(self.types.keys()): type_entry = OrderedDict() type_entry['Type'] = typ type_entry['Instances'] = list() for inst in self.types[typ]['Instances']: type_entry['Instances'].append(inst.to_dict()) types_list.append(type_entry) result['Types'] = types_list return result def reduce(self): """Reduce monolith data""" result = OrderedDict() result['Type'] = self.type result['Name'] = self.name types_list = list() for typ in list(self.types.keys()): type_entry = OrderedDict() type_entry['Type'] = typ for inst in self.types[typ]['Instances']: type_entry['Instances'] = inst.reduce() types_list.append(type_entry) result['Types'] = types_list return result def _jsonpath2jsonpointer(self, instr): """Convert json path to json pointer :param instr: input path to be converted to pointer. :type instr: str. """ outstr = instr.replace('.[', '[') outstr = outstr.replace('[', '/') outstr = outstr.replace(']', '/') if outstr.endswith('/'): outstr = outstr[:-1] return outstr def _get_current_location(self): """Return current location""" return self._current_location def _set_current_location(self, newval): """Set current location""" self._current_location = newval location = property(_get_current_location, _set_current_location) def list(self, lspath=None): """Function for list command :param lspath: path list. :type lspath: list. """ results = list() path_parts = ['Types'] # Types is always assumed if isinstance(lspath, list) and len(lspath) > 0: lspath = lspath[0] path_parts.extend(lspath.split('/')) elif not lspath: lspath = '/' else: path_parts.extend(lspath.split('/')) currpos = self.to_dict() for path_part in path_parts: if not path_part: continue if isinstance(currpos, RisMonolithMemberv100): break elif isinstance(currpos, dict) and path_part in currpos: currpos = currpos[path_part] elif isinstance(currpos, list): for positem in currpos: if 'Type' in positem and path_part == positem['Type']: currpos = positem break results.append(currpos) return results def killthreads(self): """Function to kill threads on logout""" threads = [] for thread in threading.enumerate(): if isinstance(thread, SuperDuperWorker): self.queue.put(('KILL', 'KILL', 'KILL', 'KILL')) threads.append(thread) for thread in threads: thread.join()
def query_dqsegdb(cls, flags, *args, **kwargs): """Query the advanced LIGO DQSegDB for a list of flags. Parameters ---------- flags : `iterable` A list of flag names for which to query. *args Either, two `float`-like numbers indicating the GPS [start, stop) interval, or a `SegmentList` defining a number of summary segments. on_error : `str` how to handle an error querying for one flag, one of - `'raise'` (default): raise the Exception - `'warn'`: print a warning - `'ignore'`: move onto the next flag as if nothing happened url : `str`, optional, default: ``'https://segments.ligo.org'`` URL of the segment database. Returns ------- flagdict : `DataQualityDict` An ordered `DataQualityDict` of (name, `DataQualityFlag`) pairs. """ # check on_error flag on_error = kwargs.pop('on_error', 'raise').lower() if on_error not in ['raise', 'warn', 'ignore']: raise ValueError("on_error must be one of 'raise', 'warn', " "or 'ignore'") # set up threading inq = Queue() outq = Queue() for i in range(len(flags)): t = _QueryDQSegDBThread(inq, outq, *args, **kwargs) t.setDaemon(True) t.start() for i, flag in enumerate(flags): inq.put((i, flag)) # capture output inq.join() outq.join() new = cls() results = list(zip(*sorted([outq.get() for i in range(len(flags))], key=lambda x: x[0])))[1] for result, flag in zip(results, flags): if isinstance(result, Exception): result.args = ('%s [%s]' % (str(result), str(flag)),) if on_error == 'ignore': pass elif on_error == 'warn': warnings.warn(str(result)) else: raise result else: new[flag] = result return new
class MainWin( wx.Frame ): def __init__( self, parent, id = wx.ID_ANY, title='', size=(1000,800) ): wx.Frame.__init__(self, parent, id, title, size=size) self.db = Database() self.bufferSecs = 10 self.setFPS( 30 ) self.xFinish = None self.tFrameCount = self.tLaunch = self.tLast = now() self.frameCount = 0 self.fpt = timedelta(seconds=0) self.iTriggerSelect = None self.triggerInfo = None self.tsMax = None self.captureTimer = wx.CallLater( 10, self.stopCapture ) self.tdCaptureBefore = tdCaptureBeforeDefault self.tdCaptureAfter = tdCaptureAfterDefault self.config = wx.Config() self.requestQ = Queue() # Select photos from photobuf. self.dbWriterQ = Queue() # Photos waiting to be written self.messageQ = Queue() # Collection point for all status/failure messages. self.SetBackgroundColour( wx.Colour(232,232,232) ) self.focusDialog = FocusDialog( self ) self.photoDialog = PhotoDialog( self ) self.autoCaptureDialog = AutoCaptureDialog( self ) self.triggerDialog = TriggerDialog( self ) mainSizer = wx.BoxSizer( wx.VERTICAL ) #------------------------------------------------------------------------------------------------ headerSizer = wx.BoxSizer( wx.HORIZONTAL ) self.logo = Utils.GetPngBitmap('CrossMgrHeader.png') headerSizer.Add( wx.StaticBitmap(self, wx.ID_ANY, self.logo) ) self.title = wx.StaticText(self, label='CrossMgr Video\nVersion {}'.format(AppVerName.split()[1]), style=wx.ALIGN_RIGHT ) self.title.SetFont( wx.Font( (0,28), wx.FONTFAMILY_SWISS, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL ) ) headerSizer.Add( self.title, flag=wx.ALL, border=10 ) clock = Clock( self, size=(90,90) ) clock.SetBackgroundColour( self.GetBackgroundColour() ) clock.Start() headerSizer.Add( clock, flag=wx.ALIGN_CENTER_VERTICAL|wx.LEFT|wx.RIGHT, border=4 ) #------------------------------------------------------------------------------ self.cameraDevice = wx.StaticText( self ) self.cameraResolution = wx.StaticText( self ) self.targetFPS = wx.StaticText( self, label='30 fps' ) self.actualFPS = wx.StaticText( self, label='30.0 fps' ) boldFont = self.cameraDevice.GetFont() boldFont.SetWeight( wx.BOLD ) for w in (self.cameraDevice, self.cameraResolution, self.targetFPS, self.actualFPS): w.SetFont( boldFont ) fgs = wx.FlexGridSizer( 2, 2, 2 ) # 2 Cols fgs.Add( wx.StaticText(self, label='Camera Device:'), flag=wx.ALIGN_RIGHT ) fgs.Add( self.cameraDevice ) fgs.Add( wx.StaticText(self, label='Resolution:'), flag=wx.ALIGN_RIGHT ) fgs.Add( self.cameraResolution ) fgs.Add( wx.StaticText(self, label='Target:'), flag=wx.ALIGN_RIGHT ) fgs.Add( self.targetFPS, flag=wx.ALIGN_RIGHT ) fgs.Add( wx.StaticText(self, label='Actual:'), flag=wx.ALIGN_RIGHT ) fgs.Add( self.actualFPS, flag=wx.ALIGN_RIGHT ) self.focus = wx.Button( self, label="Focus..." ) self.focus.Bind( wx.EVT_BUTTON, self.onFocus ) self.reset = wx.Button( self, label="Reset Camera" ) self.reset.Bind( wx.EVT_BUTTON, self.resetCamera ) self.manage = wx.Button( self, label="Manage Database" ) self.manage.Bind( wx.EVT_BUTTON, self.manageDatabase ) self.autoCaptureBtn = wx.Button( self, label="Config Auto Capture" ) self.autoCaptureBtn.Bind( wx.EVT_BUTTON, self.autoCaptureConfig ) self.help = wx.Button( self, wx.ID_HELP ) self.help.Bind( wx.EVT_BUTTON, self.onHelp ) self.snapshot, self.autoCapture, self.capture = CreateCaptureButtons( self ) self.snapshot.Bind( wx.EVT_LEFT_DOWN, self.onStartSnapshot ) self.focusDialog.snapshot.Bind( wx.EVT_LEFT_DOWN, self.onStartSnapshot ) self.autoCapture.Bind( wx.EVT_LEFT_DOWN, self.onStartAutoCapture ) self.focusDialog.autoCapture.Bind( wx.EVT_LEFT_DOWN, self.onStartAutoCapture ) self.capture.Bind( wx.EVT_LEFT_DOWN, self.onStartCapture ) self.capture.Bind( wx.EVT_LEFT_UP, self.onStopCapture ) self.focusDialog.capture.Bind( wx.EVT_LEFT_DOWN, self.onStartCapture ) self.focusDialog.capture.Bind( wx.EVT_LEFT_UP, self.onStopCapture ) headerSizer.Add( fgs, flag=wx.ALIGN_CENTER_VERTICAL ) fgs = wx.FlexGridSizer( rows=2, cols=0, hgap=8, vgap=4 ) fgs.Add( self.focus, flag=wx.EXPAND ) fgs.Add( self.reset, flag=wx.EXPAND ) fgs.Add( self.manage, flag=wx.EXPAND ) fgs.Add( self.autoCaptureBtn, flag=wx.EXPAND ) fgs.Add( self.help, flag=wx.EXPAND ) headerSizer.Add( fgs, flag=wx.ALIGN_CENTRE|wx.LEFT, border=4 ) headerSizer.AddStretchSpacer() headerSizer.Add( self.snapshot, flag=wx.ALIGN_CENTRE_VERTICAL|wx.LEFT, border=8 ) headerSizer.Add( self.autoCapture, flag=wx.ALIGN_CENTRE_VERTICAL|wx.LEFT, border=8 ) headerSizer.Add( self.capture, flag=wx.ALIGN_CENTRE_VERTICAL|wx.LEFT|wx.RIGHT, border=8 ) #------------------------------------------------------------------------------ mainSizer.Add( headerSizer, flag=wx.EXPAND ) #------------------------------------------------------------------------------------------------ self.finishStrip = FinishStripPanel( self, size=(-1,wx.GetDisplaySize()[1]//2) ) self.finishStrip.finish.Bind( wx.EVT_RIGHT_DOWN, self.onRightClick ) self.primaryBitmap = ScaledBitmap( self, style=wx.BORDER_SUNKEN, size=(int(imageWidth*0.75), int(imageHeight*0.75)) ) self.primaryBitmap.SetTestBitmap() self.primaryBitmap.Bind( wx.EVT_LEFT_UP, self.onFocus ) self.primaryBitmap.Bind( wx.EVT_RIGHT_UP, self.onFocus ) hsDate = wx.BoxSizer( wx.HORIZONTAL ) hsDate.Add( wx.StaticText(self, label='Show Triggers for'), flag=wx.ALIGN_CENTER_VERTICAL ) tQuery = now() self.date = wx.adv.DatePickerCtrl( self, dt=wx.DateTime.FromDMY( tQuery.day, tQuery.month-1, tQuery.year ), style=wx.adv.DP_DROPDOWN|wx.adv.DP_SHOWCENTURY ) self.date.Bind( wx.adv.EVT_DATE_CHANGED, self.onQueryDateChanged ) hsDate.Add( self.date, flag=wx.LEFT, border=2 ) self.dateSelect = wx.Button( self, label='Select Date' ) hsDate.Add( self.dateSelect, flag=wx.ALIGN_CENTER_VERTICAL|wx.LEFT, border=2 ) self.dateSelect.Bind( wx.EVT_BUTTON, self.onDateSelect ) hsDate.Add( wx.StaticText(self, label='Filter by Bib'), flag=wx.ALIGN_CENTER_VERTICAL|wx.LEFT, border=12 ) self.bib = wx.lib.intctrl.IntCtrl( self, style=wx.TE_PROCESS_ENTER, size=(64,-1), min=1, allow_none=True, value=None ) self.bib.Bind( wx.EVT_TEXT_ENTER, self.onQueryBibChanged ) hsDate.Add( self.bib, flag=wx.LEFT, border=2 ) self.tsQueryLower = datetime(tQuery.year, tQuery.month, tQuery.day) self.tsQueryUpper = self.tsQueryLower + timedelta(days=1) self.bibQuery = None self.triggerList = AutoWidthListCtrl( self, style=wx.LC_REPORT|wx.BORDER_SUNKEN|wx.LC_SORT_ASCENDING|wx.LC_HRULES ) self.il = wx.ImageList(16, 16) self.sm_close = [] for bm in getCloseFinishBitmaps(): self.sm_close.append( self.il.Add(bm) ) self.sm_up = self.il.Add( Utils.GetPngBitmap('SmallUpArrow.png')) self.sm_up = self.il.Add( Utils.GetPngBitmap('SmallUpArrow.png')) self.sm_dn = self.il.Add( Utils.GetPngBitmap('SmallDownArrow.png')) self.triggerList.SetImageList(self.il, wx.IMAGE_LIST_SMALL) self.fieldCol = {f:c for c, f in enumerate('ts bib name team wave race_name note kmh mph frames'.split())} headers = ['Time', 'Bib', 'Name', 'Team', 'Wave', 'Race', 'Note', 'km/h', 'mph', 'Frames'] for i, h in enumerate(headers): self.triggerList.InsertColumn( i, h, wx.LIST_FORMAT_RIGHT if h in ('Bib','km/h','mph','Frames') else wx.LIST_FORMAT_LEFT ) self.itemDataMap = {} self.triggerList.Bind( wx.EVT_LIST_ITEM_SELECTED, self.onTriggerSelected ) self.triggerList.Bind( wx.EVT_LIST_ITEM_ACTIVATED, self.onTriggerEdit ) self.triggerList.Bind( wx.EVT_LIST_ITEM_RIGHT_CLICK, self.onTriggerRightClick ) #self.triggerList.Bind( wx.EVT_LIST_DELETE_ITEM, self.onTriggerDelete ) vsTriggers = wx.BoxSizer( wx.VERTICAL ) vsTriggers.Add( hsDate ) vsTriggers.Add( self.triggerList, 1, flag=wx.EXPAND|wx.TOP, border=2) #------------------------------------------------------------------------------------------------ mainSizer.Add( self.finishStrip, 1, flag=wx.EXPAND ) border=2 row1Sizer = wx.BoxSizer( wx.HORIZONTAL ) row1Sizer.Add( self.primaryBitmap, flag=wx.ALL, border=border ) row1Sizer.Add( vsTriggers, 1, flag=wx.TOP|wx.BOTTOM|wx.RIGHT|wx.EXPAND, border=border ) mainSizer.Add( row1Sizer, flag=wx.EXPAND ) self.Bind(wx.EVT_CLOSE, self.onCloseWindow) self.readOptions() self.updateFPS( int(float(self.targetFPS.GetLabel().split()[0])) ) self.updateAutoCaptureLabel() self.SetSizerAndFit( mainSizer ) # Start the message reporting thread so we can see what is going on. self.messageThread = threading.Thread( target=self.showMessages ) self.messageThread.daemon = True self.messageThread.start() wx.CallLater( 300, self.refreshTriggers ) def onHelp( self, event ): OpenHelp() def setFPS( self, fps ): self.fps = int(fps if fps > 0 else 30) self.frameDelay = 1.0 / self.fps self.frameCountUpdate = int(self.fps * 2) def updateFPS( self, fps ): self.setFPS( fps ) self.targetFPS.SetLabel( u'{} fps'.format(self.fps) ) def updateActualFPS( self, actualFPS ): self.actualFPS.SetLabel( '{:.1f} fps'.format(actualFPS) ) def updateAutoCaptureLabel( self ): def f( n ): s = '{:0.1f}'.format( n ) return s[:-2] if s.endswith('.0') else s label = u'\n'.join( [u'AUTO',u'CAPTURE',u'{} .. {}'.format(f(-self.tdCaptureBefore.total_seconds()), f(self.tdCaptureAfter.total_seconds()))] ) for btn in (self.autoCapture, self.focusDialog.autoCapture): btn.SetLabel( label ) btn.SetFontToFitLabel() wx.CallAfter( btn.Refresh ) def setQueryDate( self, d ): self.tsQueryLower = d self.tsQueryUpper = self.tsQueryLower + timedelta( days=1 ) self.refreshTriggers( True ) def onDateSelect( self, event ): triggerDates = self.db.getTriggerDates() triggerDates.sort( reverse=True ) with DateSelectDialog( self, triggerDates ) as dlg: if dlg.ShowModal() == wx.ID_OK and dlg.GetDate(): self.setQueryDate( dlg.GetDate() ) def onQueryDateChanged( self, event ): v = self.date.GetValue() self.setQueryDate( datetime( v.GetYear(), v.GetMonth() + 1, v.GetDay() ) ) def onQueryBibChanged( self, event ): self.bibQuery = self.bib.GetValue() self.refreshTriggers( True ) def GetListCtrl( self ): return self.triggerList def GetSortImages(self): return (self.sm_dn, self.sm_up) def getItemData( self, i ): data = self.triggerList.GetItemData( i ) return self.itemDataMap[data] def getTriggerRowFromID( self, id ): for row in six.moves.range(self.triggerList.GetItemCount()-1, -1, -1): if self.itemDataMap[row][0] == id: return row return None def updateTriggerRow( self, row, fields ): if 'last_name' in fields and 'first_name' in fields: fields['name'] = u', '.join( n for n in (fields['last_name'], fields['first_name']) if n ) for k, v in six.iteritems(fields): if k in self.fieldCol: if k == 'bib': v = u'{:>6}'.format(v) elif k == 'frames': v = six.text_type(v) if v else u'' else: v = six.text_type(v) self.triggerList.SetItem( row, self.fieldCol[k], v ) def updateTriggerRowID( self, id, fields ): row = self.getTriggerRowFromID( id ) if row is not None: self.updateTriggerRow( row, fields ) def getTriggerInfo( self, row ): data = self.itemDataMap[self.triggerList.GetItemData(row)] return { a:data[i] for i, a in enumerate(( 'id','ts','s_before','s_after','ts_start', 'bib','name','team','wave','race_name', 'first_name','last_name','note','kmh','frames')) } def refreshTriggers( self, replace=False, iTriggerRow=None ): tNow = now() self.lastTriggerRefresh = tNow # replace = True if replace: tsLower = self.tsQueryLower tsUpper = self.tsQueryUpper self.triggerList.DeleteAllItems() self.itemDataMap = {} self.tsMax = None self.iTriggerSelect = None self.triggerInfo = {} self.finishStrip.SetTsJpgs( None, None ) else: tsLower = (self.tsMax or datetime(tNow.year, tNow.month, tNow.day)) + timedelta(seconds=0.00001) tsUpper = tsLower + timedelta(days=1) triggers = self.db.getTriggers( tsLower, tsUpper, self.bibQuery ) tsPrev = (self.tsMax or datetime(2000,1,1)) if triggers: self.tsMax = triggers[-1][1] # id,ts,s_before,s_after,ts_start,bib,first_name,last_name,team,wave,race_name,note,kmh,frames zeroFrames, tsLower, tsUpper = [], datetime.max, datetime.min for i, (id,ts,s_before,s_after,ts_start,bib,first_name,last_name,team,wave,race_name,note,kmh,frames) in enumerate(triggers): if s_before == 0.0 and s_after == 0.0: s_before,s_after = tdCaptureBeforeDefault.total_seconds(),tdCaptureAfterDefault.total_seconds() dtFinish = (ts-tsPrev).total_seconds() itemImage = self.sm_close[min(len(self.sm_close)-1, int(len(self.sm_close) * dtFinish / closeFinishThreshold))] row = self.triggerList.InsertItem( 999999, ts.strftime('%H:%M:%S.%f')[:-3], itemImage ) if not frames: tsLower = min( tsLower, ts-timedelta(seconds=s_before) ) tsU = ts + timedelta(seconds=s_after) tsUpper = max( tsUpper,tsU ) zeroFrames.append( (row, id, tsU) ) kmh_text, mph_text = (u'{:.2f}'.format(kmh), u'{:.2f}'.format(kmh * 0.621371)) if kmh else (u'', u'') fields = { 'bib': bib, 'last_name': last_name, 'first_name': first_name, 'team': team, 'wave': wave, 'race_name': race_name, 'note': note, 'kmh': kmh_text, 'mph': mph_text, 'frames': frames, } self.updateTriggerRow( row, fields ) self.triggerList.SetItemData( row, row ) self.itemDataMap[row] = (id,ts,s_before,s_after,ts_start,bib,fields['name'],team,wave,race_name,first_name,last_name,note,kmh,frames) tsPrev = ts if zeroFrames: counts = self.db.getTriggerPhotoCounts( tsLower, tsUpper ) values = {'frames':0} for row, id, tsU in zeroFrames: values['frames'] = counts[id] self.updateTriggerRow( row, values ) # Don't update the trigger if the number of frames is possibly not known yet. if (tNow - tsU).total_seconds() < 5.0*60.0: del counts[id] self.db.updateTriggerPhotoCounts( counts ) for i in six.moves.range(self.triggerList.GetColumnCount()): self.triggerList.SetColumnWidth(i, wx.LIST_AUTOSIZE) if iTriggerRow is not None: iTriggerRow = min( max(0, iTriggerRow), self.triggerList.GetItemCount()-1 ) self.triggerList.EnsureVisible( iTriggerRow ) self.triggerList.Select( iTriggerRow ) else: if self.triggerList.GetItemCount() >= 1: self.triggerList.EnsureVisible( self.triggerList.GetItemCount()-1 ) def Start( self ): self.messageQ.put( ('', '************************************************') ) self.messageQ.put( ('started', now().strftime('%Y/%m/%d %H:%M:%S')) ) self.startThreads() def updateSnapshot( self, t, f ): self.snapshotCount = getattr(self, 'snapshotCount', 0) + 1 self.dbWriterQ.put( ('photo', t, f) ) self.dbWriterQ.put( ( 'trigger', t, 0.00001, # s_before 0.00001, # s_after t, self.snapshotCount, # bib u'', # first_name u'Snapshot', # last_name u'', # team u'', # save u'', # race_name ) ) self.doUpdateAutoCapture( t, self.snapshotCount, [self.snapshot, self.focusDialog.snapshot], snapshotEnableColour ) def onStartSnapshot( self, event ): event.GetEventObject().SetForegroundColour( snapshotDisableColour ) wx.CallAfter( event.GetEventObject().Refresh ) self.camInQ.put( {'cmd':'snapshot'} ) def doUpdateAutoCapture( self, tStartCapture, count, btn, colour ): self.dbWriterQ.put( ('flush',) ) self.dbWriterQ.join() triggers = self.db.getTriggers( tStartCapture, tStartCapture, count ) if triggers: id = triggers[0][0] self.db.initCaptureTriggerData( id ) self.refreshTriggers( iTriggerRow=999999, replace=True ) self.showLastTrigger() self.onTriggerSelected( iTriggerSelect=self.triggerList.GetItemCount()-1 ) for b in (btn if isinstance(btn, list) else [btn]): b.SetForegroundColour( colour ) wx.CallAfter( b.Refresh ) def onStartAutoCapture( self, event ): tNow = now() event.GetEventObject().SetForegroundColour( autoCaptureDisableColour ) wx.CallAfter( event.GetEventObject().Refresh ) self.autoCaptureCount = getattr(self, 'autoCaptureCount', 0) + 1 s_before, s_after = self.tdCaptureBefore.total_seconds(), self.tdCaptureAfter.total_seconds() self.requestQ.put( { 'time':tNow, 's_before':s_before, 's_after':s_after, 'ts_start':tNow, 'bib':self.autoCaptureCount, 'last_name':u'Auto', } ) wx.CallLater( int(CamServer.EstimateQuerySeconds(tNow, s_before, s_after, self.fps)*1000.0) + 80, self.doUpdateAutoCapture, tNow, self.autoCaptureCount, self.autoCapture, autoCaptureEnableColour ) def onStartCapture( self, event ): tNow = self.tStartCapture = now() event.GetEventObject().SetForegroundColour( captureDisableColour ) wx.CallAfter( event.GetEventObject().Refresh ) wx.BeginBusyCursor() self.captureCount = getattr(self, 'captureCount', 0) + 1 self.requestQ.put( { 'time':tNow, 's_before':0.0, 's_after':self.tdCaptureAfter.total_seconds(), 'ts_start':tNow, 'bib':self.captureCount, 'last_name':u'Capture', } ) self.camInQ.put( {'cmd':'start_capture', 'tStart':tNow-self.tdCaptureBefore} ) def showLastTrigger( self ): iTriggerRow = self.triggerList.GetItemCount() - 1 if iTriggerRow < 0: return self.triggerList.EnsureVisible( iTriggerRow ) for r in six.moves.range(self.triggerList.GetItemCount()-1): self.triggerList.Select(r, 0) self.triggerList.Select( iTriggerRow ) def onStopCapture( self, event ): self.camInQ.put( {'cmd':'stop_capture'} ) triggers = self.db.getTriggers( self.tStartCapture, self.tStartCapture, self.captureCount ) if triggers: id = triggers[0][0] self.db.updateTriggerBeforeAfter( id, 0.0, (now() - self.tStartCapture).total_seconds() ) self.db.initCaptureTriggerData( id ) self.refreshTriggers( iTriggerRow=999999, replace=True ) self.showLastTrigger() wx.EndBusyCursor() event.GetEventObject().SetForegroundColour( captureEnableColour ) wx.CallAfter( event.GetEventObject().Refresh ) def updateFS(): # Wait for all the photos to be written. self.dbWriterQ.put( ('flush',) ) self.dbWriterQ.join() # Update the finish strip. wx.CallAfter( self.onTriggerSelected, iTriggerSelect=self.triggerList.GetItemCount() - 1 ) threading.Thread( target=updateFS ).start() def autoCaptureConfig( self, event ): self.autoCaptureDialog.set( self.tdCaptureBefore.total_seconds(), self.tdCaptureAfter.total_seconds() ) if self.autoCaptureDialog.ShowModal() == wx.ID_OK: s_before, s_after = self.autoCaptureDialog.get() self.tdCaptureBefore = timedelta(seconds=s_before) if s_before is not None else tdCaptureBeforeDefault self.tdCaptureAfter = timedelta(seconds=s_after) if s_after is not None else tdCaptureAfterDefault self.writeOptions() self.updateAutoCaptureLabel() def onFocus( self, event ): if self.focusDialog.IsShown(): return self.focusDialog.Move((4,4)) self.camInQ.put( {'cmd':'send_update', 'name':'focus', 'freq':1} ) self.focusDialog.Show() def onRightClick( self, event ): if not self.triggerInfo: return self.xFinish = event.GetX() self.photoDialog.set( self.finishStrip.finish.getIJpg(self.xFinish), self.triggerInfo, self.finishStrip.GetTsJpgs(), self.fps, self.doTriggerEdit ) self.photoDialog.CenterOnParent() self.photoDialog.Move( self.photoDialog.GetScreenPosition().x, 0 ) self.photoDialog.ShowModal() if self.triggerInfo['kmh'] != (self.photoDialog.kmh or 0.0): self.db.updateTriggerKMH( self.triggerInfo['id'], self.photoDialog.kmh or 0.0 ) self.refreshTriggers( replace=True, iTriggerRow=self.iTriggerSelect ) self.photoDialog.clear() def onTriggerSelected( self, event=None, iTriggerSelect=None ): self.iTriggerSelect = event.Index if iTriggerSelect is None else iTriggerSelect if self.iTriggerSelect >= self.triggerList.GetItemCount(): self.ts = None self.tsJpg = [] self.finishStrip.SetTsJpgs( self.tsJpg, self.ts, {} ) return data = self.itemDataMap[self.triggerList.GetItemData(self.iTriggerSelect)] self.triggerInfo = self.getTriggerInfo( self.iTriggerSelect ) self.ts = self.triggerInfo['ts'] s_before, s_after = abs(self.triggerInfo['s_before']), abs(self.triggerInfo['s_after']) if s_before == 0.0 and s_after == 0.0: s_before, s_after = tdCaptureBeforeDefault.total_seconds(), tdCaptureAfterDefault.total_seconds() # Update the screen in the background so we don't freeze the UI. def updateFS( triggerInfo ): self.ts = triggerInfo['ts'] self.tsJpg = self.db.clone().getPhotos( self.ts - timedelta(seconds=s_before), self.ts + timedelta(seconds=s_after) ) triggerInfo['frames'] = len(self.tsJpg) wx.CallAfter( self.finishStrip.SetTsJpgs, self.tsJpg, self.ts, triggerInfo ) threading.Thread( target=updateFS, args=(self.triggerInfo,) ).start() def onTriggerRightClick( self, event ): self.iTriggerSelect = event.Index if not hasattr(self, "triggerDeleteID"): self.triggerDeleteID = wx.NewId() self.triggerEditID = wx.NewId() self.Bind(wx.EVT_MENU, lambda event: self.doTriggerDelete(), id=self.triggerDeleteID) self.Bind(wx.EVT_MENU, lambda event: self.doTriggerEdit(), id=self.triggerEditID) menu = wx.Menu() menu.Append(self.triggerEditID, "Edit...") menu.Append(self.triggerDeleteID, "Delete...") self.PopupMenu(menu) menu.Destroy() def doTriggerDelete( self, confirm=True ): triggerInfo = self.getTriggerInfo( self.iTriggerSelect ) message = u', '.join( f for f in (triggerInfo['ts'].strftime('%H:%M:%S.%f')[:-3], six.text_type(triggerInfo['bib']), triggerInfo['name'], triggerInfo['team'], triggerInfo['wave'], triggerInfo['race_name']) if f ) if not confirm or wx.MessageDialog( self, u'{}:\n\n{}'.format(u'Confirm Delete', message), u'Confirm Delete', style=wx.OK|wx.CANCEL|wx.ICON_QUESTION ).ShowModal() == wx.ID_OK: self.db.deleteTrigger( triggerInfo['id'], self.tdCaptureBefore.total_seconds(), self.tdCaptureAfter.total_seconds() ) self.refreshTriggers( replace=True, iTriggerRow=self.iTriggerSelect ) def onTriggerDelete( self, event ): self.iTriggerSelect = event.Index self.doTriggerDelete() def doTriggerEdit( self ): data = self.itemDataMap[self.triggerList.GetItemData(self.iTriggerSelect)] self.triggerDialog.set( self.db, data[0] ) self.triggerDialog.CenterOnParent() if self.triggerDialog.ShowModal() == wx.ID_OK: row = self.iTriggerSelect fields = {f:v for f, v in zip(Database.triggerEditFields,self.triggerDialog.get())} self.updateTriggerRow( row, fields ) self.triggerInfo.update( fields ) return self.triggerInfo def onTriggerEdit( self, event ): self.iTriggerSelect = event.Index self.doTriggerEdit() def showMessages( self ): while 1: message = self.messageQ.get() assert len(message) == 2, 'Incorrect message length' cmd, info = message six.print_( 'Message:', '{}: {}'.format(cmd, info) if cmd else info ) #wx.CallAfter( self.messageManager.write, '{}: {}'.format(cmd, info) if cmd else info ) def delayRefreshTriggers( self ): if not hasattr(self, 'refreshTimer') or not self.refreshTimer.IsRunning(): self.resetTimer = wx.CallLater( 1000, self.refreshTriggers ) def startThreads( self ): self.grabFrameOK = False self.listenerThread = SocketListener( self.requestQ, self.messageQ ) error = self.listenerThread.test() if error: wx.MessageBox('Socket Error:\n\n{}\n\nIs another CrossMgrVideo or CrossMgrCamera running on this computer?'.format(error), "Socket Error", wx.OK | wx.ICON_ERROR ) wx.Exit() self.camInQ, self.camReader = CamServer.getCamServer( self.getCameraInfo() ) self.cameraThread = threading.Thread( target=self.processCamera ) self.cameraThread.daemon = True self.eventThread = threading.Thread( target=self.processRequests ) self.eventThread.daemon = True self.dbWriterThread = threading.Thread( target=DBWriter, args=(self.dbWriterQ, lambda: wx.CallAfter(self.delayRefreshTriggers), self.db.fname) ) self.dbWriterThread.daemon = True self.cameraThread.start() self.eventThread.start() self.dbWriterThread.start() self.listenerThread.start() self.grabFrameOK = True self.messageQ.put( ('threads', 'Successfully Launched') ) self.primaryFreq = 5 self.camInQ.put( {'cmd':'send_update', 'name':'primary', 'freq':self.primaryFreq} ) return True def stopCapture( self ): self.dbWriterQ.put( ('flush',) ) def processCamera( self ): lastFrame = None lastPrimaryTime = now() primaryCount = 0 while 1: try: msg = self.camReader.recv() except EOFError: break cmd = msg['cmd'] if cmd == 'response': for t, f in msg['ts_frames']: self.dbWriterQ.put( ('photo', t, f) ) lastFrame = f elif cmd == 'update': name, lastFrame = msg['name'], lastFrame if msg['frame'] is None else msg['frame'] if lastFrame is not None: if name == 'primary': wx.CallAfter( self.primaryBitmap.SetBitmap, CVUtil.frameToBitmap(lastFrame) ) primaryCount += self.primaryFreq primaryTime = now() primaryDelta = (primaryTime - lastPrimaryTime).total_seconds() if primaryDelta > 2.5: wx.CallAfter( self.updateActualFPS, primaryCount / primaryDelta ) lastPrimaryTime = primaryTime primaryCount = 0 elif name == 'focus': if self.focusDialog.IsShown(): wx.CallAfter( self.focusDialog.SetBitmap, CVUtil.frameToBitmap(lastFrame) ) else: self.camInQ.put( {'cmd':'cancel_update', 'name':'focus'} ) elif cmd == 'snapshot': lastFrame = lastFrame if msg['frame'] is None else msg['frame'] wx.CallAfter( self.updateSnapshot, msg['ts'], lastFrame ) elif cmd == 'terminate': break def processRequests( self ): def refresh(): self.dbWriterQ.put( ('flush',) ) while 1: msg = self.requestQ.get() tSearch = msg['time'] advanceSeconds = msg.get('advanceSeconds', 0.0) tSearch += timedelta(seconds=advanceSeconds) # Record this trigger. self.dbWriterQ.put( ( 'trigger', tSearch - timedelta(seconds=advanceSeconds), msg.get('s_before', self.tdCaptureBefore.total_seconds()), # Use the configured capture interval, not the default. msg.get('s_after', self.tdCaptureAfter.total_seconds()), msg.get('ts_start', None) or now(), msg.get('bib', 99999), msg.get('first_name',u'') or msg.get('firstName',u''), msg.get('last_name',u'') or msg.get('lastName',u''), msg.get('team',u''), msg.get('wave',u''), msg.get('race_name',u'') or msg.get('raceName',u''), ) ) # Record the video frames for the trigger. tStart, tEnd = tSearch-self.tdCaptureBefore, tSearch+self.tdCaptureAfter self.camInQ.put( { 'cmd':'query', 'tStart':tStart, 'tEnd':tEnd,} ) wx.CallAfter( wx.CallLater, max(100, int(100+1000*(tEnd-now()).total_seconds())), refresh ) def shutdown( self ): # Ensure that all images in the queue are saved. if hasattr(self, 'dbWriterThread'): self.camInQ.put( {'cmd':'terminate'} ) self.dbWriterQ.put( ('terminate', ) ) self.dbWriterThread.join( 2.0 ) def setDBName( self, dbName ): if dbName != self.db.fname: if hasattr(self, 'dbWriterThread'): self.dbWriterQ.put( ('terminate', ) ) self.dbWriterThread.join() try: self.db = Database( dbName ) except: self.db = Database() self.dbWriterQ = Queue() self.dbWriterThread = threading.Thread( target=DBWriter, args=(self.dbWriterQ, lambda: wx.CallAfter(self.delayRefreshTriggers), self.db.fname) ) self.dbWriterThread.daemon = True self.dbWriterThread.start() def resetCamera( self, event=None ): dlg = ConfigDialog( self, self.getCameraDeviceNum(), self.fps, self.getCameraResolution() ) ret = dlg.ShowModal() cameraDeviceNum = dlg.GetCameraDeviceNum() cameraResolution = dlg.GetCameraResolution() fps = dlg.GetFPS() dlg.Destroy() if ret != wx.ID_OK: return False self.setCameraDeviceNum( cameraDeviceNum ) self.setCameraResolution( *cameraResolution ) self.updateFPS( fps ) self.writeOptions() if hasattr(self, 'camInQ'): self.camInQ.put( {'cmd':'cam_info', 'info':self.getCameraInfo(),} ) self.GetSizer().Layout() return True def manageDatabase( self, event ): trigFirst, trigLast = self.db.getTimestampRange() dlg = ManageDatabase( self, self.db.getsize(), self.db.fname, trigFirst, trigLast, title='Manage Database' ) if dlg.ShowModal() == wx.ID_OK: work = wx.BusyCursor() tsLower, tsUpper, vacuum, dbName = dlg.GetValues() self.setDBName( dbName ) if tsUpper: tsUpper = datetime.combine( tsUpper, time(23,59,59,999999) ) self.db.cleanBetween( tsLower, tsUpper ) if vacuum: self.db.vacuum() wx.CallAfter( self.finishStrip.Clear ) wx.CallAfter( self.refreshTriggers, True ) dlg.Destroy() def setCameraDeviceNum( self, num ): self.cameraDevice.SetLabel( six.text_type(num) ) def setCameraResolution( self, width, height ): self.cameraResolution.SetLabel( u'{}x{}'.format(width, height) ) def getCameraDeviceNum( self ): return int(self.cameraDevice.GetLabel()) def getCameraFPS( self ): return int(float(self.targetFPS.GetLabel().split()[0])) def getCameraResolution( self ): try: resolution = [int(v) for v in self.cameraResolution.GetLabel().split('x')] return resolution[0], resolution[1] except: return 640, 480 def onCloseWindow( self, event ): self.shutdown() wx.Exit() def writeOptions( self ): self.config.Write( 'DBName', self.db.fname ) self.config.Write( 'CameraDevice', self.cameraDevice.GetLabel() ) self.config.Write( 'CameraResolution', self.cameraResolution.GetLabel() ) self.config.Write( 'FPS', self.targetFPS.GetLabel() ) self.config.Write( 'SecondsBefore', '{:.3f}'.format(self.tdCaptureBefore.total_seconds()) ) self.config.Write( 'SecondsAfter', '{:.3f}'.format(self.tdCaptureAfter.total_seconds()) ) self.config.Flush() def readOptions( self ): self.setDBName( self.config.Read('DBName', '') ) self.cameraDevice.SetLabel( self.config.Read('CameraDevice', u'0') ) self.cameraResolution.SetLabel( self.config.Read('CameraResolution', u'640x480') ) self.targetFPS.SetLabel( self.config.Read('FPS', u'30.000') ) s_before = self.config.Read('SecondsBefore', u'0.5') s_after = self.config.Read('SecondsAfter', u'2.0') try: self.tdCaptureBefore = timedelta(seconds=abs(float(s_before))) except: pass try: self.tdCaptureAfter = timedelta(seconds=abs(float(s_after))) except: pass def getCameraInfo( self ): width, height = self.getCameraResolution() return {'usb':self.getCameraDeviceNum(), 'fps':self.getCameraFPS(), 'width':width, 'height':height}