async def scan(self, root): self.worker = Worker(self.worker_handlers) try: r = await self._scan(root) finally: await self.worker.close() return r
def __init__(self, rid, pipeline_name, wd, expid, priority, due_date, flush, pool, **kwargs): # called through pool self.rid = rid self.pipeline_name = pipeline_name self.wd = wd self.expid = expid self.priority = priority self.due_date = due_date self.flush = flush self.worker = Worker(pool.worker_handlers) self.termination_requested = False self._status = RunStatus.pending notification = { "pipeline": self.pipeline_name, "expid": self.expid, "priority": self.priority, "due_date": self.due_date, "flush": self.flush, "status": self._status.name } notification.update(kwargs) self._notifier = pool.notifier self._notifier[self.rid] = notification self._state_changed = pool.state_changed
def _scan_experiments(wd, log): r = dict() for f in os.listdir(wd): if f.endswith(".py"): try: worker = Worker({"log": lambda message: log("scan", message)}) try: description = yield from worker.examine(os.path.join( wd, f)) finally: yield from worker.close() for class_name, class_desc in description.items(): name = class_desc["name"] arguments = class_desc["arguments"] if name in r: logger.warning("Duplicate experiment name: '%s'", name) basename = name i = 1 while name in r: name = basename + str(i) i += 1 entry = { "file": f, "class_name": class_name, "arguments": arguments } r[name] = entry except: logger.warning("Skipping file '%s'", f, exc_info=True) return r
async def _get_repository_entries(entry_dict, root, filename, worker_handlers): worker = Worker(worker_handlers) try: description = await worker.examine("scan", os.path.join(root, filename)) except: log_worker_exception() raise finally: await worker.close() for class_name, class_desc in description.items(): name = class_desc["name"] arginfo = class_desc["arginfo"] if "/" in name: logger.warning("Character '/' is not allowed in experiment " "name (%s)", name) name = name.replace("/", "_") if name in entry_dict: basename = name i = 1 while name in entry_dict: name = basename + str(i) i += 1 logger.warning("Duplicate experiment name: '%s'\n" "Renaming class '%s' in '%s' to '%s'", basename, class_name, filename, name) entry = { "file": filename, "class_name": class_name, "arginfo": arginfo } entry_dict[name] = entry
def __init__(self, worker_handlers): self.worker = Worker(worker_handlers) self.next_rid = 0 self.queue = Notifier([]) self.queue_modified = asyncio.Event() self.timed = Notifier(dict()) self.timed_modified = asyncio.Event()
async def examine(self, filename, use_repository=True, revision=None): if use_repository: if revision is None: revision = self.cur_rev wd, _ = self.repo_backend.request_rev(revision) filename = os.path.join(wd, filename) worker = Worker(self.worker_handlers) try: description = await worker.examine("examine", filename) finally: await worker.close() if use_repository: self.repo_backend.release_rev(revision) return description
async def _get_run_task(self, expid): logger.info("Running '%s'...", self.expurl) worker = Worker(self._area.worker_handlers) try: await worker.build(rid=None, pipeline_name="browser", wd=os.path.abspath("."), expid=expid, priority=0) await worker.analyze() except: logger.error("Failed to run '%s'", self.expurl) log_worker_exception() else: logger.info("Finished running '%s'", self.expurl) finally: await worker.close()
async def _scan(self, root, subdir=""): entry_dict = dict() for de in os.scandir(os.path.join(root, subdir)): if de.name.startswith("."): continue if de.is_file() and de.name.endswith(".py"): filename = os.path.join(subdir, de.name) try: await self.process_file(entry_dict, root, filename) except Exception as exc: logger.warning("Skipping file '%s'", filename, exc_info=not isinstance(exc, WorkerInternalException)) # restart worker await self.worker.close() self.worker = Worker(self.worker_handlers) if de.is_dir(): subentries = await self._scan( root, os.path.join(subdir, de.name)) entries = {de.name + "/" + k: v for k, v in subentries.items()} entry_dict.update(entries) return entry_dict
async def examine(self, file): worker = Worker(self.worker_handlers) try: return await worker.examine("examine", file) finally: await worker.close()