def _scan_experiments(wd, log):
r = dict()
for f in os.listdir(wd):
if f.endswith(".py"):
try:
worker = Worker({"log": lambda message: log("scan", message)})
try:
description = yield from worker.examine(os.path.join(
wd, f))
finally:
yield from worker.close()
for class_name, class_desc in description.items():
name = class_desc["name"]
arguments = class_desc["arguments"]
if name in r:
logger.warning("Duplicate experiment name: '%s'", name)
basename = name
i = 1
while name in r:
name = basename + str(i)
i += 1
entry = {
"file": f,
"class_name": class_name,
"arguments": arguments
}
r[name] = entry
except:
logger.warning("Skipping file '%s'", f, exc_info=True)
return r
def __init__(self, rid, pipeline_name, wd, expid, priority, due_date,
flush, pool, **kwargs):
# called through pool
self.rid = rid
self.pipeline_name = pipeline_name
self.wd = wd
self.expid = expid
self.priority = priority
self.due_date = due_date
self.flush = flush
self.worker = Worker(pool.worker_handlers)
self._status = RunStatus.pending
notification = {
"pipeline": self.pipeline_name,
"expid": self.expid,
"priority": self.priority,
"due_date": self.due_date,
"flush": self.flush,
"status": self._status.name
}
notification.update(kwargs)
self._notifier = pool.notifier
self._notifier[self.rid] = notification
self._state_changed = pool.state_changed
def __init__(self, worker_handlers):
self.worker = Worker(worker_handlers)
self.next_rid = 0
self.queue = Notifier([])
self.queue_modified = asyncio.Event()
self.timed = Notifier(dict())
self.timed_modified = asyncio.Event()
def _scan_experiments(wd, log):
r = dict()
for f in os.listdir(wd):
if f.endswith(".py"):
try:
worker = Worker({"log": lambda message: log("scan", message)})
try:
description = yield from worker.examine(os.path.join(wd, f))
finally:
yield from worker.close()
for class_name, class_desc in description.items():
name = class_desc["name"]
arguments = class_desc["arguments"]
if name in r:
logger.warning("Duplicate experiment name: '%s'", name)
basename = name
i = 1
while name in r:
name = basename + str(i)
i += 1
entry = {
"file": f,
"class_name": class_name,
"arguments": arguments
}
r[name] = entry
except:
logger.warning("Skipping file '%s'", f, exc_info=True)
return r
async def scan(self, root):
self.worker = Worker(self.worker_handlers)
try:
r = await self._scan(root)
finally:
await self.worker.close()
return r
async def _get_repository_entries(entry_dict,
root, filename, worker_handlers):
worker = Worker(worker_handlers)
try:
description = await worker.examine("scan", os.path.join(root, filename))
except:
log_worker_exception()
raise
finally:
await worker.close()
for class_name, class_desc in description.items():
name = class_desc["name"]
arginfo = class_desc["arginfo"]
if "/" in name:
logger.warning("Character '/' is not allowed in experiment "
"name (%s)", name)
name = name.replace("/", "_")
if name in entry_dict:
basename = name
i = 1
while name in entry_dict:
name = basename + str(i)
i += 1
logger.warning("Duplicate experiment name: '%s'\n"
"Renaming class '%s' in '%s' to '%s'",
basename, class_name, filename, name)
entry = {
"file": filename,
"class_name": class_name,
"arginfo": arginfo
}
entry_dict[name] = entry
def __init__(self, rid, pipeline_name,
wd, expid, priority, due_date, flush,
pool, **kwargs):
# called through pool
self.rid = rid
self.pipeline_name = pipeline_name
self.wd = wd
self.expid = expid
self.priority = priority
self.due_date = due_date
self.flush = flush
self.worker = Worker(pool.worker_handlers)
self._status = RunStatus.pending
notification = {
"pipeline": self.pipeline_name,
"expid": self.expid,
"priority": self.priority,
"due_date": self.due_date,
"flush": self.flush,
"status": self._status.name
}
notification.update(kwargs)
self._notifier = pool.notifier
self._notifier[self.rid] = notification
self._state_changed = pool.state_changed
async def examine(self, filename, use_repository=True, revision=None):
if use_repository:
if revision is None:
revision = self.cur_rev
wd, _ = self.repo_backend.request_rev(revision)
filename = os.path.join(wd, filename)
worker = Worker(self.worker_handlers)
try:
description = await worker.examine("examine", filename)
finally:
await worker.close()
if use_repository:
self.repo_backend.release_rev(revision)
return description
async def _get_run_task(self, expid):
logger.info("Running '%s'...", self.expurl)
worker = Worker(self._area.worker_handlers)
try:
await worker.build(rid=None,
pipeline_name="browser",
wd=os.path.abspath("."),
expid=expid,
priority=0)
await worker.analyze()
except:
logger.error("Failed to run '%s'", self.expurl)
log_worker_exception()
else:
logger.info("Finished running '%s'", self.expurl)
finally:
await worker.close()
async def _scan(self, root, subdir=""):
entry_dict = dict()
for de in os.scandir(os.path.join(root, subdir)):
if de.name.startswith("."):
continue
if de.is_file() and de.name.endswith(".py"):
filename = os.path.join(subdir, de.name)
try:
await self.process_file(entry_dict, root, filename)
except Exception as exc:
logger.warning("Skipping file '%s'", filename,
exc_info=not isinstance(exc, WorkerInternalException))
# restart worker
await self.worker.close()
self.worker = Worker(self.worker_handlers)
if de.is_dir():
subentries = await self._scan(
root, os.path.join(subdir, de.name))
entries = {de.name + "/" + k: v for k, v in subentries.items()}
entry_dict.update(entries)
return entry_dict
async def examine(self, file):
worker = Worker(self.worker_handlers)
try:
return await worker.examine("examine", file)
finally:
await worker.close()
class Run:
def __init__(self, rid, pipeline_name, wd, expid, priority, due_date,
flush, pool, **kwargs):
# called through pool
self.rid = rid
self.pipeline_name = pipeline_name
self.wd = wd
self.expid = expid
self.priority = priority
self.due_date = due_date
self.flush = flush
self.worker = Worker(pool.worker_handlers)
self._status = RunStatus.pending
notification = {
"pipeline": self.pipeline_name,
"expid": self.expid,
"priority": self.priority,
"due_date": self.due_date,
"flush": self.flush,
"status": self._status.name
}
notification.update(kwargs)
self._notifier = pool.notifier
self._notifier[self.rid] = notification
self._state_changed = pool.state_changed
@property
def status(self):
return self._status
@status.setter
def status(self, value):
self._status = value
if not self.worker.closed.is_set():
self._notifier[self.rid]["status"] = self._status.name
self._state_changed.notify()
# The run with the largest priority_key is to be scheduled first
def priority_key(self, now=None):
if self.due_date is None:
due_date_k = 0
else:
due_date_k = -self.due_date
if now is not None and self.due_date is not None:
runnable = int(now > self.due_date)
else:
runnable = 1
return (runnable, self.priority, due_date_k, -self.rid)
@asyncio.coroutine
def close(self):
# called through pool
yield from self.worker.close()
del self._notifier[self.rid]
_build = _mk_worker_method("build")
@asyncio.coroutine
def build(self):
yield from self._build(self.rid, self.pipeline_name, self.wd,
self.expid, self.priority)
prepare = _mk_worker_method("prepare")
run = _mk_worker_method("run")
resume = _mk_worker_method("resume")
analyze = _mk_worker_method("analyze")
write_results = _mk_worker_method("write_results")
class Scheduler:
def __init__(self, worker_handlers, run_cb):
self.run_cb = run_cb
self.worker = Worker(worker_handlers)
self.next_rid = 0
self.queue = Notifier([])
self.queue_modified = asyncio.Event()
self.timed = Notifier(dict())
self.timed_modified = asyncio.Event()
def new_rid(self):
r = self.next_rid
self.next_rid += 1
return r
def new_trid(self):
trids = set(range(len(self.timed.read) + 1))
trids -= set(self.timed.read.keys())
return next(iter(trids))
@asyncio.coroutine
def start(self):
self.task = asyncio.Task(self._schedule())
yield from self.worker.create_process()
@asyncio.coroutine
def stop(self):
self.task.cancel()
yield from asyncio.wait([self.task])
del self.task
yield from self.worker.end_process()
def run_queued(self, run_params, timeout):
rid = self.new_rid()
self.queue.append((rid, run_params, timeout))
self.queue_modified.set()
return rid
def cancel_queued(self, rid):
idx = next(idx for idx, (qrid, _, _) in enumerate(self.queue.read)
if qrid == rid)
if idx == 0:
# Cannot cancel when already running
raise NotImplementedError
del self.queue[idx]
def run_timed(self, run_params, timeout, next_run):
if next_run is None:
next_run = time()
trid = self.new_trid()
self.timed[trid] = next_run, run_params, timeout
self.timed_modified.set()
return trid
def cancel_timed(self, trid):
del self.timed[trid]
@asyncio.coroutine
def _run(self, rid, run_params, timeout):
self.run_cb(rid, run_params)
try:
yield from self.worker.run(run_params, timeout)
except Exception as e:
print("RID {} failed:".format(rid))
print(e)
else:
print("RID {} completed successfully".format(rid))
@asyncio.coroutine
def _run_timed(self):
while True:
min_next_run = None
min_trid = None
for trid, params in self.timed.read.items():
if min_next_run is None or params[0] < min_next_run:
min_next_run = params[0]
min_trid = trid
now = time()
if min_next_run is None:
return None
min_next_run -= now
if min_next_run > 0:
return min_next_run
next_run, run_params, timeout = self.timed.read[min_trid]
del self.timed[min_trid]
rid = self.new_rid()
self.queue.insert(0, (rid, run_params, timeout))
yield from self._run(rid, run_params, timeout)
del self.queue[0]
@asyncio.coroutine
def _schedule(self):
while True:
next_timed = yield from self._run_timed()
if self.queue.read:
rid, run_params, timeout = self.queue.read[0]
yield from self._run(rid, run_params, timeout)
del self.queue[0]
else:
self.queue_modified.clear()
self.timed_modified.clear()
t1 = asyncio.Task(self.queue_modified.wait())
t2 = asyncio.Task(self.timed_modified.wait())
try:
done, pend = yield from asyncio.wait(
[t1, t2],
timeout=next_timed,
return_when=asyncio.FIRST_COMPLETED)
except:
t1.cancel()
t2.cancel()
raise
for t in pend:
t.cancel()
class Run:
def __init__(self, rid, pipeline_name,
wd, expid, priority, due_date, flush,
pool, **kwargs):
# called through pool
self.rid = rid
self.pipeline_name = pipeline_name
self.wd = wd
self.expid = expid
self.priority = priority
self.due_date = due_date
self.flush = flush
self.worker = Worker(pool.worker_handlers)
self._status = RunStatus.pending
notification = {
"pipeline": self.pipeline_name,
"expid": self.expid,
"priority": self.priority,
"due_date": self.due_date,
"flush": self.flush,
"status": self._status.name
}
notification.update(kwargs)
self._notifier = pool.notifier
self._notifier[self.rid] = notification
self._state_changed = pool.state_changed
@property
def status(self):
return self._status
@status.setter
def status(self, value):
self._status = value
if not self.worker.closed.is_set():
self._notifier[self.rid]["status"] = self._status.name
self._state_changed.notify()
# The run with the largest priority_key is to be scheduled first
def priority_key(self, now=None):
if self.due_date is None:
due_date_k = 0
else:
due_date_k = -self.due_date
if now is not None and self.due_date is not None:
runnable = int(now > self.due_date)
else:
runnable = 1
return (runnable, self.priority, due_date_k, -self.rid)
@asyncio.coroutine
def close(self):
# called through pool
yield from self.worker.close()
del self._notifier[self.rid]
_build = _mk_worker_method("build")
@asyncio.coroutine
def build(self):
yield from self._build(self.rid, self.pipeline_name,
self.wd, self.expid,
self.priority)
prepare = _mk_worker_method("prepare")
run = _mk_worker_method("run")
resume = _mk_worker_method("resume")
analyze = _mk_worker_method("analyze")
write_results = _mk_worker_method("write_results")