def _scan_experiments(wd, log):
r = dict()
for f in os.listdir(wd):
if f.endswith(".py"):
try:
worker = Worker({"log": lambda message: log("scan", message)})
try:
description = yield from worker.examine(os.path.join(
wd, f))
finally:
yield from worker.close()
for class_name, class_desc in description.items():
name = class_desc["name"]
arguments = class_desc["arguments"]
if name in r:
logger.warning("Duplicate experiment name: '%s'", name)
basename = name
i = 1
while name in r:
name = basename + str(i)
i += 1
entry = {
"file": f,
"class_name": class_name,
"arguments": arguments
}
r[name] = entry
except:
logger.warning("Skipping file '%s'", f, exc_info=True)
return r
def _scan_experiments(wd, log):
r = dict()
for f in os.listdir(wd):
if f.endswith(".py"):
try:
worker = Worker({"log": lambda message: log("scan", message)})
try:
description = yield from worker.examine(os.path.join(wd, f))
finally:
yield from worker.close()
for class_name, class_desc in description.items():
name = class_desc["name"]
arguments = class_desc["arguments"]
if name in r:
logger.warning("Duplicate experiment name: '%s'", name)
basename = name
i = 1
while name in r:
name = basename + str(i)
i += 1
entry = {
"file": f,
"class_name": class_name,
"arguments": arguments
}
r[name] = entry
except:
logger.warning("Skipping file '%s'", f, exc_info=True)
return r
class Run:
def __init__(self, rid, pipeline_name, wd, expid, priority, due_date,
flush, pool, **kwargs):
# called through pool
self.rid = rid
self.pipeline_name = pipeline_name
self.wd = wd
self.expid = expid
self.priority = priority
self.due_date = due_date
self.flush = flush
self.worker = Worker(pool.worker_handlers)
self._status = RunStatus.pending
notification = {
"pipeline": self.pipeline_name,
"expid": self.expid,
"priority": self.priority,
"due_date": self.due_date,
"flush": self.flush,
"status": self._status.name
}
notification.update(kwargs)
self._notifier = pool.notifier
self._notifier[self.rid] = notification
self._state_changed = pool.state_changed
@property
def status(self):
return self._status
@status.setter
def status(self, value):
self._status = value
if not self.worker.closed.is_set():
self._notifier[self.rid]["status"] = self._status.name
self._state_changed.notify()
# The run with the largest priority_key is to be scheduled first
def priority_key(self, now=None):
if self.due_date is None:
due_date_k = 0
else:
due_date_k = -self.due_date
if now is not None and self.due_date is not None:
runnable = int(now > self.due_date)
else:
runnable = 1
return (runnable, self.priority, due_date_k, -self.rid)
@asyncio.coroutine
def close(self):
# called through pool
yield from self.worker.close()
del self._notifier[self.rid]
_build = _mk_worker_method("build")
@asyncio.coroutine
def build(self):
yield from self._build(self.rid, self.pipeline_name, self.wd,
self.expid, self.priority)
prepare = _mk_worker_method("prepare")
run = _mk_worker_method("run")
resume = _mk_worker_method("resume")
analyze = _mk_worker_method("analyze")
write_results = _mk_worker_method("write_results")
class Run:
def __init__(self, rid, pipeline_name,
wd, expid, priority, due_date, flush,
pool, **kwargs):
# called through pool
self.rid = rid
self.pipeline_name = pipeline_name
self.wd = wd
self.expid = expid
self.priority = priority
self.due_date = due_date
self.flush = flush
self.worker = Worker(pool.worker_handlers)
self._status = RunStatus.pending
notification = {
"pipeline": self.pipeline_name,
"expid": self.expid,
"priority": self.priority,
"due_date": self.due_date,
"flush": self.flush,
"status": self._status.name
}
notification.update(kwargs)
self._notifier = pool.notifier
self._notifier[self.rid] = notification
self._state_changed = pool.state_changed
@property
def status(self):
return self._status
@status.setter
def status(self, value):
self._status = value
if not self.worker.closed.is_set():
self._notifier[self.rid]["status"] = self._status.name
self._state_changed.notify()
# The run with the largest priority_key is to be scheduled first
def priority_key(self, now=None):
if self.due_date is None:
due_date_k = 0
else:
due_date_k = -self.due_date
if now is not None and self.due_date is not None:
runnable = int(now > self.due_date)
else:
runnable = 1
return (runnable, self.priority, due_date_k, -self.rid)
@asyncio.coroutine
def close(self):
# called through pool
yield from self.worker.close()
del self._notifier[self.rid]
_build = _mk_worker_method("build")
@asyncio.coroutine
def build(self):
yield from self._build(self.rid, self.pipeline_name,
self.wd, self.expid,
self.priority)
prepare = _mk_worker_method("prepare")
run = _mk_worker_method("run")
resume = _mk_worker_method("resume")
analyze = _mk_worker_method("analyze")
write_results = _mk_worker_method("write_results")
