Esempio n. 1
0
def _scan_experiments(wd, log):
    r = dict()
    for f in os.listdir(wd):
        if f.endswith(".py"):
            try:
                worker = Worker({"log": lambda message: log("scan", message)})
                try:
                    description = yield from worker.examine(os.path.join(
                        wd, f))
                finally:
                    yield from worker.close()
                for class_name, class_desc in description.items():
                    name = class_desc["name"]
                    arguments = class_desc["arguments"]
                    if name in r:
                        logger.warning("Duplicate experiment name: '%s'", name)
                        basename = name
                        i = 1
                        while name in r:
                            name = basename + str(i)
                            i += 1
                    entry = {
                        "file": f,
                        "class_name": class_name,
                        "arguments": arguments
                    }
                    r[name] = entry
            except:
                logger.warning("Skipping file '%s'", f, exc_info=True)
    return r
Esempio n. 2
0
    def __init__(self, rid, pipeline_name, wd, expid, priority, due_date,
                 flush, pool, **kwargs):
        # called through pool
        self.rid = rid
        self.pipeline_name = pipeline_name
        self.wd = wd
        self.expid = expid
        self.priority = priority
        self.due_date = due_date
        self.flush = flush

        self.worker = Worker(pool.worker_handlers)

        self._status = RunStatus.pending

        notification = {
            "pipeline": self.pipeline_name,
            "expid": self.expid,
            "priority": self.priority,
            "due_date": self.due_date,
            "flush": self.flush,
            "status": self._status.name
        }
        notification.update(kwargs)
        self._notifier = pool.notifier
        self._notifier[self.rid] = notification
        self._state_changed = pool.state_changed
Esempio n. 3
0
 def __init__(self, worker_handlers):
     self.worker = Worker(worker_handlers)
     self.next_rid = 0
     self.queue = Notifier([])
     self.queue_modified = asyncio.Event()
     self.timed = Notifier(dict())
     self.timed_modified = asyncio.Event()
Esempio n. 4
0
def _scan_experiments(wd, log):
    r = dict()
    for f in os.listdir(wd):
        if f.endswith(".py"):
            try:
                worker = Worker({"log": lambda message: log("scan", message)})
                try:
                    description = yield from worker.examine(os.path.join(wd, f))
                finally:
                    yield from worker.close()
                for class_name, class_desc in description.items():
                    name = class_desc["name"]
                    arguments = class_desc["arguments"]
                    if name in r:
                        logger.warning("Duplicate experiment name: '%s'", name)
                        basename = name
                        i = 1
                        while name in r:
                            name = basename + str(i)
                            i += 1
                    entry = {
                        "file": f,
                        "class_name": class_name,
                        "arguments": arguments
                    }
                    r[name] = entry
            except:
                logger.warning("Skipping file '%s'", f, exc_info=True)
    return r
Esempio n. 5
0
 async def scan(self, root):
     self.worker = Worker(self.worker_handlers)
     try:
         r = await self._scan(root)
     finally:
         await self.worker.close()
     return r
Esempio n. 6
0
async def _get_repository_entries(entry_dict,
                                  root, filename, worker_handlers):
    worker = Worker(worker_handlers)
    try:
        description = await worker.examine("scan", os.path.join(root, filename))
    except:
        log_worker_exception()
        raise
    finally:
        await worker.close()
    for class_name, class_desc in description.items():
        name = class_desc["name"]
        arginfo = class_desc["arginfo"]
        if "/" in name:
            logger.warning("Character '/' is not allowed in experiment "
                           "name (%s)", name)
            name = name.replace("/", "_")
        if name in entry_dict:
            basename = name
            i = 1
            while name in entry_dict:
                name = basename + str(i)
                i += 1
            logger.warning("Duplicate experiment name: '%s'\n"
                           "Renaming class '%s' in '%s' to '%s'",
                           basename, class_name, filename, name)
        entry = {
            "file": filename,
            "class_name": class_name,
            "arginfo": arginfo
        }
        entry_dict[name] = entry
Esempio n. 7
0
    def __init__(self, rid, pipeline_name,
                 wd, expid, priority, due_date, flush,
                 pool, **kwargs):
        # called through pool
        self.rid = rid
        self.pipeline_name = pipeline_name
        self.wd = wd
        self.expid = expid
        self.priority = priority
        self.due_date = due_date
        self.flush = flush

        self.worker = Worker(pool.worker_handlers)

        self._status = RunStatus.pending

        notification = {
            "pipeline": self.pipeline_name,
            "expid": self.expid,
            "priority": self.priority,
            "due_date": self.due_date,
            "flush": self.flush,
            "status": self._status.name
        }
        notification.update(kwargs)
        self._notifier = pool.notifier
        self._notifier[self.rid] = notification
        self._state_changed = pool.state_changed
Esempio n. 8
0
 async def examine(self, filename, use_repository=True, revision=None):
     if use_repository:
         if revision is None:
             revision = self.cur_rev
         wd, _ = self.repo_backend.request_rev(revision)
         filename = os.path.join(wd, filename)
     worker = Worker(self.worker_handlers)
     try:
         description = await worker.examine("examine", filename)
     finally:
         await worker.close()
     if use_repository:
         self.repo_backend.release_rev(revision)
     return description
Esempio n. 9
0
 async def _get_run_task(self, expid):
     logger.info("Running '%s'...", self.expurl)
     worker = Worker(self._area.worker_handlers)
     try:
         await worker.build(rid=None,
                            pipeline_name="browser",
                            wd=os.path.abspath("."),
                            expid=expid,
                            priority=0)
         await worker.analyze()
     except:
         logger.error("Failed to run '%s'", self.expurl)
         log_worker_exception()
     else:
         logger.info("Finished running '%s'", self.expurl)
     finally:
         await worker.close()
Esempio n. 10
0
 async def _scan(self, root, subdir=""):
     entry_dict = dict()
     for de in os.scandir(os.path.join(root, subdir)):
         if de.name.startswith("."):
             continue
         if de.is_file() and de.name.endswith(".py"):
             filename = os.path.join(subdir, de.name)
             try:
                 await self.process_file(entry_dict, root, filename)
             except Exception as exc:
                 logger.warning("Skipping file '%s'", filename,
                     exc_info=not isinstance(exc, WorkerInternalException))
                 # restart worker
                 await self.worker.close()
                 self.worker = Worker(self.worker_handlers)
         if de.is_dir():
             subentries = await self._scan(
                 root, os.path.join(subdir, de.name))
             entries = {de.name + "/" + k: v for k, v in subentries.items()}
             entry_dict.update(entries)
     return entry_dict
Esempio n. 11
0
 async def examine(self, file):
     worker = Worker(self.worker_handlers)
     try:
         return await worker.examine("examine", file)
     finally:
         await worker.close()
Esempio n. 12
0
class Run:
    def __init__(self, rid, pipeline_name, wd, expid, priority, due_date,
                 flush, pool, **kwargs):
        # called through pool
        self.rid = rid
        self.pipeline_name = pipeline_name
        self.wd = wd
        self.expid = expid
        self.priority = priority
        self.due_date = due_date
        self.flush = flush

        self.worker = Worker(pool.worker_handlers)

        self._status = RunStatus.pending

        notification = {
            "pipeline": self.pipeline_name,
            "expid": self.expid,
            "priority": self.priority,
            "due_date": self.due_date,
            "flush": self.flush,
            "status": self._status.name
        }
        notification.update(kwargs)
        self._notifier = pool.notifier
        self._notifier[self.rid] = notification
        self._state_changed = pool.state_changed

    @property
    def status(self):
        return self._status

    @status.setter
    def status(self, value):
        self._status = value
        if not self.worker.closed.is_set():
            self._notifier[self.rid]["status"] = self._status.name
        self._state_changed.notify()

    # The run with the largest priority_key is to be scheduled first
    def priority_key(self, now=None):
        if self.due_date is None:
            due_date_k = 0
        else:
            due_date_k = -self.due_date
        if now is not None and self.due_date is not None:
            runnable = int(now > self.due_date)
        else:
            runnable = 1
        return (runnable, self.priority, due_date_k, -self.rid)

    @asyncio.coroutine
    def close(self):
        # called through pool
        yield from self.worker.close()
        del self._notifier[self.rid]

    _build = _mk_worker_method("build")

    @asyncio.coroutine
    def build(self):
        yield from self._build(self.rid, self.pipeline_name, self.wd,
                               self.expid, self.priority)

    prepare = _mk_worker_method("prepare")
    run = _mk_worker_method("run")
    resume = _mk_worker_method("resume")
    analyze = _mk_worker_method("analyze")
    write_results = _mk_worker_method("write_results")
Esempio n. 13
0
class Scheduler:
    def __init__(self, worker_handlers, run_cb):
        self.run_cb = run_cb
        self.worker = Worker(worker_handlers)
        self.next_rid = 0
        self.queue = Notifier([])
        self.queue_modified = asyncio.Event()
        self.timed = Notifier(dict())
        self.timed_modified = asyncio.Event()

    def new_rid(self):
        r = self.next_rid
        self.next_rid += 1
        return r

    def new_trid(self):
        trids = set(range(len(self.timed.read) + 1))
        trids -= set(self.timed.read.keys())
        return next(iter(trids))

    @asyncio.coroutine
    def start(self):
        self.task = asyncio.Task(self._schedule())
        yield from self.worker.create_process()

    @asyncio.coroutine
    def stop(self):
        self.task.cancel()
        yield from asyncio.wait([self.task])
        del self.task
        yield from self.worker.end_process()

    def run_queued(self, run_params, timeout):
        rid = self.new_rid()
        self.queue.append((rid, run_params, timeout))
        self.queue_modified.set()
        return rid

    def cancel_queued(self, rid):
        idx = next(idx for idx, (qrid, _, _) in enumerate(self.queue.read)
                   if qrid == rid)
        if idx == 0:
            # Cannot cancel when already running
            raise NotImplementedError
        del self.queue[idx]

    def run_timed(self, run_params, timeout, next_run):
        if next_run is None:
            next_run = time()
        trid = self.new_trid()
        self.timed[trid] = next_run, run_params, timeout
        self.timed_modified.set()
        return trid

    def cancel_timed(self, trid):
        del self.timed[trid]

    @asyncio.coroutine
    def _run(self, rid, run_params, timeout):
        self.run_cb(rid, run_params)
        try:
            yield from self.worker.run(run_params, timeout)
        except Exception as e:
            print("RID {} failed:".format(rid))
            print(e)
        else:
            print("RID {} completed successfully".format(rid))

    @asyncio.coroutine
    def _run_timed(self):
        while True:
            min_next_run = None
            min_trid = None
            for trid, params in self.timed.read.items():
                if min_next_run is None or params[0] < min_next_run:
                    min_next_run = params[0]
                    min_trid = trid

            now = time()

            if min_next_run is None:
                return None
            min_next_run -= now
            if min_next_run > 0:
                return min_next_run

            next_run, run_params, timeout = self.timed.read[min_trid]
            del self.timed[min_trid]

            rid = self.new_rid()
            self.queue.insert(0, (rid, run_params, timeout))
            yield from self._run(rid, run_params, timeout)
            del self.queue[0]

    @asyncio.coroutine
    def _schedule(self):
        while True:
            next_timed = yield from self._run_timed()
            if self.queue.read:
                rid, run_params, timeout = self.queue.read[0]
                yield from self._run(rid, run_params, timeout)
                del self.queue[0]
            else:
                self.queue_modified.clear()
                self.timed_modified.clear()
                t1 = asyncio.Task(self.queue_modified.wait())
                t2 = asyncio.Task(self.timed_modified.wait())
                try:
                    done, pend = yield from asyncio.wait(
                        [t1, t2],
                        timeout=next_timed,
                        return_when=asyncio.FIRST_COMPLETED)
                except:
                    t1.cancel()
                    t2.cancel()
                    raise
                for t in pend:
                    t.cancel()
Esempio n. 14
0
class Run:
    def __init__(self, rid, pipeline_name,
                 wd, expid, priority, due_date, flush,
                 pool, **kwargs):
        # called through pool
        self.rid = rid
        self.pipeline_name = pipeline_name
        self.wd = wd
        self.expid = expid
        self.priority = priority
        self.due_date = due_date
        self.flush = flush

        self.worker = Worker(pool.worker_handlers)

        self._status = RunStatus.pending

        notification = {
            "pipeline": self.pipeline_name,
            "expid": self.expid,
            "priority": self.priority,
            "due_date": self.due_date,
            "flush": self.flush,
            "status": self._status.name
        }
        notification.update(kwargs)
        self._notifier = pool.notifier
        self._notifier[self.rid] = notification
        self._state_changed = pool.state_changed

    @property
    def status(self):
        return self._status

    @status.setter
    def status(self, value):
        self._status = value
        if not self.worker.closed.is_set():
            self._notifier[self.rid]["status"] = self._status.name
        self._state_changed.notify()

    # The run with the largest priority_key is to be scheduled first
    def priority_key(self, now=None):
        if self.due_date is None:
            due_date_k = 0
        else:
            due_date_k = -self.due_date
        if now is not None and self.due_date is not None:
            runnable = int(now > self.due_date)
        else:
            runnable = 1
        return (runnable, self.priority, due_date_k, -self.rid)

    @asyncio.coroutine
    def close(self):
        # called through pool
        yield from self.worker.close()
        del self._notifier[self.rid]

    _build = _mk_worker_method("build")

    @asyncio.coroutine
    def build(self):
        yield from self._build(self.rid, self.pipeline_name,
                               self.wd, self.expid,
                               self.priority)

    prepare = _mk_worker_method("prepare")
    run = _mk_worker_method("run")
    resume = _mk_worker_method("resume")
    analyze = _mk_worker_method("analyze")
    write_results = _mk_worker_method("write_results")