Beispiel #1
0
 async def process_file(self, entry_dict, root, filename):
     logger.debug("processing file %s %s", root, filename)
     try:
         description = await self.worker.examine(
             "scan", os.path.join(root, filename))
     except:
         log_worker_exception()
         raise
     for class_name, class_desc in description.items():
         name = class_desc["name"]
         arginfo = class_desc["arginfo"]
         if "/" in name:
             logger.warning(
                 "Character '/' is not allowed in experiment "
                 "name (%s)", name)
             name = name.replace("/", "_")
         if name in entry_dict:
             basename = name
             i = 1
             while name in entry_dict:
                 name = basename + str(i)
                 i += 1
             logger.warning(
                 "Duplicate experiment name: '%s'\n"
                 "Renaming class '%s' in '%s' to '%s'", basename,
                 class_name, filename, name)
         entry = {
             "file": filename,
             "class_name": class_name,
             "arginfo": arginfo
         }
         entry_dict[name] = entry
Beispiel #2
0
 async def _do(self):
     while True:
         run = self._get_run()
         if run is None:
             await self.pool.state_changed.wait()
         elif isinstance(run, float):
             await asyncio_wait_or_cancel([self.pool.state_changed.wait()],
                                               timeout=run)
         else:
             if run.flush:
                 run.status = RunStatus.flushing
                 while not all(r.status in (RunStatus.pending,
                                            RunStatus.deleting)
                               or r is run
                               for r in self.pool.runs.values()):
                     ev = [self.pool.state_changed.wait(),
                           run.worker.closed.wait()]
                     await asyncio_wait_or_cancel(
                         ev, return_when=asyncio.FIRST_COMPLETED)
                     if run.worker.closed.is_set():
                         break
                 if run.worker.closed.is_set():
                         continue
             run.status = RunStatus.preparing
             try:
                 await run.build()
                 await run.prepare()
             except:
                 logger.error("got worker exception in prepare stage, "
                              "deleting RID %d", run.rid)
                 log_worker_exception()
                 self.delete_cb(run.rid)
             else:
                 run.status = RunStatus.prepare_done
Beispiel #3
0
 async def process_file(self, entry_dict, root, filename):
     logger.debug("processing file %s %s", root, filename)
     try:
         description = await self.worker.examine(
             "scan", os.path.join(root, filename))
     except:
         log_worker_exception()
         raise
     for class_name, class_desc in description.items():
         name = class_desc["name"]
         arginfo = class_desc["arginfo"]
         if "/" in name:
             logger.warning("Character '/' is not allowed in experiment "
                            "name (%s)", name)
             name = name.replace("/", "_")
         if name in entry_dict:
             basename = name
             i = 1
             while name in entry_dict:
                 name = basename + str(i)
                 i += 1
             logger.warning("Duplicate experiment name: '%s'\n"
                            "Renaming class '%s' in '%s' to '%s'",
                            basename, class_name, filename, name)
         entry = {
             "file": filename,
             "class_name": class_name,
             "arginfo": arginfo,
             "scheduler_defaults": class_desc["scheduler_defaults"]
         }
         entry_dict[name] = entry
Beispiel #4
0
 async def _do(self):
     while True:
         run = self._get_run()
         if run is None:
             await self.pool.state_changed.wait()
         elif isinstance(run, float):
             await asyncio_wait_or_cancel([self.pool.state_changed.wait()],
                                          timeout=run)
         else:
             if run.flush:
                 run.status = RunStatus.flushing
                 while not all(r.status in (RunStatus.pending,
                                            RunStatus.deleting)
                               or r.priority < run.priority
                               or r is run
                               for r in self.pool.runs.values()):
                     ev = [self.pool.state_changed.wait(),
                           run.worker.closed.wait()]
                     await asyncio_wait_or_cancel(
                         ev, return_when=asyncio.FIRST_COMPLETED)
                     if run.worker.closed.is_set():
                         break
                 if run.worker.closed.is_set():
                     continue
             run.status = RunStatus.preparing
             try:
                 await run.build()
                 await run.prepare()
             except:
                 logger.error("got worker exception in prepare stage, "
                              "deleting RID %d", run.rid)
                 log_worker_exception()
                 self.delete_cb(run.rid)
             else:
                 run.status = RunStatus.prepare_done
Beispiel #5
0
async def _get_repository_entries(entry_dict,
                                  root, filename, worker_handlers):
    worker = Worker(worker_handlers)
    try:
        description = await worker.examine("scan", os.path.join(root, filename))
    except:
        log_worker_exception()
        raise
    finally:
        await worker.close()
    for class_name, class_desc in description.items():
        name = class_desc["name"]
        arginfo = class_desc["arginfo"]
        if "/" in name:
            logger.warning("Character '/' is not allowed in experiment "
                           "name (%s)", name)
            name = name.replace("/", "_")
        if name in entry_dict:
            logger.warning("Duplicate experiment name: '%s'", name)
            basename = name
            i = 1
            while name in entry_dict:
                name = basename + str(i)
                i += 1
        entry = {
            "file": filename,
            "class_name": class_name,
            "arginfo": arginfo
        }
        entry_dict[name] = entry
Beispiel #6
0
async def _get_repository_entries(entry_dict,
                                  root, filename, worker_handlers):
    worker = Worker(worker_handlers)
    try:
        description = await worker.examine("scan", os.path.join(root, filename))
    except:
        log_worker_exception()
        raise
    finally:
        await worker.close()
    for class_name, class_desc in description.items():
        name = class_desc["name"]
        arginfo = class_desc["arginfo"]
        if "/" in name:
            logger.warning("Character '/' is not allowed in experiment "
                           "name (%s)", name)
            name = name.replace("/", "_")
        if name in entry_dict:
            basename = name
            i = 1
            while name in entry_dict:
                name = basename + str(i)
                i += 1
            logger.warning("Duplicate experiment name: '%s'\n"
                           "Renaming class '%s' in '%s' to '%s'",
                           basename, class_name, filename, name)
        entry = {
            "file": filename,
            "class_name": class_name,
            "arginfo": arginfo
        }
        entry_dict[name] = entry
Beispiel #7
0
 async def _get_run_task(self, expid):
     worker = Worker(self._area.worker_handlers)
     try:
         await worker.build(rid=None, pipeline_name="browser",
                            wd=os.path.abspath("."),
                            expid=expid, priority=0)
         await worker.analyze()
     except:
         logger.error("Failed to run '%s'", self.expurl)
         log_worker_exception()
     finally:
         await worker.close()
Beispiel #8
0
 async def _do(self):
     while True:
         run = self._get_run()
         while run is None:
             await self.pool.state_changed.wait()
             run = self._get_run()
         run.status = RunStatus.analyzing
         try:
             await run.analyze()
         except:
             logger.error("got worker exception in analyze stage of RID %d.",
                          run.rid)
             log_worker_exception()
         self.delete_cb(run.rid)
Beispiel #9
0
 async def _do(self):
     while True:
         run = self._get_run()
         while run is None:
             await self.pool.state_changed.wait()
             run = self._get_run()
         run.status = RunStatus.analyzing
         try:
             await run.analyze()
             await run.write_results()
         except:
             logger.error("got worker exception in analyze stage, "
                          "deleting RID %d", run.rid)
             log_worker_exception()
             self.delete_cb(run.rid)
         else:
             self.delete_cb(run.rid)
Beispiel #10
0
 async def _get_run_task(self, expid):
     logger.info("Running '%s'...", self.expurl)
     worker = Worker(self._area.worker_handlers)
     try:
         await worker.build(rid=None,
                            pipeline_name="browser",
                            wd=os.path.abspath("."),
                            expid=expid,
                            priority=0)
         await worker.analyze()
     except:
         logger.error("Failed to run '%s'", self.expurl)
         log_worker_exception()
     else:
         logger.info("Finished running '%s'", self.expurl)
     finally:
         await worker.close()
Beispiel #11
0
 async def _do(self):
     while True:
         run = self._get_run()
         while run is None:
             await self.pool.state_changed.wait()
             run = self._get_run()
         run.status = RunStatus.analyzing
         try:
             await run.analyze()
         except:
             logger.error("got worker exception in analyze stage of RID %d."
                          " Results will still be saved.", run.rid)
             log_worker_exception()
         try:
             await run.write_results()
         except:
             logger.error("failed to write results of RID %d.", run.rid)
             log_worker_exception()
         self.delete_cb(run.rid)
Beispiel #12
0
    async def _do(self):
        stack = []

        while True:
            next_irun = self._get_run()
            if not stack or (
                    next_irun is not None
                    and next_irun.priority_key() > stack[-1].priority_key()):
                while next_irun is None:
                    await self.pool.state_changed.wait()
                    next_irun = self._get_run()
                stack.append(next_irun)

            run = stack.pop()
            try:
                if run.status == RunStatus.paused:
                    run.status = RunStatus.running
                    # clear "termination requested" flag now
                    # so that if it is set again during the resume, this
                    # results in another exception.
                    request_termination = run.termination_requested
                    run.termination_requested = False
                    completed = await run.resume(request_termination)
                else:
                    run.status = RunStatus.running
                    completed = await run.run()
            except:
                logger.error(
                    "got worker exception in run stage, "
                    "deleting RID %d", run.rid)
                log_worker_exception()
                self.delete_cb(run.rid)
            else:
                if completed:
                    run.status = RunStatus.run_done
                else:
                    run.status = RunStatus.paused
                    stack.append(run)
Beispiel #13
0
    async def _do(self):
        stack = []

        while True:
            next_irun = self._get_run()
            if not stack or (
                    next_irun is not None and
                    next_irun.priority_key() > stack[-1].priority_key()):
                while next_irun is None:
                    await self.pool.state_changed.wait()
                    next_irun = self._get_run()
                stack.append(next_irun)

            run = stack.pop()
            try:
                if run.status == RunStatus.paused:
                    run.status = RunStatus.running
                    # clear "termination requested" flag now
                    # so that if it is set again during the resume, this
                    # results in another exception.
                    request_termination = run.termination_requested
                    run.termination_requested = False
                    completed = await run.resume(request_termination)
                else:
                    run.status = RunStatus.running
                    completed = await run.run()
            except:
                logger.error("got worker exception in run stage, "
                             "deleting RID %d", run.rid)
                log_worker_exception()
                self.delete_cb(run.rid)
            else:
                if completed:
                    run.status = RunStatus.run_done
                else:
                    run.status = RunStatus.paused
                    stack.append(run)