def cleanup(self): mgr = JobManager() mgr.baseUrl = self.config.baseUrl rJobs = mgr.getRunningJobs() if (len(rJobs) > 0): print "Cleaning %s running jobs..." % len(rJobs) for job in rJobs: mgr.setJobFinished(job[u'Id']) print "OK" cJobs = mgr.getCancelledJobs() if (len(cJobs) > 0): print "Cleaning %s cancelled jobs..." % len(cJobs) for job in cJobs: mgr.setJobFinished(job[u'Id']) print "OK"
def Do(self): mgr = JobManager() mgr.baseUrl = self.config.baseUrl # If processing not running, get next started job and run it. If no jobs started, just loop. if (self.runner == None): jobs = mgr.getStartedJobs() for job in jobs: mgr.setJobRunning(job[u'Id']) self.runner = Runner() self.runner.workDir = self.config.workDir self.runner.baseUrl = self.config.baseUrl self.runner.jobId = job[u'Id'] self.runner.tryRun() print "Processing started" break # spawn ONLY one process return # skip rest of the loop "while 1==1" # Here we already know that processing is running. Act according to that. self.runner.checkState() self.runner.showDebug() if (self.runner.isFinished()): #if (p.result == JobResult.ERROR): # print "Error occured: " + str(p.resultData) self.runner.parseLogs() # do it before exit self.runner.runZip() mgr.setJobFinished(self.runner.jobId) self.runner = None print "Processing finished" return if (mgr.isJobCancelled(self.runner.jobId)): self.runner.runZip() self.runner.cancel() print "Processing cancelled" return self.runner.parseLogs() print "Processing is still running"
import pointstreakscraper as pss from jobmanager import JobManager PS_2012_PLAYOFF = 18269 PS_2012_SEASON = 12252 if __name__ == "__main__": regular_season_ids = pss.scrape_season_gameids(PS_2012_SEASON) post_season_ids = pss.scrape_season_gameids(PS_2012_PLAYOFF) jm = JobManager("jobs.yml") jm.add_jobs(regular_season_ids, job_group="pointstreak") jm.add_jobs(post_season_ids, job_group="pointstreak") jm.save()
random.seed(11) # for reproductivity dest_cand = ['DOOR_(1,0)', 'DOOR_(4,0)', 'DOOR_(7,0)', 'DOOR_(10,0)','DOOR_(13,0)','DOOR_(16,0)', 'DOOR_(19,0)', 'DOOR_(22,0)','DOOR_(25,0)','DOOR_(28,0)'] for i in range(50): dest = random.choices(dest_cand,k=1)[0] start = 0 + random.randrange(0, 180) due = start + random.randrange(80, 160) orders.append(methods.test_input_generation_randomly(dest, due, start)) jm = JobManager(orders, ia.ItemAttb) wm = WorkerManager() wm.set_worker(Worker("WK_1","DOOR_(1,0)",1)) wm.set_worker(Worker("WK_2","DOOR_(7,0)",1)) wm.set_worker(Worker("WK_3","DOOR_(13,0)",1)) wm.set_worker(Worker("WK_4","DOOR_(19,0)",1)) wm.set_worker(Worker("WK_5","DOOR_(25,0)",1)) wh = Warehouse() wh.set_grid_layout(30,49) wh.set_doors([(1,0), (4,0), (7,0), (10,0), (13,0), (16,0), (19,0), (22,0), (25,0), (28,0)])
def _send_comm_message(self, msg_type, content): JobManager()._send_comm_message(msg_type, content)
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id, **kwargs): """ Attemps to run the app, returns a Job with the running app info. Should *hopefully* also inject that app into the Narrative's metadata. Probably need some kind of JavaScript-foo to get that to work. Parameters: ----------- app_id - should be from the app spec, e.g. 'build_a_metabolic_model' or 'MegaHit/run_megahit'. params - the dictionary of parameters. tag - optional, one of [release|beta|dev] (default=release) version - optional, a semantic version string. Only released modules have versions, so if the tag is not 'release', and a version is given, a ValueError will be raised. **kwargs - these are the set of parameters to be used with the app. They can be found by using the app_usage function. If any non-optional apps are missing, a ValueError will be raised. """ # TODO: this needs restructuring so that we can send back validation # failure messages. Perhaps a separate function and catch the errors, # or return an error structure. # Intro tests: self.spec_manager.check_app(app_id, tag, raise_exception=True) if version is not None and tag != "release": if re.match(r'\d+\.\d+\.\d+', version) is not None: raise ValueError( "Semantic versions only apply to released app modules. " + "You can use a Git commit hash instead to specify a " + "version.") # Get the spec & params spec = self.spec_manager.get_spec(app_id, tag) # There's some branching to do here. # Cases: # app has behavior.kb_service_input_mapping - valid long-running app. # app has behavior.output_mapping - not kb_service_input_mapping or # script_module - it's a viewer and should return immediately # app has other things besides kb_service_input_mapping - not valid. if 'behavior' not in spec: raise Exception("This app appears invalid - " + "it has no defined behavior") if 'kb_service_input_mapping' not in spec['behavior']: raise Exception("This app does not appear to be a long-running " + "job! Please use 'run_local_app' to start this " + "instead.") # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params) ws_id = system_variable('workspace_id') if ws_id is None: raise ValueError('Unable to retrive current ' + 'Narrative workspace information!') input_vals = self._map_inputs( spec['behavior']['kb_service_input_mapping'], params, spec_params_map) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. function_name = service_name + '.' + service_method job_meta = {'tag': tag} if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_meta['token_id'] = agent_token['id'] # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': function_name, 'service_ver': service_ver, 'params': input_vals, 'app_id': app_id, 'wsid': ws_id, 'meta': job_meta } if len(ws_input_refs) > 0: job_runner_inputs['source_ws_objects'] = ws_input_refs # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_app", log_info) try: job_id = clients.get("job_service").run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, app_id, input_vals, system_variable('user_id'), tag=tag, app_version=service_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id']) self._send_comm_message( 'run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) JobManager().register_new_job(new_job) if cell_id is not None: return else: return new_job
session = manager.init_database(use_mysql=options.apply) # track parsing success game ids def process_one(gameid): print gameid game = manager.import_game(gameid, session=session) # for event in game.events(): # session.add(event) # session.commit() if game is not None: games[game.game_id] = game if options.game is None: jm = JobManager("pending.yml") for gameid in jm.jobs("pointstreak"): try: process_one(gameid) jm.complete_job(gameid, "pointstreak") except: jm.set_job_status(gameid, "pointstreak", "error") finally: jm.save() else: process_one(options.game) #=============================================================================== # Report Summary #===============================================================================
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run): """ Attemps to run the app, returns a Job with the running app info. Should *hopefully* also inject that app into the Narrative's metadata. Probably need some kind of JavaScript-foo to get that to work. Parameters: ----------- app_id - should be from the app spec, e.g. 'build_a_metabolic_model' or 'MegaHit/run_megahit'. params - a dictionary of parameters. tag - optional, one of [release|beta|dev] (default=release) version - optional, a semantic version string. Only released modules have versions, so if the tag is not 'release', and a version is given, a ValueError will be raised. **kwargs - these are the set of parameters to be used with the app. They can be found by using the app_usage function. If any non-optional apps are missing, a ValueError will be raised. """ ws_id = strict_system_variable('workspace_id') spec = self._get_validated_app_spec(app_id, tag, True, version=version) # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params) input_vals = self._map_inputs( spec['behavior']['kb_service_input_mapping'], params, spec_params_map) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. function_name = service_name + '.' + service_method job_meta = {'tag': tag} if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': function_name, 'service_ver': service_ver, 'params': input_vals, 'app_id': app_id, 'wsid': ws_id, 'meta': job_meta } if len(ws_input_refs) > 0: job_runner_inputs['source_ws_objects'] = ws_input_refs if dry_run: return job_runner_inputs # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_runner_inputs['meta']['token_id'] = agent_token['id'] # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_app", log_info) try: job_id = clients.get( "job_service", token=agent_token['token']).run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, app_id, input_vals, system_variable('user_id'), tag=tag, app_version=service_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id']) self._send_comm_message( 'run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) JobManager().register_new_job(new_job) if cell_id is not None: return else: return new_job
def _run_app_batch_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run): batch_method = "kb_BatchApp.run_batch" batch_app_id = "kb_BatchApp/run_batch" batch_method_ver = "dev" batch_method_tag = "dev" ws_id = strict_system_variable('workspace_id') spec = self._get_validated_app_spec(app_id, tag, True, version=version) # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) # A list of lists of UPAs, used for each subjob. batch_ws_upas = list() # The list of actual input values, post-mapping. batch_run_inputs = list() for param_set in params: spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) batch_ws_upas.append( extract_ws_refs(app_id, tag, spec_params, param_set)) batch_run_inputs.append( self._map_inputs(spec['behavior']['kb_service_input_mapping'], param_set, spec_params_map)) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. job_meta = { 'tag': batch_method_tag, 'batch_app': app_id, 'batch_tag': tag, 'batch_size': len(params), } if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # Now put these all together in a way that can be sent to the batch processing app. batch_params = [{ "module_name": service_name, "method_name": service_method, "service_ver": service_ver, "wsid": ws_id, "meta": job_meta, "batch_params": [{ "params": batch_run_inputs[i], "source_ws_objects": batch_ws_upas[i] } for i in range(len(batch_run_inputs))], }] # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_meta['token_id'] = agent_token['id'] # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': batch_method, 'service_ver': batch_method_ver, 'params': batch_params, 'app_id': batch_app_id, 'wsid': ws_id, 'meta': job_meta } # if len(ws_input_refs) > 0: # job_runner_inputs['source_ws_objects'] = ws_input_refs # if we're doing a dry run, just return the inputs that we made. if dry_run: return job_runner_inputs # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': batch_method_tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_batch_app", log_info) try: job_id = clients.get( "job_service", token=agent_token['token']).run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_batch_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, batch_app_id, batch_params, system_variable('user_id'), tag=batch_method_tag, app_version=batch_method_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id'], meta=job_meta) self._send_comm_message( 'run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) JobManager().register_new_job(new_job) if cell_id is not None: return else: return new_job
requests_log = logging.getLogger("requests.packages.urllib3") requests_log.setLevel(logging.CRITICAL) logging.basicConfig(format="%(asctime)s:%(levelname)s: %(message)s") log = logging.getLogger() if args.v: log.setLevel(logging.DEBUG) if args.mode == 'daemon': log.info("Starting up daemon") statusTracker = StatusTracker(config) reactor.listenTCP(5000, server.Site(StatusSite(statusTracker))) reactor.listenTCP(5001, server.Site(PingSite(statusTracker)), interface='127.0.0.1') reactor.run() elif args.mode == 'cron': jobManager = JobManager(config) if not args.crontime: log.warn("Did not run cron, no crontime specified") parser.print_help() sys.exit(-1) else: log.info("Running cron at frequency " + args.crontime) try: if jobManager.execute_jobs(args.crontime): jobManager.mark_jobs_ran() else: jobManager.mark_jobs_ran_with_error() except Exception as e: logging.critical("Caught an exception trying to execute jobs:" + str(e)) jobManager.mark_jobs_ran_with_error()