def cleanup(self):
mgr = JobManager()
mgr.baseUrl = self.config.baseUrl
rJobs = mgr.getRunningJobs()
if (len(rJobs) > 0):
print "Cleaning %s running jobs..." % len(rJobs)
for job in rJobs:
mgr.setJobFinished(job[u'Id'])
print "OK"
cJobs = mgr.getCancelledJobs()
if (len(cJobs) > 0):
print "Cleaning %s cancelled jobs..." % len(cJobs)
for job in cJobs:
mgr.setJobFinished(job[u'Id'])
print "OK"
def cleanup(self):
mgr = JobManager()
mgr.baseUrl = self.config.baseUrl
rJobs = mgr.getRunningJobs()
if (len(rJobs) > 0):
print "Cleaning %s running jobs..." % len(rJobs)
for job in rJobs: mgr.setJobFinished(job[u'Id'])
print "OK"
cJobs = mgr.getCancelledJobs()
if (len(cJobs) > 0):
print "Cleaning %s cancelled jobs..." % len(cJobs)
for job in cJobs: mgr.setJobFinished(job[u'Id'])
print "OK"
def Do(self):
mgr = JobManager()
mgr.baseUrl = self.config.baseUrl
# If processing not running, get next started job and run it. If no jobs started, just loop.
if (self.runner == None):
jobs = mgr.getStartedJobs()
for job in jobs:
mgr.setJobRunning(job[u'Id'])
self.runner = Runner()
self.runner.workDir = self.config.workDir
self.runner.baseUrl = self.config.baseUrl
self.runner.jobId = job[u'Id']
self.runner.tryRun()
print "Processing started"
break # spawn ONLY one process
return # skip rest of the loop "while 1==1"
# Here we already know that processing is running. Act according to that.
self.runner.checkState()
self.runner.showDebug()
if (self.runner.isFinished()):
#if (p.result == JobResult.ERROR):
# print "Error occured: " + str(p.resultData)
self.runner.parseLogs() # do it before exit
self.runner.runZip()
mgr.setJobFinished(self.runner.jobId)
self.runner = None
print "Processing finished"
return
if (mgr.isJobCancelled(self.runner.jobId)):
self.runner.runZip()
self.runner.cancel()
print "Processing cancelled"
return
self.runner.parseLogs()
print "Processing is still running"
def Do(self):
mgr = JobManager()
mgr.baseUrl = self.config.baseUrl
# If processing not running, get next started job and run it. If no jobs started, just loop.
if (self.runner == None):
jobs = mgr.getStartedJobs()
for job in jobs:
mgr.setJobRunning(job[u'Id'])
self.runner = Runner()
self.runner.workDir = self.config.workDir
self.runner.baseUrl = self.config.baseUrl
self.runner.jobId = job[u'Id']
self.runner.tryRun()
print "Processing started"
break # spawn ONLY one process
return # skip rest of the loop "while 1==1"
# Here we already know that processing is running. Act according to that.
self.runner.checkState()
self.runner.showDebug()
if (self.runner.isFinished()):
#if (p.result == JobResult.ERROR):
# print "Error occured: " + str(p.resultData)
self.runner.parseLogs() # do it before exit
self.runner.runZip()
mgr.setJobFinished(self.runner.jobId)
self.runner = None
print "Processing finished"
return
if (mgr.isJobCancelled(self.runner.jobId)):
self.runner.runZip()
self.runner.cancel()
print "Processing cancelled"
return
self.runner.parseLogs()
print "Processing is still running"
import pointstreakscraper as pss
from jobmanager import JobManager
PS_2012_PLAYOFF = 18269
PS_2012_SEASON = 12252
if __name__ == "__main__":
regular_season_ids = pss.scrape_season_gameids(PS_2012_SEASON)
post_season_ids = pss.scrape_season_gameids(PS_2012_PLAYOFF)
jm = JobManager("jobs.yml")
jm.add_jobs(regular_season_ids, job_group="pointstreak")
jm.add_jobs(post_season_ids, job_group="pointstreak")
jm.save()
random.seed(11) # for reproductivity
dest_cand = ['DOOR_(1,0)', 'DOOR_(4,0)', 'DOOR_(7,0)',
'DOOR_(10,0)','DOOR_(13,0)','DOOR_(16,0)', 'DOOR_(19,0)',
'DOOR_(22,0)','DOOR_(25,0)','DOOR_(28,0)']
for i in range(50):
dest = random.choices(dest_cand,k=1)[0]
start = 0 + random.randrange(0, 180)
due = start + random.randrange(80, 160)
orders.append(methods.test_input_generation_randomly(dest, due, start))
jm = JobManager(orders, ia.ItemAttb)
wm = WorkerManager()
wm.set_worker(Worker("WK_1","DOOR_(1,0)",1))
wm.set_worker(Worker("WK_2","DOOR_(7,0)",1))
wm.set_worker(Worker("WK_3","DOOR_(13,0)",1))
wm.set_worker(Worker("WK_4","DOOR_(19,0)",1))
wm.set_worker(Worker("WK_5","DOOR_(25,0)",1))
wh = Warehouse()
wh.set_grid_layout(30,49)
wh.set_doors([(1,0), (4,0), (7,0), (10,0), (13,0), (16,0), (19,0), (22,0), (25,0), (28,0)])
def _send_comm_message(self, msg_type, content):
JobManager()._send_comm_message(msg_type, content)
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id,
**kwargs):
"""
Attemps to run the app, returns a Job with the running app info.
Should *hopefully* also inject that app into the Narrative's metadata.
Probably need some kind of JavaScript-foo to get that to work.
Parameters:
-----------
app_id - should be from the app spec, e.g. 'build_a_metabolic_model'
or 'MegaHit/run_megahit'.
params - the dictionary of parameters.
tag - optional, one of [release|beta|dev] (default=release)
version - optional, a semantic version string. Only released modules
have versions, so if the tag is not 'release', and a version
is given, a ValueError will be raised.
**kwargs - these are the set of parameters to be used with the app.
They can be found by using the app_usage function. If any
non-optional apps are missing, a ValueError will be raised.
"""
# TODO: this needs restructuring so that we can send back validation
# failure messages. Perhaps a separate function and catch the errors,
# or return an error structure.
# Intro tests:
self.spec_manager.check_app(app_id, tag, raise_exception=True)
if version is not None and tag != "release":
if re.match(r'\d+\.\d+\.\d+', version) is not None:
raise ValueError(
"Semantic versions only apply to released app modules. " +
"You can use a Git commit hash instead to specify a " +
"version.")
# Get the spec & params
spec = self.spec_manager.get_spec(app_id, tag)
# There's some branching to do here.
# Cases:
# app has behavior.kb_service_input_mapping - valid long-running app.
# app has behavior.output_mapping - not kb_service_input_mapping or
# script_module - it's a viewer and should return immediately
# app has other things besides kb_service_input_mapping - not valid.
if 'behavior' not in spec:
raise Exception("This app appears invalid - " +
"it has no defined behavior")
if 'kb_service_input_mapping' not in spec['behavior']:
raise Exception("This app does not appear to be a long-running " +
"job! Please use 'run_local_app' to start this " +
"instead.")
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params)
ws_id = system_variable('workspace_id')
if ws_id is None:
raise ValueError('Unable to retrive current ' +
'Narrative workspace information!')
input_vals = self._map_inputs(
spec['behavior']['kb_service_input_mapping'], params,
spec_params_map)
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
function_name = service_name + '.' + service_method
job_meta = {'tag': tag}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(),
token_name=token_name)
except Exception as e:
raise
job_meta['token_id'] = agent_token['id']
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': function_name,
'service_ver': service_ver,
'params': input_vals,
'app_id': app_id,
'wsid': ws_id,
'meta': job_meta
}
if len(ws_input_refs) > 0:
job_runner_inputs['source_ws_objects'] = ws_input_refs
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_app", log_info)
try:
job_id = clients.get("job_service").run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
app_id,
input_vals,
system_variable('user_id'),
tag=tag,
app_version=service_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'])
self._send_comm_message(
'run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
JobManager().register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
session = manager.init_database(use_mysql=options.apply)
# track parsing success game ids
def process_one(gameid):
print gameid
game = manager.import_game(gameid, session=session)
# for event in game.events():
# session.add(event)
# session.commit()
if game is not None:
games[game.game_id] = game
if options.game is None:
jm = JobManager("pending.yml")
for gameid in jm.jobs("pointstreak"):
try:
process_one(gameid)
jm.complete_job(gameid, "pointstreak")
except:
jm.set_job_status(gameid, "pointstreak", "error")
finally:
jm.save()
else:
process_one(options.game)
#===============================================================================
# Report Summary
#===============================================================================
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id,
dry_run):
"""
Attemps to run the app, returns a Job with the running app info.
Should *hopefully* also inject that app into the Narrative's metadata.
Probably need some kind of JavaScript-foo to get that to work.
Parameters:
-----------
app_id - should be from the app spec, e.g. 'build_a_metabolic_model'
or 'MegaHit/run_megahit'.
params - a dictionary of parameters.
tag - optional, one of [release|beta|dev] (default=release)
version - optional, a semantic version string. Only released modules
have versions, so if the tag is not 'release', and a version
is given, a ValueError will be raised.
**kwargs - these are the set of parameters to be used with the app.
They can be found by using the app_usage function. If any
non-optional apps are missing, a ValueError will be raised.
"""
ws_id = strict_system_variable('workspace_id')
spec = self._get_validated_app_spec(app_id, tag, True, version=version)
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params)
input_vals = self._map_inputs(
spec['behavior']['kb_service_input_mapping'], params,
spec_params_map)
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
function_name = service_name + '.' + service_method
job_meta = {'tag': tag}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': function_name,
'service_ver': service_ver,
'params': input_vals,
'app_id': app_id,
'wsid': ws_id,
'meta': job_meta
}
if len(ws_input_refs) > 0:
job_runner_inputs['source_ws_objects'] = ws_input_refs
if dry_run:
return job_runner_inputs
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(),
token_name=token_name)
except Exception as e:
raise
job_runner_inputs['meta']['token_id'] = agent_token['id']
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_app", log_info)
try:
job_id = clients.get(
"job_service",
token=agent_token['token']).run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
app_id,
input_vals,
system_variable('user_id'),
tag=tag,
app_version=service_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'])
self._send_comm_message(
'run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
JobManager().register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
def _run_app_batch_internal(self, app_id, params, tag, version, cell_id,
run_id, dry_run):
batch_method = "kb_BatchApp.run_batch"
batch_app_id = "kb_BatchApp/run_batch"
batch_method_ver = "dev"
batch_method_tag = "dev"
ws_id = strict_system_variable('workspace_id')
spec = self._get_validated_app_spec(app_id, tag, True, version=version)
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
# A list of lists of UPAs, used for each subjob.
batch_ws_upas = list()
# The list of actual input values, post-mapping.
batch_run_inputs = list()
for param_set in params:
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
batch_ws_upas.append(
extract_ws_refs(app_id, tag, spec_params, param_set))
batch_run_inputs.append(
self._map_inputs(spec['behavior']['kb_service_input_mapping'],
param_set, spec_params_map))
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
job_meta = {
'tag': batch_method_tag,
'batch_app': app_id,
'batch_tag': tag,
'batch_size': len(params),
}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# Now put these all together in a way that can be sent to the batch processing app.
batch_params = [{
"module_name":
service_name,
"method_name":
service_method,
"service_ver":
service_ver,
"wsid":
ws_id,
"meta":
job_meta,
"batch_params": [{
"params": batch_run_inputs[i],
"source_ws_objects": batch_ws_upas[i]
} for i in range(len(batch_run_inputs))],
}]
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(),
token_name=token_name)
except Exception as e:
raise
job_meta['token_id'] = agent_token['id']
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': batch_method,
'service_ver': batch_method_ver,
'params': batch_params,
'app_id': batch_app_id,
'wsid': ws_id,
'meta': job_meta
}
# if len(ws_input_refs) > 0:
# job_runner_inputs['source_ws_objects'] = ws_input_refs
# if we're doing a dry run, just return the inputs that we made.
if dry_run:
return job_runner_inputs
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': batch_method_tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_batch_app", log_info)
try:
job_id = clients.get(
"job_service",
token=agent_token['token']).run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_batch_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
batch_app_id,
batch_params,
system_variable('user_id'),
tag=batch_method_tag,
app_version=batch_method_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'],
meta=job_meta)
self._send_comm_message(
'run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
JobManager().register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.CRITICAL)
logging.basicConfig(format="%(asctime)s:%(levelname)s: %(message)s")
log = logging.getLogger()
if args.v:
log.setLevel(logging.DEBUG)
if args.mode == 'daemon':
log.info("Starting up daemon")
statusTracker = StatusTracker(config)
reactor.listenTCP(5000, server.Site(StatusSite(statusTracker)))
reactor.listenTCP(5001, server.Site(PingSite(statusTracker)), interface='127.0.0.1')
reactor.run()
elif args.mode == 'cron':
jobManager = JobManager(config)
if not args.crontime:
log.warn("Did not run cron, no crontime specified")
parser.print_help()
sys.exit(-1)
else:
log.info("Running cron at frequency " + args.crontime)
try:
if jobManager.execute_jobs(args.crontime):
jobManager.mark_jobs_ran()
else:
jobManager.mark_jobs_ran_with_error()
except Exception as e:
logging.critical("Caught an exception trying to execute jobs:" + str(e))
jobManager.mark_jobs_ran_with_error()
