def cancel(self, message):
data = yaml.safe_load(message._message)
command = data['command']
if command == 'CancelCommand':
ident = data['ident']
user = data['user']
# determine class, and dispatch appropriately.
# if auditResult -> agent_dispatcher
# if analysisResult -> analyzer_dispatcher
# if resultset -> both?
dList = []
ident = identity.identity_from_string(ident)
if ident.type == 'AuditResult' or ident.type == 'ResultSet':
sName = config_data.getByPath('service_names', 'agent_dispatcher')
qName = config_data.getByPath('agent_dispatcher', 'queues', 'postqueue')
dList.append((sName,qName))
if ident.type == 'AnalysisResult' or ident.type == 'ResultSet':
sName = config_data.getByPath('service_names', 'analyzer_dispatcher')
qName = config_data.getByPath('analyzer_dispatcher', 'queues', 'postqueue')
dList.append((sName,qName))
for sName,qName in dList:
msg = { 'command': 'CancelCommand', 'ident':str(ident), 'user':user}
sIdent = wtutil.makeIdentity(sName)
qIdent = wtutil.makeQueueIdentity(sIdent, qName)
self.mbus.sendMessagetoQueue(qIdent, yaml.safe_dump(msg))
self.err_msgs.render_log_msg('f0f4ee71-49a1-4d83-81cb-248657441d9a', {'ident': ident, 'queue_ident': qIdent, 'user': user})
if ident.type =='ResultSet':
pass #TODO: abort any further chained scripts on this job.
def testInitiate(self):
"""
Test Queue Management messages and errors
"""
self.timeout = 5 # Set a reasonable timeout
######
self._setUp()
self.t_job = models.Job("ScriptRunner TestJob #1", schedule=self.job_schedule, input=self.job_input_uri_list, script=self.job_script, workspace_id=1)
self.session.save(self.t_job)
self.session.update(self.t_job)
self.session.flush()
self.session.clear()
job_rec = {
'job_identity': self.t_job.href,
}
self.job_rec = job_rec
script_def = yield self.script_runner.initiate(self.job_rec, {"initiate": False})
self.assertEqual(self.job_rec["job_url"], config_data.getByPath('mir_web', 'uri') + self.job_rec["job_identity"])
self.assertEqual(str(self.job_rec["ident"]), str(identity.identity_from_string(self.t_job.href)))
self.assertTrue(self.job_rec.has_key, "logEntries")
self.assertEqual(len(self.job_rec["logEntries"]), 0)
self.assertEqual(self.job_rec["logEntries"], [])
self.assertTrue(self.job_rec.has_key, "issues")
self.assertEqual(len(self.job_rec["issues"]), 0)
self.assertEqual(self.job_rec["issues"], [])
self.assertTrue(self.job_rec.has_key, "results_hash")
self.assertEqual(len(self.job_rec["results_hash"]), 0)
self.assertEqual(self.job_rec["results_hash"], [])
self.assertTrue(self.job_rec.has_key, "inputs")
self.assertEqual(len(self.job_rec["inputs"]), 0)
self.assertEqual(self.job_rec["inputs"], [])
self.assertTrue(self.job_rec.has_key, "documents")
self.assertEqual(len(self.job_rec["documents"].keys()), 0)
self.assertEqual(self.job_rec["documents"], {})
self.session.update(self.job_rec['job_obj'])
self.assertEqual(len(self.job_rec['job_obj'].results), 1)
stringpad = " Results Container @"
size = len(self.job_rec['job_obj'].name) + len(stringpad)
self.assertEqual(self.job_rec['job_obj'].results[0].name[:size], "%s%s" % (self.job_rec['job_obj'].name, stringpad))
self.assertEqual(self.job_rec['script'], self.job_script)
self.assertEqual(self.job_rec['input'], self.job_input_uri_list)
xx = self.job_rec['script_dom'].toxml('utf-8').replace(r'"', '"').replace(r'<?xml version="1.0" encoding="utf-8"?>', '')
yy = self.job_rec['script_dom'].toxml('utf-8').replace(r'"', '"')
self.assertEqual(yy, self.job_script)
def testInitiate(self):
"""
Test Queue Management messages and errors
"""
self.timeout = 5 # Set a reasonable timeout
######
self._setUp()
self.job_rec['job_obj'].results = []
self.assertEqual(len(self.job_rec['job_obj'].results), 0)
script_def = yield self.script_runner.initiate(self.job_rec,
{"initiate": False})
self.assertEqual(
self.job_rec["job_url"],
config_data.getByPath('mir_web', 'uri') +
self.job_rec["job_identity"])
self.assertEqual(str(self.job_rec["ident"]),
str(identity.identity_from_string(self.t_job.href)))
self.assertTrue(self.job_rec.has_key, "logEntries")
self.assertEqual(len(self.job_rec["logEntries"]), 0)
self.assertEqual(self.job_rec["logEntries"], [])
self.assertTrue(self.job_rec.has_key, "issues")
self.assertEqual(len(self.job_rec["issues"]), 0)
self.assertEqual(self.job_rec["issues"], [])
self.assertTrue(self.job_rec.has_key, "results_hash")
self.assertEqual(len(self.job_rec["results_hash"]), 0)
self.assertEqual(self.job_rec["results_hash"], [])
self.assertTrue(self.job_rec.has_key, "inputs")
self.assertEqual(len(self.job_rec["inputs"]), 0)
self.assertEqual(self.job_rec["inputs"], [])
self.assertTrue(self.job_rec.has_key, "documents")
self.assertEqual(len(self.job_rec["documents"].keys()), 0)
self.assertEqual(self.job_rec["documents"], {})
self.assertEqual(len(self.job_rec['job_obj'].results), 1)
stringpad = " Results Container @"
size = len(self.job_rec['job_obj'].name) + len(stringpad)
self.assertEqual(self.job_rec['job_obj'].results[0].name[:size],
"%s%s" % (self.job_rec['job_obj'].name, stringpad))
self.assertEqual(self.job_rec['script'], self.job_script)
self.assertEqual(self.job_rec['input'], self.job_input_uri_list)
xx = self.job_rec['script_dom'].toxml('utf-8').replace(
r'"', '"').replace(r'<?xml version="1.0" encoding="utf-8"?>',
'')
yy = self.job_rec['script_dom'].toxml('utf-8').replace(r'"', '"')
self.assertEqual(yy, self.job_script)
def select_by(self, *args, **kwargs):
if kwargs.has_key('href'):
queryset = []
for href in kwargs['href']:
id = identity.identity_from_string(href).id
address = '127.0.0.%s:22201' % id
queryset.append(models.Host('Host ##%s' % id, workspace_id=kwargs["workspace_id"], id=id, address=address))
return queryset
else:
address = '127.0.0.1:22201'
return [models.Host('Host ##1', workspace_id=kwargs["workspace_id"], id=kwargs["id"], address=address)]
def cancel(self, message):
data = yaml.safe_load(message._message)
command = data['command']
if command == 'CancelCommand':
ident = data['ident']
user = data['user']
# determine class, and dispatch appropriately.
# if auditResult -> agent_dispatcher
# if analysisResult -> analyzer_dispatcher
# if resultset -> both?
dList = []
ident = identity.identity_from_string(ident)
if ident.type == 'AuditResult' or ident.type == 'ResultSet':
sName = config_data.getByPath('service_names',
'agent_dispatcher')
qName = config_data.getByPath('agent_dispatcher', 'queues',
'postqueue')
dList.append((sName, qName))
if ident.type == 'AnalysisResult' or ident.type == 'ResultSet':
sName = config_data.getByPath('service_names',
'analyzer_dispatcher')
qName = config_data.getByPath('analyzer_dispatcher', 'queues',
'postqueue')
dList.append((sName, qName))
for sName, qName in dList:
msg = {
'command': 'CancelCommand',
'ident': str(ident),
'user': user
}
sIdent = wtutil.makeIdentity(sName)
qIdent = wtutil.makeQueueIdentity(sIdent, qName)
self.mbus.sendMessagetoQueue(qIdent, yaml.safe_dump(msg))
self.err_msgs.render_log_msg(
'f0f4ee71-49a1-4d83-81cb-248657441d9a', {
'ident': ident,
'queue_ident': qIdent,
'user': user
})
if ident.type == 'ResultSet':
pass #TODO: abort any further chained scripts on this job.
def _setUp(self, scriptfile="implicit_chaining_1.xml"):
self.job_script = file(self.database + "data/job_scripts/%s" % scriptfile).read()
self.job_schedule = "now"
self.job_input_uri_list = "<input><uri>/workspaces/1/hosts/resources/1/</uri>\n<uri>/workspaces/1/hosts/resources/2/</uri></input>"
self.job_xml = '%s%s\n<when>%s</when>\n%s\n</JobDefinition>' % (
'<?xml version="1.0"?>\n<JobDefinition xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" href="/workspaces/1/jobs/234/">',
re.sub(r'^<\?[^\?]+\?>', '', self.job_script),
re.sub(r'^<\?[^\?]+\?>', '', self.job_schedule),
re.sub(r'^<\?[^\?]+\?>', '', self.job_input_uri_list),
)
self.script_runner = sr.ScriptRunner(indexer=False, mbus=self.mbus, testing=True)
self.script_runner.indexer = testindexer.ACTestIndexer()
self.session = sa.create_session()
self.t_job = models.Job("ScriptRunner TestJob #1", schedule=self.job_schedule, input=self.job_input_uri_list, script=self.job_script, workspace_id=1)
self.session.save(self.t_job)
self.session.update(self.t_job)
self.t_res = models.ResultSet(name="%s Results Container @ %s" % (self.t_job.name, mir_datetime.strISO_now()), job=self.t_job, workspace_id=self.t_job.workspace_id)
self.session.save(self.t_res)
self.session.update(self.t_res)
self.t_job.results.append(self.t_res)
self.t_res.status = models.Status("Status for ResultSet %s" % self.t_res.id, activity='Starting', state='pending', status=None, id=self.t_res.id)
self.session.flush()
self.session.update(self.t_job)
self.session.save(self.t_job)
self.session.flush()
self.session.clear()
job_rec = {
'job_obj': self.t_job,
'result': self.t_res,
'job_identity': self.t_job.href,
'job_script': self.t_job.script,
'job_url': str(identity.identity_from_string(self.t_job.href)),
'script_dom': minidom.parseString(self.job_script),
'workspace_id': 1,
'input': self.t_job.input,
'job_elem': minidom.parseString('<?xml version="1.0" encoding="utf-8"?>%s' % self.t_job.dump_job_manifest()),
'executionMetaData': minidom.parseString('<?xml version="1.0" encoding="utf-8"?>%s' % self.t_res.dump_execution_manifest()),
}
self.job_rec = job_rec
def select_by(self, *args, **kwargs):
if kwargs.has_key('href'):
queryset = []
for href in kwargs['href']:
id = identity.identity_from_string(href).id
address = '127.0.0.%s:22201' % id
queryset.append(
models.Host('Host ##%s' % id,
workspace_id=kwargs["workspace_id"],
id=id,
address=address))
return queryset
else:
address = '127.0.0.1:22201'
return [
models.Host('Host ##1',
workspace_id=kwargs["workspace_id"],
id=kwargs["id"],
address=address)
]
def initiate(self, job_rec, std_callbacks={}):
"""Pull Job information from the database."""
self.err_msgs.render_log_msg('bee5922d-6030-45fa-a5f2-e704241f467b', {'job_ident': job_rec["job_ident"], 'queued_job_ident': job_rec['queued_job'].href})
# Set up the job execution record.
job_rec["issues"] = []
jobIdentity = job_rec["job_ident"]
jobObj = yield self.dbGet(job_rec['user_name'], jobIdentity)
if True:
base_inputs = execution_node.parseJobInput(job_rec['jobObj'].input)
# is container? is resolvable?
# input possibilities:
# 1) particle that resolves to itself only - Host
# 2) particle that resolves to something else - Label
# 3) container that resolves - Label/resources/ or AuditResul/documents/
# 4) virtualized query - hosts/all/?filter=name,startswith,Foo
# 5) 2, 3, or 4 above that resolve to nothing
# 6) URI that resolves to a 404 error - unknown resource
for t_input in base_inputs:
resolved_inputs = None
if re.search(r'\?', t_input):
(input_ident, query_args) = t_input.split('?')
input_ident = identity.identity_from_string(input_ident)
query_args = map(lambda x: x[7:].split(',', 3), filter(lambda y: re.search(r'^filter=', y), query_args.split('&')))
resolved_inputs = yield self.mir.resolve_query(job_rec['user_name'], input_ident, query_args)
else:
try:
input_ident = identity.identity_from_string(t_input)
except Exception, e:
# Bad input URI provided. Skip it. Move on.
job_rec['issues'].append(isu.Issue('Warning', 'Input Resolution', 'Invalid URI provided as input to Job.', t_input))
continue
query_args = None
mdl_type = getattr(models, input_ident.type, None)
if mdl_type:
if input_ident._type == "particle":
if getattr(mdl_type, 'resolve_method', 'resolve_identity') != 'resolve_identity' :
try:
input_obj = yield self.dbGet(job_rec['user_name'], input_ident)
resolved_inputs = yield self.mir.resolve(job_rec['user_name'], input_obj)
except Exception, e:
resolved_inputs = []
else:
resolved_inputs = [t_input]
elif input_ident._type == "container":
try:
t_objs = yield self.dbGet(job_rec['user_name'], input_ident)
except Exception, e:
t_objs = []
resolved_inputs = map(lambda xx: xx.href, t_objs)
else:
job_rec['input'] = job_rec['jobObj'].input
job_rec["input_set"].update(dict([(x, True) for x in execution_node.parseJobInput(job_rec['input'])]))
# create result set
rsName="Results for %s at %s" % (job_rec['jobObj'].name, mir_datetime.strISO_now())
rsObj = self.mir.ResultSet( name=rsName,
workspace_id=job_rec['job_ident'].workspace,
state="pendingXXX",
job=job_rec['jobObj'],
schedule_summary=job_rec['queue'].get_cron_string(),
run_time=mir_datetime.strISO_now(),
input_count=len(job_rec["input_set"]),
queued_job_ident=str(job_rec['queued_job'].ident))
rsObj = yield self.mir.create(job_rec['user_name'], rsObj, where=identity.identity_from_string('%sresultsets/' % job_rec['jobObj'].ident))
self.err_msgs.render_log_msg('c175ca2a-dd0c-4c22-8543-addf466fa771', {'resultset_ident': rsObj.ident, 'job_ident': jobIdentity})
yield self.set_state(rsObj.ident, 'queued', job_rec['user_name'])
# cache info for later use
job_rec['result_ident'] = rsObj.ident
job_rec['script'] = job_rec['jobObj'].script
job_rec['job_elem_xml'] = dump_job_manifest(job_rec['jobObj'], job_rec['queue'])
# The std_callbacks stuff is to allow me to halt the execution at this
# or some future step in the path. The two most basic situations are
# std_callbacks is empty which results in all calls being made. The other
# main situation is where this function is given a False which will stop
# processing here and not go to the next step.
def _setUp(self, scriptfile="implicit_chaining_1.xml"):
self.job_script = file(self.database +
"data/job_scripts/%s" % scriptfile).read()
self.job_schedule = "now"
self.job_input_uri_list = "<input><uri>/workspaces/1/hosts/resources/1/</uri>\n<uri>/workspaces/1/hosts/resources/2/</uri></input>"
self.job_xml = '%s%s\n<when>%s</when>\n%s\n</JobDefinition>' % (
'<?xml version="1.0"?>\n<JobDefinition xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" href="/workspaces/1/jobs/234/">',
re.sub(r'^<\?[^\?]+\?>', '', self.job_script),
re.sub(r'^<\?[^\?]+\?>', '', self.job_schedule),
re.sub(r'^<\?[^\?]+\?>', '', self.job_input_uri_list),
)
self.script_runner = sr.ScriptRunner(db=False,
indexer=False,
mbus=self.mbus,
testing=True)
self.script_runner.db = testdb.ACTestObjectDB(scriptfile=scriptfile)
self.session = self.script_runner.db.session
self.script_runner.indexer = testindexer.ACTestIndexer()
self.t_job = models.Job("ScriptRunner TestJob #1",
schedule=self.job_schedule,
input=self.job_input_uri_list,
script=self.job_script,
workspace_id=1,
id=14)
self.t_res = models.ResultSet(
name="%s Results Container @ %s" %
(self.t_job.name, mir_datetime.strISO_now()),
job=self.t_job,
workspace_id=self.t_job.workspace_id)
self.t_job.results.append(self.t_res)
self.t_res.status = models.Status("Status for ResultSet %s" %
self.t_res.id,
activity='Starting',
state='pending',
status=None,
id=self.t_res.id)
job_rec = {
'job_obj':
self.t_job,
'result':
self.t_res,
'job_identity':
self.t_job.href,
'job_script':
self.t_job.script,
'job_url':
str(identity.identity_from_string(self.t_job.href)),
'script_dom':
minidom.parseString(self.job_script),
'workspace_id':
1,
'input':
self.t_job.input,
'job_elem':
minidom.parseString('<?xml version="1.0" encoding="utf-8"?>%s' %
self.t_job.dump_job_manifest()),
'executionMetaData':
minidom.parseString('<?xml version="1.0" encoding="utf-8"?>%s' %
self.t_res.dump_execution_manifest()),
}
self.job_rec = job_rec
def initiate(self, job_rec, std_callbacks={}):
"""Pull Job information from the database."""
self.err_msgs.render_log_msg(
'bee5922d-6030-45fa-a5f2-e704241f467b', {
'job_ident': job_rec["job_ident"],
'queued_job_ident': job_rec['queued_job'].href
})
# Set up the job execution record.
job_rec["issues"] = []
jobIdentity = job_rec["job_ident"]
jobObj = yield self.dbGet(job_rec['user_name'], jobIdentity)
if True:
base_inputs = execution_node.parseJobInput(job_rec['jobObj'].input)
# is container? is resolvable?
# input possibilities:
# 1) particle that resolves to itself only - Host
# 2) particle that resolves to something else - Label
# 3) container that resolves - Label/resources/ or AuditResul/documents/
# 4) virtualized query - hosts/all/?filter=name,startswith,Foo
# 5) 2, 3, or 4 above that resolve to nothing
# 6) URI that resolves to a 404 error - unknown resource
for t_input in base_inputs:
resolved_inputs = None
if re.search(r'\?', t_input):
(input_ident, query_args) = t_input.split('?')
input_ident = identity.identity_from_string(input_ident)
query_args = map(
lambda x: x[7:].split(',', 3),
filter(lambda y: re.search(r'^filter=', y),
query_args.split('&')))
resolved_inputs = yield self.mir.resolve_query(
job_rec['user_name'], input_ident, query_args)
else:
try:
input_ident = identity.identity_from_string(t_input)
except Exception, e:
# Bad input URI provided. Skip it. Move on.
job_rec['issues'].append(
isu.Issue('Warning', 'Input Resolution',
'Invalid URI provided as input to Job.',
t_input))
continue
query_args = None
mdl_type = getattr(models, input_ident.type, None)
if mdl_type:
if input_ident._type == "particle":
if getattr(
mdl_type, 'resolve_method',
'resolve_identity') != 'resolve_identity':
try:
input_obj = yield self.dbGet(
job_rec['user_name'], input_ident)
resolved_inputs = yield self.mir.resolve(
job_rec['user_name'], input_obj)
except Exception, e:
resolved_inputs = []
else:
resolved_inputs = [t_input]
elif input_ident._type == "container":
try:
t_objs = yield self.dbGet(
job_rec['user_name'], input_ident)
except Exception, e:
t_objs = []
resolved_inputs = map(lambda xx: xx.href, t_objs)
rsName = "Results for %s at %s" % (job_rec['jobObj'].name,
mir_datetime.strISO_now())
rsObj = self.mir.ResultSet(
name=rsName,
workspace_id=job_rec['job_ident'].workspace,
state="pendingXXX",
job=job_rec['jobObj'],
schedule_summary=job_rec['queue'].get_cron_string(),
run_time=mir_datetime.strISO_now(),
input_count=len(job_rec["input_set"]),
queued_job_ident=str(job_rec['queued_job'].ident))
rsObj = yield self.mir.create(
job_rec['user_name'],
rsObj,
where=identity.identity_from_string('%sresultsets/' %
job_rec['jobObj'].ident))
self.err_msgs.render_log_msg('c175ca2a-dd0c-4c22-8543-addf466fa771', {
'resultset_ident': rsObj.ident,
'job_ident': jobIdentity
})
yield self.set_state(rsObj.ident, 'queued', job_rec['user_name'])
# cache info for later use
job_rec['result_ident'] = rsObj.ident
job_rec['script'] = job_rec['jobObj'].script
job_rec['job_elem_xml'] = dump_job_manifest(job_rec['jobObj'],
job_rec['queue'])
# The std_callbacks stuff is to allow me to halt the execution at this