def build_initial_task_descriptor(filename, block_store, task_name='root', cont_id='root_cont', output_id='root_output', env=False):
parser = SWScriptParser()
with open(filename, 'r') as script_file:
script = parser.parse(script_file.read())
if script is None:
print >>sys.stderr, 'Error parsing script in file: %s' % filename
exit(-1)
cont = SWContinuation(script, SimpleContext())
if env:
cont.context.bind_identifier('env', os.environ)
cont_id = 'root_cont'
cont_ref = block_store.ref_from_object(cont, 'pickle', cont_id)
print cont_ref
#cont_ref = SW2_ConcreteReference(cont_id, cont_len, [block_store.netloc])
expected_output_id = 'root_output'
tp_ref = block_store.ref_from_object({'cont' : cont_ref}, 'pickle', cont_id + ':tp')
task_descriptor = {'task_id' : 'root', 'dependencies': [cont_ref], 'task_private': tp_ref, 'handler': 'swi', 'expected_outputs' : [expected_output_id]}
return task_descriptor, cont_ref
def start_sw_script(self, swref, args, env):
sw_str = self.get_string_for_ref(swref)
parser = SWScriptParser()
script = parser.parse(sw_str)
if script is None:
raise Exception("Couldn't parse %s" % swref)
cont = SWContinuation(script, SimpleContext())
if env is not None:
cont.context.bind_identifier('env', env)
if args is not None:
cont.context.bind_identifier('argv', args)
return cont
def build_initial_task_descriptor(filename,
block_store,
task_name='root',
cont_id='root_cont',
output_id='root_output',
env=False):
parser = SWScriptParser()
with open(filename, 'r') as script_file:
script = parser.parse(script_file.read())
if script is None:
print >> sys.stderr, 'Error parsing script in file: %s' % filename
exit(-1)
cont = SWContinuation(script, SimpleContext())
if env:
cont.context.bind_identifier('env', os.environ)
cont_id = 'root_cont'
cont_ref = block_store.ref_from_object(cont, 'pickle', cont_id)
print cont_ref
#cont_ref = SW2_ConcreteReference(cont_id, cont_len, [block_store.netloc])
expected_output_id = 'root_output'
tp_ref = block_store.ref_from_object({'cont': cont_ref}, 'pickle',
cont_id + ':tp')
task_descriptor = {
'task_id': 'root',
'dependencies': [cont_ref],
'task_private': tp_ref,
'handler': 'swi',
'expected_outputs': [expected_output_id]
}
return task_descriptor, cont_ref
def main():
parser = OptionParser()
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("SW_MASTER"))
parser.add_option("-i",
"--id",
action="store",
dest="id",
help="Job ID",
metavar="ID",
default="default")
parser.add_option(
"-e",
"--env",
action="store_true",
dest="send_env",
help=
"Set this flag to send the current environment with the script as _env",
default=False)
(options, args) = parser.parse_args()
if not options.master:
parser.print_help()
print >> sys.stderr, "Must specify master URI with --master"
sys.exit(1)
if len(args) != 1:
parser.print_help()
print >> sys.stderr, "Must specify one script file to execute, as argument"
sys.exit(1)
script_name = args[0]
master_uri = options.master
id = options.id
print id, "STARTED", now_as_timestamp()
parser = SWScriptParser()
script = parser.parse(open(script_name, 'r').read())
print id, "FINISHED_PARSING", now_as_timestamp()
if script is None:
print "Script did not parse :("
exit()
cont = SWContinuation(script, SimpleContext())
if options.send_env:
cont.context.bind_identifier('env', os.environ)
http = httplib2.Http()
master_data_uri = urlparse.urljoin(master_uri, "/data/")
pickled_cont = pickle.dumps(cont)
(_, content) = http.request(master_data_uri, "POST", pickled_cont)
cont_id = simplejson.loads(content)
out_id = 'joboutput:%s' % cont_id
print id, "SUBMITTED_CONT", now_as_timestamp()
#print continuation_uri
master_netloc = urlparse.urlparse(master_uri).netloc
task_descriptor = {
'dependencies': {
'_cont':
SW2_ConcreteReference(cont_id, len(pickled_cont), [master_netloc])
},
'handler': 'swi',
'expected_outputs': [out_id]
}
master_task_submit_uri = urlparse.urljoin(master_uri, "/job/")
(_, content) = http.request(
master_task_submit_uri, "POST",
simplejson.dumps(task_descriptor, cls=SWReferenceJSONEncoder))
print id, "SUBMITTED_JOB", now_as_timestamp()
out = simplejson.loads(content)
notify_url = urlparse.urljoin(master_uri,
"/job/%s/completion" % out['job_id'])
job_url = urlparse.urljoin(master_uri, "/browse/job/%s" % out['job_id'])
print id, "JOB_URL", job_url
#print "Blocking to get final result"
(_, content) = http.request(notify_url)
completion_result = simplejson.loads(content,
object_hook=json_decode_object_hook)
if "error" in completion_result.keys():
print id, "ERROR", completion_result["error"]
return None
else:
print id, "GOT_RESULT", now_as_timestamp()
#print content
return completion_result["result_ref"]
