示例#1
0
文件: sw_parser.py 项目: ms705/ciel
def build_initial_task_descriptor(filename, block_store, task_name='root', cont_id='root_cont', output_id='root_output', env=False):

    parser = SWScriptParser()

    with open(filename, 'r') as script_file:
        script = parser.parse(script_file.read())

    if script is None:
        print >>sys.stderr, 'Error parsing script in file: %s' % filename
        exit(-1)
    
    cont = SWContinuation(script, SimpleContext())
    if env:
        cont.context.bind_identifier('env', os.environ)
       
    cont_id = 'root_cont'
    cont_ref = block_store.ref_from_object(cont, 'pickle', cont_id)
    print cont_ref
    #cont_ref = SW2_ConcreteReference(cont_id, cont_len, [block_store.netloc])
    expected_output_id = 'root_output'
    
    tp_ref = block_store.ref_from_object({'cont' : cont_ref}, 'pickle', cont_id + ':tp')
    
    task_descriptor = {'task_id' : 'root', 'dependencies': [cont_ref], 'task_private': tp_ref, 'handler': 'swi', 'expected_outputs' : [expected_output_id]}
    
    return task_descriptor, cont_ref
示例#2
0
文件: task.py 项目: ms705/ciel
 def start_sw_script(self, swref, args, env):
 
     sw_str = self.get_string_for_ref(swref)
     parser = SWScriptParser()
     script = parser.parse(sw_str)
 
     if script is None:
         raise Exception("Couldn't parse %s" % swref)
 
     cont = SWContinuation(script, SimpleContext())
     if env is not None:
         cont.context.bind_identifier('env', env)
     if args is not None:
         cont.context.bind_identifier('argv', args)
     return cont
示例#3
0
def build_initial_task_descriptor(filename,
                                  block_store,
                                  task_name='root',
                                  cont_id='root_cont',
                                  output_id='root_output',
                                  env=False):

    parser = SWScriptParser()

    with open(filename, 'r') as script_file:
        script = parser.parse(script_file.read())

    if script is None:
        print >> sys.stderr, 'Error parsing script in file: %s' % filename
        exit(-1)

    cont = SWContinuation(script, SimpleContext())
    if env:
        cont.context.bind_identifier('env', os.environ)

    cont_id = 'root_cont'
    cont_ref = block_store.ref_from_object(cont, 'pickle', cont_id)
    print cont_ref
    #cont_ref = SW2_ConcreteReference(cont_id, cont_len, [block_store.netloc])
    expected_output_id = 'root_output'

    tp_ref = block_store.ref_from_object({'cont': cont_ref}, 'pickle',
                                         cont_id + ':tp')

    task_descriptor = {
        'task_id': 'root',
        'dependencies': [cont_ref],
        'task_private': tp_ref,
        'handler': 'swi',
        'expected_outputs': [expected_output_id]
    }

    return task_descriptor, cont_ref
示例#4
0
def main():
    parser = OptionParser()
    parser.add_option("-m",
                      "--master",
                      action="store",
                      dest="master",
                      help="Master URI",
                      metavar="MASTER",
                      default=os.getenv("SW_MASTER"))
    parser.add_option("-i",
                      "--id",
                      action="store",
                      dest="id",
                      help="Job ID",
                      metavar="ID",
                      default="default")
    parser.add_option(
        "-e",
        "--env",
        action="store_true",
        dest="send_env",
        help=
        "Set this flag to send the current environment with the script as _env",
        default=False)
    (options, args) = parser.parse_args()

    if not options.master:
        parser.print_help()
        print >> sys.stderr, "Must specify master URI with --master"
        sys.exit(1)

    if len(args) != 1:
        parser.print_help()
        print >> sys.stderr, "Must specify one script file to execute, as argument"
        sys.exit(1)

    script_name = args[0]
    master_uri = options.master
    id = options.id

    print id, "STARTED", now_as_timestamp()

    parser = SWScriptParser()

    script = parser.parse(open(script_name, 'r').read())

    print id, "FINISHED_PARSING", now_as_timestamp()

    if script is None:
        print "Script did not parse :("
        exit()

    cont = SWContinuation(script, SimpleContext())
    if options.send_env:
        cont.context.bind_identifier('env', os.environ)

    http = httplib2.Http()

    master_data_uri = urlparse.urljoin(master_uri, "/data/")
    pickled_cont = pickle.dumps(cont)
    (_, content) = http.request(master_data_uri, "POST", pickled_cont)
    cont_id = simplejson.loads(content)

    out_id = 'joboutput:%s' % cont_id

    print id, "SUBMITTED_CONT", now_as_timestamp()

    #print continuation_uri

    master_netloc = urlparse.urlparse(master_uri).netloc
    task_descriptor = {
        'dependencies': {
            '_cont':
            SW2_ConcreteReference(cont_id, len(pickled_cont), [master_netloc])
        },
        'handler': 'swi',
        'expected_outputs': [out_id]
    }

    master_task_submit_uri = urlparse.urljoin(master_uri, "/job/")
    (_, content) = http.request(
        master_task_submit_uri, "POST",
        simplejson.dumps(task_descriptor, cls=SWReferenceJSONEncoder))

    print id, "SUBMITTED_JOB", now_as_timestamp()

    out = simplejson.loads(content)

    notify_url = urlparse.urljoin(master_uri,
                                  "/job/%s/completion" % out['job_id'])
    job_url = urlparse.urljoin(master_uri, "/browse/job/%s" % out['job_id'])

    print id, "JOB_URL", job_url

    #print "Blocking to get final result"
    (_, content) = http.request(notify_url)
    completion_result = simplejson.loads(content,
                                         object_hook=json_decode_object_hook)
    if "error" in completion_result.keys():
        print id, "ERROR", completion_result["error"]
        return None
    else:
        print id, "GOT_RESULT", now_as_timestamp()
        #print content
        return completion_result["result_ref"]