Esempio n. 1
0
 def run(self):
     assert self.version == '%s.%s' % sys.version_info[:2], "Python version mismatch"
     ensure_path(os.path.dirname(self.path('OOB_FILE', '')))
     os.chdir(self.path('CHDIR_PATH'))
     path = self.path('REQ_FILES')
     write_files(self.required_files, path)
     sys.path.insert(0, path)
     self.insert_globals(self.functions)
     if self.profile:
         self._run_profile()
     else:
         self._run()
Esempio n. 2
0
 def insert_globals(self, functions):
     write_files(self.required_files, self.lib)
     sys.path.insert(0, self.lib)
     for fn in functions:
         if isinstance(fn, partial):
             fn = fn.func
         if isinstance(fn, FunctionType):
             fn.func_globals.setdefault('Task', self)
             for module in self.required_modules:
                 mod_name = module[0] if util.iskv(module) else module
                 mod = __import__(mod_name, fromlist=[mod_name])
                 fn.func_globals.setdefault(mod_name.split('.')[-1], mod)
Esempio n. 3
0
 def insert_globals(self, functions):
     write_files(self.required_files, self.lib)
     sys.path.insert(0, self.lib)
     for fn in functions:
         if isinstance(fn, partial):
             fn = fn.func
         if isinstance(fn, FunctionType):
             fn.func_globals.setdefault('Task', self)
             for module in self.required_modules:
                 mod_name = module[0] if util.iskv(module) else module
                 mod = __import__(mod_name, fromlist=[mod_name])
                 fn.func_globals.setdefault(mod_name.split('.')[-1], mod)
Esempio n. 4
0
def init_common(job):
    global status_interval, input_stream_stack, output_stream_stack
    if 'required_files' in job:
        path = Task.path("REQ_FILES")
        write_files(util.unpack(job['required_files'], globals=globals()), path)
        sys.path.insert(0, path)

    Task.num_partitions = int(job['nr_reduces'])
    status_interval = int(job['status_interval'])

    input_stream_stack = load_stack(job, Task.mode, "input")
    output_stream_stack = load_stack(job, Task.mode, "output")

    req_mod = job['required_modules'].split()
    import_modules(req_mod)
Esempio n. 5
0
def prepare(ext_task, params, path):
    params = encode_netstring_fd(params)\
        if params and isinstance(params, dict) else "0\n"
    write_files(ext_task, path)
    open_ext(path + "/op", params)
Esempio n. 6
0
def prepare(ext_job, params, path):
    write_files(marshal.loads(ext_job), path)
    open_ext(path + "/op", params)
Esempio n. 7
0
def prepare(ext_task, params, path):
    params = encode_netstring_fd(params)\
        if params and isinstance(params, dict) else "0\n"
    write_files(ext_task, path)
    open_ext(path + "/op", params)
Esempio n. 8
0
def prepare(ext_job, params, path):
    write_files(util.unpack(ext_job), path)
    open_ext(path + "/op", params)