def run(self): assert self.version == '%s.%s' % sys.version_info[:2], "Python version mismatch" ensure_path(os.path.dirname(self.path('OOB_FILE', ''))) os.chdir(self.path('CHDIR_PATH')) path = self.path('REQ_FILES') write_files(self.required_files, path) sys.path.insert(0, path) self.insert_globals(self.functions) if self.profile: self._run_profile() else: self._run()
def insert_globals(self, functions): write_files(self.required_files, self.lib) sys.path.insert(0, self.lib) for fn in functions: if isinstance(fn, partial): fn = fn.func if isinstance(fn, FunctionType): fn.func_globals.setdefault('Task', self) for module in self.required_modules: mod_name = module[0] if util.iskv(module) else module mod = __import__(mod_name, fromlist=[mod_name]) fn.func_globals.setdefault(mod_name.split('.')[-1], mod)
def init_common(job): global status_interval, input_stream_stack, output_stream_stack if 'required_files' in job: path = Task.path("REQ_FILES") write_files(util.unpack(job['required_files'], globals=globals()), path) sys.path.insert(0, path) Task.num_partitions = int(job['nr_reduces']) status_interval = int(job['status_interval']) input_stream_stack = load_stack(job, Task.mode, "input") output_stream_stack = load_stack(job, Task.mode, "output") req_mod = job['required_modules'].split() import_modules(req_mod)
def prepare(ext_task, params, path): params = encode_netstring_fd(params)\ if params and isinstance(params, dict) else "0\n" write_files(ext_task, path) open_ext(path + "/op", params)
def prepare(ext_job, params, path): write_files(marshal.loads(ext_job), path) open_ext(path + "/op", params)
def prepare(ext_job, params, path): write_files(util.unpack(ext_job), path) open_ext(path + "/op", params)