def _unzip(self, job_name): zip_file = os.path.join(self.zip_dir, job_name + '.zip') job_path = os.path.join(self.job_dir, job_name) if os.path.exists(job_path): shutil.rmtree(job_path) if os.path.exists(zip_file): ZipHandler.uncompress(zip_file, self.job_dir)
def _unzip(self, job_name): zip_file = os.path.join(self.zip_dir, job_name+'.zip') job_path = os.path.join(self.job_dir, job_name) if os.path.exists(job_path): shutil.rmtree(job_path) if os.path.exists(zip_file): ZipHandler.uncompress(zip_file, self.job_dir)
def testZip(self): zip_file = os.path.join(self.f, 'test.zip') ZipHandler.compress(zip_file, self.src_dir) ZipHandler.uncompress(zip_file, self.dest_dir) dir_ = os.path.join(self.dest_dir, 'compress') self.assertTrue(os.path.exists(dir_)) with open(os.path.join(dir_, '1.txt')) as fp: self.assertEqual(fp.read(), self.content) dir1 = os.path.join(dir_, 'dir1') self.assertTrue(os.path.exists(dir1)) with open(os.path.join(dir1, '2.txt')) as fp: self.assertEqual(fp.read(), self.content)
def start_job(self, zip_filename, uncompress=True): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0]) job = import_job(job_dir) master_port = job.context.job.master_port master = '%s:%s' % (self.master.split(':')[0], master_port) dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, 'loader.py') subprocess.Popen('python "%s" "%s" %s' % (f, job_dir, master))
def start_job(self, zip_filename, uncompress=True, client=None): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) # transfer zip file to workers for watcher in self.nodes_watchers: if watcher.split(':')[0] == self.ip_address: continue file_trans_client = FileTransportClient(watcher, zip_file) file_trans_client.send_file() job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0]) job = import_job(job_dir) worker_port = job.context.job.port port = job.context.job.master_port nodes = [watcher.split(':')[0] for watcher in self.nodes_watchers] if len(nodes) > 0: info = MasterJobInfo(port, nodes, worker_port) self.running_jobs[job.real_name] = info dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, 'loader.py') workers = ['%s:%s' % (node, worker_port) for node in nodes] cmds = [ 'python', f, '-j', job_dir, '-i', self.ip_address, '-n', ' '.join(workers) ] if self.data_path is not None: cmds.extend(['-d', self.data_path]) if self.force: cmds.append('-f') if client is not None: cmds.extend(['-c', client]) popen = subprocess.Popen(cmds) info.popen = popen # call workers to start job for worker_watcher in self.nodes_watchers: client_call(worker_watcher, 'start_job', zip_filename, uncompress, ignore=True)
def start_job(self, zip_filename, uncompress=True, client=None): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) # transfer zip file to workers for watcher in self.nodes_watchers: if watcher.split(':')[0] == self.ip_address: continue file_trans_client = FileTransportClient(watcher, zip_file) file_trans_client.send_file() job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0]) job = import_job(job_dir) worker_port = job.context.job.port port = job.context.job.master_port nodes = [watcher.split(':')[0] for watcher in self.nodes_watchers] if len(nodes) > 0: info = MasterJobInfo(port, nodes, worker_port) self.running_jobs[job.real_name] = info dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, 'loader.py') workers = ['%s:%s'%(node, worker_port) for node in nodes] cmds = ['python', f, '-j', job_dir, '-i', self.ip_address, '-n', ' '.join(workers)] if self.data_path is not None: cmds.extend(['-d', self.data_path]) if self.force: cmds.append('-f') if client is not None: cmds.extend(['-c', client]) popen = subprocess.Popen(cmds) info.popen = popen # call workers to start job for worker_watcher in self.nodes_watchers: client_call(worker_watcher, 'start_job', zip_filename, uncompress, ignore=True)
def start_job(self, zip_filename, uncompress=True, client=None): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) # transfer zip file to workers for watcher in self.nodes_watchers: if watcher.split(":")[0] == self.ip_address: continue file_trans_client = FileTransportClient(watcher, zip_file) file_trans_client.send_file() job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit(".", 1)[0]) job = import_job(job_dir) worker_port = job.context.job.port port = job.context.job.master_port nodes = [watcher.split(":")[0] for watcher in self.nodes_watchers] if len(nodes) > 0: info = MasterJobInfo(port, nodes, worker_port) self.running_jobs[job.real_name] = info dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, "loader.py") workers = ["%s:%s" % (node, worker_port) for node in nodes] cmds = ["python", f, "-j", job_dir, "-i", self.ip_address, "-n", " ".join(workers)] if self.data_path is not None: cmds.extend(["-d", self.data_path]) if self.force: cmds.append("-f") if client is not None: cmds.extend(["-c", client]) popen = subprocess.Popen(cmds) info.popen = popen # call workers to start job for worker_watcher in self.nodes_watchers: client_call(worker_watcher, "start_job", zip_filename, uncompress)
def start_job(self, zip_filename, uncompress=True): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0]) job = import_job(job_dir) master_port = job.context.job.master_port master = '%s:%s' % (self.master.split(':')[0], master_port) dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, 'loader.py') cmds = ['python', f, '-j', job_dir, '-m', master] if self.data_path is not None: cmds.extend(['-d', self.data_path]) if self.force: cmds.append('-f') popen = subprocess.Popen(cmds) self.running_jobs[job.real_name] = WorkerJobInfo(job.context.job.port, popen)
def start_job(self, zip_filename, uncompress=True): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit(".", 1)[0]) job = import_job(job_dir) master_port = job.context.job.master_port master = "%s:%s" % (self.master.split(":")[0], master_port) dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, "loader.py") cmds = ["python", f, "-j", job_dir, "-m", master] if self.data_path is not None: cmds.extend(["-d", self.data_path]) if self.force: cmds.append("-f") popen = subprocess.Popen(cmds) self.running_jobs[job.real_name] = WorkerJobInfo(job.context.job.port, popen)
def start_job(self, zip_filename, uncompress=True): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0]) job = import_job(job_dir) master_port = job.context.job.master_port master = '%s:%s' % (self.master.split(':')[0], master_port) dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, 'loader.py') cmds = ['python', f, '-j', job_dir, '-m', master] if self.data_path is not None: cmds.extend(['-d', self.data_path]) if self.force: cmds.append('-f') popen = subprocess.Popen(cmds) self.running_jobs[job.real_name] = WorkerJobInfo( job.context.job.port, popen)
def start_job(self, zip_filename, uncompress=True): if uncompress: zip_file = os.path.join(self.zip_dir, zip_filename) # transfer zip file to workers for watcher in self.nodes_watchers: if watcher.split(':')[0] == self.ip_address: continue file_trans_client = FileTransportClient(watcher, zip_file) file_trans_client.send_file() job_dir = ZipHandler.uncompress(zip_file, self.job_dir) else: job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0]) job = import_job(job_dir) worker_port = job.context.job.port port = job.context.job.master_port nodes = [watcher.split(':')[0] for watcher in self.nodes_watchers] if len(nodes) > 0: info = MasterJobInfo(port, nodes, worker_port) self.running_jobs[job.real_name] = info dirname = os.path.dirname(os.path.abspath(__file__)) f = os.path.join(dirname, 'loader.py') workers = ['%s:%s'%(node, worker_port) for node in nodes] subprocess.Popen('python "%(py)s" "%(job_dir)s" %(nodes)s' % { 'py': f, 'job_dir': job_dir, 'nodes': ' '.join(workers) }) # call workers to start job for worker_watcher in self.nodes_watchers: client_call(worker_watcher, 'start_job', zip_filename, uncompress)
def _unzip(self, job_name): zip_file = os.path.join(self.zip_dir, job_name + '.zip') if os.path.exists(zip_file): ZipHandler.uncompress(zip_file, self.job_dir)
def _unzip(self, job_name): zip_file = os.path.join(self.zip_dir, job_name+'.zip') if os.path.exists(zip_file): ZipHandler.uncompress(zip_file, self.job_dir)