def _unzip(self, job_name):
zip_file = os.path.join(self.zip_dir, job_name + '.zip')
job_path = os.path.join(self.job_dir, job_name)
if os.path.exists(job_path):
shutil.rmtree(job_path)
if os.path.exists(zip_file):
ZipHandler.uncompress(zip_file, self.job_dir)
def _unzip(self, job_name):
zip_file = os.path.join(self.zip_dir, job_name+'.zip')
job_path = os.path.join(self.job_dir, job_name)
if os.path.exists(job_path):
shutil.rmtree(job_path)
if os.path.exists(zip_file):
ZipHandler.uncompress(zip_file, self.job_dir)
def testZip(self):
zip_file = os.path.join(self.f, 'test.zip')
ZipHandler.compress(zip_file, self.src_dir)
ZipHandler.uncompress(zip_file, self.dest_dir)
dir_ = os.path.join(self.dest_dir, 'compress')
self.assertTrue(os.path.exists(dir_))
with open(os.path.join(dir_, '1.txt')) as fp:
self.assertEqual(fp.read(), self.content)
dir1 = os.path.join(dir_, 'dir1')
self.assertTrue(os.path.exists(dir1))
with open(os.path.join(dir1, '2.txt')) as fp:
self.assertEqual(fp.read(), self.content)
def start_job(self, zip_filename, uncompress=True):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
master_port = job.context.job.master_port
master = '%s:%s' % (self.master.split(':')[0], master_port)
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
subprocess.Popen('python "%s" "%s" %s' % (f, job_dir, master))
def start_job(self, zip_filename, uncompress=True, client=None):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
# transfer zip file to workers
for watcher in self.nodes_watchers:
if watcher.split(':')[0] == self.ip_address:
continue
file_trans_client = FileTransportClient(watcher, zip_file)
file_trans_client.send_file()
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir,
zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
worker_port = job.context.job.port
port = job.context.job.master_port
nodes = [watcher.split(':')[0] for watcher in self.nodes_watchers]
if len(nodes) > 0:
info = MasterJobInfo(port, nodes, worker_port)
self.running_jobs[job.real_name] = info
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
workers = ['%s:%s' % (node, worker_port) for node in nodes]
cmds = [
'python', f, '-j', job_dir, '-i', self.ip_address, '-n',
' '.join(workers)
]
if self.data_path is not None:
cmds.extend(['-d', self.data_path])
if self.force:
cmds.append('-f')
if client is not None:
cmds.extend(['-c', client])
popen = subprocess.Popen(cmds)
info.popen = popen
# call workers to start job
for worker_watcher in self.nodes_watchers:
client_call(worker_watcher,
'start_job',
zip_filename,
uncompress,
ignore=True)
def start_job(self, zip_filename, uncompress=True):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir,
zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
master_port = job.context.job.master_port
master = '%s:%s' % (self.master.split(':')[0], master_port)
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
subprocess.Popen('python "%s" "%s" %s' % (f, job_dir, master))
def start_job(self, zip_filename, uncompress=True, client=None):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
# transfer zip file to workers
for watcher in self.nodes_watchers:
if watcher.split(':')[0] == self.ip_address:
continue
file_trans_client = FileTransportClient(watcher, zip_file)
file_trans_client.send_file()
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
worker_port = job.context.job.port
port = job.context.job.master_port
nodes = [watcher.split(':')[0] for watcher in self.nodes_watchers]
if len(nodes) > 0:
info = MasterJobInfo(port, nodes, worker_port)
self.running_jobs[job.real_name] = info
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
workers = ['%s:%s'%(node, worker_port) for node in nodes]
cmds = ['python', f, '-j', job_dir, '-i', self.ip_address,
'-n', ' '.join(workers)]
if self.data_path is not None:
cmds.extend(['-d', self.data_path])
if self.force:
cmds.append('-f')
if client is not None:
cmds.extend(['-c', client])
popen = subprocess.Popen(cmds)
info.popen = popen
# call workers to start job
for worker_watcher in self.nodes_watchers:
client_call(worker_watcher, 'start_job', zip_filename, uncompress, ignore=True)
def start_job(self, zip_filename, uncompress=True, client=None):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
# transfer zip file to workers
for watcher in self.nodes_watchers:
if watcher.split(":")[0] == self.ip_address:
continue
file_trans_client = FileTransportClient(watcher, zip_file)
file_trans_client.send_file()
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir, zip_filename.rsplit(".", 1)[0])
job = import_job(job_dir)
worker_port = job.context.job.port
port = job.context.job.master_port
nodes = [watcher.split(":")[0] for watcher in self.nodes_watchers]
if len(nodes) > 0:
info = MasterJobInfo(port, nodes, worker_port)
self.running_jobs[job.real_name] = info
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, "loader.py")
workers = ["%s:%s" % (node, worker_port) for node in nodes]
cmds = ["python", f, "-j", job_dir, "-i", self.ip_address, "-n", " ".join(workers)]
if self.data_path is not None:
cmds.extend(["-d", self.data_path])
if self.force:
cmds.append("-f")
if client is not None:
cmds.extend(["-c", client])
popen = subprocess.Popen(cmds)
info.popen = popen
# call workers to start job
for worker_watcher in self.nodes_watchers:
client_call(worker_watcher, "start_job", zip_filename, uncompress)
def start_job(self, zip_filename, uncompress=True):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
master_port = job.context.job.master_port
master = '%s:%s' % (self.master.split(':')[0], master_port)
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
cmds = ['python', f, '-j', job_dir, '-m', master]
if self.data_path is not None:
cmds.extend(['-d', self.data_path])
if self.force:
cmds.append('-f')
popen = subprocess.Popen(cmds)
self.running_jobs[job.real_name] = WorkerJobInfo(job.context.job.port, popen)
def start_job(self, zip_filename, uncompress=True):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir, zip_filename.rsplit(".", 1)[0])
job = import_job(job_dir)
master_port = job.context.job.master_port
master = "%s:%s" % (self.master.split(":")[0], master_port)
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, "loader.py")
cmds = ["python", f, "-j", job_dir, "-m", master]
if self.data_path is not None:
cmds.extend(["-d", self.data_path])
if self.force:
cmds.append("-f")
popen = subprocess.Popen(cmds)
self.running_jobs[job.real_name] = WorkerJobInfo(job.context.job.port, popen)
def start_job(self, zip_filename, uncompress=True):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir,
zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
master_port = job.context.job.master_port
master = '%s:%s' % (self.master.split(':')[0], master_port)
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
cmds = ['python', f, '-j', job_dir, '-m', master]
if self.data_path is not None:
cmds.extend(['-d', self.data_path])
if self.force:
cmds.append('-f')
popen = subprocess.Popen(cmds)
self.running_jobs[job.real_name] = WorkerJobInfo(
job.context.job.port, popen)
def start_job(self, zip_filename, uncompress=True):
if uncompress:
zip_file = os.path.join(self.zip_dir, zip_filename)
# transfer zip file to workers
for watcher in self.nodes_watchers:
if watcher.split(':')[0] == self.ip_address:
continue
file_trans_client = FileTransportClient(watcher, zip_file)
file_trans_client.send_file()
job_dir = ZipHandler.uncompress(zip_file, self.job_dir)
else:
job_dir = os.path.join(self.job_dir, zip_filename.rsplit('.', 1)[0])
job = import_job(job_dir)
worker_port = job.context.job.port
port = job.context.job.master_port
nodes = [watcher.split(':')[0] for watcher in self.nodes_watchers]
if len(nodes) > 0:
info = MasterJobInfo(port, nodes, worker_port)
self.running_jobs[job.real_name] = info
dirname = os.path.dirname(os.path.abspath(__file__))
f = os.path.join(dirname, 'loader.py')
workers = ['%s:%s'%(node, worker_port) for node in nodes]
subprocess.Popen('python "%(py)s" "%(job_dir)s" %(nodes)s' % {
'py': f,
'job_dir': job_dir,
'nodes': ' '.join(workers)
})
# call workers to start job
for worker_watcher in self.nodes_watchers:
client_call(worker_watcher, 'start_job', zip_filename, uncompress)
def _unzip(self, job_name):
zip_file = os.path.join(self.zip_dir, job_name + '.zip')
if os.path.exists(zip_file):
ZipHandler.uncompress(zip_file, self.job_dir)
def _unzip(self, job_name):
zip_file = os.path.join(self.zip_dir, job_name+'.zip')
if os.path.exists(zip_file):
ZipHandler.uncompress(zip_file, self.job_dir)
