def worker_job(host, port):
# connet to the server
pandaserver = ServerProxy("http://%s:%d" % (host, port))
while True:
try:
sim_root = path.Path(pandaserver.panda_connect())
except:
time.sleep(1)
else:
break
tmpdir = path.Path(tempfile.mkdtemp())
while True:
# get the next task from the pandaserver
while True:
try:
task = pandaserver.panda_request()
except:
time.sleep(1)
else:
break
# no more tasks left
if task is None:
logger.info("Nothing left to do, shutting down")
break
task = json.loads(task)
task_name = task["task_name"]
task['data_path'] = tmpdir.joinpath(task_name)
logger.info("Got task '%s'", task_name)
startime = datetime.now()
error = simulate(task)
dt = (datetime.now() - startime).total_seconds()
logger.info("Completed task '%s' in %s seconds", task_name, dt)
if error is not None:
logger.error("Task '%s' failed with error:\n%s", task_name, error)
while True:
try:
pandaserver.panda_error(task_name)
except:
time.sleep(1)
else:
break
else:
data_path = path.Path(task["data_path"])
src_path = data_path.dirname().joinpath("%s.tar.gz" % task_name)
# first compress the data
cmd = [
'tar', 'czf', src_path, '-C',
data_path.dirname(), data_path.name]
run_command(logger, cmd)
# then send it to the server
if host in ('localhost', '127.0.0.1'):
prefix = ""
dst_path = "%s/%s.tar.gz" % (sim_root, task_name)
options = []
else:
prefix = "ubuntu@%s:" % host
dst_path = sim_root.joinpath("%s.tar.gz" % task_name)
options = [
'-o', 'StrictHostKeyChecking=no',
'-o', 'UserKnownHostsFile=/dev/null'
]
# build the scp command
cmd = ['scp', '-q']
cmd.extend(options)
cmd.append(src_path)
cmd.append(prefix + dst_path)
while True:
try:
run_command(logger, cmd)
except:
time.sleep(1)
else:
break
# then mark it as complete
while True:
try:
pandaserver.panda_complete(task_name)
except:
time.sleep(1)
else:
break
task['data_path'].rmtree_p()
tmpdir.rmtree_p()
