def worker_job(host, port): # connet to the server pandaserver = ServerProxy("http://%s:%d" % (host, port)) while True: try: sim_root = path.Path(pandaserver.panda_connect()) except: time.sleep(1) else: break tmpdir = path.Path(tempfile.mkdtemp()) while True: # get the next task from the pandaserver while True: try: task = pandaserver.panda_request() except: time.sleep(1) else: break # no more tasks left if task is None: logger.info("Nothing left to do, shutting down") break task = json.loads(task) task_name = task["task_name"] task['data_path'] = tmpdir.joinpath(task_name) logger.info("Got task '%s'", task_name) startime = datetime.now() error = simulate(task) dt = (datetime.now() - startime).total_seconds() logger.info("Completed task '%s' in %s seconds", task_name, dt) if error is not None: logger.error("Task '%s' failed with error:\n%s", task_name, error) while True: try: pandaserver.panda_error(task_name) except: time.sleep(1) else: break else: data_path = path.Path(task["data_path"]) src_path = data_path.dirname().joinpath("%s.tar.gz" % task_name) # first compress the data cmd = [ 'tar', 'czf', src_path, '-C', data_path.dirname(), data_path.name] run_command(logger, cmd) # then send it to the server if host in ('localhost', '127.0.0.1'): prefix = "" dst_path = "%s/%s.tar.gz" % (sim_root, task_name) options = [] else: prefix = "ubuntu@%s:" % host dst_path = sim_root.joinpath("%s.tar.gz" % task_name) options = [ '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null' ] # build the scp command cmd = ['scp', '-q'] cmd.extend(options) cmd.append(src_path) cmd.append(prefix + dst_path) while True: try: run_command(logger, cmd) except: time.sleep(1) else: break # then mark it as complete while True: try: pandaserver.panda_complete(task_name) except: time.sleep(1) else: break task['data_path'].rmtree_p() tmpdir.rmtree_p()