def start(self):
darelogger.info("Creating Compute Engine service ")
self.pilot_compute_service = PilotComputeService(
coordination_url=COORDINATION_URL)
self.pilot_data_service = PilotDataService(
coordination_url=COORDINATION_URL)
for compute_pilot, desc in list(
self.workflow.compute_pilot_repo.items()):
self.pilot_compute_service.create_pilot(
pilot_compute_description=desc)
for data_pilot, desc in list(self.workflow.data_pilot_repo.items()):
self.data_pilot_service_repo.append(
self.pilot_data_service.create_pilot(
pilot_data_description=desc))
self.compute_data_service = ComputeDataServiceDecentral()
self.compute_data_service.add_pilot_compute_service(
self.pilot_compute_service)
self.compute_data_service.add_pilot_data_service(
self.pilot_data_service)
### run the steps
self.step_start_lock = threading.RLock()
self.step_run_lock = threading.RLock()
for step_id in list(self.workflow.step_units_repo.keys()):
darelogger.info(" Sumitted step %s " % step_id)
self.step_start_lock.acquire()
self.start_thread_step_id = step_id
self.step_start_lock.release()
self.step_threads[step_id] = threading.Thread(
target=self.start_step)
self.step_threads[step_id].start()
while (1):
count_step = [
v.is_alive() for k, v in list(self.step_threads.items())
]
darelogger.info('count_step %s' % count_step)
if not True in count_step and len(count_step) > 0:
break
time.sleep(10)
darelogger.info(" All Steps Done processing")
self.quit(message='quit gracefully')
# create pilot job service and initiate a pilot job
pilot_compute_description = {
"service_url": 'fork://localhost',
"number_of_processes": 1,
"working_directory": os.path.join(os.getcwd(), "work"),
'affinity_datacenter_label': "eu-de-south",
'affinity_machine_label': "mymachine"
}
pilotjob = pilot_compute_service.create_pilot(
pilot_compute_description=pilot_compute_description)
pilotjob2 = pilot_compute_service.create_pilot(
pilot_compute_description=pilot_compute_description)
compute_data_service = ComputeDataServiceDecentral()
compute_data_service.add_pilot_compute_service(pilot_compute_service)
# start work unit
compute_unit_description = {
"executable": "/bin/date",
"arguments": [""],
"number_of_processes": 1,
"output": "stdout.txt",
"error": "stderr.txt",
}
for i in range(0, 4):
compute_unit = compute_data_service.submit_compute_unit(
compute_unit_description)
