def run(self): """ Run the rocket (check out a job from the database and execute it) """ all_stored_data = {} # combined stored data for *all* the Tasks all_update_spec = {} # combined update_spec for *all* the Tasks all_mod_spec = [] # combined mod_spec for *all* the Tasks lp = self.launchpad launch_dir = os.path.abspath(os.getcwd()) # check a FW job out of the launchpad if lp: m_fw, launch_id = lp.checkout_fw(self.fworker, launch_dir, self.fw_id) else: # offline mode m_fw = FireWork.from_file(os.path.join(os.getcwd(), "FW.json")) # set the run start time with open('FW_offline.json', 'r+') as f: d = json.loads(f.read()) d['started_on'] = datetime.utcnow().isoformat() f.seek(0) f.write(json.dumps(d)) f.truncate() launch_id = None # we don't need this in offline mode... if not m_fw: print("No FireWorks are ready to run and match query! {}".format( self.fworker.query)) return False if '_launch_dir' in m_fw.spec: prev_dir = launch_dir os.chdir(m_fw.spec['_launch_dir']) launch_dir = os.path.abspath(os.getcwd()) if lp: lp._change_launch_dir(launch_id, launch_dir) if not os.listdir(prev_dir) and REMOVE_USELESS_DIRS: try: os.rmdir(prev_dir) except: pass # write FW.json and/or FW.yaml to the directory if PRINT_FW_JSON: m_fw.to_file('FW.json', indent=4) if PRINT_FW_YAML: m_fw.to_file('FW.yaml') try: my_spec = dict( m_fw.spec) # make a copy of spec, don't override original # set up heartbeat (pinging the server that we're still alive) ping_stop = start_ping_launch(lp, launch_id) # start background tasks btask_stops = [] if '_background_tasks' in my_spec: for bt in my_spec['_background_tasks']: btask_stops.append(start_background_task(bt, m_fw.spec)) # execute the FireTasks! for my_task in m_fw.tasks: m_action = my_task.run_task(my_spec) # read in a FWAction from a file, in case the task is not Python and cannot return it explicitly if os.path.exists('FWAction.json'): m_action = FWAction.from_file('FWAction.json') elif os.path.exists('FWAction.yaml'): m_action = FWAction.from_file('FWAction.yaml') if not m_action: m_action = FWAction() # update the global stored data with the data to store and update from this particular Task all_stored_data.update(m_action.stored_data) all_update_spec.update(m_action.update_spec) all_mod_spec.extend(m_action.mod_spec) # update spec for next task as well my_spec.update(m_action.update_spec) for mod in m_action.mod_spec: apply_mod(mod, my_spec) if m_action.skip_remaining_tasks: break # add job packing info if this is needed if FWData().MULTIPROCESSING and STORE_PACKING_INFO: all_stored_data[ 'multiprocess_name'] = multiprocessing.current_process( ).name # perform finishing operation stop_backgrounds(ping_stop, btask_stops) for b in btask_stops: b.set() do_ping(lp, launch_id) # one last ping, esp if there is a monitor # last background monitors if '_background_tasks' in my_spec: for bt in my_spec['_background_tasks']: if bt.run_on_finish: for task in bt.tasks: task.run_task(m_fw.spec) m_action.stored_data = all_stored_data m_action.mod_spec = all_mod_spec m_action.update_spec = all_update_spec if lp: lp.complete_launch(launch_id, m_action, 'COMPLETED') else: with open('FW_offline.json', 'r+') as f: d = json.loads(f.read()) d['fwaction'] = m_action.to_dict() d['state'] = 'COMPLETED' d['completed_on'] = datetime.utcnow().isoformat() f.seek(0) f.write(json.dumps(d)) f.truncate() return True except: stop_backgrounds(ping_stop, btask_stops) traceback.print_exc() try: m_action = FWAction(stored_data={ '_message': 'runtime error during task', '_task': my_task.to_dict(), '_exception': traceback.format_exc() }, exit=True) except: m_action = FWAction(stored_data={ '_message': 'runtime error during task', '_task': None, '_exception': traceback.format_exc() }, exit=True) if lp: lp.complete_launch(launch_id, m_action, 'FIZZLED') else: with open('FW_offline.json', 'r+') as f: d = json.loads(f.read()) d['fwaction'] = m_action.to_dict() d['state'] = 'FIZZLED' f.seek(0) f.write(json.dumps(d)) f.truncate() return True
lp.reset(args.password) elif args.command == 'detect_fizzled': # TODO: report when fixed print lp.detect_fizzled(args.time, args.fix) elif args.command == 'detect_unreserved': # TODO: report when fixed print lp.detect_unreserved(args.time, args.fix) elif args.command == 'add': # TODO: make this cleaner, e.g. make TAR option explicit # fwf = Workflow.from_FireWork(FireWork.from_file(args.wf_file)) # lp.add_wf(fwf) try: fwf = Workflow.from_FireWork(FireWork.from_file(args.wf_file)) lp.add_wf(fwf) except: try: if '.tar' in args.wf_file: fwf = Workflow.from_tarfile(args.wf_file) else: fwf = Workflow.from_file(args.wf_file) lp.add_wf(fwf) except: print 'Error reading FireWork/Workflow file.' traceback.print_exc() elif args.command == 'get_fw': fw = lp.get_fw_by_id(args.fw_id) fw_dict = fw.to_dict()
def run(self): """ Run the rocket (check out a job from the database and execute it) """ all_stored_data = {} # combined stored data for *all* the Tasks all_update_spec = {} # combined update_spec for *all* the Tasks all_mod_spec = [] # combined mod_spec for *all* the Tasks lp = self.launchpad launch_dir = os.path.abspath(os.getcwd()) # check a FW job out of the launchpad if lp: m_fw, launch_id = lp.checkout_fw(self.fworker, launch_dir, self.fw_id) else: # offline mode m_fw = FireWork.from_file(os.path.join(os.getcwd(), "FW.json")) # set the run start time with open('FW_offline.json', 'r+') as f: d = json.loads(f.read()) d['started_on'] = datetime.utcnow().isoformat() f.seek(0) f.write(json.dumps(d)) f.truncate() launch_id = None # we don't need this in offline mode... if not m_fw: print("No FireWorks are ready to run and match query! {}".format(self.fworker.query)) return False if '_launch_dir' in m_fw.spec: prev_dir = launch_dir os.chdir(m_fw.spec['_launch_dir']) launch_dir = os.path.abspath(os.getcwd()) if lp: lp.change_launch_dir(launch_id, launch_dir) if not os.listdir(prev_dir) and REMOVE_USELESS_DIRS: try: os.rmdir(prev_dir) except: pass if lp: message = 'RUNNING fw_id: {} in directory: {}'.\ format(m_fw.fw_id, os.getcwd()) lp.log_message(logging.INFO, message) # write FW.json and/or FW.yaml to the directory if PRINT_FW_JSON: m_fw.to_file('FW.json', indent=4) if PRINT_FW_YAML: m_fw.to_file('FW.yaml') try: my_spec = dict(m_fw.spec) # make a copy of spec, don't override original my_spec["_fw_env"] = self.fworker.env # set up heartbeat (pinging the server that we're still alive) ping_stop = start_ping_launch(lp, launch_id) # start background tasks btask_stops = [] if '_background_tasks' in my_spec: for bt in my_spec['_background_tasks']: btask_stops.append(start_background_task(bt, m_fw.spec)) # execute the FireTasks! for my_task in m_fw.tasks: m_action = my_task.run_task(my_spec) # read in a FWAction from a file, in case the task is not Python and cannot return it explicitly if os.path.exists('FWAction.json'): m_action = FWAction.from_file('FWAction.json') elif os.path.exists('FWAction.yaml'): m_action = FWAction.from_file('FWAction.yaml') if not m_action: m_action = FWAction() # update the global stored data with the data to store and update from this particular Task all_stored_data.update(m_action.stored_data) all_update_spec.update(m_action.update_spec) all_mod_spec.extend(m_action.mod_spec) # update spec for next task as well my_spec.update(m_action.update_spec) for mod in m_action.mod_spec: apply_mod(mod, my_spec) if m_action.skip_remaining_tasks: break # add job packing info if this is needed if FWData().MULTIPROCESSING and STORE_PACKING_INFO: all_stored_data['multiprocess_name'] = multiprocessing.current_process().name # perform finishing operation stop_backgrounds(ping_stop, btask_stops) for b in btask_stops: b.set() do_ping(lp, launch_id) # one last ping, esp if there is a monitor # last background monitors if '_background_tasks' in my_spec: for bt in my_spec['_background_tasks']: if bt.run_on_finish: for task in bt.tasks: task.run_task(m_fw.spec) m_action.stored_data = all_stored_data m_action.mod_spec = all_mod_spec m_action.update_spec = all_update_spec if lp: lp.complete_launch(launch_id, m_action, 'COMPLETED') else: with open('FW_offline.json', 'r+') as f: d = json.loads(f.read()) d['fwaction'] = m_action.to_dict() d['state'] = 'COMPLETED' d['completed_on'] = datetime.utcnow().isoformat() f.seek(0) f.write(json.dumps(d)) f.truncate() return True except: stop_backgrounds(ping_stop, btask_stops) traceback.print_exc() try: m_action = FWAction(stored_data={'_message': 'runtime error during task', '_task': my_task.to_dict(), '_exception': traceback.format_exc()}, exit=True) except: m_action = FWAction(stored_data={'_message': 'runtime error during task', '_task': None, '_exception': traceback.format_exc()}, exit=True) if lp: lp.complete_launch(launch_id, m_action, 'FIZZLED') else: with open('FW_offline.json', 'r+') as f: d = json.loads(f.read()) d['fwaction'] = m_action.to_dict() d['state'] = 'FIZZLED' f.seek(0) f.write(json.dumps(d)) f.truncate() return True
lp.reset(args.password) elif args.command == 'detect_fizzled': # TODO: report when fixed print lp.detect_fizzled(args.time, args.fix) elif args.command == 'detect_unreserved': # TODO: report when fixed print lp.detect_unreserved(args.time, args.fix) elif args.command == 'add': # TODO: make this cleaner, e.g. make TAR option explicit # fwf = Workflow.from_FireWork(FireWork.from_file(args.wf_file)) # lp.add_wf(fwf) try: fwf = Workflow.from_FireWork(FireWork.from_file(args.wf_file)) lp.add_wf(fwf) except: try: if '.tar' in args.wf_file: fwf = Workflow.from_tarfile(args.wf_file) else: fwf = Workflow.from_file(args.wf_file) lp.add_wf(fwf) except: print 'Error reading FireWork/Workflow file.' traceback.print_exc() elif args.command == 'get_fw': fw = lp.get_fw_by_id(args.fw_id) fw_dict = fw.to_dict()