def copy_du(self, du, pd_new): pd_new.create_du(du) self.__filemanager.copy_du(du, pd_new) # update meta data at pd_new pd_new.data_units[du.id] = du CoordinationAdaptor.update_pd(pd_new)
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None): """ Initialize PilotData at given service url: ssh://<hostname> gsissh://<hostname> Currently only ssh schemes are supported. In the future all SAGA URL schemes/adaptors should be supported. """ self.id = None self.url = None self.pilot_data_description = None self.service_url=None self.size = None self.data_unit_description = None self.data_units={} if pd_url==None and pilot_data_service!=None: # new pd self.id = self.PD_ID_PREFIX+str(uuid.uuid1()) self.pilot_data_description = pilot_data_description self.url = CoordinationAdaptor.add_pd(CoordinationAdaptor.get_base_url(application_id)+"/"+pilot_data_service.id, self) elif pd_url != None: logger.warn("Reconnect to PilotData: %s"%pd_url) dictionary = CoordinationAdaptor.get_pd(pd_url) pd_dict = dictionary["pilot_data"] for i in pd_dict: self.__setattr__(i, pd_dict[i]) self.initialize_pilot_data()
def copy_du(self, du, ps_new): ps_new.create_du(du) self.__filemanager.copy_du(du, ps_new) # update meta data at ps_new ps_new.pilot_data[du.id] = du CoordinationAdaptor.update_ps(ps_new)
def __init__(self, cds_url=None): """ Create a Work Data Service object. Keyword arguments: cds_url -- Reconnect to an existing WDS (optional). """ # Pilot Data self.data_units={} self.pilot_data_services=[] # Pilot Job self.compute_units={} self.pilot_job_services=[] if cds_url == None: self.id=self.CDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(pilot.application_id) self.url = CoordinationAdaptor.add_cds(application_url, self) else: self.id = self.__get_cds_id(cds_url) self.url = cds_url # Background Thread for scheduling self.scheduler = Scheduler() self.cu_queue = Queue.Queue() self.du_queue = Queue.Queue() self.stop=threading.Event() self.scheduler_thread=threading.Thread(target=self._scheduler_thread) self.scheduler_thread.start()
def __add_pilot_data(self, pilot_data): logger.debug("add du to pilot data") if len(self.pilot_data) > 0: # copy files from other pilot data self.pilot_data[0].copy_du(self, pilot_data) else: # copy files from original location pilot_data.put_du(self) self.pilot_data.append(pilot_data) CoordinationAdaptor.update_du(self)
def submit_data_unit(self, data_unit_description): """ creates a data unit object and binds it to a physical resource (a pilotdata) """ du = DataUnit(pilot_data_service=self, data_unit_description=data_unit_description) self.data_units[du.id]=du self.du_queue.put(du) # queue currently not persisted CoordinationAdaptor.update_cds(self.url, self) return du
def add_pilot_store(self, pilot_store): """ add PD to a certain pilot store data will be moved into this store """ if len(self.pilot_stores) > 0: # copy files from other pilot store self.pilot_stores[0].copy_pd(self, pilot_store) else: # copy files from original location pilot_store.put_pd(self) self.pilot_stores.append(pilot_store) CoordinationAdaptor.update_pd(self)
def add_pilot_data_service(self, pds): """ Add a PilotDataService Keyword arguments: pds -- The PilotDataService to add. Return: None """ self.pilot_data_services.append(pds) CoordinationAdaptor.update_cds(self.url, self)
def add_pilot_compute_service(self, pjs): """ Add a PilotJobService to this CDS. Keyword arguments: pilotjob_services -- The PilotJob Service(s) to which this Work Unit Service will connect. Return: Result """ self.pilot_job_services.append(pjs) CoordinationAdaptor.update_cds(self.url, self)
def remove_pilot_data_service(self, pds): """ Remove a PilotDataService Keyword arguments: pds -- The PilotDataService to remove Return: None """ self.pilot_data_services.remove(pds) CoordinationAdaptor.update_cds(self.url, self)
def cancel(self): """ Cancel the PDS. All associated PD objects are deleted and removed from the associated pilot stores. Keyword arguments: None Return: None """ # terminate background thread self.stop.set() CoordinationAdaptor.delete_cds(self.url)
def submit_compute_unit(self, compute_unit_description): """ Submit a WU to this Work Unit Service. Keyword argument: cud -- The ComputeUnitDescription from the application Return: ComputeUnit object """ cu = ComputeUnit(compute_unit_description, self) self.compute_units[cu.id]=cu self.cu_queue.put(cu) CoordinationAdaptor.update_cds(self.url, self) return cu
def __init__(self, pds_url=None): """ Create a PilotDataService Keyword arguments: pds_id -- restore from pds_id """ self.pilot_data={} if pds_url == None: self.id = self.PDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_pds(application_url, self) else: self.id = self.__get_pds_id(pds_url)
def remove_pilot_compute_service(self, pjs): """ Remove a PilotJobService from this CDS. Note that it won't cancel the PilotJobService, it will just no longer be connected to this WUS. Keyword arguments: pilotjob_services -- The PilotJob Service(s) to remove from this Work Unit Service. Return: Result """ self.pilot_job_services.remove(pjs) CoordinationAdaptor.update_cds(self.url, self)
def __init__(self, pilot_data_service=None, data_unit_description=None, pd_url=None): """ 1.) create a new Pilot Data: pilot_data_service and data_unit_description required 2.) reconnect to an existing Pilot Data: pd_url required """ if pd_url==None: self.id = self.DU_ID_PREFIX + str(uuid.uuid1()) self.data_unit_description = data_unit_description self.pilot_data=[] self.url = CoordinationAdaptor.add_pd(pilot_data_service.url, self) self.state = State.New self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"]) CoordinationAdaptor.update_pd(self) else: self.id = self.__get_pd_id(pd_url) self.url = pd_url self.__restore_state()
def create_pilot(self, pilot_data_description): """ Create a PilotData Keyword arguments: pilot_data_description -- PilotData Description { 'service_url': "ssh://<hostname>/base-url/" 'size': "1000" } Return value: A PilotData handle """ pd = PilotData(pilot_data_service=self, pilot_data_description=pilot_data_description) self.pilot_data[pd.id]=pd # store pilot data in central data space CoordinationAdaptor.add_pd(self.url, pd) return pd
def __restore_state(self): pd_dict = CoordinationAdaptor.get_pd(self.url) self.data_unit_description = pd_dict["data_unit_description"] self.state = pd_dict["state"] data_unit_dict_list = pd_dict["data_units"] self.data_unit_items = [DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list] self.pilot_stores = [] for i in pd_dict["pilot_stores"]: logger.debug("PS:"+str(i)) ps = PilotData(ps_url=str(i)) self.pilot_stores.append(ps)
def __init__(self, pilot_data_service=None, data_unit_description=None, du_url=None): """ 1.) create a new Pilot Data: pilot_data_service and data_unit_description required 2.) reconnect to an existing Pilot Data: du_url required """ if du_url==None: self.id = self.DU_ID_PREFIX + str(uuid.uuid1()) self.data_unit_description = data_unit_description self.pilot_data=[] self.url = CoordinationAdaptor.add_du(pilot_data_service.url, self) self.state = State.New self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"]) CoordinationAdaptor.update_du(self) else: self.id = self.__get_du_id(du_url) self.url = du_url logger.debug("Restore du: %s"%self.id) self.__restore_state() self.transfer_threads=[]
def __restore_state(self): du_dict = CoordinationAdaptor.get_du(self.url) self.data_unit_description = du_dict["data_unit_description"] self.state = du_dict["state"] data_unit_dict_list = du_dict["data_units"] self.data_unit_items = [DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list] # restore pilot data self.pilot_data = [] for i in du_dict["pilot_data"]: logger.debug("PD: "+str(i)) pd = PilotData(pd_url=str(i)) self.pilot_data.append(pd)
def put_du(self, du): logging.debug("Put PD: %s to PS: %s"%(du.id,self.service_url)) self.__filemanager.create_du(du.id) self.__filemanager.put_du(du) self.data_units[du.id] = du CoordinationAdaptor.update_pd(self)
def update_state(self, state): self.state=state CoordinationAdaptor.update_du(self)
def remove_data_unit(self, data_unit): self.data_unit_items.remove(data_unit) CoordinationAdaptor.update_du(self)
def add_data_unit(self, data_unit): self.data_unit_items.append(data_unit) CoordinationAdaptor.update_du(self)
def cancel(self): """ Cancel the DU. """ self.state = State.Done CoordinationAdaptor.update_du(self)
def __restore_pd(self, pds_url): pd_list=CoordinationAdaptor.list_pd(pds_url) for i in pd_list: pass
def __restore_ps(self, pss_url): ps_list=CoordinationAdaptor.list_ps(pss_url) for i in ps_list: pass
def remove_du(self, du): """ Remove pilot data from pilot data """ if self.data_units.has_key(du.id): self.__filemanager.remove_du(du) del self.data_units[du.id] CoordinationAdaptor.update_pd(self)