def add_pilot_data_service(self, pds): """ Add a PilotDataService @param pds: The PilotDataService to add. """ self.pilot_data_services.append(pds) CoordinationAdaptor.update_cds(self.url, self)
def cancel(self): """ Cancel the CDS. All associated PD and PC objects are canceled. """ # terminate background thread self.stop.set() CoordinationAdaptor.delete_cds(self.url)
def put_du(self, du): """Copy Data Unit to Pilot Data""" logger.debug("Put DU: %s to Pilot-Data: %s"%(du.id,self.service_url)) self.__filemanager.create_du(du.id) self.__filemanager.put_du(du) self.data_unit_urls.append(du.get_url()) CoordinationAdaptor.update_pd(self)
def __init__(self, cds_url=None): """ Create a ComputeDataService (Decentral) object. @param cds_url: Reconnect to an existing CDS (optional). """ # Pilot Data self.data_units={} self.pilot_data_services=[] # Pilot Compute self.compute_units={} self.pilot_job_services=[] if cds_url == None: self.id=self.CDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(pilot.application_id) self.url = CoordinationAdaptor.add_cds(application_url, self) else: self.id = self.__get_cds_id(cds_url) self.url = cds_url # Background Thread for scheduling self.scheduler = Scheduler() self.du_queue = Queue.Queue() self.stop=threading.Event() self.scheduler_thread=threading.Thread(target=self._scheduler_thread) self.scheduler_thread.daemon=True self.scheduler_thread.start() logger.debug("Created ComputeDataServiceDecentral")
def __init__(self, pilot_data=None, data_unit_description=None, du_url=None): """ 1.) create a new Pilot Data: pilot_data_service and data_unit_description required 2.) reconnect to an existing Pilot Data: du_url required """ if du_url==None: self.id = self.DU_ID_PREFIX + str(uuid.uuid1()) self.data_unit_description = data_unit_description self.pilot_data=[] self.state = State.New self.data_unit_items=[] if self.data_unit_description.has_key("file_urls"): self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"]) self.url = None # register a data unit as top-level entry in Redis application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_du(application_url, self) CoordinationAdaptor.update_du(self) # Deprecated # old method only allowed the creation of a du if a pd existed #if pilot_data!=None: # # Allow data units that are not connected to a resource! # self.url = CoordinationAdaptor.add_du(pilot_data.url, self) # CoordinationAdaptor.update_du(self) else: self.id = DataUnit._get_du_id(du_url) self.url = du_url logger.debug("Restore du: %s"%self.id) self.__restore_state() self.transfer_threads=[]
def _update_state(self, state): """ Internal method for updating state""" self.state = state logger.debug("Update DU: " + str(self.url) + " state: " + state) CoordinationAdaptor.update_du_state(self, state) logger.debug("Updated DU: " + str(self.url) + " New state: " + self.get_state())
def add_pilot_compute_service(self, pcs): """ Add a PilotComputeService to this CDS. @param pcs: The PilotComputeService to which this ComputeDataService will connect. """ self.pilot_job_services.append(pcs) CoordinationAdaptor.update_cds(self.url, self)
def submit_data_unit(self, data_unit_description): """ creates a data unit object and binds it to a physical resource (a pilotdata) """ du = DataUnit(pilot_data=None, data_unit_description=data_unit_description) self.data_units[du.id] = du self.du_queue.put(du) # queue currently not persisted CoordinationAdaptor.update_cds(self.url, self) return du
def put_du(self, du): """Copy Data Unit to Pilot Data""" logger.debug("Put DU: %s to Pilot-Data: %s" % (du.id, self.service_url)) self.__filemanager.create_du(du.id) self.__filemanager.put_du(du) self.data_unit_urls.append(du.get_url()) CoordinationAdaptor.update_pd(self)
def copy_du(self, du, pd_new): """ Copy DataUnit to another Pilot Data """ pd_new.create_du(du) self.__filemanager.copy_du(du, pd_new) # update meta data at pd_new #pd_new.data_units[du.id] = du pd_new.data_unit_urls.append(du.get_url()) CoordinationAdaptor.update_pd(pd_new)
def submit_data_unit(self, data_unit_description): """ creates a data unit object and binds it to a physical resource (a pilotdata) """ du = DataUnit(pilot_data=None, data_unit_description=data_unit_description) self.data_units[du.id]=du self.du_queue.put(du) # queue currently not persisted CoordinationAdaptor.update_cds(self.url, self) return du
def add_pilot_compute_service(self, pcs): """ Add a PilotComputeService to this CDS. @param pcs: The PilotComputeService to which this ComputeDataService will connect. """ self.pilot_job_services.append(pcs) CoordinationAdaptor.update_cds(self.url, self) if len(self.pilot_job_services)>1: logger.error("Decentral ComputeDataService only supports 1 PilotComputeService") raise PilotError("Decentral ComputeDataService only supports 1 PilotComputeService")
def remove_pilot_compute_service(self, pcs): """ Remove a PilotJobService from this CDS. Note that it won't cancel the PilotComputeService, it will just no longer be connected to this CDS. Keyword arguments: @param pcs: The PilotComputeService to remove from this ComputeDataService. """ self.pilot_job_services.remove(pcs) CoordinationAdaptor.update_cds(self.url, self)
def submit_compute_unit(self, compute_unit_description): """ Submit a CU to this Compute Data Service. @param compute_unit_description: The ComputeUnitDescription from the application @return: ComputeUnit object """ cu = ComputeUnit(compute_unit_description, self) self.compute_units[cu.id] = cu self.cu_queue.put(cu) CoordinationAdaptor.update_cds(self.url, self) return cu
def __add_pilot_data(self, pilot_data): logger.debug("add du to pilot data") if len(self.pilot_data) > 0: # copy files from other pilot data self.pilot_data[0].copy_du(self, pilot_data) else: # copy files from original location pilot_data.put_du(self) self.pilot_data.append(pilot_data) self._update_state(State.Running) #self.url = CoordinationAdaptor.add_du(pilot_data.url, self) CoordinationAdaptor.update_du(self)
def submit_compute_unit(self, compute_unit_description): """ Submit a CU to this Compute Data Service. @param compute_unit_description: The ComputeUnitDescription from the application @return: ComputeUnit object """ cu = ComputeUnit(compute_unit_description, self) self.compute_units[cu.id]=cu self.cu_queue.put(cu) CoordinationAdaptor.update_cds(self.url, self) return cu
def __add_pilot_data(self, pilot_data): logger.debug("add du to pilot data") if len(self.pilot_data) > 0: # copy files from other pilot data self.pilot_data[0].copy_du(self, pilot_data) else: # copy files from original location pilot_data.put_du(self) self.pilot_data.append(pilot_data) self.state = State.Running #self.url = CoordinationAdaptor.add_du(pilot_data.url, self) CoordinationAdaptor.update_du(self)
def add_files(self, file_url_list=[]): """Add files referenced in list to Data Unit""" self.state = State.Pending item_list = DataUnitItem.create_data_unit_from_urls(None, file_url_list) for i in item_list: self.data_unit_items.append(i) CoordinationAdaptor.update_du(self) if len(self.pilot_data) > 0: for i in self.pilot_data: logger.debug("Update Pilot Data %s" % (i.get_url())) i.put_du(self) CoordinationAdaptor.update_du(self)
def add_files(self, file_url_list=[]): """Add files referenced in list to Data Unit""" self.state = State.Pending item_list = DataUnitItem.create_data_unit_from_urls( None, file_url_list) for i in item_list: self.data_unit_items.append(i) CoordinationAdaptor.update_du(self) if len(self.pilot_data) > 0: for i in self.pilot_data: logger.debug("Update Pilot Data %s" % (i.get_url())) i.put_du(self) CoordinationAdaptor.update_du(self)
def __add_pilot_data(self, pilot_data): logger.debug("DU add_pilot_data: add DU to pilot data in Thread") self._update_state(State.Pending) if len(self.pilot_data) > 0: # copy files from other pilot data self.pilot_data[0].copy_du(self, pilot_data) else: # copy files from original location pilot_data.put_du(self) logger.debug("DU add_pilot_data: Copy/Put DU to pilot data successfull") self.pilot_data.append(pilot_data) self._update_state(State.Running) logger.debug("DU add_pilot_data: Updated State") #self.url = CoordinationAdaptor.add_du(pilot_data.url, self) CoordinationAdaptor.update_du(self)
def __init__(self, coordination_url=COORDINATION_URL, pds_url=None): """ Create a PilotDataService Keyword arguments: pds_id -- restore from pds_id """ self.pilot_data = {} CoordinationAdaptor.configure_base_url(coordination_url) if pds_url == None: self.id = self.PDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_pds(application_url, self) else: self.id = self.__get_pds_id(pds_url)
def __init__(self, coordination_url=COORDINATION_URL, pds_url=None): """ Create a PilotDataService Keyword arguments: pds_id -- restore from pds_id """ self.pilot_data={} CoordinationAdaptor.configure_base_url(coordination_url) if pds_url == None: self.id = self.PDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_pds(application_url, self) else: self.id = self.__get_pds_id(pds_url)
def __add_pilot_data(self, pilot_data): logger.debug("DU add_pilot_data: add DU to pilot data in Thread") self._update_state(State.Pending) if len(self.pilot_data) > 0: # copy files from other pilot data self.pilot_data[0].copy_du(self, pilot_data) else: # copy files from original location pilot_data.put_du(self) logger.debug( "DU add_pilot_data: Copy/Put DU to pilot data successfull") self.pilot_data.append(pilot_data) self._update_state(State.Running) logger.debug("DU add_pilot_data: Updated State") #self.url = CoordinationAdaptor.add_du(pilot_data.url, self) CoordinationAdaptor.update_du(self)
def remove_pilot_compute_service(self, pcs): """ Remove a PilotJobService from this CDS. Note that it won't cancel the PilotJobService, it will just no longer be connected to this WUS. Keyword arguments: pilotjob_services -- The PilotJob Service(s) to remove from this Work Unit Service. Return: Result """ self.pilot_job_services.remove(pcs) CoordinationAdaptor.update_cds(self.url, self) if len(self.pilot_job_services)>1: logger.error("Decentral ComputeDataService only supports 1 PilotComputeService") raise PilotError("Decentral ComputeDataService only supports 1 PilotComputeService")
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None): """ Initialize PilotData at given service url:: ssh://<hostname> gsissh://<hostname> go://<hostname> gs://google.com s3://aws.amazon.com In the future more SAGA/Bliss URL schemes/adaptors are supported. """ self.id = None self.url = pd_url self.pilot_data_description = None self.pilot_data_service = pilot_data_service self.service_url = None self.size = None self.data_unit_urls = [] self.security_context = None if pd_url == None and pilot_data_service != None: # new pd self.id = self.PD_ID_PREFIX + str(uuid.uuid1()) self.pilot_data_description = pilot_data_description self.url = CoordinationAdaptor.add_pd( CoordinationAdaptor.get_base_url(application_id) + ":" + pilot_data_service.id, self) elif pd_url != None: logger.warn("Reconnect to PilotData: %s" % pd_url) dictionary = CoordinationAdaptor.get_pd(pd_url) if dictionary.has_key("security_context"): self.security_context = dictionary["security_context"] pd_dict = eval(dictionary["pilot_data"]) for i in pd_dict: self.__setattr__(i, pd_dict[i]) # A Pilot Data does not hold a direct reference to a Data Unit (only URL refs are stored) self.data_unit_urls = eval(dictionary["data_unit_urls"]) self.__initialize_pilot_data() CoordinationAdaptor.update_pd(self)
def __refresh(self): """ Update list of data units items from coordination service """ try: if self.url != None: du_dict = CoordinationAdaptor.get_du(self.url) data_unit_dict_list = eval(du_dict["data_unit_items"]) self.data_unit_items = [DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list] except: logger.warn("Refresh of DU %s failed"%(self.get_url()))
def create_pilot(self, pilot_data_description): """ Create a PilotData Keyword arguments: pilot_data_description -- PilotData Description:: { 'service_url': "ssh://<hostname>/base-url/", 'size': "1000" } Return value: A PilotData object """ pd = PilotData(pilot_data_service=self, pilot_data_description=pilot_data_description) self.pilot_data[pd.id] = pd # store pilot data in central data space CoordinationAdaptor.add_pd(self.url, pd) return pd
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None): """ Initialize PilotData at given service url:: ssh://<hostname> gsissh://<hostname> go://<hostname> gs://google.com s3://aws.amazon.com In the future more SAGA/Bliss URL schemes/adaptors are supported. """ self.id = None self.url = pd_url self.pilot_data_description = None self.pilot_data_service = pilot_data_service self.service_url = None self.size = None self.data_unit_urls = [] self.security_context = None if pd_url == None and pilot_data_service != None: # new pd self.id = self.PD_ID_PREFIX + str(uuid.uuid1()) self.pilot_data_description = pilot_data_description self.url = CoordinationAdaptor.add_pd( CoordinationAdaptor.get_base_url(application_id) + ":" + pilot_data_service.id, self ) elif pd_url != None: logger.warn("Reconnect to PilotData: %s" % pd_url) dictionary = CoordinationAdaptor.get_pd(pd_url) if dictionary.has_key("security_context"): self.security_context = dictionary["security_context"] pd_dict = eval(dictionary["pilot_data"]) for i in pd_dict: self.__setattr__(i, pd_dict[i]) # A Pilot Data does not hold a direct reference to a Data Unit (only URL refs are stored) self.data_unit_urls = eval(dictionary["data_unit_urls"]) self.__initialize_pilot_data() CoordinationAdaptor.update_pd(self)
def __refresh(self): """ Update list of data units items from coordination service """ try: if self.url != None: du_dict = CoordinationAdaptor.get_du(self.url) data_unit_dict_list = eval(du_dict["data_unit_items"]) self.data_unit_items = [ DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list ] except: logger.warn("Refresh of DU %s failed" % (self.get_url()))
def __init__(self, pilot_data=None, data_unit_description=None, du_url=None): """ 1.) create a new Pilot Data: pilot_data_service and data_unit_description required 2.) reconnect to an existing Pilot Data: du_url required """ if du_url == None: self.id = self.DU_ID_PREFIX + str(uuid.uuid1()) self.data_unit_description = data_unit_description self.pilot_data = [] self.state = State.New self.data_unit_items = [] if self.data_unit_description.has_key("file_urls"): self.data_unit_items = DataUnitItem.create_data_unit_list( self, self.data_unit_description["file_urls"]) self.url = None # register a data unit as top-level entry in Redis application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_du(application_url, self) CoordinationAdaptor.update_du(self) # Deprecated # old method only allowed the creation of a du if a pd existed #if pilot_data!=None: # # Allow data units that are not connected to a resource! # self.url = CoordinationAdaptor.add_du(pilot_data.url, self) # CoordinationAdaptor.update_du(self) else: self.id = DataUnit._get_du_id(du_url) self.url = du_url logger.debug("Restore du: %s" % self.id) self.__restore_state() self.transfer_threads = []
def __restore_state(self): du_dict = CoordinationAdaptor.get_du(self.url) # Restore Data Unit self.data_unit_description = eval(du_dict["data_unit_description"]) self.state = du_dict["state"] # Restore DataUnitItems data_unit_dict_list = eval(du_dict["data_unit_items"]) self.data_unit_items = [DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list] # restore Pilot Data pd_list = eval(du_dict["pilot_data"]) self.pilot_data = [] for i in pd_list: logger.debug("PD: "+str(i)) pd = PilotData(pd_url=str(i)) self.pilot_data.append(pd)
def __restore_state(self): du_dict = CoordinationAdaptor.get_du(self.url) # Restore Data Unit self.data_unit_description = eval(du_dict["data_unit_description"]) self.state = du_dict["state"] # Restore DataUnitItems data_unit_dict_list = eval(du_dict["data_unit_items"]) self.data_unit_items = [ DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list ] # restore Pilot Data pd_list = eval(du_dict["pilot_data"]) self.pilot_data = [] for i in pd_list: logger.debug("PD: " + str(i)) pd = PilotData(pd_url=str(i)) self.pilot_data.append(pd)
def _update_state(self, state): """ Internal method for updating state""" self.state=state logger.debug("Update DU state to " + state + " Number of PD: " + str(len(self.pilot_data))) if len(self.pilot_data) > 0: CoordinationAdaptor.update_du(self)
def get_state(self): """ Return current state of DataUnit """ # update remote state du_dict = CoordinationAdaptor.get_du(self.url) self.state = du_dict["state"] return self.state
def remove_files(self, file_urls): """Remove files from Data Unit (NOT implemented yet""" # TODO #self.data_unit_items.remove(input_data_unit) if len(self.pilot_data) > 0: CoordinationAdaptor.update_du(self)
def cancel(self): """ Cancel the Data Unit. """ self.state = State.Done if len(self.pilot_data) > 0: CoordinationAdaptor.update_du(self)
def __restore_pd(self, pds_url): pd_list=CoordinationAdaptor.list_pd(pds_url) for i in pd_list: pass
def _update_state(self, state): """ Internal method for updating state""" self.state=state logger.debug("Update DU: "+ str(self.url) + " state: " + state) CoordinationAdaptor.update_du_state(self, state) logger.debug("Updated DU: "+ str(self.url) + " New state: " + self.get_state())
def remove_du(self, du): """ Remove Data Unit from Pilot Data """ if self.data_unit_urls.count(du.get_url())>0: self.__filemanager.remove_du(du) self.data_unit_urls.remove(du.get_url()) CoordinationAdaptor.update_pd(self)
def _update_state(self, state): """ Internal method for updating state""" self.state = state if len(self.pilot_data) > 0: CoordinationAdaptor.update_du(self)
def remove_du(self, du): """ Remove Data Unit from Pilot Data """ if self.data_unit_urls.count(du.get_url()) > 0: self.__filemanager.remove_du(du) self.data_unit_urls.remove(du.get_url()) CoordinationAdaptor.update_pd(self)
def __restore_pd(self, pds_url): pd_list = CoordinationAdaptor.list_pd(pds_url) for i in pd_list: pass
def remove_pilot_data_service(self, pds): """ Remove a PilotDataService @param pds: The PilotDataService to remove """ self.pilot_data_services.remove(pds) CoordinationAdaptor.update_cds(self.url, self)