def __init__(self, pilot_data=None, data_unit_description=None, du_url=None): """ 1.) create a new Pilot Data: pilot_data_service and data_unit_description required 2.) reconnect to an existing Pilot Data: du_url required """ if du_url==None: self.id = self.DU_ID_PREFIX + str(uuid.uuid1()) self.data_unit_description = data_unit_description self.pilot_data=[] self.state = State.New self.data_unit_items=[] if self.data_unit_description.has_key("file_urls"): self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"]) self.url = None # register a data unit as top-level entry in Redis application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_du(application_url, self) CoordinationAdaptor.update_du(self) # Deprecated # old method only allowed the creation of a du if a pd existed #if pilot_data!=None: # # Allow data units that are not connected to a resource! # self.url = CoordinationAdaptor.add_du(pilot_data.url, self) # CoordinationAdaptor.update_du(self) else: self.id = DataUnit._get_du_id(du_url) self.url = du_url logger.debug("Restore du: %s"%self.id) self.__restore_state() self.transfer_threads=[]
def __init__(self, cds_url=None): """ Create a ComputeDataService (Decentral) object. @param cds_url: Reconnect to an existing CDS (optional). """ # Pilot Data self.data_units={} self.pilot_data_services=[] # Pilot Compute self.compute_units={} self.pilot_job_services=[] if cds_url == None: self.id=self.CDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(pilot.application_id) self.url = CoordinationAdaptor.add_cds(application_url, self) else: self.id = self.__get_cds_id(cds_url) self.url = cds_url # Background Thread for scheduling self.scheduler = Scheduler() self.du_queue = Queue.Queue() self.stop=threading.Event() self.scheduler_thread=threading.Thread(target=self._scheduler_thread) self.scheduler_thread.daemon=True self.scheduler_thread.start() logger.debug("Created ComputeDataServiceDecentral")
def __init__(self, coordination_url=COORDINATION_URL, pds_url=None): """ Create a PilotDataService Keyword arguments: pds_id -- restore from pds_id """ self.pilot_data={} CoordinationAdaptor.configure_base_url(coordination_url) if pds_url == None: self.id = self.PDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_pds(application_url, self) else: self.id = self.__get_pds_id(pds_url)
def __init__(self, coordination_url=COORDINATION_URL, pds_url=None): """ Create a PilotDataService Keyword arguments: pds_id -- restore from pds_id """ self.pilot_data = {} CoordinationAdaptor.configure_base_url(coordination_url) if pds_url == None: self.id = self.PDS_ID_PREFIX + str(uuid.uuid1()) application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_pds(application_url, self) else: self.id = self.__get_pds_id(pds_url)
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None): """ Initialize PilotData at given service url:: ssh://<hostname> gsissh://<hostname> go://<hostname> gs://google.com s3://aws.amazon.com In the future more SAGA/Bliss URL schemes/adaptors are supported. """ self.id = None self.url = pd_url self.pilot_data_description = None self.pilot_data_service = pilot_data_service self.service_url = None self.size = None self.data_unit_urls = [] self.security_context = None if pd_url == None and pilot_data_service != None: # new pd self.id = self.PD_ID_PREFIX + str(uuid.uuid1()) self.pilot_data_description = pilot_data_description self.url = CoordinationAdaptor.add_pd( CoordinationAdaptor.get_base_url(application_id) + ":" + pilot_data_service.id, self) elif pd_url != None: logger.warn("Reconnect to PilotData: %s" % pd_url) dictionary = CoordinationAdaptor.get_pd(pd_url) if dictionary.has_key("security_context"): self.security_context = dictionary["security_context"] pd_dict = eval(dictionary["pilot_data"]) for i in pd_dict: self.__setattr__(i, pd_dict[i]) # A Pilot Data does not hold a direct reference to a Data Unit (only URL refs are stored) self.data_unit_urls = eval(dictionary["data_unit_urls"]) self.__initialize_pilot_data() CoordinationAdaptor.update_pd(self)
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None): """ Initialize PilotData at given service url:: ssh://<hostname> gsissh://<hostname> go://<hostname> gs://google.com s3://aws.amazon.com In the future more SAGA/Bliss URL schemes/adaptors are supported. """ self.id = None self.url = pd_url self.pilot_data_description = None self.pilot_data_service = pilot_data_service self.service_url = None self.size = None self.data_unit_urls = [] self.security_context = None if pd_url == None and pilot_data_service != None: # new pd self.id = self.PD_ID_PREFIX + str(uuid.uuid1()) self.pilot_data_description = pilot_data_description self.url = CoordinationAdaptor.add_pd( CoordinationAdaptor.get_base_url(application_id) + ":" + pilot_data_service.id, self ) elif pd_url != None: logger.warn("Reconnect to PilotData: %s" % pd_url) dictionary = CoordinationAdaptor.get_pd(pd_url) if dictionary.has_key("security_context"): self.security_context = dictionary["security_context"] pd_dict = eval(dictionary["pilot_data"]) for i in pd_dict: self.__setattr__(i, pd_dict[i]) # A Pilot Data does not hold a direct reference to a Data Unit (only URL refs are stored) self.data_unit_urls = eval(dictionary["data_unit_urls"]) self.__initialize_pilot_data() CoordinationAdaptor.update_pd(self)
def __init__(self, pilot_data=None, data_unit_description=None, du_url=None): """ 1.) create a new Pilot Data: pilot_data_service and data_unit_description required 2.) reconnect to an existing Pilot Data: du_url required """ if du_url == None: self.id = self.DU_ID_PREFIX + str(uuid.uuid1()) self.data_unit_description = data_unit_description self.pilot_data = [] self.state = State.New self.data_unit_items = [] if self.data_unit_description.has_key("file_urls"): self.data_unit_items = DataUnitItem.create_data_unit_list( self, self.data_unit_description["file_urls"]) self.url = None # register a data unit as top-level entry in Redis application_url = CoordinationAdaptor.get_base_url(application_id) self.url = CoordinationAdaptor.add_du(application_url, self) CoordinationAdaptor.update_du(self) # Deprecated # old method only allowed the creation of a du if a pd existed #if pilot_data!=None: # # Allow data units that are not connected to a resource! # self.url = CoordinationAdaptor.add_du(pilot_data.url, self) # CoordinationAdaptor.update_du(self) else: self.id = DataUnit._get_du_id(du_url) self.url = du_url logger.debug("Restore du: %s" % self.id) self.__restore_state() self.transfer_threads = []