コード例 #1
0
 def copy_pd(self, pd, ps_new):
     ps_new.create_pd(pd)
     self.__filemanager.copy_pd(pd, ps_new)
     
     # update meta data at ps_new
     ps_new.pilot_data[pd.id] = pd
     CoordinationAdaptor.update_ps(ps_new)
コード例 #2
0
    def __init__(self, wds_url=None):
        """ Create a Work Data Service object.

            Keyword arguments:
            wds_url -- Reconnect to an existing WDS (optional).
        """
        # Pilot Data
        self.pilot_data={}
        self.pilot_store_services=[]
        
        # Pilot Job
        self.pilot_job_services=[]
        self.work_units={}
            
        if wds_url == None:
            self.id=self.WDS_ID_PREFIX + str(uuid.uuid1())
            application_url = CoordinationAdaptor.get_base_url(bigdata.application_id)
            self.url = CoordinationAdaptor.add_wds(application_url, self)
            
        else:
            self.id = self.__get_wds_id(wds_url)
            self.url = wds_url
           
        # Background Thread for scheduling
        self.scheduler = Scheduler()
        self.wu_queue = Queue.Queue()
        self.pd_queue = Queue.Queue()
        self.stop=threading.Event()
        self.scheduler_thread=threading.Thread(target=self._scheduler_thread)
        self.scheduler_thread.start()
コード例 #3
0
 def __init__(self, pilot_store_service=None, pilot_store_description=None, ps_url=None):    
     """ 
         Initialize PilotStore at given service url:
         
         ssh://<hostname>
         gsissh://<hostname>
         
         Currently only ssh schemes are supported. In the future all 
         SAGA URL schemes/adaptors should be supported.        
     """ 
     self.id = None
     self.url = None
     self.pilot_store_description = None
     self.service_url=None
     self.size = None
     self.pilot_store_description = None
     self.pilot_data={}
     
     if ps_url==None and pilot_store_service!=None:      # new ps          
         self.id = self.PS_ID_PREFIX+str(uuid.uuid1())
         self.pilot_store_description = pilot_store_description
         self.url = CoordinationAdaptor.add_ps(CoordinationAdaptor.get_base_url(bigdata.application_id)+"/"+pilot_store_service.id, self)
     elif ps_url != None:
         logger.warn("Reconnect to PilotStore: %s"%ps_url)
         dictionary = CoordinationAdaptor.get_ps(ps_url)
         ps_dict = dictionary["pilot_store"]
         for i in ps_dict:
             self.__setattr__(i, ps_dict[i])
                     
     self.initialize_pilot_store()
コード例 #4
0
 def submit_pilot_data(self, pilot_data_description):
     """ creates a pilot data object and binds it to a physical resource (a pilotstore) """
     pd = PilotData(pilot_data_service=self, 
                    pilot_data_description=pilot_data_description)
     self.pilot_data[pd.id]=pd
     self.pd_queue.put(pd)
     # queue currently not persisted
     CoordinationAdaptor.update_wds(self.url, self)
     return pd
コード例 #5
0
 def add_pilot_store(self, pilot_store):
     """ add PD to a certain pilot store 
         data will be moved into this store
     """
     if len(self.pilot_stores) > 0: # copy files from other pilot store
         self.pilot_stores[0].copy_pd(self, pilot_store)
     else: # copy files from original location
         pilot_store.put_pd(self)
     self.pilot_stores.append(pilot_store)
     CoordinationAdaptor.update_pd(self)  
コード例 #6
0
    def add_pilot_store_service(self, pss):
        """ Add a PilotStoreService 

            Keyword arguments:
            pss -- The PilotStoreService to add.

            Return:
            None
        """
        self.pilot_store_services.append(pss)
        CoordinationAdaptor.update_wds(self.url, self)
コード例 #7
0
    def add_pilot_job_service(self, pjs):
        """ Add a PilotJobService to this WUS.

            Keyword arguments:
            pilotjob_services -- The PilotJob Service(s) to which this 
                                 Work Unit Service will connect.

            Return:
            Result
        """
        self.pilot_job_services.append(pjs)
        CoordinationAdaptor.update_wds(self.url, self)
コード例 #8
0
    def remove_pilot_store_service(self, pss):

        """ Remove a PilotStoreService 
            
            Keyword arguments:
            pss -- The PilotStoreService to remove 
            
            Return:
            None
        """
        self.pilot_store_services.remove(pss)
        CoordinationAdaptor.update_wds(self.url, self)
コード例 #9
0
    def cancel(self):
        """ Cancel the PDS. 
            All associated PD objects are deleted and removed from the associated pilot stores.            
            
            Keyword arguments:
            None

            Return:
            None
        """
        # terminate background thread
        self.stop.set()
        CoordinationAdaptor.delete_wds(self.url)
コード例 #10
0
    def submit_work_unit(self, work_unit_description):
        """ Submit a WU to this Work Unit Service.

            Keyword argument:
            wud -- The WorkUnitDescription from the application

            Return:
            WorkUnit object
        """
        wu = WorkUnit(work_unit_description, self)
        self.work_units[wu.id]=wu
        self.wu_queue.put(wu)
        CoordinationAdaptor.update_wds(self.url, self)
        return wu
コード例 #11
0
    def __init__(self, pss_url=None):
        """ Create a PilotStoreService

            Keyword arguments:
            pss_id -- restore from pss_id
        """        
        self.pilot_stores={}
        
        if pss_url == None:
            self.id = self.PSS_ID_PREFIX + str(uuid.uuid1())
            application_url = CoordinationAdaptor.get_base_url(bigdata.application_id)
            self.url = CoordinationAdaptor.add_pss(application_url, self)
        else:
            self.id = self.__get_pss_id(pss_url)
コード例 #12
0
    def remove_pilot_job_service(self, pjs):
        """ Remove a PilotJobService from this WUS.

            Note that it won't cancel the PilotJobService, it will just no
            longer be connected to this WUS.

            Keyword arguments:
            pilotjob_services -- The PilotJob Service(s) to remove from this
                                 Work Unit Service. 

            Return:
            Result
        """
        self.pilot_job_services.remove(pjs)
        CoordinationAdaptor.update_wds(self.url, self)
コード例 #13
0
 def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None):
     """
         1.) create a new Pilot Data: pilot_data_service and pilot_data_description required
         2.) reconnect to an existing Pilot Data: pd_url required 
         
     """
     if pd_url==None:
         self.id = self.PD_ID_PREFIX + str(uuid.uuid1())
         self.pilot_data_description = pilot_data_description        
         self.pilot_stores=[]
         self.url = CoordinationAdaptor.add_pd(pilot_data_service.url, self)
         self.state = State.New
         self.data_units = DataUnit.create_data_unit_list(self, self.pilot_data_description["file_urls"]) 
         CoordinationAdaptor.update_pd(self)
     else:
         self.id = self.__get_pd_id(pd_url)
         self.url = pd_url            
         self.__restore_state()
コード例 #14
0
    def create_pilotstore(self, pilot_store_description):
        """ Create a PilotStore 

            Keyword arguments:
            pilot_store_description -- PilotStore Description    
            {
                'service_url': "ssh://<hostname>/base-url/"                
                'size': "1000"
            }
            Return value:
            A PilotStore handle
        """
        ps = PilotStore(pilot_store_service=self, 
                        pilot_store_description=pilot_store_description)
        self.pilot_stores[ps.id]=ps
        
        # store pilot store in central data space
        CoordinationAdaptor.add_ps(self.url, ps)        
        return ps
コード例 #15
0
 def __restore_state(self):
     pd_dict = CoordinationAdaptor.get_pd(self.url)
     self.pilot_data_description = pd_dict["pilot_data_description"]
     self.state = pd_dict["state"]
     data_unit_dict_list = pd_dict["data_units"]
     self.data_units = [DataUnit.create_data_unit_from_dict(i) for i in data_unit_dict_list]
     self.pilot_stores = [] 
     for i in pd_dict["pilot_stores"]:
         logger.debug("PS:"+str(i)) 
         ps = PilotStore(ps_url=str(i))
         self.pilot_stores.append(ps) 
コード例 #16
0
 def cancel(self):
     """ Cancel the PD. """
     self.state = State.Done    
     CoordinationAdaptor.update_pd(self)
コード例 #17
0
 def __restore_ps(self, pss_url):
     ps_list=CoordinationAdaptor.list_ps(pss_url) 
     for i in ps_list:
        pass
コード例 #18
0
 def remove_pd(self, pd):
     """ Remove pilot data from pilot store """
     if self.pilot_data.has_key(pd.id):
         self.__filemanager.remove_pd(pd)
         del self.pilot_data[pd.id]
     CoordinationAdaptor.update_ps(self)
コード例 #19
0
 def put_pd(self, pd):
     logging.debug("Put PD: %s to PS: %s"%(pd.id,self.service_url))
     self.__filemanager.create_pd(pd.id)
     self.__filemanager.put_pd(pd)
     self.pilot_data[pd.id] = pd
     CoordinationAdaptor.update_ps(self)
コード例 #20
0
 def update_state(self, state):
     self.state=state
     CoordinationAdaptor.update_pd(self)
コード例 #21
0
 def remove_data_unit(self, data_unit):
     self.data_units.remove(data_unit)
     CoordinationAdaptor.update_pd(self)
コード例 #22
0
 def add_data_unit(self, data_unit):
     self.data_units.append(data_unit)    
     CoordinationAdaptor.update_pd(self)