def copy_pd(self, pd, ps_new):
ps_new.create_pd(pd)
self.__filemanager.copy_pd(pd, ps_new)
# update meta data at ps_new
ps_new.pilot_data[pd.id] = pd
CoordinationAdaptor.update_ps(ps_new)
def __init__(self, wds_url=None):
""" Create a Work Data Service object.
Keyword arguments:
wds_url -- Reconnect to an existing WDS (optional).
"""
# Pilot Data
self.pilot_data={}
self.pilot_store_services=[]
# Pilot Job
self.pilot_job_services=[]
self.work_units={}
if wds_url == None:
self.id=self.WDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(bigdata.application_id)
self.url = CoordinationAdaptor.add_wds(application_url, self)
else:
self.id = self.__get_wds_id(wds_url)
self.url = wds_url
# Background Thread for scheduling
self.scheduler = Scheduler()
self.wu_queue = Queue.Queue()
self.pd_queue = Queue.Queue()
self.stop=threading.Event()
self.scheduler_thread=threading.Thread(target=self._scheduler_thread)
self.scheduler_thread.start()
def __init__(self, pilot_store_service=None, pilot_store_description=None, ps_url=None):
"""
Initialize PilotStore at given service url:
ssh://<hostname>
gsissh://<hostname>
Currently only ssh schemes are supported. In the future all
SAGA URL schemes/adaptors should be supported.
"""
self.id = None
self.url = None
self.pilot_store_description = None
self.service_url=None
self.size = None
self.pilot_store_description = None
self.pilot_data={}
if ps_url==None and pilot_store_service!=None: # new ps
self.id = self.PS_ID_PREFIX+str(uuid.uuid1())
self.pilot_store_description = pilot_store_description
self.url = CoordinationAdaptor.add_ps(CoordinationAdaptor.get_base_url(bigdata.application_id)+"/"+pilot_store_service.id, self)
elif ps_url != None:
logger.warn("Reconnect to PilotStore: %s"%ps_url)
dictionary = CoordinationAdaptor.get_ps(ps_url)
ps_dict = dictionary["pilot_store"]
for i in ps_dict:
self.__setattr__(i, ps_dict[i])
self.initialize_pilot_store()
def submit_pilot_data(self, pilot_data_description):
""" creates a pilot data object and binds it to a physical resource (a pilotstore) """
pd = PilotData(pilot_data_service=self,
pilot_data_description=pilot_data_description)
self.pilot_data[pd.id]=pd
self.pd_queue.put(pd)
# queue currently not persisted
CoordinationAdaptor.update_wds(self.url, self)
return pd
def add_pilot_store(self, pilot_store):
""" add PD to a certain pilot store
data will be moved into this store
"""
if len(self.pilot_stores) > 0: # copy files from other pilot store
self.pilot_stores[0].copy_pd(self, pilot_store)
else: # copy files from original location
pilot_store.put_pd(self)
self.pilot_stores.append(pilot_store)
CoordinationAdaptor.update_pd(self)
def add_pilot_store_service(self, pss):
""" Add a PilotStoreService
Keyword arguments:
pss -- The PilotStoreService to add.
Return:
None
"""
self.pilot_store_services.append(pss)
CoordinationAdaptor.update_wds(self.url, self)
def add_pilot_job_service(self, pjs):
""" Add a PilotJobService to this WUS.
Keyword arguments:
pilotjob_services -- The PilotJob Service(s) to which this
Work Unit Service will connect.
Return:
Result
"""
self.pilot_job_services.append(pjs)
CoordinationAdaptor.update_wds(self.url, self)
def remove_pilot_store_service(self, pss):
""" Remove a PilotStoreService
Keyword arguments:
pss -- The PilotStoreService to remove
Return:
None
"""
self.pilot_store_services.remove(pss)
CoordinationAdaptor.update_wds(self.url, self)
def cancel(self):
""" Cancel the PDS.
All associated PD objects are deleted and removed from the associated pilot stores.
Keyword arguments:
None
Return:
None
"""
# terminate background thread
self.stop.set()
CoordinationAdaptor.delete_wds(self.url)
def submit_work_unit(self, work_unit_description):
""" Submit a WU to this Work Unit Service.
Keyword argument:
wud -- The WorkUnitDescription from the application
Return:
WorkUnit object
"""
wu = WorkUnit(work_unit_description, self)
self.work_units[wu.id]=wu
self.wu_queue.put(wu)
CoordinationAdaptor.update_wds(self.url, self)
return wu
def __init__(self, pss_url=None):
""" Create a PilotStoreService
Keyword arguments:
pss_id -- restore from pss_id
"""
self.pilot_stores={}
if pss_url == None:
self.id = self.PSS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(bigdata.application_id)
self.url = CoordinationAdaptor.add_pss(application_url, self)
else:
self.id = self.__get_pss_id(pss_url)
def remove_pilot_job_service(self, pjs):
""" Remove a PilotJobService from this WUS.
Note that it won't cancel the PilotJobService, it will just no
longer be connected to this WUS.
Keyword arguments:
pilotjob_services -- The PilotJob Service(s) to remove from this
Work Unit Service.
Return:
Result
"""
self.pilot_job_services.remove(pjs)
CoordinationAdaptor.update_wds(self.url, self)
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None):
"""
1.) create a new Pilot Data: pilot_data_service and pilot_data_description required
2.) reconnect to an existing Pilot Data: pd_url required
"""
if pd_url==None:
self.id = self.PD_ID_PREFIX + str(uuid.uuid1())
self.pilot_data_description = pilot_data_description
self.pilot_stores=[]
self.url = CoordinationAdaptor.add_pd(pilot_data_service.url, self)
self.state = State.New
self.data_units = DataUnit.create_data_unit_list(self, self.pilot_data_description["file_urls"])
CoordinationAdaptor.update_pd(self)
else:
self.id = self.__get_pd_id(pd_url)
self.url = pd_url
self.__restore_state()
def create_pilotstore(self, pilot_store_description):
""" Create a PilotStore
Keyword arguments:
pilot_store_description -- PilotStore Description
{
'service_url': "ssh://<hostname>/base-url/"
'size': "1000"
}
Return value:
A PilotStore handle
"""
ps = PilotStore(pilot_store_service=self,
pilot_store_description=pilot_store_description)
self.pilot_stores[ps.id]=ps
# store pilot store in central data space
CoordinationAdaptor.add_ps(self.url, ps)
return ps
def __restore_state(self):
pd_dict = CoordinationAdaptor.get_pd(self.url)
self.pilot_data_description = pd_dict["pilot_data_description"]
self.state = pd_dict["state"]
data_unit_dict_list = pd_dict["data_units"]
self.data_units = [DataUnit.create_data_unit_from_dict(i) for i in data_unit_dict_list]
self.pilot_stores = []
for i in pd_dict["pilot_stores"]:
logger.debug("PS:"+str(i))
ps = PilotStore(ps_url=str(i))
self.pilot_stores.append(ps)
def cancel(self):
""" Cancel the PD. """
self.state = State.Done
CoordinationAdaptor.update_pd(self)
def __restore_ps(self, pss_url):
ps_list=CoordinationAdaptor.list_ps(pss_url)
for i in ps_list:
pass
def remove_pd(self, pd):
""" Remove pilot data from pilot store """
if self.pilot_data.has_key(pd.id):
self.__filemanager.remove_pd(pd)
del self.pilot_data[pd.id]
CoordinationAdaptor.update_ps(self)
def put_pd(self, pd):
logging.debug("Put PD: %s to PS: %s"%(pd.id,self.service_url))
self.__filemanager.create_pd(pd.id)
self.__filemanager.put_pd(pd)
self.pilot_data[pd.id] = pd
CoordinationAdaptor.update_ps(self)
def update_state(self, state):
self.state=state
CoordinationAdaptor.update_pd(self)
def remove_data_unit(self, data_unit):
self.data_units.remove(data_unit)
CoordinationAdaptor.update_pd(self)
def add_data_unit(self, data_unit):
self.data_units.append(data_unit)
CoordinationAdaptor.update_pd(self)