def copy_du(self, du, pd_new):
pd_new.create_du(du)
self.__filemanager.copy_du(du, pd_new)
# update meta data at pd_new
pd_new.data_units[du.id] = du
CoordinationAdaptor.update_pd(pd_new)
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None):
"""
Initialize PilotData at given service url:
ssh://<hostname>
gsissh://<hostname>
Currently only ssh schemes are supported. In the future all
SAGA URL schemes/adaptors should be supported.
"""
self.id = None
self.url = None
self.pilot_data_description = None
self.service_url=None
self.size = None
self.data_unit_description = None
self.data_units={}
if pd_url==None and pilot_data_service!=None: # new pd
self.id = self.PD_ID_PREFIX+str(uuid.uuid1())
self.pilot_data_description = pilot_data_description
self.url = CoordinationAdaptor.add_pd(CoordinationAdaptor.get_base_url(application_id)+"/"+pilot_data_service.id, self)
elif pd_url != None:
logger.warn("Reconnect to PilotData: %s"%pd_url)
dictionary = CoordinationAdaptor.get_pd(pd_url)
pd_dict = dictionary["pilot_data"]
for i in pd_dict:
self.__setattr__(i, pd_dict[i])
self.initialize_pilot_data()
def copy_du(self, du, ps_new):
ps_new.create_du(du)
self.__filemanager.copy_du(du, ps_new)
# update meta data at ps_new
ps_new.pilot_data[du.id] = du
CoordinationAdaptor.update_ps(ps_new)
def __init__(self, cds_url=None):
""" Create a Work Data Service object.
Keyword arguments:
cds_url -- Reconnect to an existing WDS (optional).
"""
# Pilot Data
self.data_units={}
self.pilot_data_services=[]
# Pilot Job
self.compute_units={}
self.pilot_job_services=[]
if cds_url == None:
self.id=self.CDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(pilot.application_id)
self.url = CoordinationAdaptor.add_cds(application_url, self)
else:
self.id = self.__get_cds_id(cds_url)
self.url = cds_url
# Background Thread for scheduling
self.scheduler = Scheduler()
self.cu_queue = Queue.Queue()
self.du_queue = Queue.Queue()
self.stop=threading.Event()
self.scheduler_thread=threading.Thread(target=self._scheduler_thread)
self.scheduler_thread.start()
def __add_pilot_data(self, pilot_data):
logger.debug("add du to pilot data")
if len(self.pilot_data) > 0: # copy files from other pilot data
self.pilot_data[0].copy_du(self, pilot_data)
else: # copy files from original location
pilot_data.put_du(self)
self.pilot_data.append(pilot_data)
CoordinationAdaptor.update_du(self)
def submit_data_unit(self, data_unit_description):
""" creates a data unit object and binds it to a physical resource (a pilotdata) """
du = DataUnit(pilot_data_service=self,
data_unit_description=data_unit_description)
self.data_units[du.id]=du
self.du_queue.put(du)
# queue currently not persisted
CoordinationAdaptor.update_cds(self.url, self)
return du
def add_pilot_store(self, pilot_store):
""" add PD to a certain pilot store
data will be moved into this store
"""
if len(self.pilot_stores) > 0: # copy files from other pilot store
self.pilot_stores[0].copy_pd(self, pilot_store)
else: # copy files from original location
pilot_store.put_pd(self)
self.pilot_stores.append(pilot_store)
CoordinationAdaptor.update_pd(self)
def add_pilot_data_service(self, pds):
""" Add a PilotDataService
Keyword arguments:
pds -- The PilotDataService to add.
Return:
None
"""
self.pilot_data_services.append(pds)
CoordinationAdaptor.update_cds(self.url, self)
def add_pilot_compute_service(self, pjs):
""" Add a PilotJobService to this CDS.
Keyword arguments:
pilotjob_services -- The PilotJob Service(s) to which this
Work Unit Service will connect.
Return:
Result
"""
self.pilot_job_services.append(pjs)
CoordinationAdaptor.update_cds(self.url, self)
def remove_pilot_data_service(self, pds):
""" Remove a PilotDataService
Keyword arguments:
pds -- The PilotDataService to remove
Return:
None
"""
self.pilot_data_services.remove(pds)
CoordinationAdaptor.update_cds(self.url, self)
def cancel(self):
""" Cancel the PDS.
All associated PD objects are deleted and removed from the associated pilot stores.
Keyword arguments:
None
Return:
None
"""
# terminate background thread
self.stop.set()
CoordinationAdaptor.delete_cds(self.url)
def submit_compute_unit(self, compute_unit_description):
""" Submit a WU to this Work Unit Service.
Keyword argument:
cud -- The ComputeUnitDescription from the application
Return:
ComputeUnit object
"""
cu = ComputeUnit(compute_unit_description, self)
self.compute_units[cu.id]=cu
self.cu_queue.put(cu)
CoordinationAdaptor.update_cds(self.url, self)
return cu
def __init__(self, pds_url=None):
""" Create a PilotDataService
Keyword arguments:
pds_id -- restore from pds_id
"""
self.pilot_data={}
if pds_url == None:
self.id = self.PDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(application_id)
self.url = CoordinationAdaptor.add_pds(application_url, self)
else:
self.id = self.__get_pds_id(pds_url)
def remove_pilot_compute_service(self, pjs):
""" Remove a PilotJobService from this CDS.
Note that it won't cancel the PilotJobService, it will just no
longer be connected to this WUS.
Keyword arguments:
pilotjob_services -- The PilotJob Service(s) to remove from this
Work Unit Service.
Return:
Result
"""
self.pilot_job_services.remove(pjs)
CoordinationAdaptor.update_cds(self.url, self)
def __init__(self, pilot_data_service=None, data_unit_description=None, pd_url=None):
"""
1.) create a new Pilot Data: pilot_data_service and data_unit_description required
2.) reconnect to an existing Pilot Data: pd_url required
"""
if pd_url==None:
self.id = self.DU_ID_PREFIX + str(uuid.uuid1())
self.data_unit_description = data_unit_description
self.pilot_data=[]
self.url = CoordinationAdaptor.add_pd(pilot_data_service.url, self)
self.state = State.New
self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"])
CoordinationAdaptor.update_pd(self)
else:
self.id = self.__get_pd_id(pd_url)
self.url = pd_url
self.__restore_state()
def create_pilot(self, pilot_data_description):
""" Create a PilotData
Keyword arguments:
pilot_data_description -- PilotData Description
{
'service_url': "ssh://<hostname>/base-url/"
'size': "1000"
}
Return value:
A PilotData handle
"""
pd = PilotData(pilot_data_service=self,
pilot_data_description=pilot_data_description)
self.pilot_data[pd.id]=pd
# store pilot data in central data space
CoordinationAdaptor.add_pd(self.url, pd)
return pd
def __restore_state(self):
pd_dict = CoordinationAdaptor.get_pd(self.url)
self.data_unit_description = pd_dict["data_unit_description"]
self.state = pd_dict["state"]
data_unit_dict_list = pd_dict["data_units"]
self.data_unit_items = [DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list]
self.pilot_stores = []
for i in pd_dict["pilot_stores"]:
logger.debug("PS:"+str(i))
ps = PilotData(ps_url=str(i))
self.pilot_stores.append(ps)
def __init__(self, pilot_data_service=None, data_unit_description=None, du_url=None):
"""
1.) create a new Pilot Data: pilot_data_service and data_unit_description required
2.) reconnect to an existing Pilot Data: du_url required
"""
if du_url==None:
self.id = self.DU_ID_PREFIX + str(uuid.uuid1())
self.data_unit_description = data_unit_description
self.pilot_data=[]
self.url = CoordinationAdaptor.add_du(pilot_data_service.url, self)
self.state = State.New
self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"])
CoordinationAdaptor.update_du(self)
else:
self.id = self.__get_du_id(du_url)
self.url = du_url
logger.debug("Restore du: %s"%self.id)
self.__restore_state()
self.transfer_threads=[]
def __restore_state(self):
du_dict = CoordinationAdaptor.get_du(self.url)
self.data_unit_description = du_dict["data_unit_description"]
self.state = du_dict["state"]
data_unit_dict_list = du_dict["data_units"]
self.data_unit_items = [DataUnitItem.create_data_unit_from_dict(i) for i in data_unit_dict_list]
# restore pilot data
self.pilot_data = []
for i in du_dict["pilot_data"]:
logger.debug("PD: "+str(i))
pd = PilotData(pd_url=str(i))
self.pilot_data.append(pd)
def put_du(self, du):
logging.debug("Put PD: %s to PS: %s"%(du.id,self.service_url))
self.__filemanager.create_du(du.id)
self.__filemanager.put_du(du)
self.data_units[du.id] = du
CoordinationAdaptor.update_pd(self)
def update_state(self, state):
self.state=state
CoordinationAdaptor.update_du(self)
def remove_data_unit(self, data_unit):
self.data_unit_items.remove(data_unit)
CoordinationAdaptor.update_du(self)
def add_data_unit(self, data_unit):
self.data_unit_items.append(data_unit)
CoordinationAdaptor.update_du(self)
def cancel(self):
""" Cancel the DU. """
self.state = State.Done
CoordinationAdaptor.update_du(self)
def __restore_pd(self, pds_url):
pd_list=CoordinationAdaptor.list_pd(pds_url)
for i in pd_list:
pass
def __restore_ps(self, pss_url):
ps_list=CoordinationAdaptor.list_ps(pss_url)
for i in ps_list:
pass
def remove_du(self, du):
""" Remove pilot data from pilot data """
if self.data_units.has_key(du.id):
self.__filemanager.remove_du(du)
del self.data_units[du.id]
CoordinationAdaptor.update_pd(self)
