def __init__(self, pilot_data=None, data_unit_description=None, du_url=None):
"""
1.) create a new Pilot Data: pilot_data_service and data_unit_description required
2.) reconnect to an existing Pilot Data: du_url required
"""
if du_url==None:
self.id = self.DU_ID_PREFIX + str(uuid.uuid1())
self.data_unit_description = data_unit_description
self.pilot_data=[]
self.state = State.New
self.data_unit_items=[]
if self.data_unit_description.has_key("file_urls"):
self.data_unit_items = DataUnitItem.create_data_unit_list(self, self.data_unit_description["file_urls"])
self.url = None
# register a data unit as top-level entry in Redis
application_url = CoordinationAdaptor.get_base_url(application_id)
self.url = CoordinationAdaptor.add_du(application_url, self)
CoordinationAdaptor.update_du(self)
# Deprecated
# old method only allowed the creation of a du if a pd existed
#if pilot_data!=None:
# # Allow data units that are not connected to a resource!
# self.url = CoordinationAdaptor.add_du(pilot_data.url, self)
# CoordinationAdaptor.update_du(self)
else:
self.id = DataUnit._get_du_id(du_url)
self.url = du_url
logger.debug("Restore du: %s"%self.id)
self.__restore_state()
self.transfer_threads=[]
def __init__(self, cds_url=None):
""" Create a ComputeDataService (Decentral) object.
@param cds_url: Reconnect to an existing CDS (optional).
"""
# Pilot Data
self.data_units={}
self.pilot_data_services=[]
# Pilot Compute
self.compute_units={}
self.pilot_job_services=[]
if cds_url == None:
self.id=self.CDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(pilot.application_id)
self.url = CoordinationAdaptor.add_cds(application_url, self)
else:
self.id = self.__get_cds_id(cds_url)
self.url = cds_url
# Background Thread for scheduling
self.scheduler = Scheduler()
self.du_queue = Queue.Queue()
self.stop=threading.Event()
self.scheduler_thread=threading.Thread(target=self._scheduler_thread)
self.scheduler_thread.daemon=True
self.scheduler_thread.start()
logger.debug("Created ComputeDataServiceDecentral")
def __init__(self, cds_url=None):
""" Create a ComputeDataService (Decentral) object.
@param cds_url: Reconnect to an existing CDS (optional).
"""
# Pilot Data
self.data_units={}
self.pilot_data_services=[]
# Pilot Compute
self.compute_units={}
self.pilot_job_services=[]
if cds_url == None:
self.id=self.CDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(pilot.application_id)
self.url = CoordinationAdaptor.add_cds(application_url, self)
else:
self.id = self.__get_cds_id(cds_url)
self.url = cds_url
# Background Thread for scheduling
self.scheduler = Scheduler()
self.du_queue = Queue.Queue()
self.stop=threading.Event()
self.scheduler_thread=threading.Thread(target=self._scheduler_thread)
self.scheduler_thread.daemon=True
self.scheduler_thread.start()
logger.debug("Created ComputeDataServiceDecentral")
def __init__(self, coordination_url=COORDINATION_URL, pds_url=None):
""" Create a PilotDataService
Keyword arguments:
pds_id -- restore from pds_id
"""
self.pilot_data={}
CoordinationAdaptor.configure_base_url(coordination_url)
if pds_url == None:
self.id = self.PDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(application_id)
self.url = CoordinationAdaptor.add_pds(application_url, self)
else:
self.id = self.__get_pds_id(pds_url)
def __init__(self, coordination_url=COORDINATION_URL, pds_url=None):
""" Create a PilotDataService
Keyword arguments:
pds_id -- restore from pds_id
"""
self.pilot_data = {}
CoordinationAdaptor.configure_base_url(coordination_url)
if pds_url == None:
self.id = self.PDS_ID_PREFIX + str(uuid.uuid1())
application_url = CoordinationAdaptor.get_base_url(application_id)
self.url = CoordinationAdaptor.add_pds(application_url, self)
else:
self.id = self.__get_pds_id(pds_url)
def __init__(self,
pilot_data_service=None,
pilot_data_description=None,
pd_url=None):
"""
Initialize PilotData at given service url::
ssh://<hostname>
gsissh://<hostname>
go://<hostname>
gs://google.com
s3://aws.amazon.com
In the future more SAGA/Bliss URL schemes/adaptors are supported.
"""
self.id = None
self.url = pd_url
self.pilot_data_description = None
self.pilot_data_service = pilot_data_service
self.service_url = None
self.size = None
self.data_unit_urls = []
self.security_context = None
if pd_url == None and pilot_data_service != None: # new pd
self.id = self.PD_ID_PREFIX + str(uuid.uuid1())
self.pilot_data_description = pilot_data_description
self.url = CoordinationAdaptor.add_pd(
CoordinationAdaptor.get_base_url(application_id) + ":" +
pilot_data_service.id, self)
elif pd_url != None:
logger.warn("Reconnect to PilotData: %s" % pd_url)
dictionary = CoordinationAdaptor.get_pd(pd_url)
if dictionary.has_key("security_context"):
self.security_context = dictionary["security_context"]
pd_dict = eval(dictionary["pilot_data"])
for i in pd_dict:
self.__setattr__(i, pd_dict[i])
# A Pilot Data does not hold a direct reference to a Data Unit (only URL refs are stored)
self.data_unit_urls = eval(dictionary["data_unit_urls"])
self.__initialize_pilot_data()
CoordinationAdaptor.update_pd(self)
def __init__(self, pilot_data_service=None, pilot_data_description=None, pd_url=None):
"""
Initialize PilotData at given service url::
ssh://<hostname>
gsissh://<hostname>
go://<hostname>
gs://google.com
s3://aws.amazon.com
In the future more SAGA/Bliss URL schemes/adaptors are supported.
"""
self.id = None
self.url = pd_url
self.pilot_data_description = None
self.pilot_data_service = pilot_data_service
self.service_url = None
self.size = None
self.data_unit_urls = []
self.security_context = None
if pd_url == None and pilot_data_service != None: # new pd
self.id = self.PD_ID_PREFIX + str(uuid.uuid1())
self.pilot_data_description = pilot_data_description
self.url = CoordinationAdaptor.add_pd(
CoordinationAdaptor.get_base_url(application_id) + ":" + pilot_data_service.id, self
)
elif pd_url != None:
logger.warn("Reconnect to PilotData: %s" % pd_url)
dictionary = CoordinationAdaptor.get_pd(pd_url)
if dictionary.has_key("security_context"):
self.security_context = dictionary["security_context"]
pd_dict = eval(dictionary["pilot_data"])
for i in pd_dict:
self.__setattr__(i, pd_dict[i])
# A Pilot Data does not hold a direct reference to a Data Unit (only URL refs are stored)
self.data_unit_urls = eval(dictionary["data_unit_urls"])
self.__initialize_pilot_data()
CoordinationAdaptor.update_pd(self)
def __init__(self,
pilot_data=None,
data_unit_description=None,
du_url=None):
"""
1.) create a new Pilot Data: pilot_data_service and data_unit_description required
2.) reconnect to an existing Pilot Data: du_url required
"""
if du_url == None:
self.id = self.DU_ID_PREFIX + str(uuid.uuid1())
self.data_unit_description = data_unit_description
self.pilot_data = []
self.state = State.New
self.data_unit_items = []
if self.data_unit_description.has_key("file_urls"):
self.data_unit_items = DataUnitItem.create_data_unit_list(
self, self.data_unit_description["file_urls"])
self.url = None
# register a data unit as top-level entry in Redis
application_url = CoordinationAdaptor.get_base_url(application_id)
self.url = CoordinationAdaptor.add_du(application_url, self)
CoordinationAdaptor.update_du(self)
# Deprecated
# old method only allowed the creation of a du if a pd existed
#if pilot_data!=None:
# # Allow data units that are not connected to a resource!
# self.url = CoordinationAdaptor.add_du(pilot_data.url, self)
# CoordinationAdaptor.update_du(self)
else:
self.id = DataUnit._get_du_id(du_url)
self.url = du_url
logger.debug("Restore du: %s" % self.id)
self.__restore_state()
self.transfer_threads = []
