def create_pilot_data_from_dict(cls, ps_dict):
ps = PilotData()
for i in ps_dict.keys():
ps.__setattr__(i, ps_dict[i])
ps.initialize_pilot_data()
logger.debug("created ps " + str(ps))
return ps
def create_pilot_data_from_dict(cls, pd_dict):
pd = PilotData()
for i in pd_dict.keys():
pd.__setattr__(i, pd_dict[i])
pd.initialize_pilot_data()
logger.debug("created pd " + str(pd))
return pd
def test_pd_reconnect():
du_url = "redis://localhost/bigdata:pds-f31a670c-e3f6-11e1-afaf-705681b3df0f:pd-f31c47b8-e3f6-11e1-af44-705681b3df0f:du-f4debce8-e3f6-11e1-8399-705681b3df0f"
pd_url = __get_pd_url(du_url)
pd = PilotData(pd_url=pd_url)
print str(pd.list_data_units())
du = pd.get_du(du_url)
#du = DataUnit(du_url="redis://localhost/bigdata:pds-32d63b2e-df05-11e1-a329-705681b3df0f:pd-37674138-df05-11e1-80d0-705681b3df0f:du-3b8d428c-df05-11e1-af2a-705681b3df0f")
logger.debug(str(du.list()))
def test_pd_reconnect():
du_url = "redis://localhost/bigdata:pds-f31a670c-e3f6-11e1-afaf-705681b3df0f:pd-f31c47b8-e3f6-11e1-af44-705681b3df0f:du-f4debce8-e3f6-11e1-8399-705681b3df0f"
pd_url = __get_pd_url(du_url)
pd = PilotData(pd_url=pd_url)
print str(pd.list_data_units())
du = pd.get_du(du_url)
#du = DataUnit(du_url="redis://localhost/bigdata:pds-32d63b2e-df05-11e1-a329-705681b3df0f:pd-37674138-df05-11e1-80d0-705681b3df0f:du-3b8d428c-df05-11e1-af2a-705681b3df0f")
logger.debug(str(du.list()))
def test_data_unit_add_file():
pilot_data_service = PilotDataService(coordination_url="redis://localhost/")
pilot_data_description = {"service_url": "ssh://localhost/tmp/pilot-" + str(uuid.uuid1()), "size": 100}
pd = pilot_data_service.create_pilot(pilot_data_description=pilot_data_description)
# create data unit for output data
output_data_unit_description = {"file_urls": [], "file_url_patterns": ["test.txt"]}
output_data_unit = pd.submit_data_unit(output_data_unit_description)
output_data_unit.wait()
logger.debug("Output DU: " + output_data_unit.get_url())
pd_reconnect_url = pd.get_url()
du_url = output_data_unit.get_url()
pd_reconnect = PilotData(pd_url=pd_reconnect_url)
du_reconnect = pd_reconnect.get_du(du_url)
du_reconnect.add_files(["test.txt"])
def create_pilot_data_from_dict(cls, pd_dict):
"""Restore Pilot Data from dictionary"""
pd = PilotData()
for i in pd_dict.keys():
pd.__setattr__(i, pd_dict[i])
pd.__initialize_pilot_data()
logger.debug("created pd " + str(pd))
return pd
def test_data_unit_add_file():
pilot_data_service = PilotDataService(
coordination_url="redis://localhost/")
pilot_data_description = {
"service_url": "ssh://localhost/tmp/pilot-" + str(uuid.uuid1()),
"size": 100
}
pd = pilot_data_service.create_pilot(
pilot_data_description=pilot_data_description)
# create data unit for output data
output_data_unit_description = {
"file_urls": [],
"file_url_patterns": ["test.txt"]
}
output_data_unit = pd.submit_data_unit(output_data_unit_description)
output_data_unit.wait()
logger.debug("Output DU: " + output_data_unit.get_url())
pd_reconnect_url = pd.get_url()
du_url = output_data_unit.get_url()
pd_reconnect = PilotData(pd_url=pd_reconnect_url)
du_reconnect = pd_reconnect.get_du(du_url)
du_reconnect.add_files(["test.txt"])
def __restore_state(self):
du_dict = CoordinationAdaptor.get_du(self.url)
# Restore Data Unit
self.data_unit_description = eval(du_dict["data_unit_description"])
self.state = du_dict["state"]
# Restore DataUnitItems
data_unit_dict_list = eval(du_dict["data_unit_items"])
self.data_unit_items = [
DataUnitItem.create_data_unit_from_dict(i)
for i in data_unit_dict_list
]
# restore Pilot Data
pd_list = eval(du_dict["pilot_data"])
self.pilot_data = []
for i in pd_list:
logger.debug("PD: " + str(i))
pd = PilotData(pd_url=str(i))
self.pilot_data.append(pd)
def create_pilot(self, pilot_data_description):
""" Create a PilotData
Keyword arguments:
pilot_data_description -- PilotData Description::
{
'service_url': "ssh://<hostname>/base-url/",
'size': "1000"
}
Return value:
A PilotData object
"""
pd = PilotData(pilot_data_service=self,
pilot_data_description=pilot_data_description)
self.pilot_data[pd.id] = pd
# store pilot data in central data space
CoordinationAdaptor.add_pd(self.url, pd)
return pd
