def __init__(self, generic_scheduler_arguments, baseurl, username, password, login_baseurl = None, experiments_map = None, uuid = None, **kwargs):
super(ExternalWebLabDeustoScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.baseurl = baseurl
self.login_baseurl = login_baseurl
self.username = username
self.password = password
if experiments_map is None:
self.experiments_map = {}
else:
self.experiments_map = experiments_map
if uuid is None:
self.uuids = []
elif isinstance(uuid, basestring):
human = baseurl
self.uuids = [ (uuid, human) ]
else:
self.uuids = [uuid]
from weblab.core.coordinator.coordinator import POST_RESERVATION_EXPIRATION_TIME, DEFAULT_POST_RESERVATION_EXPIRATION_TIME
post_reservation_expiration_time = self.cfg_manager.get_value(POST_RESERVATION_EXPIRATION_TIME, DEFAULT_POST_RESERVATION_EXPIRATION_TIME)
self.expiration_delta = datetime.timedelta(seconds=post_reservation_expiration_time)
period = self.cfg_manager.get_value(RETRIEVAL_PERIOD_PROPERTY_NAME, DEFAULT_RETRIEVAL_PERIOD)
self.retriever = ResultsRetriever(self, period, self._create_logged_in_client)
self.retriever.start()
def __init__(
self,
generic_scheduler_arguments,
baseurl,
username,
password,
login_baseurl=None,
experiments_map=None,
**kwargs
):
super(ExternalWebLabDeustoScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.baseurl = baseurl
self.login_baseurl = login_baseurl
self.username = username
self.password = password
if experiments_map is None:
self.experiments_map = {}
else:
self.experiments_map = experiments_map
from weblab.core.coordinator.coordinator import (
POST_RESERVATION_EXPIRATION_TIME,
DEFAULT_POST_RESERVATION_EXPIRATION_TIME,
)
post_reservation_expiration_time = self.cfg_manager.get_value(
POST_RESERVATION_EXPIRATION_TIME, DEFAULT_POST_RESERVATION_EXPIRATION_TIME
)
self.expiration_delta = datetime.timedelta(seconds=post_reservation_expiration_time)
period = self.cfg_manager.get_value(RETRIEVAL_PERIOD_PROPERTY_NAME, DEFAULT_RETRIEVAL_PERIOD)
self.retriever = ResultsRetriever(self, period, self._create_logged_in_client)
self.retriever.start()
class ExternalWebLabDeustoScheduler(Scheduler):
def __init__(
self,
generic_scheduler_arguments,
baseurl,
username,
password,
login_baseurl=None,
experiments_map=None,
**kwargs
):
super(ExternalWebLabDeustoScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.baseurl = baseurl
self.login_baseurl = login_baseurl
self.username = username
self.password = password
if experiments_map is None:
self.experiments_map = {}
else:
self.experiments_map = experiments_map
from weblab.core.coordinator.coordinator import (
POST_RESERVATION_EXPIRATION_TIME,
DEFAULT_POST_RESERVATION_EXPIRATION_TIME,
)
post_reservation_expiration_time = self.cfg_manager.get_value(
POST_RESERVATION_EXPIRATION_TIME, DEFAULT_POST_RESERVATION_EXPIRATION_TIME
)
self.expiration_delta = datetime.timedelta(seconds=post_reservation_expiration_time)
period = self.cfg_manager.get_value(RETRIEVAL_PERIOD_PROPERTY_NAME, DEFAULT_RETRIEVAL_PERIOD)
self.retriever = ResultsRetriever(self, period, self._create_logged_in_client)
self.retriever.start()
def stop(self):
self.retriever.stop()
@Override(Scheduler)
def is_remote(self):
return True
@logged()
@Override(Scheduler)
def removing_current_resource_slot(self, session, resource_instance_id):
# Will in fact never be called
return False
# TODO: pooling
def _create_client(self, cookies=None):
client = WebLabDeustoClient(self.baseurl)
if cookies is not None:
client.set_cookies(cookies)
return client
def _create_login_client(self, cookies=None):
client = WebLabDeustoClient(self.login_baseurl or self.baseurl)
if cookies is not None:
client.set_cookies(cookies)
return client
def _create_logged_in_client(self, cookies):
login_client = self._create_login_client(cookies)
session_id = login_client.login(self.username, self.password)
client = self._create_client(login_client.get_cookies())
return session_id, client
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@logged()
@Override(Scheduler)
def reserve_experiment(
self,
reservation_id,
experiment_id,
time,
priority,
initialization_in_accounting,
client_initial_data,
request_info,
):
server_uuids = list(request_info.get(SERVER_UUIDS, []))
server_uuids.append((self.core_server_uuid, self.core_server_uuid_human))
consumer_data = {
"time_allowed": time,
"priority": priority,
"initialization_in_accounting": initialization_in_accounting,
"external_user": request_info.get("username", ""),
SERVER_UUIDS: server_uuids,
}
for forwarded_key in FORWARDED_KEYS:
if forwarded_key in request_info:
consumer_data[forwarded_key] = request_info[forwarded_key]
# TODO: identifier of the server
login_client = self._create_login_client()
session_id = login_client.login(self.username, self.password)
client = self._create_client(login_client.get_cookies())
serialized_client_initial_data = json.dumps(client_initial_data)
serialized_consumer_data = json.dumps(consumer_data)
# If the administrator has mapped that this experiment_id is other, take that other. Otherwide, take the same one
requested_experiment_id_str = self.experiments_map.get(
experiment_id.to_weblab_str(), experiment_id.to_weblab_str()
)
requested_experiment_id = ExperimentId.parse(requested_experiment_id_str)
external_reservation = client.reserve_experiment(
session_id, requested_experiment_id, serialized_client_initial_data, serialized_consumer_data
)
if external_reservation.is_null():
return None
remote_reservation_id = external_reservation.reservation_id.id
log.log(
ExternalWebLabDeustoScheduler,
log.level.Info,
"Local reservation_id %s is linked to remote reservation %s" % (reservation_id, remote_reservation_id),
)
cookies = client.get_cookies()
serialized_cookies = pickle.dumps(cookies)
session = self.session_maker()
try:
reservation = ExternalWebLabDeustoReservation(
reservation_id, remote_reservation_id, serialized_cookies, time_mod.time()
)
pending_results = ExternalWebLabDeustoReservationPendingResults(
reservation_id,
remote_reservation_id,
self.resource_type_name,
self.core_server_route,
request_info.get("username", ""),
pickle.dumps(request_info),
experiment_id.to_weblab_str(),
)
session.add(reservation)
session.add(pending_results)
session.commit()
finally:
session.close()
reservation_status = self._convert_reservation_to_status(
external_reservation, reservation_id, remote_reservation_id
)
return reservation_status
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@logged()
@Override(Scheduler)
def get_reservation_status(self, reservation_id):
reservation_found = False
max_iterations = 15
while not reservation_found and max_iterations >= 0:
session = self.session_maker()
try:
reservation = (
session.query(ExternalWebLabDeustoReservation)
.filter_by(local_reservation_id=reservation_id)
.first()
)
if reservation is None:
pending_result = (
session.query(ExternalWebLabDeustoReservationPendingResults)
.filter_by(
resource_type_name=self.resource_type_name,
server_route=self.core_server_route,
reservation_id=reservation_id,
)
.first()
)
if pending_result is None:
# reservation not yet stored in local database
pass
else:
return WSS.PostReservationStatus(reservation_id, False, "", "")
else:
reservation_found = True
remote_reservation_id = reservation.remote_reservation_id
serialized_cookies = reservation.cookies
finally:
session.close()
# Introduce a delay to let the system store the reservation in the local database
if not reservation_found:
time_mod.sleep(0.1)
max_iterations -= 1
if not reservation_found:
return WSS.PostReservationStatus(reservation_id, False, "", "")
cookies = pickle.loads(str(serialized_cookies))
client = self._create_client(cookies)
reservation = client.get_reservation_status(SessionId(remote_reservation_id))
return self._convert_reservation_to_status(reservation, reservation_id, remote_reservation_id)
def _convert_reservation_to_status(self, reservation, local_reservation_id, remote_reservation_id):
reservation_status = reservation.to_status()
reservation_status.set_reservation_id(local_reservation_id)
if reservation_status.status == WSS.WebLabSchedulingStatus.RESERVED_REMOTE:
#
# If it has been successfully reserved in a remote server, it can mean two things:
#
# a) the remote_reservation_id can be empty, and therefore the remote server we're
# contacting is the one with the resource
#
# b) the remote_reservation_id is other address, and therefore the remote server is
# proxying the communications another server
#
# We have to change the remote reservation id if it is empty
#
if reservation_status.remote_reservation_id == "":
reservation_status.set_remote_reservation_id(remote_reservation_id)
reservation_id_with_route = "%s;%s.%s" % (local_reservation_id, local_reservation_id, self.core_server_route)
reservation_status.reservation_id = reservation_id_with_route
return reservation_status
################################################################
#
# Called when it is confirmed by the Laboratory Server.
#
@logged()
@Override(Scheduler)
def confirm_experiment(self, reservation_id, lab_session_id, initial_configuration):
# At some point, we must call the upper level to say that we want to confirm
# at this point, it's normal that they call us back, even if there is nothing
# to do
pass
################################################################
#
# Called when the user disconnects or finishes the resource.
#
@logged()
@Override(Scheduler)
def finish_reservation(self, reservation_id):
session = self.session_maker()
try:
reservation = (
session.query(ExternalWebLabDeustoReservation).filter_by(local_reservation_id=reservation_id).first()
)
if reservation is not None:
remote_reservation_id = reservation.remote_reservation_id
serialized_cookies = reservation.cookies
else:
log.log(
ExternalWebLabDeustoScheduler,
log.level.Info,
"Not finishing reservation %s since somebody already did it" % reservation_id,
)
return
finally:
session.close()
cookies = pickle.loads(str(serialized_cookies))
client = self._create_client(cookies)
client.finished_experiment(SessionId(remote_reservation_id))
try:
client.get_reservation_status(SessionId(remote_reservation_id))
except:
# TODO: Actually check that the reservation was expired
pass # Expired reservation
else:
now = self.time_provider.get_datetime()
self.post_reservation_data_manager.create(
reservation_id, now, now + self.expiration_delta, json.dumps("''")
)
session = self.session_maker()
try:
reservation = (
session.query(ExternalWebLabDeustoReservation).filter_by(local_reservation_id=reservation_id).first()
)
if reservation is not None:
try:
session.delete(reservation)
session.commit()
except StaleDataError:
log.log(
ExternalWebLabDeustoScheduler,
log.level.Info,
"Could not remove reservation_id %s from ExternalWebLabDeustoReservation since somebody already did it"
% reservation_id,
)
else:
log.log(
ExternalWebLabDeustoScheduler,
log.level.Info,
"Not deleting reservation %s from ExternalWebLabDeustoReservation since somebody already did it"
% reservation_id,
)
return
finally:
session.close()
##############################################################
#
# ONLY FOR TESTING: It completely removes the whole database
#
@Override(Scheduler)
def _clean(self):
session = self.session_maker()
try:
for reservation in session.query(ExternalWebLabDeustoReservation).all():
session.delete(reservation)
for pending_reservation in session.query(ExternalWebLabDeustoReservationPendingResults).all():
session.delete(pending_reservation)
session.commit()
finally:
session.close()
class ExternalWebLabDeustoScheduler(Scheduler):
def __init__(self, generic_scheduler_arguments, baseurl, username, password, login_baseurl = None, experiments_map = None, uuid = None, **kwargs):
super(ExternalWebLabDeustoScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.baseurl = baseurl
self.login_baseurl = login_baseurl
self.username = username
self.password = password
if experiments_map is None:
self.experiments_map = {}
else:
self.experiments_map = experiments_map
if uuid is None:
self.uuids = []
elif isinstance(uuid, basestring):
human = baseurl
self.uuids = [ (uuid, human) ]
else:
self.uuids = [uuid]
from weblab.core.coordinator.coordinator import POST_RESERVATION_EXPIRATION_TIME, DEFAULT_POST_RESERVATION_EXPIRATION_TIME
post_reservation_expiration_time = self.cfg_manager.get_value(POST_RESERVATION_EXPIRATION_TIME, DEFAULT_POST_RESERVATION_EXPIRATION_TIME)
self.expiration_delta = datetime.timedelta(seconds=post_reservation_expiration_time)
period = self.cfg_manager.get_value(RETRIEVAL_PERIOD_PROPERTY_NAME, DEFAULT_RETRIEVAL_PERIOD)
self.retriever = ResultsRetriever(self, period, self._create_logged_in_client)
self.retriever.start()
def stop(self):
self.retriever.stop()
@Override(Scheduler)
def is_remote(self):
return True
@Override(Scheduler)
def get_uuids(self):
return self.uuids
@logged()
@Override(Scheduler)
def removing_current_resource_slot(self, session, resource_instance_id):
# Will in fact never be called
return False
# TODO: pooling
def _create_client(self, cookies = None):
client = WebLabDeustoClient(self.baseurl)
if cookies is not None:
client.set_cookies(cookies)
return client
def _create_login_client(self, cookies = None):
client = WebLabDeustoClient(self.login_baseurl or self.baseurl)
if cookies is not None:
client.set_cookies(cookies)
return client
def _create_logged_in_client(self, cookies):
login_client = self._create_login_client(cookies)
session_id = login_client.login(self.username, self.password)
client = self._create_client(login_client.get_cookies())
return session_id, client
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@logged()
@Override(Scheduler)
def reserve_experiment(self, reservation_id, experiment_id, time, priority, initialization_in_accounting, client_initial_data, request_info):
server_uuids = list(request_info.get(SERVER_UUIDS, []))
server_uuids.append((self.core_server_uuid, self.core_server_uuid_human))
consumer_data = {
'time_allowed' : time,
'priority' : priority,
'initialization_in_accounting' : initialization_in_accounting,
'external_user' : request_info.get('username', ''),
SERVER_UUIDS : server_uuids,
}
for forwarded_key in FORWARDED_KEYS:
if forwarded_key in request_info:
consumer_data[forwarded_key] = request_info[forwarded_key]
consumer_data['external_user_unique'] = request_info.get('username_unique', request_info.get('username', ''))
# TODO: identifier of the server
login_client = self._create_login_client()
session_id = login_client.login(self.username, self.password)
client = self._create_client(login_client.get_cookies())
serialized_client_initial_data = json.dumps(client_initial_data)
serialized_consumer_data = json.dumps(consumer_data)
# If the administrator has mapped that this experiment_id is other, take that other. Otherwide, take the same one
requested_experiment_id_str = self.experiments_map.get(experiment_id.to_weblab_str(), experiment_id.to_weblab_str())
requested_experiment_id = ExperimentId.parse(requested_experiment_id_str)
external_reservation = client.reserve_experiment(session_id, requested_experiment_id, serialized_client_initial_data, serialized_consumer_data)
if external_reservation.is_null():
return None
remote_reservation_id = external_reservation.reservation_id.id
log.log(ExternalWebLabDeustoScheduler, log.level.Info, "Local reservation_id %s is linked to remote reservation %s" % (reservation_id, remote_reservation_id))
cookies = client.get_cookies()
serialized_cookies = pickle.dumps(cookies)
session = self.session_maker()
try:
reservation = ExternalWebLabDeustoReservation(reservation_id, remote_reservation_id, serialized_cookies, time_mod.time())
pending_results = ExternalWebLabDeustoReservationPendingResults(reservation_id, remote_reservation_id, self.resource_type_name, self.core_server_route, request_info.get('username', ''), pickle.dumps(request_info), experiment_id.to_weblab_str())
session.add(reservation)
session.add(pending_results)
session.commit()
finally:
session.close()
reservation_status = self._convert_reservation_to_status(external_reservation, reservation_id, remote_reservation_id)
return reservation_status
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@logged()
@Override(Scheduler)
def get_reservation_status(self, reservation_id):
reservation_found = False
max_iterations = 15
while not reservation_found and max_iterations >= 0:
session = self.session_maker()
try:
reservation = session.query(ExternalWebLabDeustoReservation).filter_by(local_reservation_id = reservation_id).first()
if reservation is None:
pending_result = session.query(ExternalWebLabDeustoReservationPendingResults).filter_by(resource_type_name = self.resource_type_name, server_route = self.core_server_route, reservation_id = reservation_id).first()
if pending_result is None:
# reservation not yet stored in local database
pass
else:
return WSS.PostReservationStatus(reservation_id, False, '', '')
else:
reservation_found = True
remote_reservation_id = reservation.remote_reservation_id
serialized_cookies = reservation.cookies
finally:
session.close()
# Introduce a delay to let the system store the reservation in the local database
if not reservation_found:
time_mod.sleep(0.1)
max_iterations -= 1
if not reservation_found:
return WSS.PostReservationStatus(reservation_id, False, '', '')
cookies = pickle.loads(str(serialized_cookies))
client = self._create_client(cookies)
reservation = client.get_reservation_status(SessionId(remote_reservation_id))
return self._convert_reservation_to_status(reservation, reservation_id, remote_reservation_id)
def _convert_reservation_to_status(self, reservation, local_reservation_id, remote_reservation_id):
reservation_status = reservation.to_status()
reservation_status.set_reservation_id(local_reservation_id)
if reservation_status.status == WSS.WebLabSchedulingStatus.RESERVED_REMOTE:
#
# If it has been successfully reserved in a remote server, it can mean two things:
#
# a) the remote_reservation_id can be empty, and therefore the remote server we're
# contacting is the one with the resource
#
# b) the remote_reservation_id is other address, and therefore the remote server is
# proxying the communications another server
#
# We have to change the remote reservation id if it is empty
#
if reservation_status.remote_reservation_id == '':
reservation_status.set_remote_reservation_id(remote_reservation_id)
reservation_id_with_route = '%s;%s.%s' % (local_reservation_id, local_reservation_id, self.core_server_route)
reservation_status.reservation_id = reservation_id_with_route
return reservation_status
################################################################
#
# Called when it is confirmed by the Laboratory Server.
#
@logged()
@Override(Scheduler)
def confirm_experiment(self, reservation_id, lab_session_id, initial_configuration, exp_info):
# At some point, we must call the upper level to say that we want to confirm
# at this point, it's normal that they call us back, even if there is nothing
# to do
pass
################################################################
#
# Called when the user disconnects or finishes the resource.
#
@logged()
@Override(Scheduler)
def finish_reservation(self, reservation_id):
session = self.session_maker()
try:
reservation = session.query(ExternalWebLabDeustoReservation).filter_by(local_reservation_id = reservation_id).first()
if reservation is not None:
remote_reservation_id = reservation.remote_reservation_id
serialized_cookies = reservation.cookies
else:
log.log(ExternalWebLabDeustoScheduler, log.level.Info, "Not finishing reservation %s since somebody already did it" % reservation_id)
return
finally:
session.close()
cookies = pickle.loads(str(serialized_cookies))
client = self._create_client(cookies)
client.finished_experiment(SessionId(remote_reservation_id))
try:
client.get_reservation_status(SessionId(remote_reservation_id))
except:
# TODO: Actually check that the reservation was expired
pass # Expired reservation
else:
now = self.time_provider.get_datetime()
self.post_reservation_data_manager.create(reservation_id, now, now + self.expiration_delta, json.dumps("''"))
session = self.session_maker()
try:
reservation = session.query(ExternalWebLabDeustoReservation).filter_by(local_reservation_id = reservation_id).first()
if reservation is not None:
try:
session.delete(reservation)
session.commit()
except StaleDataError:
log.log(ExternalWebLabDeustoScheduler, log.level.Info, "Could not remove reservation_id %s from ExternalWebLabDeustoReservation since somebody already did it" % reservation_id)
else:
log.log(ExternalWebLabDeustoScheduler, log.level.Info, "Not deleting reservation %s from ExternalWebLabDeustoReservation since somebody already did it" % reservation_id)
return
finally:
session.close()
##############################################################
#
# ONLY FOR TESTING: It completely removes the whole database
#
@Override(Scheduler)
def _clean(self):
session = self.session_maker()
try:
for reservation in session.query(ExternalWebLabDeustoReservation).all():
session.delete(reservation)
for pending_reservation in session.query(ExternalWebLabDeustoReservationPendingResults).all():
session.delete(pending_reservation)
session.commit()
finally:
session.close()
