def _test_server(self, server, address):
""" _test_server(self,server,address) -> bool
It returns True (if we could perform a call to "test_me"), or False (if we couldn't)
"""
# Check if the server is up and running
try:
random_msg = str(random.random())
result_msg = server.test_me(random_msg)
if random_msg != result_msg:
# This was not a valid server, try another
log.log(
ServerLocator, log.level.Info,
"Test message received from server %s different from the message sent (%s vs %s). Trying another server"
% (address.address, random_msg, result_msg))
return False
except Exception as e:
#There was a exception: this is not a valid server, try another
log.log(
ServerLocator, log.level.Info,
"Testing server %s raised exception %s. Trying another server"
% (address.address, e))
log.log_exc(ServerLocator, log.level.Info)
return False
else:
return True
def _repr_impl(self):
"""__repr__: it takes all the arguments of the constructor and
checks for their value in the object"""
try:
my_class = type(self)
ctor_arguments = _extract_ctor_args(my_class)
repr_str = "%s(" % my_class.__name__
var_names = {}
for var_name in ctor_arguments:
if not hasattr(self, var_name) and hasattr(self, '_%s' % var_name):
var_names[var_name] = '_%s' % var_name
elif not hasattr(self, var_name) and hasattr(
self, '_%s__%s' % (my_class.__name__, var_name)):
var_names[var_name] = '_%s__%s' % (my_class.__name__, var_name)
else:
var_names[var_name] = var_name
repr_str += ', '.join([
'%s = %r' % (v, getattr(self, var_names[v]))
for v in ctor_arguments
])
repr_str += ")"
except:
log.log(
Representable, log.level.Error,
"Could not generate a representation of object of type %s" %
type(self))
log.log_exc(Representable, log.level.Error)
repr_str = '"Could not generate a representation. See the logs"'
return repr_str
def _retrieve_session_id_from_coordinator(self, original_server_address,
server_type, restrictions):
try:
return self._coordinator.new_query(original_server_address,
server_type, restrictions)
except ProtocolErrors.ProtocolError as pe:
log.log(
ServerLocator, log.level.Error,
"Problem while asking for new session id to the coordinator server. %s"
% pe)
log.log_exc(ServerLocator, log.level.Warning)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Couldn't retrieve new session id from coordinator server: " +
str(pe), pe)
except Exception as e:
log.log(
ServerLocator, log.level.Error,
"Unexpected exception while asking for new session id to the coordinator server. %s"
% e)
log.log_exc(ServerLocator, log.level.Warning)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Unexpected exception while asking new session id from coordinator server: "
+ str(e), e)
def wrapper(*args, **kwargs):
try:
for _ in xrange(10):
try:
return func(*args, **kwargs)
except OperationalError as oe:
# XXX MySQL dependent!!!
if oe.orig.args[0] == 1213:
log.log(
PriorityQueueScheduler, log.level.Error,
"Deadlock found, restarting...%s" % func.__name__ )
log.log_exc(PriorityQueueScheduler, log.level.Warning)
continue
else:
raise
except:
if DEBUG:
print("Error in exc_checker: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Error,
"Unexpected exception while running %s" % func.__name__ )
log.log_exc(PriorityQueueScheduler, log.level.Warning)
raise
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
def _call_from_coordaddr(self, *args, **kwargs):
try:
servers = self._easy_locator._get_server_from_coordaddr(self._other_coord, server_type, self._how_many)
except LocatorErrors.NoServerFoundError:
log.log( EasyLocator, log.level.Error, "Can't get %s servers! Error in get_server_from_coordaddr " % server_type)
raise LocatorErrors.UnableToCompleteOperationError(
"Couldn't connect to %s" % server_type
)
tested = 0
for server in servers:
tested += 1
try:
return getattr(server, method)(*args, **kwargs)
except ProtocolErrors.RemoteError:
log.log(
EasyLocator,
log.level.Warning,
"%s failed in reserve_session" % server_type
)
log.log_exc( EasyLocator, log.level.Warning )
self._easy_locator.inform_server_not_working(
server,
server_type,
()
)
log.log( EasyLocator, log.level.Error, "Can't get a %s server! Error in get_server after testing %s servers " % (server_type, tested))
raise LocatorErrors.UnableToCompleteOperationError(
"Couldn't connect to any %s" % server_type
)
def _iterate(self, element):
elements = [element]
while True:
try:
new_element = self.queue.get_nowait()
elements.append(new_element)
except Queue.Empty:
break
if not self.stopped:
execute = True
with self.period_lock:
if time.time(
) - self._latest_update <= self.next_period_between_updates:
execute = False
else:
self._latest_update = time.time()
self._update_period_between_updates()
if execute:
try:
self.scheduler.update()
except:
log.log(SchedulerTransactionsSynchronizer,
log.level.Critical, "Exception updating scheduler")
log.log_exc(SchedulerTransactionsSynchronizer,
log.level.Critical)
self._notify_elements(elements)
def _program_file_t(self, file_content):
"""
Running in its own thread, this method will program the board
while updating the state of the experiment appropriately.
"""
try:
start_time = time.time() # To track the time it takes
self._current_state = STATE_PROGRAMMING
self._program_file(file_content)
self._current_state = STATE_READY
elapsed = time.time() - start_time # Calculate the time the programming process took
# Remember when real usage starts, so that we can enforce use-time specific limits.
self._use_time_start = time.time()
if DEBUG:
print "[DBG]: STATE became STATE_READY. UseTimeStart = %s." % self._use_time_start
# If we are in adaptive mode, change the programming time appropriately.
# TODO: Consider limiting the variation range to dampen anomalies.
if self._adaptive_time:
self._programmer_time = elapsed
except Exception as e:
# Note: Currently, running the fake xilinx will raise this exception when
# trying to do a CleanInputs, for which apparently serial is needed.
self._current_state = STATE_FAILED
log.log(UdXilinxExperiment, log.level.Warning, "Error programming file: " + str(e))
log.log_exc(UdXilinxExperiment, log.level.Warning)
def _program_file(self, file_content):
try:
fd, file_name = tempfile.mkstemp(
prefix="ud_xilinx_experiment_program", suffix="." + self._xilinx_impact.get_suffix()
)
try:
try:
# TODO: encode? utf8?
if isinstance(file_content, unicode):
file_content_encoded = file_content.encode("utf8")
else:
file_content_encoded = file_content
file_content_recovered = ExperimentUtil.deserialize(file_content_encoded)
os.write(fd, file_content_recovered)
finally:
os.close(fd)
self._programmer.program(file_name)
finally:
os.remove(file_name)
# print file_name
# import sys
# sys.stdout.flush()
except Exception as e:
# TODO: test me
log.log(UdXilinxExperiment, log.level.Info, "Exception joining sending program to device: %s" % e.args[0])
log.log_exc(UdXilinxExperiment, log.level.Debug)
raise ExperimentErrors.SendingFileFailureError("Error sending file to device: %s" % e)
self._clear()
def _free_experiment(self, lab_coordaddress, reservation_id,
lab_session_id, experiment_instance_id):
try:
initial_time = datetime.datetime.now()
try:
labserver = self.locator[lab_coordaddress]
experiment_response = labserver.free_experiment(
SessionId.SessionId(lab_session_id))
except Exception as e:
if DEBUG:
traceback.print_exc()
log.log(ReservationConfirmer, log.level.Error,
"Exception freeing experiment: %s" % e)
log.log_exc(ReservationConfirmer, log.level.Warning)
self.coordinator.mark_experiment_as_broken(
experiment_instance_id, [str(e)])
else: # Everything went fine
end_time = datetime.datetime.now()
self.coordinator.confirm_resource_disposal(
lab_coordaddress.address, reservation_id, lab_session_id,
experiment_instance_id, experiment_response, initial_time,
end_time)
except:
if DEBUG:
traceback.print_exc()
log.log(ReservationConfirmer, log.level.Critical,
"Unexpected exception freeing experiment")
log.log_exc(ReservationConfirmer, log.level.Critical)
def _retrieve_networks_from_coordinator(self, original_server_address,
server_coord_address):
try:
return self._coordinator.get_networks(original_server_address,
server_coord_address)
except ProtocolErrors.ProtocolError as pe:
# TODO: not unittested
log.log(
ServerLocator, log.level.Error,
"Problem while asking for networks to the coordinator server. %s"
% pe)
log.log_exc(ServerLocator, log.level.Warning)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Couldn't retrieve networks from coordinator server: " +
str(pe), pe)
except Exception as e:
# TODO: not unittested
log.log(
ServerLocator, log.level.Error,
"Unexpected exception while asking for networks to the coordinator server. %s"
% e)
log.log_exc(ServerLocator, log.level.Warning)
import traceback
traceback.print_exc()
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Unexpected exception while asking for networks from coordinator server: "
+ str(e), e)
def _iterate(self, element):
elements = [element]
while True:
try:
new_element = self.queue.get_nowait()
elements.append(new_element)
except Queue.Empty:
break
if not self.stopped:
execute = True
with self.period_lock:
if time.time() - self._latest_update <= self.next_period_between_updates:
execute = False
else:
self._latest_update = time.time()
self._update_period_between_updates()
if execute:
try:
self.scheduler.update()
except:
log.log(SchedulerTransactionsSynchronizer, log.level.Critical, "Exception updating scheduler")
log.log_exc(SchedulerTransactionsSynchronizer, log.level.Critical)
self._notify_elements(elements)
def _free_experiment(self, lab_session_id):
if not self._session_manager.has_session(lab_session_id):
return
session = self._session_manager.get_session_locking(lab_session_id)
finished = True
experiment_response = None
try:
# Remove the async requests whose results we have not retrieved.
# It seems that they might still be running when free gets called.
# TODO: Consider possible issues.
session_id = session['session_id']
if session_id in self._async_requests:
del self._async_requests[session_id]
experiment_instance_id = session['experiment_instance_id']
try:
experiment_response = self._free_experiment_from_assigned_experiments(experiment_instance_id, lab_session_id)
except Exception as e:
log.log( LaboratoryServer, log.level.Error, "Exception freeing experiment" % e )
log.log_exc(LaboratoryServer, log.level.Error)
experiment_response = ''
if experiment_response is not None and experiment_response.lower() != 'ok' and experiment_response != '':
try:
response = json.loads(experiment_response)
finished = response.get(Coordinator.FINISH_FINISHED_MESSAGE)
except:
traceback.print_exc()
finally:
if finished:
self._session_manager.delete_session_unlocking(lab_session_id)
else:
self._session_manager.modify_session_unlocking(lab_session_id, session)
return experiment_response
def run(self):
while not self.stopping:
try:
sleep(1)
if self.frequency is None:
continue
# Here self.frequency is configured, so wait the rest of the required time
if self.frequency > 1:
sleep(self.frequency - 1)
if self.stopping:
break
if self.coordinator is None:
continue
coordinator = self.coordinator()
if coordinator is None or coordinator.locator is None:
continue # coordinator not configured yet
checker = self.Checker(coordinator)
checker.check()
except Exception as e:
log.log(ResourcesCheckerThread, log.level.Critical,
"Exception checking resources: %s" % e )
log.log_exc(ResourcesCheckerThread, log.level.Error)
def _retrieve_all_servers_from_coordinator(self,original_server_address,server_type,restrictions):
try:
return self._coordinator.get_all_servers(original_server_address,server_type,restrictions)
except ProtocolErrors.ProtocolError as pe:
# TODO: not unittested
log.log(
ServerLocator,
log.level.Error,
"Problem while asking for all servers to the coordinator server. %s" % pe
)
log.log_exc(
ServerLocator,
log.level.Warning
)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Couldn't retrieve all servers from coordinator server: " + str(pe),
pe
)
except Exception as e:
# TODO: not unittested
log.log(
ServerLocator,
log.level.Error,
"Unexpected exception while asking for all servers to the coordinator server. %s" % e
)
log.log_exc(
ServerLocator,
log.level.Warning
)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Unexpected exception while asking all servers from coordinator server: " + str(e),
e
)
def _test_server(self,server,address):
""" _test_server(self,server,address) -> bool
It returns True (if we could perform a call to "test_me"), or False (if we couldn't)
"""
# Check if the server is up and running
try:
random_msg = str(random.random())
result_msg = server.test_me(random_msg)
if random_msg != result_msg:
# This was not a valid server, try another
log.log(
ServerLocator,
log.level.Warning,
"Test message received from server %s different from the message sent (%s vs %s). Trying another server" %(
address.address,
random_msg,
result_msg
)
)
return False
except Exception as e:
#There was a exception: this is not a valid server, try another
log.log(
ServerLocator,
log.level.Warning,
"Testing server %s raised exception %s. Trying another server" % (
address.address,
e
)
)
log.log_exc(ServerLocator, log.level.Info)
return False
else:
return True
def create_external_user(self, external_user, external_id, system, group_names):
session = self.Session()
try:
try:
auth_type = session.query(model.DbAuthType).filter_by(name=system).one()
auth = auth_type.auths[0]
except (NoResultFound, KeyError):
raise DbErrors.DbUserNotFoundError("System '%s' not found in database" % system)
groups = []
for group_name in group_names:
try:
group = session.query(model.DbGroup).filter_by(name=group_name).one()
except NoResultFound:
raise DbErrors.DbUserNotFoundError("Group '%s' not found in database" % group_name)
groups.append(group)
try:
role = session.query(model.DbRole).filter_by(name=external_user.role.name).one()
user = model.DbUser(external_user.login, external_user.full_name, external_user.email, role = role)
user_auth = model.DbUserAuth(user, auth, configuration = external_id)
for group in groups:
group.users.append(user)
session.add(user)
session.add(user_auth)
session.commit()
except Exception as e:
log.log( DatabaseGateway, log.level.Warning, "Couldn't create user: %s" % e)
log.log_exc(DatabaseGateway, log.level.Info)
raise DbErrors.DatabaseError("Couldn't create user! Contact administrator")
finally:
session.close()
def run(self):
try:
try:
self.result = self._func(
self._self,
*self._args,
**self._kargs
)
self.finished_ok = True
finally:
if self._resource_manager != None:
self._resource_manager.remove_resource(self)
except Exception as e:
self.raised_exc = e
if self.logging:
log.log(
_ThreadedFunc,
log.level.Warning,
"threaded: exception caught while running %s: %s" % (
self._func.__name__,
e
)
)
log.log_exc( _ThreadedFunc, log.level.Warning)
sio = StringIO.StringIO()
traceback.print_exc(file=sio)
self.raised_exc_traceback = sio.getvalue()
def _retrieve_all_servers_from_coordinator(self, original_server_address,
server_type, restrictions):
try:
return self._coordinator.get_all_servers(original_server_address,
server_type, restrictions)
except ProtocolErrors.ProtocolError as pe:
# TODO: not unittested
log.log(
ServerLocator, log.level.Error,
"Problem while asking for all servers to the coordinator server. %s"
% pe)
log.log_exc(ServerLocator, log.level.Warning)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Couldn't retrieve all servers from coordinator server: " +
str(pe), pe)
except Exception as e:
# TODO: not unittested
log.log(
ServerLocator, log.level.Error,
"Unexpected exception while asking for all servers to the coordinator server. %s"
% e)
log.log_exc(ServerLocator, log.level.Warning)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Unexpected exception while asking all servers from coordinator server: "
+ str(e), e)
def _get_server_from_coordinator(self, session_id):
try:
return self._coordinator.get_server(session_id)
except CoordinatorServerErrors.NoServerFoundError as nsfe:
raise nsfe
except ProtocolErrors.ProtocolError as pe:
log.log(
ServerLocator,
log.level.Error,
"Problem while asking for other server to the coordinator server. %s" % pe
)
log.log_exc(
ServerLocator,
log.level.Warning
)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Couldn't ask for other server to coordinator server: " + str(pe),
pe
)
except Exception as e:
log.log(
ServerLocator,
log.level.Error,
"Unexpected exception while asking for other server to the coordinator server. %s" % e
)
log.log_exc(
ServerLocator,
log.level.Warning
)
raise LocatorErrors.ProblemCommunicatingWithCoordinatorError(
"Unexpected exception while asking for other server to the coordinator server: " + str(e),
e
)
def confirm_resource_disposal(self, lab_coordaddress, reservation_id, lab_session_id, experiment_instance_id, experiment_response, initial_time, end_time):
experiment_finished = True
information_to_store = None
time_remaining = 0.5 # Every half a second by default
if experiment_response is None or experiment_response == 'ok' or experiment_response == '':
pass # Default value
else:
try:
response = json.loads(experiment_response)
experiment_finished = response.get(FINISH_FINISHED_MESSAGE, experiment_finished)
time_remaining = response.get(FINISH_ASK_AGAIN_MESSAGE, time_remaining)
information_to_store = response.get(FINISH_DATA_MESSAGE, information_to_store)
except Exception as e:
log.log( AbstractCoordinator, log.level.Error, "Could not parse experiment server finishing response: %s; %s" % (e, experiment_response) )
log.log_exc( AbstractCoordinator, log.level.Warning )
if not experiment_finished:
time.sleep(time_remaining)
# We just ignore the data retrieved, if any, and perform the query again
self.confirmer.enqueue_free_experiment(lab_coordaddress, reservation_id, lab_session_id, experiment_instance_id)
return
else:
# Otherwise we mark it as finished
self.post_reservation_data_manager.finish(reservation_id, json.dumps(information_to_store))
try:
# and we remove the resource
# print "AT CONFIRM_RESOURCE_DISPOSAL"
self._release_resource_instance(experiment_instance_id)
finally:
self.finished_store.put(reservation_id, information_to_store, initial_time, end_time)
# It's done here so it's called often enough
self.post_reservation_data_manager.clean_expired()
def _send_async_command_t(self, session, command):
"""
This method is used for asynchronously calling the experiment server's
send_command_to_device, and for that purpose runs on its own thread.
This implies that its response will arrive asynchronously to the client.
"""
lab_session_id = session['session_id']
experiment_instance_id = session['experiment_instance_id']
api = self._assigned_experiments.get_api(experiment_instance_id)
experiment_coord_address = session['experiment_coord_address']
experiment_server = self._locator[experiment_coord_address]
try:
if api.endswith("concurrent"):
response = experiment_server.send_command_to_device(lab_session_id, command.get_command_string())
else:
response = experiment_server.send_command_to_device(command.get_command_string())
except Exception as e:
log.log( LaboratoryServer, log.level.Warning, "Exception sending async command to experiment: %s" % e )
log.log_exc(LaboratoryServer, log.level.Info)
raise LaboratoryErrors.FailedToSendCommandError("Couldn't send async command: %s" % str(e))
return Command.Command(str(response))
def do_GET(self):
create_context(self.server, self.client_address, self.headers)
try:
first_question_mark = self.path.find("?")
if first_question_mark >= 0:
path = self.path[0:first_question_mark]
options = urllib.unquote(self.path[first_question_mark + 1:])
else:
path = self.path
options = ""
parameters = [
field for field in options.split("&") if field.find("=") >= 0
]
session_id_param = [
field[field.find("=") + 1:] for field in parameters
if field.startswith("sessionid")
]
if len(session_id_param) == 0:
self._write(400, "sessionid not provided")
return
session_id = session_id_param[0]
last_slash = path.rfind("/")
if last_slash >= 0:
method_name = path[last_slash + 1:]
else:
method_name = path
if 'get_%s' % method_name in dir(Methods):
method = getattr(Methods, 'get_%s' % method_name)
try:
response = method(self, session_id, parameters)
except MethodError as me:
# import traceback
# traceback.print_exc()
log.log(self, log.level.Error, str(me))
log.log_exc(self, log.level.Warning)
self._write(400, "Error: %s" % me)
return
json_response = json.dumps(response)
else:
self._write(
400,
"method %s not implemented" % urllib2.quote(method_name))
return
self._write(200, json_response)
except Exception as e:
import traceback
traceback.print_exc()
log.log(self, log.level.Error, str(e))
log.log_exc(self, log.level.Warning)
self._write(500, 'Error in server. Contact administrator')
finally:
delete_context()
def _validate_simple_authn(self, username, credentials):
"""
When the login() method is called, this method is used with the
username and credentials (e.g., password, IP address, etc.). This
method will only check the SimpleAuthn instances.
"""
try:
login, role_name, user_auths = self._db.retrieve_role_and_user_auths(
username)
except DbUserNotFoundError:
return self._process_invalid()
# login could be different to username.
# For example, in MySQL, where login = '******' is equivalent to where login = '******'
# For this reason, we don't trust "username", and retrieve login from the database
errors = False
for user_auth in user_auths:
# Take only those auth types that use a simple interface
if user_auth.is_simple_authn():
# With each user auth, try to authenticate the user.
try:
authenticated = user_auth.authenticate(login, credentials)
except:
# If there is an error, the user could not be authenticated.
log.log(
LoginManager, log.level.Warning,
"Username: %s with user_auth %s: ERROR" %
(login, user_auth))
log.log_exc(LoginManager, log.level.Warning)
errors = True
traceback.print_exc()
continue
if authenticated:
# If authenticated, return that it was correctly authenticated.
log.log(
LoginManager, log.level.Debug,
"Username: %s with user_auth %s: SUCCESS" %
(login, user_auth))
return ValidDatabaseSessionId(login, role_name)
else:
# If not authenticated, log it and continue with the next user_auth.
log.log(
LoginManager, log.level.Warning,
"Username: %s with user_auth %s: FAIL" %
(login, user_auth))
if errors:
# Raise error: there was a server problem and this might be the reason for not
# authenticating the user. Examples: LDAP server is down, there is an error in the
# local database or so.
raise LoginErrors.LoginError(
"Error checking credentials. Contact administrators!")
return self._process_invalid()
def check(self):
try:
experiments_per_laboratory = self.coordinator.list_laboratories_addresses(
)
# Use a common broken_resources to avoid endless loops if a resource is registered
# in labs in more than one laboratory server (and one might state that it works while
# other might state that it doesn't).
broken_resources = {}
for laboratory_address_str in experiments_per_laboratory:
self.current_lab = laboratory_address_str
new_broken_resources = self.check_laboratory(
laboratory_address_str,
experiments_per_laboratory[laboratory_address_str])
for broken_resource in new_broken_resources:
if broken_resource in broken_resources:
broken_resources[
broken_resource] += ';' + new_broken_resources[
broken_resource]
else:
broken_resources[
broken_resource] = new_broken_resources[
broken_resource]
all_notifications = {
# (recipient1, recipient2) : [message1, message2, message3],
# (recipient1, ) : [message4, message5],
# (recipient3, ) : [message6, message7],
}
for laboratory_address_str in experiments_per_laboratory:
experiments = experiments_per_laboratory[
laboratory_address_str]
for experiment in experiments:
laboratory_resource = experiments[experiment]
if laboratory_resource in broken_resources:
notifications = self.coordinator.mark_resource_as_broken(
laboratory_resource,
broken_resources[laboratory_resource])
else:
notifications = self.coordinator.mark_resource_as_fixed(
laboratory_resource)
for recipients in notifications:
if recipients in all_notifications:
all_notifications[recipients].extend(
notifications[recipients])
else:
all_notifications[recipients] = list(
notifications[recipients])
if all_notifications:
self.coordinator.notify_status(all_notifications)
except:
traceback.print_exc()
log.log(ResourcesChecker, log.level.Critical,
"Error checking resources.")
log.log_exc(ResourcesChecker, log.level.Critical)
def _skeleton(self, *parameters, **kparameters):
""" Dynamically generated method. Protocol: BaseSocket.
Method name: METHOD_NAME. Documentation: DOCUMENTATION """
try:
return getattr(self._parent, "do_" + METHOD_NAME)(*parameters, **kparameters)
except Exception:
# TODO: watch out, if server gets a Control + C, the exception is going to propagate
log.log_exc(self, log.level.Info)
raise
def _logout_from_coordinator(self, session_id):
try:
self._coordinator.logout(session_id)
except Exception as e:
log.log(
ServerLocator, log.level.Warning,
"Unexpected exception while logging out from Coordinator Server. %s "
% e)
log.log_exc(ServerLocator, log.level.Info)
def _skeleton(self, *parameters, **kparameters):
""" Dynamically generated method. Protocol: BaseSocket.
Method name: METHOD_NAME. Documentation: DOCUMENTATION """
try:
return getattr(self._parent, "do_"+METHOD_NAME)(*parameters, **kparameters)
except Exception:
# TODO: watch out, if server gets a Control + C, the exception is going to propagate
log.log_exc(self, log.level.Info)
raise
def iterate_initial(self):
initial_information = self.initial_store.get(timeout=self.timeout)
if initial_information is not None:
initial_timestamp = (
time.mktime(initial_information.initial_time.timetuple())
+ initial_information.initial_time.microsecond / 10e6
)
end_timestamp = (
time.mktime(initial_information.end_time.timetuple()) + initial_information.end_time.microsecond / 10e6
)
request_info = initial_information.request_info
from_ip = request_info.pop("from_ip", "<address not found>")
try:
username = request_info.pop("username")
role = request_info.pop("role")
except:
log.log(
TemporalInformationRetriever,
log.level.Critical,
"Provided information did not contain some required fields (such as username or role). This usually means that the reservation has previously been expired. Provided request_info: %r; provided data: %r"
% (request_info, initial_information),
max_size=10000,
)
log.log_exc(TemporalInformationRetriever, log.level.Critical)
return
usage = ExperimentUsage()
usage.start_date = initial_timestamp
usage.from_ip = from_ip
usage.experiment_id = initial_information.experiment_id
usage.reservation_id = initial_information.reservation_id
usage.coord_address = initial_information.exp_coordaddr
usage.request_info = initial_information.request_info
command_request = CommandSent(
Command.Command("@@@initial::request@@@"),
initial_timestamp,
Command.Command(str(initial_information.client_initial_data)),
end_timestamp,
)
command_response = CommandSent(
Command.Command("@@@initial::response@@@"),
initial_timestamp,
Command.Command(str(initial_information.initial_configuration)),
end_timestamp,
)
usage.append_command(command_request)
usage.append_command(command_response)
self.db_manager.store_experiment_usage(DbSession.ValidDatabaseSessionId(username, role), usage)
def run(self):
while self.keep_running:
try:
self.iterations += 1
self.iterate()
except:
if self.PRINT_ERRORS:
import traceback
traceback.print_exc()
log.log( TemporalInformationRetriever, log.level.Critical, "Exception iterating in TemporalInformationRetriever!!!")
log.log_exc( TemporalInformationRetriever, log.level.Critical )
def check_schema(self, xmlfile_path, xsdfile_path):
if not LXML_AVAILABLE:
global MESSAGE_SHOWN
if not MESSAGE_SHOWN:
msg = "The optional library 'lxml' is not available. The syntax of the configuration files will not be checked."
print >> sys.stderr, msg
log.log(SchemaChecker, log.level.Warning, msg)
MESSAGE_SHOWN = True
return
xmlfile_content = self._read_xml_file(xmlfile_path)
xsdfile_full_path = data_filename(
os.path.join(module_directory, 'xsd', xsdfile_path))
try:
xsdfile_content = self._read_xsd_file(xsdfile_full_path)
except:
msg = "The XSD file %s could not be loaded. The syntax of the configuration files will not be checked." % xsdfile_full_path
print >> sys.stderr, msg
log.log(SchemaChecker, log.level.Warning, msg)
return
try:
sio_xsd = StringIO(xsdfile_content)
xmlschema_doc = etree.parse(sio_xsd)
xmlschema = etree.XMLSchema(xmlschema_doc)
except Exception as e:
log.log(
SchemaChecker, log.level.Warning,
'Invalid syntax file configuration: File %s: %s' %
(xsdfile_path, e))
log.log_exc(SchemaChecker, log.level.Info)
raise LoaderErrors.InvalidSyntaxFileConfigurationError(
e, xsdfile_path)
try:
sio_xml = StringIO(xmlfile_content)
xml_doc = etree.parse(sio_xml)
xmlschema.assertValid(xml_doc)
except etree.DocumentInvalid as di:
log.log(
SchemaChecker, log.level.Warning,
'Not a valid configuration file. Check it with a XML Schema validator: File %s'
% (xmlfile_path))
raise LoaderErrors.InvalidSyntaxFileConfigurationError(
'Not a valid configuration file. Check it with a XML Schema validator. %s'
% di.args, xmlfile_path)
except Exception as e:
log.log(
SchemaChecker, log.level.Warning,
'Invalid syntax file configuration: File %s: %s' %
(xmlfile_path, e))
log.log_exc(SchemaChecker, log.level.Info)
raise LoaderErrors.InvalidSyntaxFileConfigurationError(
e, xmlfile_path)
def run(self):
while self.keep_running:
try:
self.iterations += 1
self.iterate()
except:
if self.PRINT_ERRORS:
import traceback
traceback.print_exc()
log.log( TemporalInformationRetriever, log.level.Critical, "Exception iterating in TemporalInformationRetriever!!!")
log.log_exc( TemporalInformationRetriever, log.level.Critical )
def _program_file(self, file_content):
try:
fd, file_name = tempfile.mkstemp(
prefix='ud_xilinx_experiment_program',
suffix='.' + self._programmer.get_suffix()
) # Originally the Programmer wasn't the one to contain the suffix info.
if DEBUG:
print "[DBG]: 2"
df2 = open("/tmp/orig_content", "w")
df2.write("---begin---\n")
df2.write(file_content)
df2.close()
# For debugging purposes write the file to tmp
df = open("/tmp/toprogram_dbg", "w")
try:
try:
# TODO: encode? utf8?
if isinstance(file_content, unicode):
if DEBUG: print "[DBG]: Encoding file content in utf8"
file_content_encoded = file_content.encode('utf8')
else:
if DEBUG: print "[DBG]: Not encoding file content"
file_content_encoded = file_content
file_content_recovered = ExperimentUtil.deserialize(
file_content_encoded)
os.write(fd, file_content_recovered)
if DEBUG:
df.write(file_content_recovered)
finally:
os.close(fd)
self._programmer.program(file_name)
finally:
os.remove(file_name)
# print file_name
# import sys
# sys.stdout.flush()
except Exception as e:
if DEBUG:
tb = traceback.format_exc()
print "FULL EXCEPTION IS: {0}".format(tb)
# TODO: test me
log.log(
UdXilinxExperiment, log.level.Info,
"Exception joining sending program to device: %s" % e.args[0])
log.log_exc(UdXilinxExperiment, log.level.Debug)
raise ExperimentErrors.SendingFileFailureError(
"Error sending file to device: %s" % e)
self._clear()
def _validate_simple_authn(self, username, credentials):
"""
When the login() method is called, this method is used with the
username and credentials (e.g., password, IP address, etc.). This
method will only check the SimpleAuthn instances.
"""
try:
login, role_name, user_auths = self._db.retrieve_role_and_user_auths(username)
except DbUserNotFoundError:
return self._process_invalid()
# login could be different to username.
# For example, in MySQL, where login = '******' is equivalent to where login = '******'
# For this reason, we don't trust "username", and retrieve login from the database
errors = False
for user_auth in user_auths:
# Take only those auth types that use a simple interface
if user_auth.is_simple_authn():
# With each user auth, try to authenticate the user.
try:
authenticated = user_auth.authenticate(login, credentials)
except:
# If there is an error, the user could not be authenticated.
log.log(
LoginManager, log.level.Warning, "Username: %s with user_auth %s: ERROR" % (login, user_auth)
)
log.log_exc(LoginManager, log.level.Warning)
errors = True
traceback.print_exc()
continue
if authenticated:
# If authenticated, return that it was correctly authenticated.
log.log(
LoginManager, log.level.Debug, "Username: %s with user_auth %s: SUCCESS" % (login, user_auth)
)
return ValidDatabaseSessionId(login, role_name)
else:
# If not authenticated, log it and continue with the next user_auth.
log.log(
LoginManager, log.level.Warning, "Username: %s with user_auth %s: FAIL" % (login, user_auth)
)
if errors:
# Raise error: there was a server problem and this might be the reason for not
# authenticating the user. Examples: LDAP server is down, there is an error in the
# local database or so.
raise LoginErrors.LoginError("Error checking credentials. Contact administrators!")
return self._process_invalid()
def _remove_expired_reservations(self):
session = self.session_maker()
try:
now = self.time_provider.get_time()
current_expiration_time = datetime.datetime.utcfromtimestamp(now - EXPIRATION_TIME)
reservations_removed = False
enqueue_free_experiment_args_retrieved = []
expired_query = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.expired_timestamp != 0).filter(ConcreteCurrentReservation.expired_timestamp < self.time_provider.get_time())
for expired_concrete_current_reservation in expired_query.all():
expired_reservation = expired_concrete_current_reservation.current_reservation_id
if expired_reservation is None:
continue # Maybe it's not an expired_reservation anymore
enqueue_free_experiment_args = self._clean_current_reservation(session, expired_concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(expired_concrete_current_reservation)
self.reservations_manager.delete(session, expired_reservation)
reservations_removed = True
for expired_reservation_id in self.reservations_manager.list_expired_reservations(session, current_expiration_time):
concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == expired_reservation_id).first()
if concrete_current_reservation is not None:
enqueue_free_experiment_args = self._clean_current_reservation(session, concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(concrete_current_reservation)
waiting_reservation = session.query(WaitingReservation).filter(WaitingReservation.reservation_id == expired_reservation_id).first()
if waiting_reservation is not None:
session.delete(waiting_reservation)
self.reservations_manager.delete(session, expired_reservation_id)
reservations_removed = True
if reservations_removed:
try:
session.commit()
except ConcurrentModificationError as e:
if DEBUG:
print("Other error when commiting when reservations_removed: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"IntegrityError: %s" % e )
log.log_exc(PriorityQueueScheduler, log.level.Info)
pass # Someone else removed these users before us.
else:
for enqueue_free_experiment_args in enqueue_free_experiment_args_retrieved:
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
else:
session.rollback()
finally:
session.close()
def _process(self):
redis_client = self.redis_maker()
pending_results = []
for reservation_id in redis_client.hkeys(self.external_weblabdeusto_pending):
pending_result_str = redis_client.hget(self.external_weblabdeusto_pending, reservation_id)
if pending_result_str is not None:
pending_result = json.loads(pending_result_str)
pending_result['reservation_id'] = reservation_id
pending_results.append(pending_result)
if len(pending_results) > 0:
try:
session_id, client = self.create_client_func(None)
except urllib2.URLError:
# Remote server is down, try later
return
remote_reservation_ids = [ SessionId(pending_result['remote_reservation_id']) for pending_result in pending_results ]
results = client.get_experiment_uses_by_id(session_id, remote_reservation_ids)
for pending_result, result in zip(pending_results, results):
if result.is_alive():
continue
username = pending_result['username']
try:
request_info = pickle.loads(pending_result['serialized_request_info'].encode('utf-8'))
except Exception as e:
log.log(ResultsRetriever, log.level.Critical, "Probably serialized_request_info was truncated in %s" % pending_result)
log.log_exc(ResultsRetriever, log.level.Error)
request_info = {'error' : 'could not be stored: %s' % e}
reservation_id = pending_result['reservation_id']
remote_reservation_id = pending_result['remote_reservation_id']
if not redis_client.hdel(self.external_weblabdeusto_pending, reservation_id):
log.log(ResultsRetriever, log.level.Info, "Pending reservation %r not found. Assuming it is managed by other thread" % pending_result)
continue
if result.is_finished():
use = result.experiment_use
use.experiment_id = ExperimentId.parse(pending_result['experiment_id_str'])
use.remote_reservation_id = remote_reservation_id
use.reservation_id = reservation_id
for key in [ key for key in request_info ]:
if not isinstance(request_info[key], (basestring, numbers.Number)):
request_info.pop(key)
use.request_info = request_info
callback = lambda : self.post_reservation_data_manager.delete(reservation_id)
self.completed_store.put(username, use, callback)
else:
log.log(ResultsRetriever, log.level.Info, "Reservation id %s was cancelled and therefore not stored" % reservation_id)
def _remove_expired_reservations(self):
session = self.session_maker()
try:
now = self.time_provider.get_time()
current_expiration_time = datetime.datetime.utcfromtimestamp(now - EXPIRATION_TIME)
reservations_removed = False
enqueue_free_experiment_args_retrieved = []
expired_query = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.expired_timestamp != 0).filter(ConcreteCurrentReservation.expired_timestamp < self.time_provider.get_time())
for expired_concrete_current_reservation in expired_query.all():
expired_reservation = expired_concrete_current_reservation.current_reservation_id
if expired_reservation is None:
continue # Maybe it's not an expired_reservation anymore
enqueue_free_experiment_args = self._clean_current_reservation(session, expired_concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(expired_concrete_current_reservation)
self.reservations_manager.delete(session, expired_reservation)
reservations_removed = True
for expired_reservation_id in self.reservations_manager.list_expired_reservations(session, current_expiration_time):
concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == expired_reservation_id).first()
if concrete_current_reservation is not None:
enqueue_free_experiment_args = self._clean_current_reservation(session, concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(concrete_current_reservation)
waiting_reservation = session.query(WaitingReservation).filter(WaitingReservation.reservation_id == expired_reservation_id).first()
if waiting_reservation is not None:
session.delete(waiting_reservation)
self.reservations_manager.delete(session, expired_reservation_id)
reservations_removed = True
if reservations_removed:
try:
session.commit()
except ConcurrentModificationError as e:
if DEBUG:
print("Other error when commiting when reservations_removed: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"IntegrityError: %s" % e )
log.log_exc(PriorityQueueScheduler, log.level.Info)
pass # Someone else removed these users before us.
else:
for enqueue_free_experiment_args in enqueue_free_experiment_args_retrieved:
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
else:
session.rollback()
finally:
session.close()
def get_json():
rv = request.get_json(force=True, silent=True)
if rv:
return rv
data = request.data
print("Error: could not deserialize:", data, file=sys.stderr)
sys.stderr.flush()
log(__name__, level.Warning, "Error retrieving JSON contents")
log(__name__, level.Warning, data)
log_exc(__name__, level.Info)
return None
def get_json():
rv = request.get_json(force=True, silent=True)
if rv:
return rv
data = request.data
print("Error: could not deserialize:", data, file=sys.stderr)
sys.stderr.flush()
log(__name__, level.Warning, "Error retrieving JSON contents")
log(__name__, level.Warning, data)
log_exc(__name__, level.Info)
return None
def do_reserve_experiment(self, experiment_instance_id, client_initial_data, server_initial_data):
lab_sess_id = self._session_manager.create_session()
try:
experiment_coord_address = self._assigned_experiments.reserve_experiment(experiment_instance_id, lab_sess_id)
except LaboratoryErrors.BusyExperimentError:
# If it was already busy, free it and reserve it again
try:
old_lab_sess_id = self._assigned_experiments.get_lab_session_id(experiment_instance_id)
self._free_experiment(old_lab_sess_id)
except Exception as e:
# If there is an error freeing the experiment, we don't want to propagate it to the User Processing Server:
# our focus is to reserve the new session.
log.log( LaboratoryServer, log.level.Warning, "Exception freeing already reserved experiment: %s" % e )
log.log_exc(LaboratoryServer, log.level.Info)
try:
experiment_coord_address = self._assigned_experiments.reserve_experiment(experiment_instance_id, lab_sess_id)
except LaboratoryErrors.BusyExperimentError:
# The session might have expired and that's why this experiment is still reserved. Free it directly from
# assigned_experiments.
self._free_experiment_from_assigned_experiments(experiment_instance_id, lab_sess_id)
experiment_coord_address = self._assigned_experiments.reserve_experiment(experiment_instance_id, lab_sess_id)
self._session_manager.modify_session(lab_sess_id, {
'experiment_instance_id' : experiment_instance_id,
'experiment_coord_address' : experiment_coord_address,
'session_id' : lab_sess_id
})
# Obtain the API of the experiment.
api = self._find_api(experiment_instance_id, experiment_coord_address)
experiment_server = self._locator[experiment_coord_address]
if api == ExperimentApiLevel.level_1:
experiment_server.start_experiment()
experiment_server_response = "ok"
elif api == ExperimentApiLevel.level_2:
experiment_server_response = experiment_server.start_experiment(client_initial_data, server_initial_data)
# If the API version is concurrent, we will also send the session id, to be able to identify the user for each request.
elif api == ExperimentApiLevel.level_2_concurrent:
experiment_server_response = experiment_server.start_experiment(lab_sess_id, client_initial_data, server_initial_data)
else:
# ERROR: Unrecognized version.
experiment_server_response = experiment_server.start_experiment(lab_sess_id, client_initial_data, server_initial_data)
experiment_info = {
'address' : experiment_coord_address.address,
'manages_polling' : self._assigned_experiments.manages_polling(experiment_instance_id),
}
return lab_sess_id, experiment_server_response, experiment_info
def do_GET(self):
create_context(self.server, self.client_address, self.headers)
try:
first_question_mark = self.path.find("?")
if first_question_mark >= 0:
path = self.path[0:first_question_mark]
options = urllib.unquote(self.path[first_question_mark + 1:])
else:
path = self.path
options = ""
parameters = [ field for field in options.split("&") if field.find("=") >= 0 ]
session_id_param = [ field[field.find("=") + 1:] for field in parameters if field.startswith("sessionid") ]
if len(session_id_param) == 0:
self._write(400, "sessionid not provided")
return
session_id = session_id_param[0]
last_slash = path.rfind("/")
if last_slash >= 0:
method_name = path[last_slash + 1:]
else:
method_name = path
if 'get_%s' % method_name in dir(Methods):
method = getattr(Methods, 'get_%s' % method_name)
try:
response = method(self, session_id, parameters)
except MethodError as me:
# import traceback
# traceback.print_exc()
log.log( self, log.level.Error, str(me))
log.log_exc( self, log.level.Warning)
self._write(400, "Error: %s" % me)
return
json_response = json.dumps(response)
else:
self._write(400, "method %s not implemented" % urllib2.quote(method_name))
return
self._write(200, json_response)
except Exception as e:
import traceback
traceback.print_exc()
log.log( self, log.level.Error, str(e))
log.log_exc( self, log.level.Warning)
self._write(500, 'Error in server. Contact administrator')
finally:
delete_context()
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
if DEBUG:
import traceback
traceback.print_exc()
log.log(PriorityQueueScheduler, log.level.Error,
"Unexpected exception while running %s" % func.__name__)
log.log_exc(PriorityQueueScheduler, log.level.Warning)
raise
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
if DEBUG:
import traceback
traceback.print_exc()
log.log(
PriorityQueueScheduler, log.level.Error,
"Unexpected exception while running %s" % func.__name__ )
log.log_exc(PriorityQueueScheduler, log.level.Warning)
raise
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
def setup(self):
""" Configures the VM """
self.load_user_manager()
try:
self.user_manager.configure(self.session_id)
if DEBUG:
print "t_configured"
except Exception as ex:
self.error = ex
self.is_error = True
log.log(VMExperiment, log.level.Error,
"Error configuring user manager: %s" % ex)
log.log_exc(VMExperiment, log.level.Warning)
def _retrieve_server_instances_from_networks(self, networks, server_type, how_many = ALL):
server_instances = []
for network in networks:
address = network.address
cur_server = self._get_server_from_registry(address)
if cur_server is not None:
# TODO: not unittested
# First check if the server is up and running
if not self._test_server(cur_server,address):
# There was some error
continue
server_instances.append(cur_server)
continue
# Server was not in the ServerRegistry
methods = self._server_type_handler.retrieve_methods(
server_type
)
try:
cur_server = address.create_client(methods)
except ProtocolErrors.ClientClassCreationError as ccce:
# TODO: not unittested
# There was some error creating the client
log.log(
ServerLocator,
log.level.Warning,
"Generating client for server with %s raised exception %s. Trying another server..." % (
address.address,
ccce
)
)
log.log_exc(
ServerLocator,
log.level.Info
)
continue
self._save_server_in_registry(address,cur_server)
# Check if the server is up and running
if not self._test_server(cur_server,address):
# TODO: not unittested
# There was some error
continue
# Server up and running :-)
server_instances.append(cur_server)
if how_many != ALL and len(server_instances) == how_many:
break
return server_instances
def _purge_expired_users(self, expired_users):
for expired_reservation in expired_users:
if self._stopping:
return
try:
expired_session = self._reservations_session_manager.get_session_locking(expired_reservation)
try:
reservation_processor = self._load_reservation(expired_session)
reservation_processor.finish()
finally:
self._reservations_session_manager.modify_session_unlocking(expired_reservation, expired_session)
except Exception as e:
log.log( UserProcessingServer, log.level.Error,
"Exception freeing experiment of %s: %s" % (expired_reservation, e))
log.log_exc( UserProcessingServer, log.level.Warning)
def _program_file_t(self, file_content):
"""
Running in its own thread, this method will program the board
while updating the state of the experiment appropriately.
"""
try:
self._current_state = STATE_PROGRAMMING
self._program_file(file_content)
self._current_state = STATE_READY
except Exception as e:
# Note: Currently, running the fake xilinx will raise this exception when
# trying to do a CleanInputs, for which apparently serial is needed.
self._current_state = STATE_FAILED
log.log(UdXilinxExperiment, log.level.Warning, "Error programming file: " + str(e) )
log.log_exc(UdXilinxExperiment, log.level.Warning )
def _purge_expired_users(self, expired_users):
for expired_reservation in expired_users:
if self._stopping:
return
try:
expired_session = self._reservations_session_manager.get_session_locking(expired_reservation)
try:
reservation_processor = self._load_reservation(expired_session)
reservation_processor.finish()
finally:
self._reservations_session_manager.modify_session_unlocking(expired_reservation, expired_session)
except Exception as e:
log.log( UserProcessingServer, log.level.Error,
"Exception freeing experiment of %s: %s" % (expired_reservation, e))
log.log_exc( UserProcessingServer, log.level.Warning)
def _program_file(self, file_content):
try:
fd, file_name = tempfile.mkstemp(prefix='ud_xilinx_experiment_program',
suffix='.' + self._programmer.get_suffix()) # Originally the Programmer wasn't the one to contain the suffix info.
if DEBUG:
print "[DBG]: 2"
df2 = open("/tmp/orig_content", "w")
df2.write("---begin---\n")
df2.write(file_content)
df2.close()
# For debugging purposes write the file to tmp
df = open("/tmp/toprogram_dbg", "w")
try:
try:
# TODO: encode? utf8?
if isinstance(file_content, unicode):
if DEBUG: print "[DBG]: Encoding file content in utf8"
file_content_encoded = file_content.encode('utf8')
else:
if DEBUG: print "[DBG]: Not encoding file content"
file_content_encoded = file_content
file_content_recovered = ExperimentUtil.deserialize(file_content_encoded)
os.write(fd, file_content_recovered)
if DEBUG:
df.write(file_content_recovered)
finally:
os.close(fd)
self._programmer.program(file_name)
finally:
os.remove(file_name)
# print file_name
# import sys
# sys.stdout.flush()
except Exception as e:
if DEBUG:
tb = traceback.format_exc()
print "FULL EXCEPTION IS: {0}".format(tb)
# TODO: test me
log.log(UdXilinxExperiment, log.level.Info,
"Exception joining sending program to device: %s" % e.args[0])
log.log_exc(UdXilinxExperiment, log.level.Debug)
raise ExperimentErrors.SendingFileFailureError("Error sending file to device: %s" % e)
self._clear()
def confirm_resource_disposal(self, lab_coordaddress, reservation_id,
lab_session_id, experiment_instance_id,
experiment_response, initial_time, end_time):
experiment_finished = True
information_to_store = None
time_remaining = 0.5 # Every half a second by default
if experiment_response is None or experiment_response == 'ok' or experiment_response == '':
pass # Default value
else:
try:
response = json.loads(experiment_response)
experiment_finished = response.get(FINISH_FINISHED_MESSAGE,
experiment_finished)
time_remaining = response.get(FINISH_ASK_AGAIN_MESSAGE,
time_remaining)
information_to_store = response.get(FINISH_DATA_MESSAGE,
information_to_store)
except Exception as e:
log.log(
AbstractCoordinator, log.level.Error,
"Could not parse experiment server finishing response: %s; %s"
% (e, experiment_response))
log.log_exc(AbstractCoordinator, log.level.Warning)
if not experiment_finished:
time.sleep(time_remaining)
# We just ignore the data retrieved, if any, and perform the query again
self.confirmer.enqueue_free_experiment(lab_coordaddress,
reservation_id,
lab_session_id,
experiment_instance_id)
return
else:
# Otherwise we mark it as finished
self.post_reservation_data_manager.finish(
reservation_id, json.dumps(information_to_store))
try:
# and we remove the resource
# print "AT CONFIRM_RESOURCE_DISPOSAL"
self._release_resource_instance(experiment_instance_id)
finally:
self.finished_store.put(reservation_id, information_to_store,
initial_time, end_time)
# It's done here so it's called often enough
self.post_reservation_data_manager.clean_expired()
def check_laboratory(self, address_str, experiments):
""" Checks in that laboratory address which experiments are broken and which ones are working.
:param address_str: laboratory address, e.g. "laboratory:general_laboratory@server1"
:param experiments: dictionary of experiments: resources, e.g. { "exp1|ud-fpga|FPGA experiments" : "fpga1@fpga boards"}
"""
try:
laboratory_resources = set()
for experiment in experiments:
laboratory_resources.add(experiments[experiment])
broken_resources = {}
address = CoordAddress.CoordAddress.translate_address(address_str)
server = self.locator.get_server_from_coordaddr(address, ServerType.Laboratory)
failing_experiments = server.check_experiments_resources()
#
# failing_experiments is a dictionary such as:
# {
# experiment_instance_id : error_message
# }
#
for failing_experiment in failing_experiments:
if not failing_experiment in experiments:
log.log( ResourcesChecker, log.level.Error,
"Laboratory server %s reported that experiment %s was failing; however this laboratory does NOT manage this experiment. Attack?" % (address_str, failing_experiment))
continue
#
# The error for a resource will be concatenated
#
broken_resource = experiments[failing_experiment]
error_message = failing_experiments[failing_experiment]
if broken_resource in broken_resources:
broken_resources[broken_resource] = broken_resources[broken_resource] + ';' + error_message
else:
broken_resources[broken_resource] = error_message
for laboratory_resource in laboratory_resources:
if laboratory_resource in broken_resources:
self.coordinator.mark_resource_as_broken(laboratory_resource, broken_resources[laboratory_resource])
else:
self.coordinator.mark_resource_as_fixed(laboratory_resource)
except:
traceback.print_exc()
log.log( ResourcesChecker, log.level.Critical,
"Error checking resources of laboratory %s " % address_str)
log.log_exc(ResourcesChecker, log.level.Critical)
def _program_file_t(self, file_content):
"""
Running in its own thread, this method will program the board
while updating the state of the experiment appropriately.
"""
try:
self._current_state = STATE_PROGRAMMING
self._program_file(file_content)
self._current_state = STATE_READY
except Exception as e:
# Note: Currently, running the fake xilinx will raise this exception when
# trying to do a CleanInputs, for which apparently serial is needed.
self._current_state = STATE_FAILED
log.log(UdXilinxExperiment, log.level.Warning,
"Error programming file: " + str(e))
log.log_exc(UdXilinxExperiment, log.level.Warning)
def confirm_experiment(self, experiment_coordaddress, experiment_id, reservation_id, lab_coordaddress_str, lab_session_id, server_initialization_response, initial_time, end_time):
default_still_initialing = False
default_batch = False
default_initial_configuration = "{}"
if server_initialization_response is None or server_initialization_response == 'ok' or server_initialization_response == '':
still_initializing = default_still_initialing
batch = default_batch
initial_configuration = default_initial_configuration
else:
try:
response = json.loads(server_initialization_response)
still_initializing = response.get('keep_initializing', default_still_initialing)
batch = response.get('batch', default_batch)
initial_configuration = response.get('initial_configuration', default_initial_configuration)
except Exception as e:
log.log( AbstractCoordinator, log.level.Error, "Could not parse experiment server response: %s; %s; using default values" % (e, server_initialization_response) )
log.log_exc( AbstractCoordinator, log.level.Warning )
still_initializing = default_still_initialing
batch = default_batch
initial_configuration = default_initial_configuration
serialized_request_info, serialized_client_initial_data = self.reservations_manager.get_request_info_and_client_initial_data(reservation_id)
request_info = json.loads(serialized_request_info)
# Put the entry into a queue that is continuosly storing information into the db
initial_information_entry = TemporalInformationStore.InitialInformationEntry(
reservation_id, experiment_id, experiment_coordaddress,
initial_configuration, initial_time, end_time, request_info,
serialized_client_initial_data )
self.initial_store.put(initial_information_entry)
now = self.time_provider.get_datetime()
self.post_reservation_data_manager.create(reservation_id, now, now + self.expiration_delta, json.dumps(initial_configuration))
if still_initializing:
# TODO XXX
raise NotImplementedError("Not yet implemented: still_initializing")
aggregator = self._get_scheduler_aggregator_per_reservation(reservation_id)
aggregator.confirm_experiment(reservation_id, lab_session_id, initial_configuration)
if batch: # It has already finished, so make this experiment available to others
self.finish_reservation(reservation_id)
return
self.confirmer.enqueue_should_finish(lab_coordaddress_str, lab_session_id, reservation_id)
def iterate_initial(self):
initial_information = self.initial_store.get(timeout=self.timeout)
if initial_information is not None:
initial_timestamp = time.mktime(
initial_information.initial_time.timetuple(
)) + initial_information.initial_time.microsecond / 1e6
end_timestamp = time.mktime(initial_information.end_time.timetuple(
)) + initial_information.end_time.microsecond / 1e6
request_info = initial_information.request_info
from_ip = request_info.pop('from_ip', '<address not found>')
try:
username = request_info.pop('username')
except:
log.log(
TemporalInformationRetriever,
log.level.Critical,
"Provided information did not contain some required fields (such as username or role). This usually means that the reservation has previously been expired. Provided request_info: %r; provided data: %r"
% (request_info, initial_information),
max_size=10000)
log.log_exc(TemporalInformationRetriever, log.level.Critical)
return
usage = ExperimentUsage()
usage.start_date = initial_timestamp
usage.from_ip = from_ip
usage.experiment_id = initial_information.experiment_id
usage.reservation_id = initial_information.reservation_id
usage.coord_address = initial_information.exp_coordaddr
usage.request_info = initial_information.request_info
command_request = CommandSent(
Command.Command("@@@initial::request@@@"), initial_timestamp,
Command.Command(str(initial_information.client_initial_data)),
end_timestamp)
command_response = CommandSent(
Command.Command("@@@initial::response@@@"), initial_timestamp,
Command.Command(str(
initial_information.initial_configuration)), end_timestamp)
usage.append_command(command_request)
usage.append_command(command_response)
self.db_manager.store_experiment_usage(username, usage)
def check_laboratory(self, address_str, experiments):
""" Checks in that laboratory address which experiments are broken and which ones are working.
:param address_str: laboratory address, e.g. "laboratory:general_laboratory@server1"
:param experiments: dictionary of experiments: resources, e.g. { "exp1|ud-fpga|FPGA experiments" : "fpga1@fpga boards"}
"""
broken_resources = {
# resource_id : error_message
}
try:
address = CoordAddress.translate(address_str)
server = self.locator.get(
address, timeout=1800) # Extended timeout for this method
failing_experiments = server.check_experiments_resources()
#
# failing_experiments is a dictionary such as:
# {
# experiment_instance_id : error_message
# }
#
for failing_experiment in failing_experiments:
if not failing_experiment in experiments:
log.log(
ResourcesChecker, log.level.Error,
"Laboratory server %s reported that experiment %s was failing; however this laboratory does NOT manage this experiment. Attack?"
% (address_str, failing_experiment))
continue
#
# The error for a resource will be concatenated
#
broken_resource = experiments[failing_experiment]
error_message = failing_experiments[failing_experiment]
if broken_resource in broken_resources:
broken_resources[broken_resource] = broken_resources[
broken_resource] + ';' + error_message
else:
broken_resources[broken_resource] = error_message
except:
traceback.print_exc()
log.log(ResourcesChecker, log.level.Critical,
"Error checking resources of laboratory %s " % address_str)
log.log_exc(ResourcesChecker, log.level.Critical)
return broken_resources
def do_GET(self):
self.weblab_cookie = None
self.login_weblab_cookie = None
for current_cookie in (self.headers.getheader('cookie') or '').split('; '):
if current_cookie.startswith('weblabsessionid'):
self.weblab_cookie = current_cookie
if current_cookie.startswith('loginweblabsessionid'):
self.login_weblab_cookie = current_cookie
if self.weblab_cookie is None:
if self.server_route is not None:
self.weblab_cookie = "weblabsessionid=sessidnotfound.%s" % self.server_route
else:
self.weblab_cookie = "weblabsessionid=sessidnotfound"
if self.login_weblab_cookie is None:
if self.server_route is not None:
self.login_weblab_cookie = "loginweblabsessionid=loginsessid.not.found.%s" % self.server_route
else:
self.login_weblab_cookie = "loginweblabsessionid=sessidnotfound"
create_context(self.server, self.client_address, self.headers)
try:
for method in self.methods:
if method.matches(self.path):
m = method(self, self.cfg_manager, self.original_server)
message = m.run()
self._write(m.get_status(), m.get_content_type(), m.get_other_cookies(), message, m.get_other_headers(), m.avoid_weblab_cookies())
break
else:
NotFoundMethod(self, self.cfg_manager, self.original_server).run()
except RequestManagedError as e:
return
except MethodError as e:
log.log( self, log.level.Error, str(e))
log.log_exc( self, log.level.Warning)
self._write(e.status, 'text/html', [], e.msg)
except Exception as e:
import traceback
traceback.print_exc()
log.log( self, log.level.Error, str(e))
log.log_exc( self, log.level.Warning)
self._write(500, 'text/html', [], 'Error in server. Contact administrator')
finally:
delete_context()
def finish_reservation(self, reservation_id):
client = self.redis_maker()
ilab_batch = ILAB_BATCH % self.lab_server_url
reservation = client.hget(ilab_batch, reservation_id)
if reservation is not None:
remote_experiment_id = reservation['remote_experiment_id']
if not client.hdel(ilab_batch, reservation_id):
return
else:
return
client = self._create_client()
try:
client.cancel(remote_experiment_id)
except:
log.log(ILabBatchScheduler, log.level.Error, "Skipping error when cancelling iLab reservation")
log.log_exc(ILabBatchScheduler, log.level.Error)
def create_external_user(self, external_user, external_id, system,
group_names):
session = self.Session()
try:
try:
auth_type = session.query(
Model.DbAuthType).filter_by(name=system).one()
auth = auth_type.auths[0]
except (NoResultFound, KeyError):
raise DbErrors.DbUserNotFoundError(
"System '%s' not found in database" % system)
groups = []
for group_name in group_names:
try:
group = session.query(
Model.DbGroup).filter_by(name=group_name).one()
except NoResultFound:
raise DbErrors.DbUserNotFoundError(
"Group '%s' not found in database" % group_name)
groups.append(group)
try:
role = session.query(Model.DbRole).filter_by(
name=external_user.role.name).one()
user = Model.DbUser(external_user.login,
external_user.full_name,
external_user.email,
role=role)
user_auth = Model.DbUserAuth(user,
auth,
configuration=external_id)
for group in groups:
group.users.append(user)
session.add(user)
session.add(user_auth)
session.commit()
except Exception as e:
log.log(AuthDatabaseGateway, log.level.Warning,
"Couldn't create user: %s" % e)
log.log_exc(AuthDatabaseGateway, log.level.Info)
raise DbErrors.DatabaseError(
"Couldn't create user! Contact administrator")
finally:
session.close()
def dispose_resource(self, resource):
try:
if self._cancel:
resource.cancel()
except Exception as e:
log.log(CancelAndJoinResourceManager, self._log_level,
"Exception joining resource at %s: %s" % (self._name, e))
log.log_exc(CancelAndJoinResourceManager, self._log_exc_level)
try:
if self._timeout is not None:
resource.join(self._timeout)
else:
resource.join()
except Exception as e:
log.log(CancelAndJoinResourceManager, self._log_level,
"Exception joining resource at %s: %s" % (self._name, e))
log.log_exc(CancelAndJoinResourceManager, self._log_exc_level)
def delete_expired_sessions(self):
if self._timeout is None:
return
session_ids = self.list_sessions()
for session_id in session_ids:
try:
session_obj = self.get_session_obj(session_id)
if session_obj.is_expired(self._timeout):
self.delete_session(session_id)
except SessionErrors.SessionNotFoundError:
# The session was removed during the iteration of this loop
continue
except:
exc, inst, _ = sys.exc_info()
log.log( self, log.level.Error, "Unexpected exception (%s, %s) while trying to remove session_id %s" % (exc, inst, session_id))
log.log_exc( self, log.level.Warning )
continue