def create_usage(gateway, reservation_id = 'my_reservation_id'):
session = gateway.Session()
student1 = gateway._get_user(session, 'student1')
initial_usage = ExperimentUsage()
initial_usage.start_date = time.time()
initial_usage.end_date = time.time()
initial_usage.from_ip = "130.206.138.16"
initial_usage.experiment_id = ExperimentId("ud-dummy","Dummy experiments")
initial_usage.coord_address = CoordAddress.CoordAddress("machine1","instance1","server1") #.translate_address("server1:instance1@machine1")
initial_usage.reservation_id = reservation_id
file1 = FileSent(
'path/to/file1',
'{sha}12345',
time.time()
)
file2 = FileSent(
'path/to/file2',
'{sha}123456',
time.time(),
Command.Command('response'),
time.time(),
file_info = 'program'
)
command1 = CommandSent(
Command.Command("your command1"),
time.time()
)
command2 = CommandSent(
Command.Command("your command2"),
time.time(),
Command.Command("your response2"),
time.time()
)
initial_usage.append_command(command1)
initial_usage.append_command(command2)
initial_usage.append_file(file1)
initial_usage.append_file(file2)
initial_usage.request_info = {'facebook' : False}
gateway.store_experiment_usage(student1.login, initial_usage)
return student1, initial_usage, command1, command2, file1, file2
def _append_file(self, file_path, file_hash, timestamp_before, response,
timestamp_after, file_info):
file_sent = FileSent(file_path, file_hash, timestamp_before, response,
timestamp_after, file_info)
self._session['files'].append(file_sent)
def test_update_file(self):
student1 = self.gateway._get_user(self.session, 'student1')
RESERVATION_ID1 = 'my_reservation_id1'
RESERVATION_ID2 = 'my_reservation_id2'
usage1 = ExperimentUsage()
usage1.start_date = time.time()
usage1.end_date = time.time()
usage1.from_ip = "130.206.138.16"
usage1.experiment_id = ExperimentId("ud-dummy", "Dummy experiments")
usage1.coord_address = CoordAddress("machine1", "instance1", "server1")
usage1.reservation_id = RESERVATION_ID1
usage1.request_info = {
'facebook': False,
'permission_scope': 'user',
'permission_id': student1.id
}
usage2 = ExperimentUsage()
usage2.start_date = time.time()
usage2.end_date = time.time()
usage2.from_ip = "130.206.138.17"
usage2.experiment_id = ExperimentId("ud-dummy", "Dummy experiments")
usage2.coord_address = CoordAddress("machine1", "instance1", "server1")
usage2.reservation_id = RESERVATION_ID2
usage2.request_info = {
'facebook': False,
'permission_scope': 'user',
'permission_id': student1.id
}
self.gateway.store_experiment_usage(student1.login, usage1)
self.gateway.store_experiment_usage(student1.login, usage2)
file_sent1 = FileSent('path/to/file2',
'{sha}123456',
time.time(),
file_info='program')
usages = self.gateway.list_usages_per_user(student1.login)
self.assertEquals(2, len(usages))
full_usage1 = self.gateway.retrieve_usage(usages[0].experiment_use_id)
full_usage2 = self.gateway.retrieve_usage(usages[1].experiment_use_id)
self.assertEquals(0, len(full_usage1.commands))
self.assertEquals(0, len(full_usage2.commands))
self.assertEquals(0, len(full_usage1.sent_files))
self.assertEquals(0, len(full_usage2.sent_files))
file_sent_id = self.gateway.append_file(RESERVATION_ID1, file_sent1)
full_usage1 = self.gateway.retrieve_usage(usages[0].experiment_use_id)
full_usage2 = self.gateway.retrieve_usage(usages[1].experiment_use_id)
self.assertEquals(0, len(full_usage1.commands))
self.assertEquals(0, len(full_usage2.commands))
self.assertEquals(1, len(full_usage1.sent_files))
self.assertEquals(0, len(full_usage2.sent_files))
self.assertEquals(None,
full_usage1.sent_files[0].response.commandstring)
self.gateway.update_file(file_sent_id, Command.Command("response"),
time.time())
full_usage1 = self.gateway.retrieve_usage(usages[0].experiment_use_id)
full_usage2 = self.gateway.retrieve_usage(usages[1].experiment_use_id)
self.assertEquals(0, len(full_usage1.commands))
self.assertEquals(0, len(full_usage2.commands))
self.assertEquals(1, len(full_usage1.sent_files))
self.assertEquals(0, len(full_usage2.sent_files))
self.assertEquals("response",
full_usage1.sent_files[0].response.commandstring)
def test_add_file(self):
session = self.gateway.Session()
student1 = self.gateway._get_user(session, 'student1')
RESERVATION_ID1 = 'my_reservation_id1'
RESERVATION_ID2 = 'my_reservation_id2'
usage1 = ExperimentUsage()
usage1.start_date = time.time()
usage1.end_date = time.time()
usage1.from_ip = "130.206.138.16"
usage1.experiment_id = ExperimentId("ud-dummy","Dummy experiments")
usage1.coord_address = CoordAddress.CoordAddress("machine1","instance1","server1") #.translate_address("server1:instance1@machine1")
usage1.reservation_id = RESERVATION_ID1
command1 = CommandSent(
Command.Command("your command1"),
time.time(),
Command.Command("your response1"),
time.time()
)
usage1.append_command(command1)
usage1.request_info = {'facebook' : False}
usage2 = ExperimentUsage()
usage2.start_date = time.time()
usage2.end_date = time.time()
usage2.from_ip = "130.206.138.17"
usage2.experiment_id = ExperimentId("ud-dummy","Dummy experiments")
usage2.coord_address = CoordAddress.CoordAddress("machine1","instance1","server1") #.translate_address("server1:instance1@machine1")
usage2.reservation_id = RESERVATION_ID2
command2 = CommandSent(
Command.Command("your command2"),
time.time(),
Command.Command("your response2"),
time.time()
)
usage2.append_command(command2)
usage2.request_info = {'facebook' : False}
self.gateway.store_experiment_usage(student1.login, usage1)
self.gateway.store_experiment_usage(student1.login, usage2)
file_sent1 = FileSent(
'path/to/file2',
'{sha}123456',
time.time(),
Command.Command('response'),
time.time(),
file_info = 'program'
)
usages = self.gateway.list_usages_per_user(student1.login)
self.assertEquals(2, len(usages))
full_usage1 = self.gateway.retrieve_usage(usages[0].experiment_use_id)
full_usage2 = self.gateway.retrieve_usage(usages[1].experiment_use_id)
self.assertEquals(1, len(full_usage1.commands))
self.assertEquals(1, len(full_usage2.commands))
self.assertEquals(0, len(full_usage1.sent_files))
self.assertEquals(0, len(full_usage2.sent_files))
self.gateway.append_file(RESERVATION_ID1, file_sent1)
full_usage1 = self.gateway.retrieve_usage(usages[0].experiment_use_id)
full_usage2 = self.gateway.retrieve_usage(usages[1].experiment_use_id)
self.assertEquals(1, len(full_usage1.commands))
self.assertEquals(1, len(full_usage2.commands))
self.assertEquals(1, len(full_usage1.sent_files))
self.assertEquals(0, len(full_usage2.sent_files))
self.assertEquals("response", full_usage1.sent_files[0].response.commandstring)
def iterate_command(self):
information = self.commands_store.get(timeout=self.timeout)
if information is not None:
all_information = [ information ]
# Retrieve all the remaining information to ensure that it it finally empty,
# with a maximum of 1000 registries per request
max_registries = 1000
counter = 0
while not self.commands_store.empty() and counter < max_registries:
counter += 1
information = self.commands_store.get(timeout=0)
if information is not None:
all_information.append(information)
command_pairs = []
command_responses = []
command_requests = {}
file_pairs = []
file_responses = []
file_requests = {}
backup_information = {}
backup_information_responses = {}
# Process
for information in all_information:
if information.is_command:
if information.is_before:
backup_information[information.entry_id] = information
command_requests[information.entry_id] = (information.reservation_id, CommandSent( information.payload, information.timestamp))
else:
backup_information_responses[information.entry_id] = information
command_request = command_requests.pop(information.entry_id, None)
if command_request is not None:
reservation_id, command_sent = command_request
complete_command = CommandSent(
command_sent.command, command_sent.timestamp_before,
information.payload, information.timestamp)
command_pairs.append((reservation_id, information.entry_id, complete_command))
else:
with self.entry_id2command_id_lock:
command_id = self.entry_id2command_id.pop(information.entry_id, None)
if command_id is None:
self.commands_store.put(information)
else:
command_responses.append((information.entry_id, command_id, information.payload, information.timestamp))
else:
if information.is_before:
backup_information[information.entry_id] = information
file_requests[information.entry_id] = (information.reservation_id, information.payload)
else:
backup_information_responses[information.entry_id] = information
file_request = file_requests.pop(information.entry_id, None)
if file_request is not None:
reservation_id, file_sent = file_request
if file_sent.is_loaded():
storer = file_storer.FileStorer(self.cfg_manager, reservation_id)
stored = storer.store_file(self, file_sent.file_content, file_sent.file_info)
file_path = stored.file_path
file_hash = stored.file_hash
else:
file_path = file_sent.file_path
file_hash = file_sent.file_hash
complete_file = FileSent(file_path, file_hash, file_sent.timestamp_before,
information.payload, information.timestamp)
file_pairs.append((reservation_id, information.entry_id, complete_file))
else:
with self.entry_id2command_id_lock:
command_id = self.entry_id2command_id.pop(information.entry_id, None)
if command_id is None:
self.commands_store.put(information)
else:
file_responses.append((information.entry_id, command_id, information.payload, information.timestamp))
# At this point, we have all the information processed and
# ready to be passed to the database in a single commit
mappings = self.db_manager.store_commands(command_pairs, command_requests, command_responses, file_pairs, file_requests, file_responses)
elements_to_backup = []
with self.entry_id2command_id_lock:
for entry_id in mappings:
command_id = mappings[entry_id]
if command_id is not None and command_id is not False:
self.entry_id2command_id[entry_id] = mappings[entry_id]
else:
elements_to_backup.append(entry_id)
for entry_id in elements_to_backup:
if entry_id in backup_information:
self.commands_store.put(backup_information[entry_id])
if entry_id in backup_information_responses:
self.commands_store.put(backup_information_responses[entry_id])
def _store_two_reservations(self):
#
# Two users: student2, that started before "any" but finished after "any", and "any" then. Both use
# the same experiment.
#
reservation_id1 = SessionId.SessionId(u'5')
initial_usage1 = ExperimentUsage()
initial_usage1.start_date = time.time()
initial_usage1.end_date = time.time()
initial_usage1.from_ip = u"130.206.138.16"
initial_usage1.experiment_id = ExperimentId(u"ud-dummy",u"Dummy experiments")
initial_usage1.coord_address = CoordAddress.CoordAddress(u"machine1",u"instance1",u"server1") #.translate_address("server1:instance1@machine1")
initial_usage1.reservation_id = reservation_id1.id
valid_file_path = os.path.relpath(os.sep.join(('test','__init__.py')))
file1 = FileSent( valid_file_path, u'{sha}12345', time.time())
file2 = FileSent( valid_file_path, u'{sha}123456',
time.time(), Command(u'response'),
time.time(), file_info = u'program')
command1 = CommandSent( Command(u"your command1"), time.time())
command2 = CommandSent( Command(u"your command2"), time.time(),
Command(u"your response2"), time.time())
initial_usage1.append_command(command1)
initial_usage1.append_command(command2)
initial_usage1.append_file(file1)
initial_usage1.append_file(file2)
reservation_id2 = SessionId.SessionId(u'6')
initial_usage2 = ExperimentUsage()
initial_usage2.start_date = time.time()
initial_usage2.end_date = time.time()
initial_usage2.from_ip = u"130.206.138.16"
initial_usage2.experiment_id = ExperimentId(u"ud-dummy",u"Dummy experiments")
initial_usage2.coord_address = CoordAddress.CoordAddress(u"machine1",u"instance1",u"server1") #.translate_address("server1:instance1@machine1")
initial_usage2.reservation_id = reservation_id2.id
file1 = FileSent( valid_file_path, u'{sha}12345', time.time())
file2 = FileSent( valid_file_path, u'{sha}123456',
time.time(), Command(u'response'),
time.time(), file_info = u'program')
command1 = CommandSent( Command(u"your command1"), time.time())
command2 = CommandSent( Command(u"your command2"), time.time(),
Command(u"your response2"), time.time())
initial_usage2.append_command(command1)
initial_usage2.append_command(command2)
initial_usage2.append_file(file1)
initial_usage2.append_file(file2)
self.ups._db_manager._gateway.store_experiment_usage('student1', initial_usage1)
self.ups._db_manager._gateway.store_experiment_usage('student2', initial_usage2)
return (reservation_id1, reservation_id2), (initial_usage1, initial_usage2)