def test_idle_state(self):
a_project = Project('project name')
listener = ServerListener( project = a_project )
listener.current_time = lambda : "1000-10-10-10-10-10"
listener.listen_found_new_commits(True, next_run_in_seconds=60)
self.assertEquals("{'date': '1000-10-10-10-10-10', 'new_commits_found': True, 'next_run_in_seconds': 60}",
open(listener.idle_file).read())
def test_two_clients(self):
a_project = Project('project name')
a_client = Client('client 1')
a_client2 = Client('client 2')
listener1 = ServerListener(
client=a_client,
logs_base_dir='/tmp/clients_testdir',
project=a_project)
listener2 = ServerListener(
client=a_client2,
logs_base_dir='/tmp/clients_testdir',
project=a_project)
listener1.current_time = lambda : "some date"
listener2.current_time = lambda : "some other date"
generator = WebGenerator(logs_base_dir='/tmp/clients_testdir', project_name= a_project.name)
task = Task(a_project, a_client, 'task')
task.add_subtask('subtask1', [])
Runner(task, testinglisteners=[listener1])
task = Task(a_project, a_client2, 'task')
task.add_subtask('subtask1', [])
Runner(task, testinglisteners=[listener2])
self.assertEquals(
{'client 1':[('some date', 'some date', 'task', 'stable')],
'client 2':[('some other date', 'some other date', 'task', 'stable')]},
generator.get_executions() )
listener1.clean_log_files()
listener2.clean_log_files()
def test_stats_single_client_multiple_key(self):
a_project = Project('project name')
a_client = Client('a_client')
listener = ServerListener( client=a_client, project=a_project )
generator = WebGenerator(project_name= a_project.name)
listener.current_time = lambda : "2006-04-04-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'key1':5, 'key2':0} }] )
Runner(task, testinglisteners=[listener])
listener.current_time = lambda : "2006-04-05-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'key1':-1, 'key2':4} }] )
Runner(task, testinglisteners=[listener])
self.assertEquals(
{
'a_client':
{
'subtask':
[
('2006-04-04-00-00-00', {'key2': 0, 'key1': 5}),
('2006-04-05-00-00-00', {'key2': 4, 'key1': -1})
]
}
}, generator.collect_stats() )
def __init__(self, port = 12800, serv_dir = "~/Public"):
self.host = self.getlocalip()
self.port = port
self.server_directory = serv_dir
self.client_list = []
# Initiate connexion.
self.connexion = sck.socket(sck.AF_INET, sck.SOCK_STREAM)
self.serverlistener = ServerListener(self.connexion, self.client_list)
def test_executions__one_green_execution(self):
a_project = Project('project name')
listener = ServerListener( project = a_project)
listener.current_time = lambda : "a date"
generator = WebGenerator(project_name= a_project.name)
a_client = Client('client name')
task = Task(a_project, a_client, 'task')
task.add_subtask('subtask1', [])
Runner(task, testinglisteners=[listener])
self.assertEquals(
{'testing_client' : [('a date', 'a date', 'task', 'stable')]},
generator.get_executions() )
def test_no_stats(self):
a_project = Project('project name')
a_client = Client('a_client')
listener = ServerListener( client=a_client, project=a_project )
generator = WebGenerator(project_name= a_project.name)
listener.current_time = lambda : "2006-04-04-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", ["echo no stats"] )
Runner(task, testinglisteners=[listener])
self.assertEquals( {'a_client': {} }, generator.collect_stats() )
def test_idles(self):
a_project = Project('project name')
a_client = Client('a_client')
listener = ServerListener( client=a_client, project=a_project )
listener.current_time = lambda : "a date"
generator = WebGenerator(project_name= a_project.name)
task = Task(a_project, a_client, 'task')
task.set_check_for_new_commits( checking_cmd="echo P patched_file | grep ^[UP]", minutes_idle=1 )
Runner(task, testinglisteners=[listener])
self.assertEquals(
{'a_client' :
{'date':'a date',
'new_commits_found': True,
'next_run_in_seconds':60
}
},
generator.idle() )
def test_executions__day_executions__multiple_clients__first_client_with_last_day_empty(self):
a_project = Project('project name')
a_client = Client('a_client')
a_client2 = Client('a_client2')
listener1 = ServerListener( client=a_client, project=a_project )
listener1.current_time = lambda : "2006-04-29-12-12-00"
listener2 = ServerListener( client=a_client2, project=a_project )
listener2.current_time = lambda : "2006-04-29-12-00-00"
generator = WebGenerator(project_name= a_project.name)
task = Task(a_project, a_client, 'task')
task.add_subtask('subtask1', [])
Runner(task, testinglisteners=[listener1])
task = Task(a_project, a_client2, 'task')
task.add_subtask('subtask1', [])
Runner(task, testinglisteners=[listener2])
listener2.current_time = lambda : "2006-04-30-12-00-00"
Runner(task, testinglisteners=[listener2])
self.assertEquals(
{'2006-04-30':
{'a_client2': [('2006-04-30-12-00-00', '2006-04-30-12-00-00', 'task', 'stable')]},
'2006-04-29':
{'a_client2': [('2006-04-29-12-00-00', '2006-04-29-12-00-00', 'task', 'stable')],
'a_client': [('2006-04-29-12-12-00', '2006-04-29-12-12-00', 'task', 'stable')]
}
}, generator.day_executions(generator.get_executions()) )
def test_executions__day_executions__single_client(self):
a_project = Project('project name')
listener = ServerListener( project = a_project)
listener.current_time = lambda : "2006-04-29-12-00-00"
a_project = Project('project name')
generator = WebGenerator(project_name= a_project.name)
a_client = Client('client name')
task = Task(a_project, a_client, 'task')
task.add_subtask('subtask1', [])
Runner(task, testinglisteners=[listener])
listener.current_time = lambda : "2006-04-30-12-00-00"
Runner(task, testinglisteners=[listener])
self.assertEquals(
{'2006-04-30':
{'testing_client': [('2006-04-30-12-00-00', '2006-04-30-12-00-00', 'task', 'stable')]},
'2006-04-29':
{'testing_client': [('2006-04-29-12-00-00', '2006-04-29-12-00-00', 'task', 'stable')]}
}, generator.day_executions(generator.get_executions()) )
def test_send_task_info(self):
a_project = Project('project name')
a_client = Client('client name')
listener = ServerListener( client = a_client, project = a_project)
generator = WebGenerator(project_name = a_project.name)
task = Task(a_project, a_client, 'task')
task.add_subtask('subtask1', ["echo subtask1"])
task.add_subtask('subtask2', [{CMD:"echo something echoed", INFO: lambda x:x}])
listener.listen_task_info(task)
self.assertEquals(
[('BEGIN_TASK', 'task'),
('BEGIN_SUBTASK', 'subtask1'),
('CMD', 'echo subtask1'),
('END_SUBTASK', 'subtask1'),
('BEGIN_SUBTASK', 'subtask2'),
('CMD', 'echo something echoed'),
('END_SUBTASK', 'subtask2'),
('END_TASK', 'task'),]
, generator.load_task_info(a_client.name))#, task.name))
def test_project_info__more_info(self):
a_project = Project('project name')
a_client = Client('client name')
a_project.brief_description = "a brief description"
a_project.set_attribute('one more attribute', 'one_more_value')
a_project.set_attribute('another_attribute', 'another value')
listener = ServerListener( client = a_client, project = a_project)
generator = WebGenerator(project_name = a_project.name)
self.assertEquals(
[('Brief description', 'a brief description'),
('another_attribute', 'another value'),
('one more attribute', 'one_more_value')]
, generator.load_project_info())
def test_plot_data_file(self):
a_project = Project('project name')
a_client = Client('client1')
a_client2 = Client('client2')
listener1 = ServerListener( client=a_client, project=a_project )
listener2 = ServerListener( client=a_client2, project=a_project )
generator = WebGenerator(project_name= a_project.name)
listener1.current_time = lambda : "2006-04-04-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'key':5} }] )
Runner(task, testinglisteners=[listener1])
listener1.current_time = lambda : "2006-04-05-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'kk':3} }, {STATS:lambda x: {'key':1, 'key2':2} }] )
Runner(task, testinglisteners=[listener1])
listener2.current_time = lambda : "2000-01-01-12-54-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'clau 1':0, 'clau 2':10} }] )
task.add_subtask("another subtask", [{STATS:lambda x: {'clau 1':2, 'clau 2':13} }] )
Runner(task, testinglisteners=[listener2])
generator.plot_stats()
self.assertEquals('''\
time kk key2 key
2006/04/04.00:00 - - 5
2006/04/05.00:00 3 - -
2006/04/05.00:00 - 2 1
''', open("%s/%s/client1_1.plot" % (generator.logs_base_dir, generator.project_name)).read() )
self.assertEquals('''\
time clau_1 clau_2
2000/01/01.12:54 0 10
''', open("%s/%s/client2_1.plot" % (generator.logs_base_dir, generator.project_name)).read() )
self.assertEquals('''\
time clau_1 clau_2
2000/01/01.12:54 2 13
''', open("%s/%s/client2_2.plot" % (generator.logs_base_dir, generator.project_name)).read() )
def test_stats_multiple_client_single_key(self):
a_project = Project('project name')
a_client = Client('client1')
a_client2 = Client('client2')
listener1 = ServerListener( client=a_client, project=a_project )
listener2 = ServerListener( client=a_client2, project=a_project )
generator = WebGenerator(project_name= a_project.name)
listener1.current_time = lambda : "2006-04-04-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'key':5} }] )
Runner(task, testinglisteners=[listener1])
listener1.current_time = lambda : "2006-04-05-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'key':1} }] )
Runner(task, testinglisteners=[listener1])
listener2.current_time = lambda : "1000-00-00-00-00-00"
task = Task(a_project, a_client, 'task')
task.add_subtask("subtask", [{STATS:lambda x: {'clau':0} }] )
Runner(task, testinglisteners=[listener2])
self.assertEquals(
{
'client1':
{
'subtask':
[
('2006-04-04-00-00-00', {'key': 5}),
('2006-04-05-00-00-00', {'key': 1})
]
},
'client2':
{'subtask':
[
('1000-00-00-00-00-00', {'clau': 0})
]
}
}, generator.collect_stats() )
def test_multiple_repositories_multiple_tasks(self):
a_project = Project('project name')
id = lambda txt : txt
listener1 = ServerListener( project=a_project)
listener2 = ServerListener( project=a_project)
listener1.current_time = lambda : "2006-03-17-13-26-20"
listener2.current_time = lambda : "2006-03-17-13-26-20"
a_client = Client('a_client')
task1 = Task(a_project, a_client, 'task')
task2 = Task(a_project, a_client, 'task')
task1.add_subtask('subtask1', ["echo subtask1"])
task1.add_subtask('subtask2', [{CMD:"echo something echoed", INFO:id}, "./lalala gh"])
task2.add_subtask('subtask1', [])
task2.add_subtask('subtask2', ["ls"])
Runner(task1, testinglisteners=[listener1])
Runner(task2, testinglisteners=[listener2])
self.assertEquals("""\
('BEGIN_TASK', 'task', '2006-03-17-13-26-20'),
('BEGIN_SUBTASK', 'subtask1'),
('BEGIN_CMD', 'echo subtask1'),
('END_CMD', 'echo subtask1', True, '', '', {}),
('END_SUBTASK', 'subtask1'),
('BEGIN_SUBTASK', 'subtask2'),
('BEGIN_CMD', 'echo something echoed'),
('END_CMD', 'echo something echoed', True, '', 'something echoed\\n', {}),
('BEGIN_CMD', './lalala gh'),
('END_CMD', './lalala gh', False, '\\x1b[31m/bin/sh: 1: ./lalala: not found\\n\\x1b[0m', '', {}),
('END_SUBTASK', 'subtask2'),
('END_TASK', 'task', '2006-03-17-13-26-20', 'False'),
('BEGIN_TASK', 'task', '2006-03-17-13-26-20'),
('BEGIN_SUBTASK', 'subtask1'),
('END_SUBTASK', 'subtask1'),
('BEGIN_SUBTASK', 'subtask2'),
('BEGIN_CMD', 'ls'),
('END_CMD', 'ls', True, '', '', {}),
('END_SUBTASK', 'subtask2'),
('END_TASK', 'task', '2006-03-17-13-26-20', 'True'),
""", open( listener2.logfile ).read() )
def setUp(self):
a_project = Project('project name')
listener = ServerListener( project = a_project )
listener.clean_log_files()
class Server:
def __init__(self, port = 12800, serv_dir = "~/Public"):
self.host = self.getlocalip()
self.port = port
self.server_directory = serv_dir
self.client_list = []
# Initiate connexion.
self.connexion = sck.socket(sck.AF_INET, sck.SOCK_STREAM)
self.serverlistener = ServerListener(self.connexion, self.client_list)
#self.process = ServerProcess(self.connexion)
def getlocalip(self):
s = sck.socket(sck.AF_INET, sck.SOCK_DGRAM)
# connect() for UDP doesn't send packets
s.connect(('8.8.8.8', 1))
ip = s.getsockname()[0]
s.close()
return ip
def infos(self):
infos = os.uname()
print("# Server infos #")
print("hostname: ", sck.gethostname())
print("ip: ", self.connexion.getsockname()[0])
print("port: ", self.connexion.getsockname()[1])
print("hardware: ", infos.machine)
print("operating system: ", infos.sysname)
print("system release: ", infos.release)
print("system version: ", infos.version)
def listclient(self):
id = 0
for client in self.client_list:
print("[" + str(id) + "]: ",client[1])
id += 1
def start(self):
""" Start the python server """
bind = False
while not bind:
try:
self.connexion.bind((self.host,self.port))
bind = True
except:
self.port += 1
if self.port > 65535:
raise Exception("No port found.")
print("Try another port: " + str(self.port))
self.serverlistener.start()
def runterminal(self):
term = Terminal(self)
term.run()
def close(self):
print("Server end 3.")
self.serverlistener.stop_listening()
print("Server end 1.")
self.serverlistener.join(timeout=1)
print("Server end. 2")
self.connexion.close()
print("Server end.")
def buildmetadata(self, path):
metadata = {}
metadata["name"] = os.path.basename(path)
if os.path.isdir(path):
metadata["type"] = "dir"
underdata = {}
for o in os.listdir(path):
underdata[o] = self.buildmetadata(os.path.join(path,o))
metadata["content"] = underdata
else:
metadata["type"] = "file"
metadata["size"] = str(os.path.getsize(path))
return metadata
def send(self, path = "./", clientID = "all"):
clientID = int(clientID)
objectdata = self.buildmetadata(path)
metadata = json.dumps(objectdata).encode()
self.client_list[clientID][0].send(metadata)
response = self.client_list[clientID][0].recv(4096)
if response == b"N":
return
if objectdata["type"] == "dir":
self.senddir(clientID, path, objectdata)
else:
self.sendfile(clientID, path, objectdata["size"])
return
def senddir(self, clientID, path, objectdata):
content = objectdata["content"]
for o in content:
if content[o]["type"] == "file":
self.sendfile(clientID, os.path.join(path, content[o]["name"]), content[o]["size"])
else:
self.senddir(clientID, os.path.join(path, content[o]["name"]), content[o])
def sendfile(self, clientID, file_path, size):
print("Sending file: ", file_path)
size = int(size)
chunksize = calculatechunk(size)
nbsend = int(size / chunksize) + 1
with open(file_path, "rb") as f:
for i in range(1,nbsend+1):
byte = f.read(chunksize)
try:
self.client_list[clientID][0].send(byte)
except BrokenPipeError:
print("Connection interrupted.")
f.close()
return
printprogress(i, nbsend, status="sending")
f.close()
# Needed to print after printprogress:
print("")
time.sleep(0.1)
print("File transmited.")
def test_project_info__only_name(self):
a_project = Project('project name')
a_client = Client('client name')
listener = ServerListener( client = a_client, project = a_project)
generator = WebGenerator(project_name = a_project.name)
self.assertEquals(None, generator.load_project_info())
def __init__(self, task, continuous=False, first_run_always=True, local_base_dir = None, remote_server_url = None, verbose = False, testinglisteners = [], extra_listeners = [], ) : "Runs a task defined in user's script" self.listeners = [ ConsoleResultListener() ] serverlistener = None # for keyboard interrupt purpose if remote_server_url: listenerproxy = ServerListenerProxy( client=task.client, service_url=remote_server_url, project=task.project ) self.listeners.append( listenerproxy ) if local_base_dir : serverlistener = ServerListener( client=task.client, logs_base_dir=local_base_dir + "/logs", project=task.project ) self.listeners.append( serverlistener ) server_to_push = WebGenerator( logs_base_dir=local_base_dir + "/logs", html_base_dir=local_base_dir + "/html", project_name=task.project.name) else: server_to_push = None if testinglisteners: self.listeners = testinglisteners self.listeners += extra_listeners try : #do_subtasks at least one time new_commits_found = task.do_checking_for_new_commits( self.listeners, verbose=verbose ) #this creates a valid .idle file print "first_run_always ",first_run_always print "new commits_found ", new_commits_found if first_run_always or new_commits_found : task.do_subtasks( self.listeners, server_to_push = server_to_push, verbose=verbose ) while continuous : new_commits_found = task.do_checking_for_new_commits( self.listeners, verbose=verbose ) if new_commits_found: time.sleep(2) #avoid having executions with the same time task.do_subtasks( self.listeners, server_to_push = server_to_push, verbose=verbose ) else: if server_to_push: #update idle time display server_to_push.update_static_html_files() time.sleep( task.seconds_idle ) except KeyboardInterrupt : task.stop_execution_gently(self.listeners, server_to_push = server_to_push)
def test_purged_details(self):
a_project = Project('project name')
a_client = Client('a_client')
listener = ServerListener( client=a_client, project=a_project )
generator = WebGenerator(project_name= a_project.name)
listener.current_time = lambda : "1999-99-99-99-99-99"
listener.listen_begin_task("we want this one")
listener.listen_begin_subtask("subtask")
listener.listen_begin_command("a command")
listener.listen_end_command("a command", False, "some output", "some info", {'a':1})
listener.listen_begin_command("a command")
listener.listen_end_command("a command", False, "some more output", "some more info", {'a':1})
listener.listen_end_subtask("subtask")
listener.current_time = lambda : "2000-00-00-00-00-00"
listener.listen_end_task("we want this one", False)
expected = [
('BEGIN_TASK', 'we want this one', '1999-99-99-99-99-99'),
('BEGIN_SUBTASK', 'subtask'),
('BEGIN_CMD', 'a command'),
('END_CMD', 'a command', False, 'some output', 'some info', {'a':1}),
('BEGIN_CMD', 'a command'),
('END_CMD', 'a command', False, 'some more output','some more info', {'a':1}),
('END_SUBTASK', 'subtask'),
('END_TASK', 'we want this one', '2000-00-00-00-00-00', 'False'),
]
generator.purge_client_logfile('a_client','1999-99-99-99-99-99')
self.assertEquals( expected, generator.single_execution_details('a_client', '1999-99-99-99-99-99') )
