def test_idle_state(self): a_project = Project('project name') listener = ServerListener( project = a_project ) listener.current_time = lambda : "1000-10-10-10-10-10" listener.listen_found_new_commits(True, next_run_in_seconds=60) self.assertEquals("{'date': '1000-10-10-10-10-10', 'new_commits_found': True, 'next_run_in_seconds': 60}", open(listener.idle_file).read())
def test_two_clients(self): a_project = Project('project name') a_client = Client('client 1') a_client2 = Client('client 2') listener1 = ServerListener( client=a_client, logs_base_dir='/tmp/clients_testdir', project=a_project) listener2 = ServerListener( client=a_client2, logs_base_dir='/tmp/clients_testdir', project=a_project) listener1.current_time = lambda : "some date" listener2.current_time = lambda : "some other date" generator = WebGenerator(logs_base_dir='/tmp/clients_testdir', project_name= a_project.name) task = Task(a_project, a_client, 'task') task.add_subtask('subtask1', []) Runner(task, testinglisteners=[listener1]) task = Task(a_project, a_client2, 'task') task.add_subtask('subtask1', []) Runner(task, testinglisteners=[listener2]) self.assertEquals( {'client 1':[('some date', 'some date', 'task', 'stable')], 'client 2':[('some other date', 'some other date', 'task', 'stable')]}, generator.get_executions() ) listener1.clean_log_files() listener2.clean_log_files()
def test_stats_single_client_multiple_key(self): a_project = Project('project name') a_client = Client('a_client') listener = ServerListener( client=a_client, project=a_project ) generator = WebGenerator(project_name= a_project.name) listener.current_time = lambda : "2006-04-04-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'key1':5, 'key2':0} }] ) Runner(task, testinglisteners=[listener]) listener.current_time = lambda : "2006-04-05-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'key1':-1, 'key2':4} }] ) Runner(task, testinglisteners=[listener]) self.assertEquals( { 'a_client': { 'subtask': [ ('2006-04-04-00-00-00', {'key2': 0, 'key1': 5}), ('2006-04-05-00-00-00', {'key2': 4, 'key1': -1}) ] } }, generator.collect_stats() )
def __init__(self, port = 12800, serv_dir = "~/Public"): self.host = self.getlocalip() self.port = port self.server_directory = serv_dir self.client_list = [] # Initiate connexion. self.connexion = sck.socket(sck.AF_INET, sck.SOCK_STREAM) self.serverlistener = ServerListener(self.connexion, self.client_list)
def test_executions__one_green_execution(self): a_project = Project('project name') listener = ServerListener( project = a_project) listener.current_time = lambda : "a date" generator = WebGenerator(project_name= a_project.name) a_client = Client('client name') task = Task(a_project, a_client, 'task') task.add_subtask('subtask1', []) Runner(task, testinglisteners=[listener]) self.assertEquals( {'testing_client' : [('a date', 'a date', 'task', 'stable')]}, generator.get_executions() )
def test_no_stats(self): a_project = Project('project name') a_client = Client('a_client') listener = ServerListener( client=a_client, project=a_project ) generator = WebGenerator(project_name= a_project.name) listener.current_time = lambda : "2006-04-04-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", ["echo no stats"] ) Runner(task, testinglisteners=[listener]) self.assertEquals( {'a_client': {} }, generator.collect_stats() )
def test_idles(self): a_project = Project('project name') a_client = Client('a_client') listener = ServerListener( client=a_client, project=a_project ) listener.current_time = lambda : "a date" generator = WebGenerator(project_name= a_project.name) task = Task(a_project, a_client, 'task') task.set_check_for_new_commits( checking_cmd="echo P patched_file | grep ^[UP]", minutes_idle=1 ) Runner(task, testinglisteners=[listener]) self.assertEquals( {'a_client' : {'date':'a date', 'new_commits_found': True, 'next_run_in_seconds':60 } }, generator.idle() )
def test_executions__day_executions__multiple_clients__first_client_with_last_day_empty(self): a_project = Project('project name') a_client = Client('a_client') a_client2 = Client('a_client2') listener1 = ServerListener( client=a_client, project=a_project ) listener1.current_time = lambda : "2006-04-29-12-12-00" listener2 = ServerListener( client=a_client2, project=a_project ) listener2.current_time = lambda : "2006-04-29-12-00-00" generator = WebGenerator(project_name= a_project.name) task = Task(a_project, a_client, 'task') task.add_subtask('subtask1', []) Runner(task, testinglisteners=[listener1]) task = Task(a_project, a_client2, 'task') task.add_subtask('subtask1', []) Runner(task, testinglisteners=[listener2]) listener2.current_time = lambda : "2006-04-30-12-00-00" Runner(task, testinglisteners=[listener2]) self.assertEquals( {'2006-04-30': {'a_client2': [('2006-04-30-12-00-00', '2006-04-30-12-00-00', 'task', 'stable')]}, '2006-04-29': {'a_client2': [('2006-04-29-12-00-00', '2006-04-29-12-00-00', 'task', 'stable')], 'a_client': [('2006-04-29-12-12-00', '2006-04-29-12-12-00', 'task', 'stable')] } }, generator.day_executions(generator.get_executions()) )
def test_executions__day_executions__single_client(self): a_project = Project('project name') listener = ServerListener( project = a_project) listener.current_time = lambda : "2006-04-29-12-00-00" a_project = Project('project name') generator = WebGenerator(project_name= a_project.name) a_client = Client('client name') task = Task(a_project, a_client, 'task') task.add_subtask('subtask1', []) Runner(task, testinglisteners=[listener]) listener.current_time = lambda : "2006-04-30-12-00-00" Runner(task, testinglisteners=[listener]) self.assertEquals( {'2006-04-30': {'testing_client': [('2006-04-30-12-00-00', '2006-04-30-12-00-00', 'task', 'stable')]}, '2006-04-29': {'testing_client': [('2006-04-29-12-00-00', '2006-04-29-12-00-00', 'task', 'stable')]} }, generator.day_executions(generator.get_executions()) )
def test_send_task_info(self): a_project = Project('project name') a_client = Client('client name') listener = ServerListener( client = a_client, project = a_project) generator = WebGenerator(project_name = a_project.name) task = Task(a_project, a_client, 'task') task.add_subtask('subtask1', ["echo subtask1"]) task.add_subtask('subtask2', [{CMD:"echo something echoed", INFO: lambda x:x}]) listener.listen_task_info(task) self.assertEquals( [('BEGIN_TASK', 'task'), ('BEGIN_SUBTASK', 'subtask1'), ('CMD', 'echo subtask1'), ('END_SUBTASK', 'subtask1'), ('BEGIN_SUBTASK', 'subtask2'), ('CMD', 'echo something echoed'), ('END_SUBTASK', 'subtask2'), ('END_TASK', 'task'),] , generator.load_task_info(a_client.name))#, task.name))
def test_project_info__more_info(self): a_project = Project('project name') a_client = Client('client name') a_project.brief_description = "a brief description" a_project.set_attribute('one more attribute', 'one_more_value') a_project.set_attribute('another_attribute', 'another value') listener = ServerListener( client = a_client, project = a_project) generator = WebGenerator(project_name = a_project.name) self.assertEquals( [('Brief description', 'a brief description'), ('another_attribute', 'another value'), ('one more attribute', 'one_more_value')] , generator.load_project_info())
def test_plot_data_file(self): a_project = Project('project name') a_client = Client('client1') a_client2 = Client('client2') listener1 = ServerListener( client=a_client, project=a_project ) listener2 = ServerListener( client=a_client2, project=a_project ) generator = WebGenerator(project_name= a_project.name) listener1.current_time = lambda : "2006-04-04-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'key':5} }] ) Runner(task, testinglisteners=[listener1]) listener1.current_time = lambda : "2006-04-05-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'kk':3} }, {STATS:lambda x: {'key':1, 'key2':2} }] ) Runner(task, testinglisteners=[listener1]) listener2.current_time = lambda : "2000-01-01-12-54-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'clau 1':0, 'clau 2':10} }] ) task.add_subtask("another subtask", [{STATS:lambda x: {'clau 1':2, 'clau 2':13} }] ) Runner(task, testinglisteners=[listener2]) generator.plot_stats() self.assertEquals('''\ time kk key2 key 2006/04/04.00:00 - - 5 2006/04/05.00:00 3 - - 2006/04/05.00:00 - 2 1 ''', open("%s/%s/client1_1.plot" % (generator.logs_base_dir, generator.project_name)).read() ) self.assertEquals('''\ time clau_1 clau_2 2000/01/01.12:54 0 10 ''', open("%s/%s/client2_1.plot" % (generator.logs_base_dir, generator.project_name)).read() ) self.assertEquals('''\ time clau_1 clau_2 2000/01/01.12:54 2 13 ''', open("%s/%s/client2_2.plot" % (generator.logs_base_dir, generator.project_name)).read() )
def test_stats_multiple_client_single_key(self): a_project = Project('project name') a_client = Client('client1') a_client2 = Client('client2') listener1 = ServerListener( client=a_client, project=a_project ) listener2 = ServerListener( client=a_client2, project=a_project ) generator = WebGenerator(project_name= a_project.name) listener1.current_time = lambda : "2006-04-04-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'key':5} }] ) Runner(task, testinglisteners=[listener1]) listener1.current_time = lambda : "2006-04-05-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'key':1} }] ) Runner(task, testinglisteners=[listener1]) listener2.current_time = lambda : "1000-00-00-00-00-00" task = Task(a_project, a_client, 'task') task.add_subtask("subtask", [{STATS:lambda x: {'clau':0} }] ) Runner(task, testinglisteners=[listener2]) self.assertEquals( { 'client1': { 'subtask': [ ('2006-04-04-00-00-00', {'key': 5}), ('2006-04-05-00-00-00', {'key': 1}) ] }, 'client2': {'subtask': [ ('1000-00-00-00-00-00', {'clau': 0}) ] } }, generator.collect_stats() )
def test_multiple_repositories_multiple_tasks(self): a_project = Project('project name') id = lambda txt : txt listener1 = ServerListener( project=a_project) listener2 = ServerListener( project=a_project) listener1.current_time = lambda : "2006-03-17-13-26-20" listener2.current_time = lambda : "2006-03-17-13-26-20" a_client = Client('a_client') task1 = Task(a_project, a_client, 'task') task2 = Task(a_project, a_client, 'task') task1.add_subtask('subtask1', ["echo subtask1"]) task1.add_subtask('subtask2', [{CMD:"echo something echoed", INFO:id}, "./lalala gh"]) task2.add_subtask('subtask1', []) task2.add_subtask('subtask2', ["ls"]) Runner(task1, testinglisteners=[listener1]) Runner(task2, testinglisteners=[listener2]) self.assertEquals("""\ ('BEGIN_TASK', 'task', '2006-03-17-13-26-20'), ('BEGIN_SUBTASK', 'subtask1'), ('BEGIN_CMD', 'echo subtask1'), ('END_CMD', 'echo subtask1', True, '', '', {}), ('END_SUBTASK', 'subtask1'), ('BEGIN_SUBTASK', 'subtask2'), ('BEGIN_CMD', 'echo something echoed'), ('END_CMD', 'echo something echoed', True, '', 'something echoed\\n', {}), ('BEGIN_CMD', './lalala gh'), ('END_CMD', './lalala gh', False, '\\x1b[31m/bin/sh: 1: ./lalala: not found\\n\\x1b[0m', '', {}), ('END_SUBTASK', 'subtask2'), ('END_TASK', 'task', '2006-03-17-13-26-20', 'False'), ('BEGIN_TASK', 'task', '2006-03-17-13-26-20'), ('BEGIN_SUBTASK', 'subtask1'), ('END_SUBTASK', 'subtask1'), ('BEGIN_SUBTASK', 'subtask2'), ('BEGIN_CMD', 'ls'), ('END_CMD', 'ls', True, '', '', {}), ('END_SUBTASK', 'subtask2'), ('END_TASK', 'task', '2006-03-17-13-26-20', 'True'), """, open( listener2.logfile ).read() )
def setUp(self): a_project = Project('project name') listener = ServerListener( project = a_project ) listener.clean_log_files()
class Server: def __init__(self, port = 12800, serv_dir = "~/Public"): self.host = self.getlocalip() self.port = port self.server_directory = serv_dir self.client_list = [] # Initiate connexion. self.connexion = sck.socket(sck.AF_INET, sck.SOCK_STREAM) self.serverlistener = ServerListener(self.connexion, self.client_list) #self.process = ServerProcess(self.connexion) def getlocalip(self): s = sck.socket(sck.AF_INET, sck.SOCK_DGRAM) # connect() for UDP doesn't send packets s.connect(('8.8.8.8', 1)) ip = s.getsockname()[0] s.close() return ip def infos(self): infos = os.uname() print("# Server infos #") print("hostname: ", sck.gethostname()) print("ip: ", self.connexion.getsockname()[0]) print("port: ", self.connexion.getsockname()[1]) print("hardware: ", infos.machine) print("operating system: ", infos.sysname) print("system release: ", infos.release) print("system version: ", infos.version) def listclient(self): id = 0 for client in self.client_list: print("[" + str(id) + "]: ",client[1]) id += 1 def start(self): """ Start the python server """ bind = False while not bind: try: self.connexion.bind((self.host,self.port)) bind = True except: self.port += 1 if self.port > 65535: raise Exception("No port found.") print("Try another port: " + str(self.port)) self.serverlistener.start() def runterminal(self): term = Terminal(self) term.run() def close(self): print("Server end 3.") self.serverlistener.stop_listening() print("Server end 1.") self.serverlistener.join(timeout=1) print("Server end. 2") self.connexion.close() print("Server end.") def buildmetadata(self, path): metadata = {} metadata["name"] = os.path.basename(path) if os.path.isdir(path): metadata["type"] = "dir" underdata = {} for o in os.listdir(path): underdata[o] = self.buildmetadata(os.path.join(path,o)) metadata["content"] = underdata else: metadata["type"] = "file" metadata["size"] = str(os.path.getsize(path)) return metadata def send(self, path = "./", clientID = "all"): clientID = int(clientID) objectdata = self.buildmetadata(path) metadata = json.dumps(objectdata).encode() self.client_list[clientID][0].send(metadata) response = self.client_list[clientID][0].recv(4096) if response == b"N": return if objectdata["type"] == "dir": self.senddir(clientID, path, objectdata) else: self.sendfile(clientID, path, objectdata["size"]) return def senddir(self, clientID, path, objectdata): content = objectdata["content"] for o in content: if content[o]["type"] == "file": self.sendfile(clientID, os.path.join(path, content[o]["name"]), content[o]["size"]) else: self.senddir(clientID, os.path.join(path, content[o]["name"]), content[o]) def sendfile(self, clientID, file_path, size): print("Sending file: ", file_path) size = int(size) chunksize = calculatechunk(size) nbsend = int(size / chunksize) + 1 with open(file_path, "rb") as f: for i in range(1,nbsend+1): byte = f.read(chunksize) try: self.client_list[clientID][0].send(byte) except BrokenPipeError: print("Connection interrupted.") f.close() return printprogress(i, nbsend, status="sending") f.close() # Needed to print after printprogress: print("") time.sleep(0.1) print("File transmited.")
def test_project_info__only_name(self): a_project = Project('project name') a_client = Client('client name') listener = ServerListener( client = a_client, project = a_project) generator = WebGenerator(project_name = a_project.name) self.assertEquals(None, generator.load_project_info())
def __init__(self, task, continuous=False, first_run_always=True, local_base_dir = None, remote_server_url = None, verbose = False, testinglisteners = [], extra_listeners = [], ) : "Runs a task defined in user's script" self.listeners = [ ConsoleResultListener() ] serverlistener = None # for keyboard interrupt purpose if remote_server_url: listenerproxy = ServerListenerProxy( client=task.client, service_url=remote_server_url, project=task.project ) self.listeners.append( listenerproxy ) if local_base_dir : serverlistener = ServerListener( client=task.client, logs_base_dir=local_base_dir + "/logs", project=task.project ) self.listeners.append( serverlistener ) server_to_push = WebGenerator( logs_base_dir=local_base_dir + "/logs", html_base_dir=local_base_dir + "/html", project_name=task.project.name) else: server_to_push = None if testinglisteners: self.listeners = testinglisteners self.listeners += extra_listeners try : #do_subtasks at least one time new_commits_found = task.do_checking_for_new_commits( self.listeners, verbose=verbose ) #this creates a valid .idle file print "first_run_always ",first_run_always print "new commits_found ", new_commits_found if first_run_always or new_commits_found : task.do_subtasks( self.listeners, server_to_push = server_to_push, verbose=verbose ) while continuous : new_commits_found = task.do_checking_for_new_commits( self.listeners, verbose=verbose ) if new_commits_found: time.sleep(2) #avoid having executions with the same time task.do_subtasks( self.listeners, server_to_push = server_to_push, verbose=verbose ) else: if server_to_push: #update idle time display server_to_push.update_static_html_files() time.sleep( task.seconds_idle ) except KeyboardInterrupt : task.stop_execution_gently(self.listeners, server_to_push = server_to_push)
def test_purged_details(self): a_project = Project('project name') a_client = Client('a_client') listener = ServerListener( client=a_client, project=a_project ) generator = WebGenerator(project_name= a_project.name) listener.current_time = lambda : "1999-99-99-99-99-99" listener.listen_begin_task("we want this one") listener.listen_begin_subtask("subtask") listener.listen_begin_command("a command") listener.listen_end_command("a command", False, "some output", "some info", {'a':1}) listener.listen_begin_command("a command") listener.listen_end_command("a command", False, "some more output", "some more info", {'a':1}) listener.listen_end_subtask("subtask") listener.current_time = lambda : "2000-00-00-00-00-00" listener.listen_end_task("we want this one", False) expected = [ ('BEGIN_TASK', 'we want this one', '1999-99-99-99-99-99'), ('BEGIN_SUBTASK', 'subtask'), ('BEGIN_CMD', 'a command'), ('END_CMD', 'a command', False, 'some output', 'some info', {'a':1}), ('BEGIN_CMD', 'a command'), ('END_CMD', 'a command', False, 'some more output','some more info', {'a':1}), ('END_SUBTASK', 'subtask'), ('END_TASK', 'we want this one', '2000-00-00-00-00-00', 'False'), ] generator.purge_client_logfile('a_client','1999-99-99-99-99-99') self.assertEquals( expected, generator.single_execution_details('a_client', '1999-99-99-99-99-99') )