def test_prepare_reports_dir(self): """Test test_control.run_test where the exec command returns non-zero""" previous_directory = os.getcwd() reports_dir = os.path.join(self.reports_path, 'reports') reports_tarball = os.path.join(self.reports_container, 'reports.tgz') def _test_prepare_reports_dir(): try: os.chdir(self.reports_path) prepare_reports_dir(reports_dir=reports_dir) finally: os.chdir(previous_directory) self.assertFalse(os.path.exists(reports_tarball)) self.assertTrue(os.path.exists(reports_dir)) self.assertTrue(os.path.islink(reports_dir)) _test_prepare_reports_dir() touch(reports_tarball) _test_prepare_reports_dir() os.remove(reports_dir) mkdir_p(reports_dir) self.assertRaises(OSError, _test_prepare_reports_dir)
def checkout_repos(self, source, target, branch=None, verbose=False): """ Clone repository from GitHub into target directory. :param str source: Link to GitHub repository :param str target: Path to target directory :param branch: Specific branch to clone, if None clones default branch. :types branch: str, None :param bool verbose: Use the --quiet flag for clone and checkout if verbose is False. Defaults to False. """ quiet_arg = '' if verbose else '--quiet' if not os.path.isdir(target): LOG.info('checkout_repos target directory %s does not exist', target) mkdir_p(os.path.dirname(target)) self.exec_command(['git', 'clone', quiet_arg, source, target]) if branch is not None: self.exec_command( ['cd', target, '&&', 'git', 'checkout', quiet_arg, branch]) elif self.exec_command(['cd', target, '&&', 'git', 'status']) != 0: raise UserWarning( '{} exists and is not a git repository'.format(target)) else: LOG.info( 'checkout_repos target directory %s exists and is a git repository', target)
def jstest_one_host(config, mongo_uri, reports_dir, current_test_id, name): """ Run a jstest against one host. :param dict(ConfigDict) config: The system configuration. :param string mongo_uri: The mongo URI of the host to check :param string reports_dir: the report directory. :param string current_test_id: The identifier for this test. :param string name: The name of the jstest to run. Valid names are the keys from SCRIPT_NAMES. """ directory = os.path.join(reports_dir, current_test_id, 'db-correctness', name) filename = os.path.join(directory, mongo_uri) mkdir_p(directory) client_host = make_workload_runner_host(config) script_path = os.path.join(config['test_control']['jstests_dir'], SCRIPT_NAMES[name]) with open(filename, 'w') as out: if name == 'db-hash-check' and mongodb_tls_configured( config['mongodb_setup']['meta']): script_template = jinja2.Template(''' TestData = new Object(); TestData.clusterAuthMode = "x509"; TestData.auth = true; TestData.keyFile = "dummyKeyFile"; TestData.authUser = {{user|tojson}}; TestData.keyFileData = {{password|tojson}}; load({{jstests_script_file|tojson}}); ''') jstests_script = script_template.render( user=config['mongodb_setup']['authentication']['username'], password=config['mongodb_setup']['authentication']['password'], jstests_script_file=script_path) error = client_host.exec_mongo_command( script=jstests_script, remote_file_name='jstests_script.js', connection_string=mongo_uri, stdout=out, stderr=out) else: error = client_host.exec_command('bin/mongo {} {}'.format( mongo_uri, script_path), stdout=out, stderr=out) if error: # The return code of the script call is significant. If it is non-zero we put 1 at the # end of the file to indicate failure. The analysis script rules.py checks the number of # the last line. out.write("ERROR\n1") LOG.error("Failed %s correctness check on %s", name, mongo_uri) else: # Indicate that the script returned 0. This is checked by the analysis script rules.py. out.write("0") client_host.close()
def get(self, request, *args, **kwargs): """ used in get method calls """ u_id = str(uuid.uuid4()) kwargs['u_id'] = u_id if kwargs['optionset'] == 'sftp': server_object = get_object_or_404(ServerInfor, hostname=kwargs['start_path']) optinon_sets = self.get_optionset(**kwargs) optinon_sets['roots'][u_id][0]['alias'] = '{0}-{1}'.format( server_object.name, server_object.ip) key_label = "%s::%s" % (server_object.ip, server_object.credential.username) if server_object.credential.method == 'password': optinon_sets['roots'][u_id][0]['storageKwArgs'] = { 'host': server_object.ip, 'params': { 'port': server_object.credential.port, 'username': server_object.credential.username, 'password': server_object.credential.password, 'timeout': 30 }, 'root_path': '/', 'interactive': False, 'key_label': key_label } else: optinon_sets['roots'][u_id][0]['storageKwArgs'] = { 'host': server_object.ip, 'params': { 'port': server_object.credential.port, 'username': server_object.credential.username, 'key_filename': server_object.credential.key, 'timeout': 30 }, 'root_path': '/', 'interactive': False, 'key_label': key_label } self.elfinder = ElfinderConnector(optinon_sets, u_id, request.session) else: optinon_sets = self.get_optionset(**kwargs) optinon_sets['roots'][u_id][0]['alias'] = '{0}_tmp_dir'.format( request.user.username) optinon_sets['roots'][u_id][0]['path'] = os.path.join( settings.MEDIA_ROOT, request.user.username, 'Download') optinon_sets['roots'][u_id][0]['URL'] = '{0}{1}/{2}/'.format( settings.MEDIA_URL, request.user.username, 'Download') mkdir_p(os.path.join(settings.MEDIA_ROOT, request.user.username)) self.elfinder = ElfinderConnector(optinon_sets, u_id, request.session) return self.output(self.get_command(request.GET), request.GET)
def run_test(test, config, reports_dir='reports'): """ Run one test. This creates a Host object, runs the command, and saves the output to a file. :param test ConfigDict: The ConfigDict object for the test to run :param config ConfigDict: The top level ConfigDict :param str reports_dir: The report directory """ directory = os.path.join(reports_dir, test['id']) filename = os.path.join(directory, 'test_output.log') mkdir_p(directory) client_host = common.command_runner.make_workload_runner_host(config) no_output_timeout_ms = config['test_control']['timeouts']['no_output_ms'] # Generate and upload the test's configuration file if there is one generate_config_file(test, directory, client_host) with open(filename, 'wb+', 0) as out: safe_out = common.log.UTF8WrapperStream(out) tee_out = common.log.TeeStream(INFO_ADAPTER, safe_out) try: exit_status = client_host.exec_command( test['cmd'], stdout=tee_out, stderr=tee_out, no_output_timeout_ms=no_output_timeout_ms, get_pty=True) error = ExitStatus(exit_status, test['cmd']) except Exception as e: # pylint: disable=broad-except error = get_error_from_exception(e) # Old analysis/*check.py code picks up exit codes from the test_output.log write_exit_status(tee_out, error) # New DSI way for bin/analysis.py config['test_control']['out']['exit_codes'][test['id']] = { 'status': error.status, 'message': error.message } # Automatically retrieve output files, if specified, and put them into the reports directory if 'output_files' in test: for output_file in test['output_files']: # TODO: TIG-1130: if remote file doesn't exist, this will silently fail client_host.retrieve_path( output_file, os.path.join(directory, os.path.basename(output_file))) client_host.close() if error.status != EXIT_STATUS_OK: raise subprocess.CalledProcessError(error.status, test['id'], output=error.message)
def run(self): """ run the command. no checking is done with this instance. If it is called multiple times it will attempt to run multiple times. As host is closed, only the first call will succeed. The file is opened with 'wb+' so previous contents will be lost. """ # 0 => no buffering mkdir_p(os.path.dirname(self.filename)) with open(self.filename, 'wb+', 0) as out: self.host.exec_command(self.command, stdout=out, stderr=out, get_pty=True)
def setUp(self): """ Init a ConfigDict object and load the configuration files from docs/config-specs/ """ self.old_dir = os.getcwd() # Save the old path to restore # Note that this chdir only works without breaking relative imports # because it's at the same directory depth os.chdir(os.path.dirname(os.path.abspath(__file__)) + '/../../docs/config-specs/') self.config = ConfigDict('mongodb_setup') self.config.load() self.parent_dir = os.path.join(os.path.expanduser('~'), 'checkout_repos_test') self._delete_fixtures() self.reports_container = FIXTURE_FILES.fixture_file_path('container') self.reports_path = os.path.join(self.reports_container, 'reports_tests') mkdir_p(self.reports_path)
def download_logs(self, log_job_id, local_path): """ Download a tar.gz file with logs from the given job id. GET groups/{groupId}/logCollectionJobs/{jobId}/download :param str log_job_id: A string previously returned by create_log_collection_job(). :param str local_path: The local file where to store the downloaded file. """ url = "{}groups/{}/logCollectionJobs/{}/download".format( self.root, self.group_id, log_job_id) with requests.get(url, auth=self.auth, stream=True) as response: if not response.ok: LOG.error("HTTP error in download_logs", response=response.json()) response.raise_for_status() else: LOG.info("Downloading Atlas log file.", local_path=local_path) mkdir_p(os.path.dirname(local_path)) with open(local_path, "w") as file_handle: shutil.copyfileobj(response.raw, file_handle)
def posix_shell(chan, channel, channelid): stdout = list() begin_time = time.time() last_write_time = {'last_activity_time': begin_time} command = list() logobj = Log.objects.get(log=channelid) vim_flag = False vim_data = '' try: directory_date_time = timezone.now() log_name = os.path.join( '{0}-{1}-{2}'.format(directory_date_time.year, directory_date_time.month, directory_date_time.day), '{0}'.format(channelid)) from webterminal.asgi import channel_layer # terminate ssh session control redis_instance = get_redis_instance() redis_sub = redis_instance.pubsub() redis_sub.subscribe(logobj.channel) while True: text = redis_sub.get_message() if text: try: if isinstance(text['data'], bytes): text['data'] = smart_unicode(text['data']) data = json.loads(text['data']) except: data = [] if len(data) >= 1 and str(data[0]) == 'close': if not channel.closed: channel.close() r, w, x = select.select([channel], [], []) if channel in r: data = channel.recv(1024 * 24) if len(data) == 0: if not channel.closed: channel.send('\r\n*** EOF\r\n') break if data == "exit\r\n" or data == "logout\r\n" or data == 'logout': channel.close() now = time.time() delay = now - last_write_time['last_activity_time'] last_write_time['last_activity_time'] = now if vim_flag: vim_data += data if '\r' not in smart_unicode(data): command.append(smart_unicode(data)) else: command_result = CommandDeal().deal_command( ''.join(command)) if len(command_result) != 0: # vim command record patch if command_result.strip().startswith( 'vi') or command_result.strip().startswith( 'fg'): CommandLog.objects.create( log=logobj, command=command_result[0:255]) vim_flag = True else: if vim_flag: if re.compile('\[.*@.*\][\$#]').search( vim_data): vim_flag = False vim_data = '' else: CommandLog.objects.create( log=logobj, command=command_result[0:255]) command = list() if isinstance(data, unicode): stdout.append([delay, data]) else: stdout.append([ delay, codecs.getincrementaldecoder('UTF-8')( 'replace').decode(data) ]) channel_layer.send_group( smart_unicode('monitor-{0}'.format(channelid)), {'text': json.dumps(['stdout', smart_unicode(data)])}) if not chan.closed: chan.send(data) else: print('else') finally: attrs = { "version": 1, "width": 180, "height": 40, "duration": round(time.time() - begin_time, 6), "command": os.environ.get('SHELL', None), 'title': None, "env": { "TERM": os.environ.get('TERM'), "SHELL": os.environ.get('SHELL', 'sh') }, 'stdout': list(map(lambda frame: [round(frame[0], 6), frame[1]], stdout)) } mkdir_p('/'.join(os.path.join(MEDIA_ROOT, log_name).rsplit('/')[0:-1])) with open(os.path.join(MEDIA_ROOT, log_name), "a") as f: f.write( json.dumps(attrs, ensure_ascii=True, cls=CustomeFloatEncoder, indent=2)) logobj.is_finished = True logobj.end_time = timezone.now() logobj.save() chan.close() channel.close()
def posix_shell(chan, channel, log_name=None, width=90, height=40, elementid=None): from webterminal.asgi import channel_layer stdout = list() begin_time = time.time() last_write_time = {'last_activity_time': begin_time} command = list() if elementid: logobj = Log.objects.get(channel=elementid) else: logobj = Log.objects.get(channel=channel) vim_flag = False vim_data = '' try: chan.settimeout(0.0) while True: try: x = u(chan.recv(1024)) if len(x) == 0: if elementid: channel_layer.send( channel, { 'text': json.dumps([ 'disconnect', smart_unicode('\r\n*** EOF\r\n'), elementid.rsplit('_')[0] ]) }) else: channel_layer.send( channel, { 'text': json.dumps([ 'disconnect', smart_unicode('\r\n*** EOF\r\n') ]) }) break now = time.time() delay = now - last_write_time['last_activity_time'] last_write_time['last_activity_time'] = now if x == "exit\r\n" or x == "logout\r\n" or x == 'logout': chan.close() else: if vim_flag: vim_data += x logger.debug('raw data {0}'.format(command)) if '\r\n' not in x: command.append(x) else: command = CommandDeal().deal_command(''.join(command)) if len(command) != 0: #vim command record patch logger.debug('command {0}'.format(command)) if command.strip().startswith( 'vi') or command.strip().startswith('fg'): CommandLog.objects.create(log=logobj, command=command) vim_flag = True else: if vim_flag: if re.compile('\[.*@.*\][\$#]').search( vim_data): vim_flag = False vim_data = '' else: CommandLog.objects.create(log=logobj, command=command) command = list() if isinstance(x, unicode): stdout.append([delay, x]) else: stdout.append([ delay, codecs.getincrementaldecoder('UTF-8')( 'replace').decode(x) ]) if isinstance(x, unicode): if elementid: channel_layer.send( channel, { 'text': json.dumps( ['stdout', x, elementid.rsplit('_')[0]]) }) else: channel_layer.send(channel, {'text': json.dumps(['stdout', x])}) else: if elementid: channel_layer.send( channel, { 'text': json.dumps([ 'stdout', smart_unicode(x), elementid.rsplit('_')[0] ]) }) else: channel_layer.send( channel, {'text': json.dumps(['stdout', smart_unicode(x)])}) #send message to monitor group if log_name: channel_layer.send_group( u'monitor-{0}'.format(log_name.rsplit('/')[1]), {'text': json.dumps(['stdout', smart_unicode(x)])}) except socket.timeout: pass except Exception, e: logger.error(traceback.print_exc()) if elementid: channel_layer.send( channel, { 'text': json.dumps([ 'stdout', 'A bug find,You can report it to me' + smart_unicode(e), elementid.rsplit('_')[0] ]) }) else: channel_layer.send( channel, { 'text': json.dumps([ 'stdout', 'A bug find,You can report it to me' + smart_unicode(e) ]) }) finally: attrs = { "version": 1, "width": width, #int(subprocess.check_output(['tput', 'cols'])), "height": height, #int(subprocess.check_output(['tput', 'lines'])), "duration": round(time.time() - begin_time, 6), "command": os.environ.get('SHELL', None), 'title': None, "env": { "TERM": os.environ.get('TERM'), "SHELL": os.environ.get('SHELL', 'sh') }, 'stdout': list(map(lambda frame: [round(frame[0], 6), frame[1]], stdout)) } mkdir_p('/'.join(os.path.join(MEDIA_ROOT, log_name).rsplit('/')[0:-1])) with open(os.path.join(MEDIA_ROOT, log_name), "a") as f: f.write( json.dumps(attrs, ensure_ascii=True, cls=CustomeFloatEncoder, indent=2)) if elementid: audit_log = Log.objects.get(channel=elementid, log=log_name.rsplit('/')[-1]) else: audit_log = Log.objects.get(channel=channel, log=log_name.rsplit('/')[-1]) audit_log.is_finished = True audit_log.end_time = timezone.now() audit_log.save() #hand ssh terminal exit queue = get_redis_instance() redis_channel = queue.pubsub() queue.publish(channel, json.dumps(['close']))
def connect(self, message, id): self.message.reply_channel.send({"accept": True}) user = get_redis_instance().get(id) if not self.authenticate(id): self.message.reply_channel.send({"text": json.dumps( {'status': False, 'message': 'You must login to the system!'})}, immediately=True) self.message.reply_channel.send({"accept": False}) else: # permission auth username = get_redis_instance().get(id) try: Permission.objects.get( user__username=username, groups__servers__id=int(id[-1])) except ObjectDoesNotExist: self.message.reply_channel.send({"text": json.dumps( '\033[1;3;31mYou have not permission to connect server !\033[0m')}, immediately=True) self.message.reply_channel.send({"accept": False}) return except MultipleObjectsReturned: pass client = GuacamoleClient( settings.GUACD_HOST, settings.GUACD_PORT) try: data = ServerInfor.objects.get(id=int(id[-1])) if data.credential.protocol in ['vnc', 'rdp', 'telnet']: pass else: self.message.reply_channel.send({"accept": False}) except ObjectDoesNotExist: # server info not exist self.message.reply_channel.send({"accept": False}) cache_key = str(uuid.uuid4()) directory_date_time = now() recording_path = os.path.join(MEDIA_ROOT, '{0}-{1}-{2}'.format( directory_date_time.year, directory_date_time.month, directory_date_time.day)) if isinstance(username, bytes): username = username.decode() drive_path = os.path.join(MEDIA_ROOT, str(username), 'Download') """ Create recording media file and drive path """ mkdir_p(recording_path) mkdir_p(drive_path) try: client.handshake(width=data.credential.width, height=data.credential.height, protocol=data.credential.protocol, hostname=data.ip, port=data.credential.port, username=data.credential.username, password=data.credential.password, recording_path=recording_path, recording_name=cache_key, create_recording_path='true', enable_wallpaper='true', ignore_cert='true', enable_drive='true', drive_path=drive_path, create_drive_path='true', security=data.credential.security, # console_audio='true', # enable_audio_input='true', # disable_audio='false', # console='true', enable_full_window_drag='true', resize_method="reconnect" # display-update ) except Exception as e: print(e) print(traceback.print_exc()) self.message.reply_channel.send({"accept": False}) return self.message.reply_channel.send( {"text": '0.,{0}.{1};'.format(len(cache_key), cache_key)}, immediately=True) audit_log = Log.objects.create(user=User.objects.get(username=username), server=data, channel=self.message.reply_channel.name, width=data.credential.width, height=data.credential.height, log=cache_key, gucamole_client_id=client._id) audit_log.save() guacamolethread = GuacamoleThread(self.message, client) guacamolethread.setDaemon = True guacamolethread.start() guacamolethreadwrite = GuacamoleThreadWrite(self.message, client) guacamolethreadwrite.setDaemon = True guacamolethreadwrite.start()
def connect(self, message, id): self.message.reply_channel.send({"accept": True}) if not self.authenticate: self.message.reply_channel.send( { "text": json.dumps({ 'status': False, 'message': 'You must login to the system!' }) }, immediately=True) self.message.reply_channel.send({"accept": False}) else: #permission auth try: Permission.objects.get( user__username=self.message.user.username, groups__servers__id=id) except ObjectDoesNotExist: self.message.reply_channel.send( { "text": json.dumps( '\033[1;3;31mYou have not permission to connect server !\033[0m' ) }, immediately=True) self.message.reply_channel.send({"accept": False}) return client = GuacamoleClient(settings.GUACD_HOST, settings.GUACD_PORT) try: data = ServerInfor.objects.get(id=id) if data.credential.protocol in ['vnc', 'rdp', 'telnet']: pass else: self.message.reply_channel.send({"accept": False}) except ObjectDoesNotExist: #server info not exist self.message.reply_channel.send({"accept": False}) cache_key = str(uuid.uuid4()) directory_date_time = now() recording_path = os.path.join( MEDIA_ROOT, '{0}-{1}-{2}'.format(directory_date_time.year, directory_date_time.month, directory_date_time.day)) drive_path = os.path.join(MEDIA_ROOT, self.message.user.username) """ Create recording media file and drive path """ mkdir_p(recording_path) mkdir_p(drive_path) try: client.handshake(width=data.credential.width, height=data.credential.height, protocol=data.credential.protocol, hostname=data.ip, port=data.credential.port, username=data.credential.username, password=data.credential.password, recording_path=recording_path, recording_name=cache_key, create_recording_path='true', enable_wallpaper='true', ignore_cert='true', enable_drive='true', drive_path=drive_path, create_drive_path='true') #security='tls',) except Exception: self.message.reply_channel.send({"accept": False}) return self.message.reply_channel.send( {"text": '0.,{0}.{1};'.format(len(cache_key), cache_key)}, immediately=True) audit_log = Log.objects.create( user=User.objects.get(username=self.message.user), server=data, channel=self.message.reply_channel.name, width=data.credential.width, height=data.credential.height, log=cache_key, gucamole_client_id=client._id) audit_log.save() guacamolethread = GuacamoleThread(self.message, client) guacamolethread.setDaemon = True guacamolethread.start() guacamolethreadwrite = GuacamoleThreadWrite(self.message, client) guacamolethreadwrite.setDaemon = True guacamolethreadwrite.start()
def setUp(self): """Create a dict that looks like a ConfigDict object """ self.config = { 'infrastructure_provisioning': { 'tfvars': { 'ssh_user': '******', 'ssh_key_file': 'mock/ssh/key/file' }, 'out': { 'mongod': [ {'public_ip': '53.1.1.1', 'private_ip': '10.2.1.1'}, {'public_ip': '53.1.1.9', 'private_ip': '10.2.1.9'} ], 'mongos': [ {'public_ip': '53.1.1.102', 'private_ip': '10.2.1.102'} ], 'configsvr': [ {'public_ip': '53.1.1.53', 'private_ip': '10.2.1.53'} ], 'workload_client': [ {'public_ip': '53.1.1.101', 'private_ip': '10.2.1.200'} ] } }, 'mongodb_setup': { 'post_test': [ {'on_all_servers': { 'retrieve_files': { 'data/logs/': './'} } }, {'on_mongod': { 'retrieve_files': { 'data/dbs/diagnostic.data': './diagnostic.data'} } }, {'on_configsvr': { 'retrieve_files': { 'data/dbs/diagnostic.data': './diagnostic.data'} } } ], 'mongod_config_file': { 'storage': { 'engine': 'wiredTiger' } } }, 'runtime': { 'task_id': 'STAY IN YOUR VEHICLE CITIZEN' }, 'test_control': { 'task_name': 'test_config', 'reports_dir_basename': 'reports', 'perf_json': { 'path': 'perf.json' }, 'output_file': { 'mongoshell': 'test_output.log', 'ycsb': 'test_output.log', 'fio': 'fio.json', 'iperf': 'iperf.json' }, 'timeouts': { 'no_output_ms': 5000, }, 'run': [ {'id': 'benchRun', 'type': 'shell', 'cmd': '$DSI_PATH/workloads/run_workloads.py -c workloads.yml', 'config_filename': 'workloads.yml', 'output_files': ['mock_output0.txt', 'mock_output0.txt'], 'workload_config': 'mock_workload_config' }, {'id': 'ycsb_load', 'type': 'ycsb', 'cmd': 'cd YCSB/ycsb-mongodb; ./bin/ycsb load mongodb -s -P ' + 'workloads/workloadEvergreen -threads 8; sleep 1;', 'config_filename': 'workloadEvergreen', 'workload_config': 'mock_workload_config', 'skip_validate': True }, {'id': 'fio', 'type': 'fio', 'cmd': './fio-test.sh some_hostname', 'skip_validate': True } ], 'jstests_dir': './jstests/hooks', 'post_test': [ {'on_workload_client': { 'retrieve_files': { 'workloads/workload_timestamps.csv': '../workloads_timestamps.csv'} } } ], 'out': {'exit_codes': {}} } } # yapf: disable self.reports_container = FIXTURE_FILES.fixture_file_path('container') self.reports_path = os.path.join(self.reports_container, 'reports_tests') mkdir_p(self.reports_path)