def bootcd(self): "all nodes: invoke GetBootMedium and store result locally" utils.header("Calling GetBootMedium for {}".format(self.name())) # this would clearly belong in the config but, well .. options = self.node_spec['bootmedium_options'] if 'bootmedium_options' in self.node_spec else [] encoded = self.test_plc.apiserver.GetBootMedium( self.test_plc.auth_root(), self.name(), 'node-iso', '', options) if encoded == '': raise Exception('GetBootmedium failed') filename = "{}/{}.iso".format(self.nodedir(), self.name()) utils.header('Storing boot medium into {}'.format(filename)) # xxx discovered with python3, but a long stading issue: # encoded at this point is a str instead of a bytes # Quick & dirty : we convert this explicitly to a bytearray # Longer run : clearly it seems like the plcapi server side should # tag its result with <base64></base64> rather than as a string bencoded = str.encode(encoded) if self.dry_run(): print("Dry_run: skipped writing of iso image") return True else: # with python3 we need to call decodestring here with open(filename,'wb') as storage: storage.write(base64.decodestring(bencoded)) return True
def ssh_tasks(self,options, expected=True, command=None): # timeout_minutes=20, silent_minutes=10, period_seconds=15): # timeout = timedelta(minutes=timeout_minutes) # graceout = timedelta(minutes=silent_minutes) # period = timedelta(seconds=period_seconds) if not command: command = "echo hostname ; hostname; echo id; id; echo uname -a ; uname -a" # locate a key private_key = self.locate_private_key() if not private_key : utils.header("WARNING: Cannot find a valid key for slice {}".format(self.name())) return False # convert nodenames to real hostnames if expected: msg="ssh slice access enabled" else: msg="ssh slice access disabled" utils.header("checking for {} -- slice {}".format(msg, self.name())) tasks=[] slicename=self.name() dry_run = getattr(options, 'dry_run', False) for nodename in self.slice_spec['nodenames']: site_spec, node_spec = self.test_plc.locate_node(nodename) tasks.append( CompleterTaskSliceSsh(self.test_plc, node_spec['node_fields']['hostname'], slicename, private_key, command, expected, dry_run)) return tasks
def __finish(self): tests = self.__gather_tests # Normally you don't have to use Fore.BLUE before each line, but # Travis-CI seems to reset color codes on newline (see travis-ci/travis-ci#2692) # or stream flush, so we have to ensure that the color code is printed repeatedly prefix = Fore.CYAN for line in header("Verification Summary", top='=', bottom='').split('\n'): print prefix + line for test in tests: print prefix + "| Test: %s" % test.name if test.name in self.results['verify'].keys(): for test_type, result in self.results['verify'][test.name].iteritems(): if result.upper() == "PASS": color = Fore.GREEN elif result.upper() == "WARN": color = Fore.YELLOW else: color = Fore.RED print prefix + "| " + test_type.ljust(11) + ' : ' + color + result.upper() else: print prefix + "| " + Fore.RED + "NO RESULTS (Did framework launch?)" print prefix + header('', top='', bottom='=') + Style.RESET_ALL print "Time to complete: " + str(int(time.time() - self.start_time)) + " seconds" print "Results are saved in " + os.path.join(self.result_directory, self.timestamp)
def scp_to_webroot(self, localfiles, recursive = False): if self.config.verbose: utils.header("Copying %s to %s webroot" % (localfiles, self['name']), logfile = self.config.logfile) self.scp_to("%(localfiles)s" % locals(), "/var/www/html/") url = 'http://%s/%s' % (self['ip'], localfiles) return url
def sfi_configure (self,dir_name): plc_spec=self.test_plc.plc_spec # cheat a bit: retrieve the global SFA spec from the plc obj sfa_spec=self.test_plc.plc_spec['sfa'] # fetch keys in config spec and expose to sfi for spec_name in ['pi_spec','user_spec']: user_spec=self.auth_sfa_spec[spec_name] user_leaf=user_spec['name'] key_name=user_spec['key_name'] key_spec = self.test_plc.locate_key (key_name) for (kind,ext) in [ ('private', 'pkey'), ('public', 'pub') ] : contents=key_spec[kind] file_name=os.path.join(dir_name,self.obj_hrn(user_leaf))+"."+ext fileconf=open(file_name,'w') fileconf.write (contents) fileconf.close() utils.header ("(Over)wrote {}".format(file_name)) # file_name=dir_name + os.sep + 'sfi_config' fileconf=open(file_name,'w') SFI_AUTH=self.auth_hrn() fileconf.write ("SFI_AUTH='{}'".format(SFI_AUTH)) fileconf.write('\n') # default is to run as a PI SFI_USER=self.obj_hrn(self.auth_sfa_spec['pi_spec']['name']) fileconf.write("SFI_USER='******'".format(SFI_USER)) fileconf.write('\n') SFI_REGISTRY='http://{}:{}/'.format(sfa_spec['settings']['SFA_REGISTRY_HOST'], 12345) fileconf.write("SFI_REGISTRY='{}'".format(SFI_REGISTRY)) fileconf.write('\n') SFI_SM='http://{}:{}/'.format(sfa_spec['settings']['SFA_AGGREGATE_HOST'], sfa_spec['sfi-connects-to-port']) fileconf.write("SFI_SM='{}'".format(SFI_SM)) fileconf.write('\n') fileconf.close() utils.header ("(Over)wrote {}".format(file_name))
def run(self, command, message=None, background=False, dry_run=False): local_command = self.actual_command(command, dry_run=dry_run) if dry_run: utils.header("DRY RUN " + local_command) return 0 else: self.header(message) return utils.system(local_command, background)
def is_ready(self, timeout=30): # Node is considered ready when Node Manager has started avuseradd processes have stopped log = self.config.logfile class test: def __init__(self, name, description, system, cmd, check, inverse = False, logfile = log): self.system = system self.cmd = cmd self.check = check self.name = name self.description = description self.inverse = inverse self.logfile = logfile def run(self, logfile, verbose = True): if verbose: utils.header(self.description, logfile = self.logfile) (status, output) = self.system(self.cmd) if self.inverse and output.find(self.check) == -1: if verbose: utils.header("%s Passed Test" % self.name, logfile = self.logfile) return True elif not self.inverse and output and output.find(self.check) -1: if verbose: utils.header("%s Passed Test" % self.name, logfile = self.logfile) return True if verbose: utils.header("%s Failed Test" % self.name, logfile = self.logfile) return False ready = False start_time = time.time() end_time = start_time + 60 * timeout vcheck_cmd = "ps -elfy | grep vuseradd | grep -v grep" grep_cmd = "grep 'Starting Node Manager' %s" % self.logfile.filename tests = { '1': test("NodeManager", "Checking if NodeManager has started", utils.commands, grep_cmd, "OK", logfile = self.config.logfile), '2': test("vuseradd", "Checking if vuseradd is done", self.commands, vcheck_cmd, "vuseradd", True, logfile = self.config.logfile) } while time.time() < end_time and ready == False: # Run tests in order steps = tests.keys() steps.sort() results = {} for step in steps: test = tests[step] results[step] = result = test.run(self.config.verbose) if not result: break # Check results. We are ready if all passed if not set(results.values()).intersection([False, None]): utils.header("%s is ready" % (self['hostname'])) ready = True else: if self.config.verbose: utils.header("%s not ready. Waiting 30 seconds. %s seconds left" % \ (self['hostname'], int(end_time - time.time())) ) time.sleep(30) return ready
def delete_nodes (self): auth = self.owner_auth() slice_name = self.slice_name() print('retrieving slice {}'.format(slice_name)) slice=self.test_plc.apiserver.GetSlices(auth, slice_name)[0] node_ids=slice['node_ids'] utils.header ("Deleting {} nodes from slice {}"\ .format(len(node_ids), slice_name)) self.test_plc.apiserver.DeleteSliceFromNodes (auth, slice_name, node_ids)
def host_box(self): if self.is_real(): return 'localhost' else: try: return self.node_spec['host_box'] except: utils.header("WARNING : qemu nodes need a host box") return 'localhost'
def kill_qemu(self): #Prepare the log file before killing the nodes test_box = self.test_box() # kill the right processes utils.header("Stopping qemu for node {} on box {}"\ .format(self.name(), self.test_box().hostname())) command = "{}/qemu-kill-node {}".format(self.nodedir(),self.name()) self.test_box().run_in_buildname(command, dry_run=self.dry_run()) return True
def qemu_start(self): "all nodes: start the qemu instance (also runs qemu-bridge-init start)" model = self.node_spec['node_fields']['model'] #starting the Qemu nodes before if self.is_qemu(): self.start_qemu() else: utils.header("TestNode.qemu_start : {} model {} taken as real node"\ .format(self.name(), model)) return True
def start_qemu(self): test_box = self.test_box() utils.header("Starting qemu node {} on {}".format(self.name(), test_box.hostname())) test_box.run_in_buildname("{}/qemu-bridge-init start >> {}/log.txt"\ .format(self.nodedir(), self.nodedir()), dry_run=self.dry_run()) # kick it off in background, as it would otherwise hang test_box.run_in_buildname("{}/qemu-start-node 2>&1 >> {}/log.txt"\ .format(self.nodedir(), self.nodedir()))
def add_nodes (self): auth = self.owner_auth() slice_name = self.slice_name() hostnames=[] for nodename in self.slice_spec['nodenames']: node_spec=self.test_site.locate_node(nodename) test_node=TestNode(self.test_plc, self.test_site, node_spec) hostnames += [test_node.name()] utils.header("Adding {} in {}".format(hostnames, slice_name)) self.test_plc.apiserver.AddSliceToNodes(auth, slice_name, hostnames)
def __run_tests(self, tests): if len(tests) == 0: return 0 logging.debug("Start __run_tests.") logging.debug("__name__ = %s",__name__) error_happened = False if self.os.lower() == 'windows': logging.debug("Executing __run_tests on Windows") for test in tests: with open(self.current_benchmark, 'w') as benchmark_resume_file: benchmark_resume_file.write(test.name) with self.quiet_out.enable(): if self.__run_test(test) != 0: error_happened = True else: logging.debug("Executing __run_tests on Linux") # Setup a nice progressbar and ETA indicator widgets = [self.mode, ': ', progressbar.Percentage(), ' ', progressbar.Bar(), ' Rough ', progressbar.ETA()] pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(tests)).start() pbar_test = 0 # These features do not work on Windows for test in tests: pbar.update(pbar_test) pbar_test = pbar_test + 1 if __name__ == 'benchmark.benchmarker': print header("Running Test: %s" % test.name) with open(self.current_benchmark, 'w') as benchmark_resume_file: benchmark_resume_file.write(test.name) with self.quiet_out.enable(): test_process = Process(target=self.__run_test, name="Test Runner (%s)" % test.name, args=(test,)) test_process.start() test_process.join(self.run_test_timeout_seconds) self.__load_results() # Load intermediate result from child process if(test_process.is_alive()): logging.debug("Child process for {name} is still alive. Terminating.".format(name=test.name)) self.__write_intermediate_results(test.name,"__run_test timeout (="+ str(self.run_test_timeout_seconds) + " seconds)") test_process.terminate() test_process.join() if test_process.exitcode != 0: error_happened = True pbar.finish() if os.path.isfile(self.current_benchmark): os.remove(self.current_benchmark) logging.debug("End __run_tests.") if error_happened: return 1 return 0
def qemu_export(self): "all nodes: push local node-dep directory on the qemu box" # if relevant, push the qemu area onto the host box if self.test_box().is_local(): return True dry_run = self.dry_run() utils.header("Cleaning any former sequel of {} on {}"\ .format(self.name(), self.host_box())) utils.header("Transferring configuration files for node {} onto {}"\ .format(self.name(), self.host_box())) return self.test_box().copy(self.nodedir(), recursive=True, dry_run=dry_run) == 0
def is_local_hostname(hostname): if hostname == "localhost": return True import socket try: local_ip = socket.gethostbyname(socket.gethostname()) remote_ip = socket.gethostbyname(hostname) return local_ip == remote_ip except: utils.header("WARNING : something wrong in is_local_hostname with hostname={}".format(hostname)) return False
def update_api(self): # Set up API acccess # If plc is specified, find its configuration # and use its API self.update_ip() name, ip, port, path = self['name'], self['ip'], self['port'], self['api_path'] if self.config.verbose: utils.header("Updating %(name)s's api to https://%(ip)s:%(port)s/%(path)s" % locals(), logfile = self.config.logfile) api_server = "https://%(ip)s:%(port)s/%(path)s" % locals() self.config.api = xmlrpclib.Server(api_server, allow_none = 1) self.config.api_type = 'xmlrpc'
def create_user (self): user_spec = self.user_spec fields = user_spec['user_fields'] auth = self.test_plc.auth_root() utils.header('Adding user {} - roles {}'.format(fields['email'], user_spec['roles'])) self.test_plc.apiserver.AddPerson(auth, fields) self.test_plc.apiserver.UpdatePerson(auth, fields['email'], {'enabled': True}) for role in user_spec['roles']: self.test_plc.apiserver.AddRoleToPerson(auth,role,fields['email']) self.test_plc.apiserver.AddPersonToSite(auth, self.name(), self.test_site.name())
def store_key(self): pub = self.publicpath() priv = self.privatepath() utils.header("Storing key {} in {}".format(self.name(), pub)) dir = os.path.dirname(pub) if not os.path.isdir(dir): os.mkdir(dir) with open(pub,"w") as f: f.write(self.key_spec['key_fields']['key']) with open(priv,"w") as f: f.write(self.key_spec['private']) os.chmod(priv,0o400) os.chmod(pub,0o444)
def sfa_check_slice_plc (self, options): "check the slice has been created at the plc - all local nodes should be in slice" slice = self.test_plc.apiserver.GetSlices(self.test_plc.auth_root(), self.plc_name())[0] nodes = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), {'peer_id':None}) result = True for node in nodes: if node['node_id'] in slice['node_ids']: utils.header("local node {} found in slice {}".format(node['hostname'], slice['name'])) else: utils.header("ERROR - local node {} NOT FOUND in slice {}"\ .format(node['hostname'], slice['name'])) result = False return result
def create_node(self): ownername = self.node_spec['owner'] user_spec = self.test_site.locate_user(ownername) test_user = TestUser(self.test_plc,self.test_site,user_spec) userauth = test_user.auth() utils.header("node {} created by user {}".format(self.name(), test_user.name())) rootauth = self.test_plc.auth_root() server = self.test_plc.apiserver node_id = server.AddNode(userauth, self.test_site.site_spec['site_fields']['login_base'], self.node_spec['node_fields']) # create as reinstall to avoid user confirmation server.UpdateNode(userauth, self.name(), { 'boot_state' : 'reinstall' }) # you are supposed to make sure the tags exist for tagname, tagvalue in self.node_spec['tags'].items(): server.AddNodeTag(userauth, node_id, tagname, tagvalue) if not self.test_plc.has_addresses_api(): # print 'USING OLD INTERFACE' # populate network interfaces - primary server.AddInterface(userauth, self.name(), self.node_spec['interface_fields']) else: # print 'USING NEW INTERFACE with separate ip addresses' # this is for setting the 'dns' stuff that now goes with the node server.UpdateNode(userauth, self.name(), self.node_spec['node_fields_nint']) interface_id = server.AddInterface(userauth, self.name(),self.node_spec['interface_fields_nint']) server.AddIpAddress(userauth, interface_id, self.node_spec['ipaddress_fields']) route_fields = self.node_spec['route_fields'] route_fields['interface_id'] = interface_id server.AddRoute(userauth, node_id, self.node_spec['route_fields']) pass # populate network interfaces - others if 'extra_interfaces' in self.node_spec: for interface in self.node_spec['extra_interfaces']: server.AddInterface(userauth, self.name(), interface['interface_fields']) if 'settings' in interface: for attribute, value in interface['settings'].items(): # locate node network interface = server.GetInterfaces( userauth, {'ip' : interface['interface_fields']['ip']})[0] interface_id = interface['interface_id'] # locate or create node network attribute type try: interface_tagtype = server.GetTagTypes(userauth, {'name' : attribute})[0] except: interface_tagtype = server.AddTagType(rootauth,{'category' : 'test', 'tagname' : attribute}) # attach value server.AddInterfaceTag(userauth, interface_id, attribute, value)
def wget(self, url, targetdir, user = '******'): if self.config.verbose: utils.header("Downloading %(url)s to %(targetdir)s" % locals()) cmd_prefix = "" if user not in ['root']: cmd_prefix = " su - user -c " fileparts = url.split(os.sep) filename = fileparts[-1:][0] cleanup_cmd = "%(cmd_prefix)s rm -f %(targetdir)s/%(filename)s" % locals() print >> self.logfile, cleanup_cmd self.commands(cleanup_cmd, False) wget_cmd = "%(cmd_prefix)s wget -nH -P %(targetdir)s %(url)s" % locals() print >> self.logfile, wget_cmd self.commands(wget_cmd)
def copy_abs(self, local_file, remote_file, recursive=False, dry_run=False): if self.is_local(): dest = "" else: dest = "{}:".format(self.hostname_part()) scp_command = "scp " scp_command += TestSsh.std_options if recursive: scp_command += "-r " scp_command += self.key_part() scp_command += "{} {}{}".format(local_file, dest, remote_file) if dry_run: utils.header("DRY RUN TestSsh.copy {}".format(scp_command)) # need to be consistent with the non-dry-run mode return 0 return utils.system(scp_command)
def archive_scripts(self, prefix): valid_prefix = ['slice', 'node'] if prefix not in valid_prefix: raise "Error. Invalid prefix %s. Must be in %s" % (prefix, valid_prefix) scripts_dir = self.path + os.sep + 'tests' +os.sep + prefix + os.sep workdir = '/tmp' + os.sep archive_path = workdir + os.sep + prefix + os.sep archive_filename = prefix + ".tar.gz" if self.verbose: utils.header("Creating/Updating %s archive %s" % (prefix, archive_path + archive_filename), logfile = self.logfile) utils.commands("mkdir -p %(archive_path)s" % locals(), logfile = self.logfile) utils.commands("cp -Rf %(scripts_dir)s* %(archive_path)s" % locals(), logfile = self.logfile) tar_cmd = "cd %(workdir)s && tar -czf %(workdir)s/%(archive_filename)s %(prefix)s" % locals() utils.commands(tar_cmd, logfile = self.logfile) return (archive_filename, workdir+archive_filename)
def copy(self, local_file, recursive=False, dry_run=False): if self.is_local(): return 0 self.create_buildname_once(dry_run) scp_command = "scp " if not dry_run: scp_command += TestSsh.std_options if recursive: scp_command += "-r " scp_command += self.key_part() scp_command += "{} {}:{}/{}".format(local_file, self.hostname_part(), self.fullname(self.buildname), os.path.basename(local_file) or ".") if dry_run: utils.header("DRY RUN TestSsh.copy {}".format(scp_command)) # need to be consistent with the non-dry-run mode return 0 return utils.system(scp_command)
def run (self, timeout_timedelta, silent_timedelta, period): begin = datetime.now() timeout = begin+timeout_timedelta timeout_seconds = timeout_timedelta.total_seconds() timeout_minutes = timeout_seconds/60 graceout = datetime.now()+silent_timedelta silent_seconds = silent_timedelta.total_seconds() silent_minutes = silent_seconds/60 period_seconds = int(period.total_seconds()) if self.verbose: if timeout_seconds >= 120: utils.header("Completer [{} tasks]: max timeout is {} minutes, " "silent for {} minutes (period is {} s)"\ .format(len(self.tasks), timeout_minutes, silent_minutes, period_seconds)) else: utils.header("Completer [{} tasks]: max timeout is {} seconds, " "silent for {} seconds (period is {} s)"\ .format(len(self.tasks), timeout_seconds, silent_seconds, period_seconds)) tasks = self.tasks while tasks: fine = [] for task in tasks: success = task.run (silent=datetime.now() <= graceout) if success: fine.append(task) for task in fine: tasks.remove(task) if not tasks: if self.verbose: duration = datetime.now() - begin print("total completer {} {}s".format(self.message, int(duration.total_seconds()))) return True if datetime.now() > timeout: for task in tasks: task.failure_epilogue() return False if self.verbose: print('{}s..'.format(period_seconds), end=' ') time.sleep(period_seconds) # in case we're empty return True
def verify_type(test_type): with open(os.path.join(verificationPath, (test_type + '.txt')), 'w') as verification: test = self.runTests[test_type] test.setup_out(verification) verification.write(header("VERIFYING %s" % test_type.upper())) base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port) try: results = test.verify(base_url) except ConnectionError as e: results = [('fail',"Server did not respond to request", base_url)] logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e) except Exception as e: results = [('fail',"""Caused Exception in TFB This almost certainly means your return value is incorrect, but also that you have found a bug. Please submit an issue including this message: %s\n%s""" % (e, traceback.format_exc()), base_url)] logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e) traceback.format_exc() test.failed = any(result == 'fail' for (result, reason, url) in results) test.warned = any(result == 'warn' for (result, reason, url) in results) test.passed = all(result == 'pass' for (result, reason, url) in results) def output_result(result, reason, url): specific_rules_url = "http://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/#specific-test-requirements" color = Fore.GREEN if result.upper() == "WARN": color = Fore.YELLOW elif result.upper() == "FAIL": color = Fore.RED verification.write((" " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url)) print (" " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url) if reason is not None and len(reason) != 0: for line in reason.splitlines(): verification.write(" " + line + '\n') print " " + line if not test.passed: verification.write(" See %s\n" % specific_rules_url) print " See %s\n" % specific_rules_url [output_result(r1,r2,url) for (r1, r2, url) in results] if test.failed: self.benchmarker.report_verify_results(self, test_type, 'fail') elif test.warned: self.benchmarker.report_verify_results(self, test_type, 'warn') elif test.passed: self.benchmarker.report_verify_results(self, test_type, 'pass') else: raise Exception("Unknown error - test did not pass,warn,or fail") verification.flush()
def analysis(): """PDB Tools menu""" utils.header(__VERSION__) print(""" 1) Proximity Analysis (PDB) 2) Catalytic Pocket Selector (PDB) b) Back to Main Menu q) Exit """) choice = input("\nSelection:\t") if choice == "1": nei_fin.main() elif choice == "2": pro_pru.main() elif choice == "b": main() elif choice == "q": sys.exit()
def run(self, logfile, verbose = True): if verbose: utils.header(self.description, logfile = self.logfile) (status, output) = self.system(self.cmd) if self.inverse and output.find(self.check) == -1: if verbose: utils.header("%s Passed Test" % self.name, logfile = self.logfile) return True elif not self.inverse and output and output.find(self.check) -1: if verbose: utils.header("%s Passed Test" % self.name, logfile = self.logfile) return True if verbose: utils.header("%s Failed Test" % self.name, logfile = self.logfile) return False
def run(self): ########################## # Generate metadata ########################## self.run_list_test_metadata() ########################## # Get a list of all known # tests that we can run. ########################## all_tests = self.__gather_tests ########################## # Setup client/server ########################## print(header("Preparing Server, Database, and Client ...", top='=', bottom='=')) with self.quiet_out.enable(): self.__setup_server() self.__setup_database() self.__setup_client() ## Check if wrk (and wrk-pipeline) is installed and executable, if not, raise an exception #if not (os.access("/usr/local/bin/wrk", os.X_OK) and os.access("/usr/local/bin/wrk-pipeline", os.X_OK)): # raise Exception("wrk and/or wrk-pipeline are not properly installed. Not running tests.") ########################## # Run tests ########################## print(header("Running Tests...", top='=', bottom='=')) result = self.__run_tests(all_tests) ########################## # Parse results ########################## if self.mode == "benchmark": print(header("Parsing Results ...", top='=', bottom='=')) self.__parse_results(all_tests) self.__set_completion_time() self.__upload_results() self.__finish() return result
def ssh_slice_sfa(self, options, timeout_minutes=40, silent_minutes=0, period_seconds=15): "tries to ssh-enter the SFA slice" timeout = timedelta(minutes=timeout_minutes) graceout = timedelta(minutes=silent_minutes) period = timedelta(seconds=period_seconds) # locate a key private_key=self.locate_private_key() if not private_key : utils.header("WARNING: Cannot find a valid key for slice {}".format(self.hrn())) return False command="echo hostname ; hostname; echo id; id; echo uname -a ; uname -a" tasks = [] slicename = self.plc_name() dry_run = getattr(options,'dry_run',False) for nodename in self.slice_spec['nodenames']: (site_spec,node_spec) = self.test_plc.locate_node(nodename) tasks.append( CompleterTaskSliceSsh(self.test_plc, node_spec['node_fields']['hostname'], slicename, private_key, command, expected=True, dry_run=dry_run)) return Completer (tasks, message='ssh_slice_sfa').run(timeout, graceout, period)
def run(self): ########################## # Get a list of all known # tests that we can run. ########################## all_tests = self.__gather_tests ########################## # Setup client/server ########################## print header("Preparing Server, Database, and Client ...", top='=', bottom='=') self.__setup_server() self.__setup_database() self.__setup_client() ## Check if wrk (and wrk-pipeline) is installed and executable, if not, raise an exception #if not (os.access("/usr/local/bin/wrk", os.X_OK) and os.access("/usr/local/bin/wrk-pipeline", os.X_OK)): # raise Exception("wrk and/or wrk-pipeline are not properly installed. Not running tests.") ########################## # Run tests ########################## print header("Running Tests...", top='=', bottom='=') result = self.__run_tests(all_tests) ########################## # Parse results ########################## if self.mode == "benchmark": print header("Parsing Results ...", top='=', bottom='=') self.__parse_results(all_tests) self.__finish() return result
def main(): utils.header(__VERSION__) print(""" 1) Trajectory Manipulation 2) PDB Tools 3) Others 4) Molecular properties and vibrations q) Exit """) choice = input("\nSelection:\t") if choice == "1": data_manipulation() elif choice == "2": analysis() elif choice == "3": input_generation() elif choice == "4": molecule_interface() elif choice == "q": sys.exit()
def main(): load_bank_data() cleaner() header() account_auth = auth_account() if account_auth: cleaner() header() welcome(account_auth) while True: option_typed = get_menu_options_typed(account_auth) do_operation(option_typed, account_auth) print() loop_typed = input('Deseja sair da conta S/n? ') if not loop_typed == 'n': print('Bye') break else: print('Conta inválida')
def __stop_test(self, test, out): # self.__process may not be set if the user hit ctrl+c prior to the test # starting properly. if self.__process is not None: out.write(header("Stopping %s" % test.name)) out.flush() # Ask TFBReaper to nicely terminate itself self.__process.terminate() slept = 0 returnCode = None # Check once a second to see if TFBReaper has exited while(slept < 30 and returnCode is None): time.sleep(1) slept += 1 returnCode = self.__process.poll() # If TFBReaper has not exited at this point, we have a problem if returnCode is None: self.__write_intermediate_results(test.name, "port " + str(test.port) + " was not released by stop") out.write(header("Error: Port %s was not released by stop - %s" % (test.port, test.name))) out.write(header("Running Processes")) out.write(subprocess.check_output(['ps -aux'], shell=True)) out.flush() return exit_with_code(1)
def verify_type(test_type): test = self.runTests[test_type] test.setup_out_err(out, err) out.write(header("VERIFYING %s" % test_type.upper())) base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port) try: results = test.verify(base_url) except Exception as e: results = [('fail', """Caused Exception in TFB This almost certainly means your return value is incorrect, but also that you have found a bug. Please submit an issue including this message: %s\n%s""" % (e, traceback.format_exc()), base_url)] logging.warning( "Verifying test %s for %s caused an exception: %s", test_type, self.name, e) traceback.format_exc() test.failed = any(result is 'fail' for (result, reason, url) in results) test.warned = any(result is 'warn' for (result, reason, url) in results) test.passed = all(result is 'pass' for (result, reason, url) in results) def output_result(result, reason, url): out.write(" %s for %s\n" % (result.upper(), url)) print " %s for %s" % (result.upper(), url) if reason is not None and len(reason) != 0: for line in reason.splitlines(): out.write(" " + line + '\n') print " " + line [output_result(r1, r2, url) for (r1, r2, url) in results] if test.failed: self.benchmarker.report_verify_results(self, test_type, 'fail') elif test.warned: self.benchmarker.report_verify_results(self, test_type, 'warn') elif test.passed: self.benchmarker.report_verify_results(self, test_type, 'pass') else: raise Exception( "Unknown error - test did not pass,warn,or fail")
def text_generator(seed, unconditional=False, nsamples=1, batch_size=-1, length=-1, temperature=0.7, top_k=40): enc = get_encoder() context_tokens = enc.encode(seed) if batch_size == -1: batch_size = 1 assert nsamples % batch_size == 0 if length == -1: length = config.n_ctx // 2 elif length > config.n_ctx: raise ValueError("Can't get samples longer than window size: %s" % config.n_ctx) out = sample_sequence( model=model, length=length, context=context_tokens if not unconditional else None, start_token=enc.encoder['<|endoftext|>'] if unconditional else None, batch_size=batch_size, temperature=temperature, top_k=top_k, device=device) text = '' out = out[:, len(context_tokens):].tolist() for i in range(batch_size): text += enc.decode(out[i]) html = '' html = add_content( html, header('Input Seed ', color='black', gen_text='Network Output')) html = add_content(html, box(seed, text)) return f'<div>{html}</div>'
def login(): if "username" in session and session["username"] != "": return "<script>window.location='/'</script>" #FIX header = utils.header("") footer = utils.footer() if request.method == 'POST': return_code = user_actions.login(request.form["username"], request.form["password"]) if return_code == 1: session["username"] = request.form["username"] return "<script>window.location='/'</script>" #FIX else: return render_template("login.html", return_code=return_code, header=header, footer=footer) else: return render_template("login.html", header=header, footer=footer)
def generate_output(s, words_to_generate=50, diversity=0.75): """Generate output from a sequence""" # Mapping of words to integers word_idx = json.load(open('../data/word-index.json')) idx_word = {idx: word for word, idx in word_idx.items()} # Original formated text start = format_sequence(s).split() gen = [] s = start[:] with graph.as_default(): # Generate output for i in range(words_to_generate): # Conver to array x = np.array([word_idx.get(word, 0) for word in s]).reshape( (1, -1)) # Make predictions preds = model.predict(x)[0].astype(float) # Diversify preds = np.log(preds) / diversity exp_preds = np.exp(preds) # Softmax preds = exp_preds / np.sum(exp_preds) # Pick next index next_idx = np.argmax(np.random.multinomial(1, preds, size=1)) s.append(idx_word[next_idx]) gen.append(idx_word[next_idx]) # Formatting in html start = remove_spaces(' '.join(start)) + ' ' gen = remove_spaces(' '.join(gen)) html = '' html = addContent( html, header('Input Seed ', color='black', gen_text='Network Output')) html = addContent(html, box(start, gen)) return html
import pandas as pd from utils import wdir,ddir,header,normalize_txt df = pd.read_csv(ddir+'rayon.csv',sep=';') df['txt'] = df.Categorie1_Name+' '+df.Categorie2_Name+' '+df.Categorie3_Name df.txt = df.apply(lambda r:normalize_txt(r.txt),axis=1) dfrayon = df wrayon = set((' '.join(list(dfrayon.txt))).split()) df = pd.read_csv(ddir+'training_shuffled_normed.csv',sep=';',names=header()) df['txt'] = (df.Marque+' ')*3+(df.Libelle+' ')*2+df.Description*1 dftrain = df wtrain = set((' '.join(list(dftrain.txt))).split()) df = pd.read_csv(ddir+'test_normed.csv',sep=';',names=header(test=True)) df['txt'] = [df.Marque]*3+[df.libelle]*2+[df.Description]*1 dftest = df wtest = set((' '.join(list(dftest.txt))).split()) df = pd.read_csv(ddir+'validation_normed.csv',sep=';',names=header()) df['txt'] = [df.Marque]*3+[df.libelle]*2+[df.Description]*1 dfvalid = df wvalid = set((' '.join(list(dfvalid.txt))).split())
# create sample set # from training set ##################### # NOTE : USE analyse_test.py and perfect_sampling.py to get perfect training & validation set # NOTE : USE analyse_test.py and perfect_sampling.py to get perfect training & validation set # NOTE : USE analyse_test.py and perfect_sampling.py to get perfect training & validation set # NOTE : USE analyse_test.py and perfect_sampling.py to get perfect training & validation set ####################### # training # stage1 : Categorie1 # stage3 : Categorie3|Categorie1 ####################### dftrain = pd.read_csv(ddir+'training_perfect_200.csv',sep=';',names = header()).fillna('') dfvalid = pd.read_csv(ddir+'validation_perfect.csv',sep=';',names = header()).fillna('') dftest = pd.read_csv(ddir+'test_normed.csv',sep=';',names = header(test=True)).fillna('') add_txt(dftrain) add_txt(dfvalid) add_txt(dftest) dftrain = dftrain[['Categorie3','Categorie1','txt']] dfvalid = dfvalid[['Categorie3','Categorie1','txt']] dftest = dftest[['Identifiant_Produit','txt']] # training stage1 dt = -time.time() sct,scv = training_stage1(dftrain,dfvalid)
utils.header(__VERSION__) list_molecules() input("\nPress enter to return to the menu ... ") molecule_interface() elif choice == "b": main() elif choice == "q": sys.exit() MOL_LIST = [] if len(sys.argv) == 1: main() else: utils.header(__VERSION__) if sys.argv[1].split(".")[-1] == "interp": print( "\nQUICK MODE:\tDetected .interp file. Opening as fast as I can..." ) neb_vis.main(sys.argv[1]) elif sys.argv[1].split(".")[-1] == "out": print("\nQUICK MODE:\tDetected .out file. Opening as fast as I can...") out_sum.main(sys.argv[1]) elif sys.argv[1] == "load": print( "\nQUICK MODE:\tDetected load instruction.\nLoading .xyz files...") print("------------------------------------------------") for i, argument in enumerate(sys.argv): if i > 1 and argument.split(".")[-1] == "xyz": MOL_LIST.append(mol.MOL(argument))
if Xv.shape[0]==0: scv = -1 else: scv = cla.score(Xv,Yv) print 'Stage 3.'+str(i)+':',cat,'score',sct,scv joblib.dump((labels,cla),fname) del cla return (sct,scv) ################################################# # SAMPLING START HERE ################################################# print 'loading...' dftrain = pd.read_csv(ddir+'training_shuffled_normed.csv',sep=';',names = header()).fillna('') print 'txting...' add_txt(dftrain) dfsample = get_sample(dftrain) dfsample = dfsample[['Identifiant_Produit','Categorie3','Categorie1','txt']] dfsample.to_csv(ddir+'training_sup9.csv',sep=';',index=False,header=False) Y = dfsample.Categorie3.values ID = dfsample.Identifiant_Produit.values print 'vectorizing...' vec,X = vectorizer(dfsample.txt) print 'dumping...' joblib.dump((vec,ID,X,Y),ddir+'joblib/vecIDXY')
def molecule_interface(): """Molecule class menu""" def ask_ID(): ID = int( input("Select the molecule ID (from 1 to " + str(len(MOL_LIST)) + "):\t")) if ID < 1 and ID > len(MOL_LIST): print("ERROR: The selected ID does not exist") molecule_interface() print("Selected ID: " + str(ID) + "\tMolecule name: " + str(MOL_LIST[ID - 1].name) + "\n") return ID def list_molecules(): print("\n LIST OF LOADED MOLECULES") print("------------------------------------------------\n") print("ID\tNAME\n") for i, myclass in enumerate(MOL_LIST): print(str(i + 1) + "\t" + myclass.name) utils.header(__VERSION__) print(""" 1) Load molecule from .xyz file 2) Load vibrational data from .hess file (ORCA) 3) Plot IR spectra 4) Compute properties 5) Linear transit l) List loaded molecules b) Back to Main Menu q) Exit """) choice = input("\nSelection:\t") if choice == "1": utils.header(__VERSION__) path = input("\nSelect the path of the .xyz file:\t") MOL_LIST.append(mol.MOL(path)) print("\nMolecule saved with the ID:\t" + str(len(MOL_LIST))) input("\nPress enter to return to the menu ... ") molecule_interface() elif choice == "2": utils.header(__VERSION__) path = input("\nSelect the path of the .hess file:\t") ID = ask_ID() MOL_LIST[ID - 1].load_hess_file(path, verbose=True) input("\nPress enter to return to the menu ... ") molecule_interface() elif choice == "3": utils.clear() print("================================================") print(" VIBRATIONAL SPECTRUM") print("================================================\n") ID = ask_ID() type_of_plot = input( "Select the type of plot (gaussian, lorentzian, bar):\t") if type_of_plot == "gaussian" or type_of_plot == "lorentzian": width = input("Select the amplitude parameter (in cm^-1): ") mol.plot_ir_spectrum(MOL_LIST[ID - 1], width, style=type_of_plot) save_state = input("\nDo you want to save the plot (y/n):\t") if save_state == "y": path = input("Select the path to the destination folder:\t") mol.plot_ir_spectrum(MOL_LIST[ID - 1], width, style=type_of_plot, path=path, show=False) else: mol.plot_ir_spectrum(MOL_LIST[ID - 1], style=type_of_plot) save_state = input("\nDo you want to save the plot (y/n):\t") if save_state == "y": path = input("Select the path to the destination folder:\t") mol.plot_ir_spectrum(MOL_LIST[ID - 1], style=type_of_plot, path=path, show=False) molecule_interface() elif choice == "4": utils.clear() print("================================================") print(" MOLECULAR PROPERTIES COMPUTATION") print("================================================\n") print(""" 1) General informations b) Back q) Exit """) calculation_choice = input("\nSelection:\t") if calculation_choice == "1": utils.clear() print("================================================") print(" GENERAL PROPERTIES") print("================================================") list_molecules() print("\n================================================") molecule_selection = input(""" Select the ID of the molecules divided by comma (type "all" if you want all the entries):\t""") if molecule_selection == "all": molecule_selection = list(range(0, len(MOL_LIST))) else: molecule_selection = [ int(i) for i in molecule_selection.split(",") ] utils.clear() print("================================================") print(" GENERAL PROPERTIES") print("================================================") print("\nSelected molecules: ", molecule_selection) for myID in molecule_selection: myMol = MOL_LIST[myID - 1] print("\n Molecule: " + myMol.name + "\t(ID: " + str(myID) + ")") print("------------------------------------------------\n") print("Composition:") comp_list = myMol.composition() print("ATOM\tN\t%MASS") for line in comp_list: print( str(line[0]) + "\t" + str(line[1]) + "\t" + str(line[2])) print("\n") print("Center of mass (Angstrom):") print("\tx:\t" + str(myMol.rcm()[0])) print("\ty:\t" + str(myMol.rcm()[1])) print("\tz:\t" + str(myMol.rcm()[2])) print("\nMoment of Inertia (amu*Angstrom**2):") inert = myMol.inertia_tensor() print("\t{:.3e}\t{:.3e}\t{:.3e}".format( inert[0][0][0], inert[0][0][1], inert[0][0][2])) print("\t{:.3e}\t{:.3e}\t{:.3e}".format( inert[0][1][0], inert[0][1][1], inert[0][1][2])) print("\t{:.3e}\t{:.3e}\t{:.3e}".format( inert[0][2][0], inert[0][2][1], inert[0][2][2])) print("\nEigenvalues:") print("\t{:.3e}\t{:.3e}\t{:.3e}".format( inert[1][0], inert[1][1], inert[1][2])) print("\nRotor type: {}".format(inert[2])) input("\nPress enter to return to the menu ... ") molecule_interface() elif calculation_choice == "b": molecule_interface() elif calculation_choice == "q": sys.exit() elif choice == "5": utils.clear() print("================================================") print(" LINEAR TRANSIT MODULE") print("================================================") print("Select the ID for the start and end structures\n") ID_start = ask_ID() ID_end = ask_ID() n_steps = int(input("Select the number of intermediate structures:\t")) path = input( "Select the destination path (without final extension):\t") print('\n') mol.rigid_linear_transit(MOL_LIST[ID_start - 1], MOL_LIST[ID_end - 1], n_steps, path) molecule_interface() elif choice == "l": utils.header(__VERSION__) list_molecules() input("\nPress enter to return to the menu ... ") molecule_interface() elif choice == "b": main() elif choice == "q": sys.exit()
Xv = vec.transform(dfv.txt) Yv = dfv['Categorie3'].values scv = cla.score(Xv,Yv) print 'training',cat1,'\t\t(',i,') : N=',len(df),'K=',len(labels) print 'training',cat1,'\t\t(',i,') : training=',sct,'validation=',scv joblib.dump((labels,vec,cla),fname) del vec,cla return (sct,scv) ####################### # training # stage1 : Categorie1 # stage3 : Categorie3|Categorie1 ####################### dftrain = pd.read_csv(ddir+'training_sample.csv'+ext,sep=';',names = header()).fillna('') dfvalid = pd.read_csv(ddir+'validation_sample.csv'+ext,sep=';',names = header()).fillna('') dftest = pd.read_csv(ddir+'test_normed.csv',sep=';',names = header(test=True)).fillna('') add_txt(dftrain) add_txt(dfvalid) add_txt(dftest) dftrain = dftrain[['Categorie3','Categorie1','txt']] dfvalid = dfvalid[['Categorie3','Categorie1','txt']] dftest = dftest[['Identifiant_Produit','txt']] # training stage1 dt = -time.time()
cla.fit(X,Y) scv = (-1,0) else: # performs a gridsearch Xvs = [ vec.transform(dfv.txt) for dfv in dfvs] Yvs = [ dfv['Categorie3'].values for dfv in dfvs] cla,scv = best_classifier(X,Y,Xvs,Yvs) print 'training',cat1,'\t\t(',i,') N=',len(dft),'K=',len(labels),': mean =',scv[0],'dev=',scv[1] joblib.dump((labels,vec,cla,scv),fname+ext) del vec,cla return scv ################# # prepare train # ################# dftrain = pd.read_csv(ddir+'training_random.csv'+ext,sep=';',names = header()).fillna('') add_txt(dftrain) dftrain = dftrain[['Categorie3','Categorie1','txt']] ################# # prepare valid # ################# dfvs = [pd.read_csv(ddir+'validation_random.csv.'+str(i),sep=';',names = header()).fillna('') for i in range(9)] for i in range(9): add_txt(dfvs[i]) dfvs[i] = dfvs[i][['Identifiant_Produit','Categorie3','Categorie1','txt']] ################# # prepare test # #################
# from training set ##################### # NOTE : reference model is limited to ~1M rows balanced train set with ~4500 unique Categorie3 labels # # dftrain = pd.read_csv(ddir+'training_shuffled_normed.csv',sep=';',names = header()).fillna('') # create_sample(dftrain,'Categorie3',200,10) #~1M rows # del dftrain ####################### # training # stage1 : Categorie1 # stage3 : Categorie3|Categorie1 ####################### dftrain = pd.read_csv(ddir+'training_sampled_Categorie3_200.csv',sep=';',names = header()).fillna('') dfvalid = pd.read_csv(ddir+'validation_normed.csv',sep=';',names = header()).fillna('') dftest = pd.read_csv(ddir+'test_normed.csv',sep=';',names = header(test=True)).fillna('') add_txt(dftrain) add_txt(dfvalid) add_txt(dftest) dftrain = dftrain[['Categorie3','Categorie1','txt']] dfvalid = dfvalid[['Categorie3','Categorie1','txt']] dftest = dftest[['Identifiant_Produit','txt']] # training stage1 dt = -time.time() vec,sct,scv = training_stage1(dftrain,dfvalid)
def __run_test(self, test): # Used to capture return values def exit_with_code(code): if self.os.lower() == 'windows': return code else: sys.exit(code) logDir = os.path.join(self.full_results_directory(), test.name.lower()) try: os.makedirs(logDir) except Exception: pass with open(os.path.join(logDir, 'out.txt'), 'w') as out: if test.os.lower() != self.os.lower() or test.database_os.lower( ) != self.database_os.lower(): out.write( "OS or Database OS specified in benchmark_config.json does not match the current environment. Skipping.\n" ) return exit_with_code(0) # If the test is in the excludes list, we skip it if self.exclude != None and test.name in self.exclude: out.write( "Test {name} has been added to the excludes list. Skipping.\n" .format(name=test.name)) return exit_with_code(0) out.write( "test.os.lower() = {os} test.database_os.lower() = {dbos}\n". format(os=test.os.lower(), dbos=test.database_os.lower())) out.write("self.results['frameworks'] != None: {val}\n".format( val=str(self.results['frameworks'] != None))) out.write("test.name: {name}\n".format(name=str(test.name))) out.write("self.results['completed']: {completed}\n".format( completed=str(self.results['completed']))) if self.results['frameworks'] != None and test.name in self.results[ 'completed']: out.write( 'Framework {name} found in latest saved data. Skipping.\n'. format(name=str(test.name))) print 'WARNING: Test {test} exists in the results directory; this must be removed before running a new test.\n'.format( test=str(test.name)) return exit_with_code(1) out.flush() out.write(header("Beginning %s" % test.name, top='=')) out.flush() ########################## # Start this test ########################## out.write(header("Starting %s" % test.name)) out.flush() try: self.__cleanup_leftover_processes_before_test() if self.__is_port_bound(test.port): time.sleep(60) if self.__is_port_bound(test.port): # We gave it our all self.__write_intermediate_results( test.name, "port " + str(test.port) + " is not available before start") out.write( header( "Error: Port %s is not available, cannot start %s" % (test.port, test.name))) out.flush() print "Error: Unable to recover port, cannot start test" return exit_with_code(1) result, process = test.start(out) self.__process = process if result != 0: self.__process.terminate() time.sleep(5) out.write("ERROR: Problem starting {name}\n".format( name=test.name)) out.flush() self.__write_intermediate_results( test.name, "<setup.py>#start() returned non-zero") return exit_with_code(1) logging.info( "Sleeping %s seconds to ensure framework is ready" % self.sleep) time.sleep(self.sleep) ########################## # Verify URLs ########################## if self.mode == "debug": logging.info( "Entering debug mode. Server has started. CTRL-c to stop." ) while True: time.sleep(1) else: logging.info("Verifying framework URLs") passed_verify = test.verify_urls(logDir) ########################## # Benchmark this test ########################## if self.mode == "benchmark": logging.info("Benchmarking") out.write(header("Benchmarking %s" % test.name)) out.flush() test.benchmark(logDir) ########################## # Stop this test ########################## self.__stop_test(test, out) out.write(header("Stopped %s" % test.name)) out.flush() ########################################################## # Remove contents of /tmp folder ########################################################## try: subprocess.check_call('sudo rm -rf /tmp/*', shell=True, stderr=out, stdout=out) except Exception: out.write(header("Error: Could not empty /tmp")) ########################################################## # Remove apt sources to avoid pkg errors and collisions ########################################################## os.system("sudo rm -rf /etc/apt/sources.list.d/*") ########################################################## # Save results thus far into the latest results directory ########################################################## out.write(header("Saving results through %s" % test.name)) out.flush() self.__write_intermediate_results( test.name, time.strftime("%Y%m%d%H%M%S", time.localtime())) ########################################################## # Upload the results thus far to another server (optional) ########################################################## self.__upload_results() if self.mode == "verify" and not passed_verify: print "Failed verify!" return exit_with_code(1) except KeyboardInterrupt: self.__stop_test(test, out) except (OSError, IOError, subprocess.CalledProcessError) as e: self.__write_intermediate_results( test.name, "<setup.py> raised an exception") out.write(header("Subprocess Error %s" % test.name)) traceback.print_exc(file=out) out.flush() out.close() return exit_with_code(1) out.close() return exit_with_code(0)
def main(): header() make_money_slips()
dfsample = dfsample.reset_index(drop=True) dfsample = dfsample.reindex(np.random.permutation(dfsample.index), copy=False) return dfsample # NOTE : training_head is the first 1500000 rows of shuffled normalized data set used for training only # NOTE : training_tail is the last 786885 rows of shuffled normalized data set used for validation only ########################## # building the training set ########################## class_ratio = joblib.load(ddir + 'joblib/class_ratio') df = pd.read_csv(ddir + 'training_head.csv', sep=';', names=header()).fillna('') for i in range(9): print i dfsample = training_sample_random(df, N=456, class_ratio=class_ratio) dfsample.to_csv(ddir + 'training_random.csv.' + str(i), sep=';', index=False, header=False) ########################## # building the validation set ########################## class_ratio = joblib.load(ddir + 'joblib/class_ratio') df = pd.read_csv(ddir + 'training_tail.csv', sep=';', names=header()).fillna('')
##################### # create sample set # from training set ##################### # NOTE : USE class_ratio to estimate class balance and random_sampling.py to get training sample set ####################### # training # stage1 : Categorie1 # stage3 : Categorie3|Categorie1 ####################### dftrain = pd.read_csv(ddir + 'training_random.csv' + ext, sep=';', names=header()).fillna('') dfvalid = pd.read_csv(ddir + 'validation_random.csv' + ext, sep=';', names=header()).fillna('') dftest = pd.read_csv(ddir + 'test_normed.csv', sep=';', names=header(test=True)).fillna('') add_txt(dftrain) add_txt(dfvalid) add_txt(dftest) dftrain = dftrain[['Categorie3', 'Categorie1', 'txt']] dfvalid = dfvalid[['Categorie3', 'Categorie1', 'txt']] dftest = dftest[['Identifiant_Produit', 'txt']]
def __run_tests(self, tests): if len(tests) == 0: return 0 logging.debug("Start __run_tests.") logging.debug("__name__ = %s", __name__) error_happened = False if self.os.lower() == 'windows': logging.debug("Executing __run_tests on Windows") for test in tests: with open(self.current_benchmark, 'w') as benchmark_resume_file: benchmark_resume_file.write(test.name) with self.quiet_out.enable(): if self.__run_test(test) != 0: error_happened = True else: logging.debug("Executing __run_tests on Linux") # Setup a nice progressbar and ETA indicator widgets = [ self.mode, ': ', progressbar.Percentage(), ' ', progressbar.Bar(), ' Rough ', progressbar.ETA() ] pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(tests)).start() pbar_test = 0 # These features do not work on Windows for test in tests: pbar.update(pbar_test) pbar_test = pbar_test + 1 if __name__ == 'benchmark.benchmarker': print header("Running Test: %s" % test.name) with open(self.current_benchmark, 'w') as benchmark_resume_file: benchmark_resume_file.write(test.name) with self.quiet_out.enable(): test_process = Process(target=self.__run_test, name="Test Runner (%s)" % test.name, args=(test, )) test_process.start() test_process.join(self.run_test_timeout_seconds) self.__load_results( ) # Load intermediate result from child process if (test_process.is_alive()): logging.debug( "Child process for {name} is still alive. Terminating." .format(name=test.name)) self.__write_intermediate_results( test.name, "__run_test timeout (=" + str(self.run_test_timeout_seconds) + " seconds)") test_process.terminate() test_process.join() if test_process.exitcode != 0: error_happened = True pbar.finish() if os.path.isfile(self.current_benchmark): os.remove(self.current_benchmark) logging.debug("End __run_tests.") if error_happened: return 1 return 0
def main(): clear() header() CashMachineConsole.call_operation()
import random from sklearn.externals import joblib from utils import itocat1,itocat2,itocat3 from utils import cat1toi,cat2toi,cat3toi from utils import cat3tocat2,cat3tocat1,cat2tocat1 from utils import cat1count,cat2count,cat3count import sys ddir = '/home/ngaude/workspace/data/cdiscount/' assert len(sys.argv) == 2 ##### usage guess.py $RESULTAT.CSV #### rname = sys.argv[1] assert isfile(ddir+rname) ##### usage guess.py $RESULTAT.CSV #### test_normed = pd.read_csv(ddir+'test_normed.csv',sep=';',names=header(True)).fillna('') add_txt(test_normed) test_num_word = map(lambda t:len(set(t.split())),test_normed.txt) test_nn = pd.read_csv(ddir+'test_nn.csv',sep=';').fillna('') test_nn['Marque'] = test_nn.Marque_nn test_nn['Libelle'] = test_nn.Libelle_nn test_nn['Description'] = test_nn.Description_nn add_txt(test_nn) nn_num_word = map(lambda t:len(set(t.split())),test_nn.txt) test_nn.drop('Marque', axis=1, inplace=True) test_nn.drop('Libelle', axis=1, inplace=True) test_nn.drop('Description', axis=1, inplace=True) best = pd.read_csv(ddir+rname,sep=';') #best = pd.read_csv('proba.auto.merging.60.csv',sep=';')
__credits__ = [] __version__ = "1.0" __status__ = "Development" os.environ['FOR_IGNORE_EXCEPTIONS'] = '1' os.environ['FOR_DISABLE_CONSOLE_CTRL_HANDLER'] = '1' if __name__ == "__main__": import sys while True: try: while True: system('cls') # limpa a tela header() # exibe nome do sistema, nomes dos desenvolvedores e telefone da FATEC Ourinhos # escolha de qual problema ser executado option = input( ' 1. Determinar quantos hectares devem ser cultivados para cada uma das plantações e ' 'tendo lucro máximo.\n' ' 2. Determinar quantos hectares devem ser cultivados para cada uma das plantações e ' 'quanto de cada criação deve ser mantido para maximizar a receita líquida.\n' ' 3. Maximizar o valor presente das plantações e escolher quais cultivar.\n\n' ' Qual problema deve ser executado? ') system('cls') # limpa a tela header() # exibe nome do sistema, nomes dos desenvolvedores e telefone da FATEC Ourinhos if option.__eq__('1'): Problem1() break elif option.__eq__('2'): Problem2()
assert which in ("mc", "lc"), "Only most or least common" data = arr.copy() for i in range(arr.shape[1]): mc = most_common(data[:, i]) if which == "lc": mc = 1 - mc data = data[data[:, i] == mc] if data.shape[0] == 1: break return data[0] if __name__ == "__main__": data = read_array("data/03/input.txt") print(header("Part 1")) mc = most_common(data) lc = 1 - mc print("γ rate =", mc, bits2int(mc)) print("ε rate =", lc, bits2int(lc)) print("Product =", bits2int(mc) * bits2int(lc)) print(header("Part 2")) o2 = iterative_mc_lc(data) co2 = iterative_mc_lc(data, which="lc") print("O2 rating =", o2, bits2int(o2)) print("CO2 rating =", co2, bits2int(co2)) print("Product =", bits2int(o2) * bits2int(co2)) print()
def __run_test(self, test): # Used to capture return values def exit_with_code(code): if self.os.lower() == 'windows': return code else: sys.exit(code) logDir = os.path.join(self.full_results_directory(), test.name.lower()) try: os.makedirs(logDir) except Exception: pass with open(os.path.join(logDir, 'out.txt'), 'w') as out: if test.os.lower() != self.os.lower() or test.database_os.lower() != self.database_os.lower(): out.write("OS or Database OS specified in benchmark_config.json does not match the current environment. Skipping.\n") return exit_with_code(0) # If the test is in the excludes list, we skip it if self.exclude != None and test.name in self.exclude: out.write("Test {name} has been added to the excludes list. Skipping.\n".format(name=test.name)) return exit_with_code(0) out.write("test.os.lower() = {os} test.database_os.lower() = {dbos}\n".format(os=test.os.lower(),dbos=test.database_os.lower())) out.write("self.results['frameworks'] != None: {val}\n".format(val=str(self.results['frameworks'] != None))) out.write("test.name: {name}\n".format(name=str(test.name))) out.write("self.results['completed']: {completed}\n".format(completed=str(self.results['completed']))) if self.results['frameworks'] != None and test.name in self.results['completed']: out.write('Framework {name} found in latest saved data. Skipping.\n'.format(name=str(test.name))) print 'WARNING: Test {test} exists in the results directory; this must be removed before running a new test.\n'.format(test=str(test.name)) return exit_with_code(1) out.flush() out.write(header("Beginning %s" % test.name, top='=')) out.flush() ########################## # Start this test ########################## out.write(header("Starting %s" % test.name)) out.flush() try: if test.requires_database(): p = subprocess.Popen(self.database_ssh_string, stdin=subprocess.PIPE, stdout=out, stderr=out, shell=True) p.communicate(""" sudo restart mysql sudo restart mongod sudo service postgresql restart sudo service cassandra restart /opt/elasticsearch/elasticsearch restart """) time.sleep(10) st = verify_database_connections([ ("mysql", self.database_host, 3306), ("mongodb", self.database_host, 27017), ("postgresql", self.database_host, 5432), ("cassandra", self.database_host, 9160), ("elasticsearch", self.database_host, 9200) ]) print "database connection test results:\n" + "\n".join(st[1]) self.__cleanup_leftover_processes_before_test(); if self.__is_port_bound(test.port): # We gave it our all self.__write_intermediate_results(test.name, "port " + str(test.port) + " is not available before start") out.write(header("Error: Port %s is not available, cannot start %s" % (test.port, test.name))) out.flush() print "Error: Unable to recover port, cannot start test" return exit_with_code(1) result, process = test.start(out) if result != 0: self.__stop_test(out, process) time.sleep(5) out.write( "ERROR: Problem starting {name}\n".format(name=test.name) ) out.flush() self.__write_intermediate_results(test.name,"<setup.py>#start() returned non-zero") return exit_with_code(1) logging.info("Sleeping %s seconds to ensure framework is ready" % self.sleep) time.sleep(self.sleep) ########################## # Verify URLs ########################## logging.info("Verifying framework URLs") passed_verify = test.verify_urls(logDir) ########################## # Nuke /tmp ########################## try: subprocess.check_call('sudo rm -rf /tmp/*', shell=True, stderr=out, stdout=out) except Exception: out.write(header("Error: Could not empty /tmp")) ########################## # Benchmark this test ########################## if self.mode == "benchmark": logging.info("Benchmarking") out.write(header("Benchmarking %s" % test.name)) out.flush() test.benchmark(logDir) ########################## # Stop this test ########################## out.write(header("Stopping %s" % test.name)) out.flush() self.__stop_test(out, process) out.flush() time.sleep(5) if self.__is_port_bound(test.port): # This can happen sometimes - let's try again self.__stop_test(out, process) out.flush() time.sleep(5) if self.__is_port_bound(test.port): # We gave it our all self.__write_intermediate_results(test.name, "port " + str(test.port) + " was not released by stop") out.write(header("Error: Port %s was not released by stop %s" % (test.port, test.name))) out.flush() return exit_with_code(1) out.write(header("Stopped %s" % test.name)) out.flush() ########################################################## # Remove contents of /tmp folder ########################################################## if self.clear_tmp: try: filelist = [ f for f in os.listdir("/tmp") ] for f in filelist: try: os.remove("/tmp/" + f) except OSError as err: print "Failed to remove " + str(f) + " from /tmp directory: " + str(err) except OSError: print "Failed to remove contents of /tmp directory." ########################################################## # Save results thus far into the latest results directory ########################################################## out.write(header("Saving results through %s" % test.name)) out.flush() self.__write_intermediate_results(test.name,time.strftime("%Y%m%d%H%M%S", time.localtime())) if self.mode == "verify" and not passed_verify: print "Failed verify!" return exit_with_code(1) except (OSError, IOError, subprocess.CalledProcessError) as e: self.__write_intermediate_results(test.name,"<setup.py> raised an exception") out.write(header("Subprocess Error %s" % test.name)) traceback.print_exc(file=out) out.flush() try: self.__stop_test(out, process) except (subprocess.CalledProcessError) as e: self.__write_intermediate_results(test.name,"<setup.py>#stop() raised an error") out.write(header("Subprocess Error: Test .stop() raised exception %s" % test.name)) traceback.print_exc(file=out) out.flush() out.close() return exit_with_code(1) # TODO - subprocess should not catch this exception! # Parent process should catch it and cleanup/exit except (KeyboardInterrupt) as e: self.__stop_test(out, process) out.write(header("Cleaning up...")) out.flush() self.__finish() sys.exit(1) out.close() return exit_with_code(0)
def verify_type(test_type): verificationPath = os.path.join(logPath, test_type) try: os.makedirs(verificationPath) except OSError: pass with open(os.path.join(verificationPath, 'verification.txt'), 'w') as verification: test = self.runTests[test_type] test.setup_out(verification) verification.write(header("VERIFYING %s" % test_type.upper())) base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port) try: # Verifies headers from the server. This check is made from the # App Server using Pythons requests module. Will do a second check from # the client to make sure the server isn't only accepting connections # from localhost on a multi-machine setup. results = test.verify(base_url) # Now verify that the url is reachable from the client machine, unless # we're already failing if not any(result == 'fail' for (result, reason, url) in results): p = subprocess.call(["ssh", "TFB-client", "curl -sSf %s" % base_url + test.get_url()], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p is not 0: results = [('fail', "Server did not respond to request from client machine.", base_url)] logging.warning("""This error usually means your server is only accepting requests from localhost.""") except ConnectionError as e: results = [('fail',"Server did not respond to request", base_url)] logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e) except Exception as e: results = [('fail',"""Caused Exception in TFB This almost certainly means your return value is incorrect, but also that you have found a bug. Please submit an issue including this message: %s\n%s""" % (e, traceback.format_exc()), base_url)] logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e) traceback.format_exc() test.failed = any(result == 'fail' for (result, reason, url) in results) test.warned = any(result == 'warn' for (result, reason, url) in results) test.passed = all(result == 'pass' for (result, reason, url) in results) def output_result(result, reason, url): specific_rules_url = "http://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/#specific-test-requirements" color = Fore.GREEN if result.upper() == "WARN": color = Fore.YELLOW elif result.upper() == "FAIL": color = Fore.RED verification.write((" " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url)) print (" " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url) if reason is not None and len(reason) != 0: for line in reason.splitlines(): verification.write(" " + line + '\n') print " " + line if not test.passed: verification.write(" See %s\n" % specific_rules_url) print " See %s\n" % specific_rules_url [output_result(r1,r2,url) for (r1, r2, url) in results] if test.failed: self.benchmarker.report_verify_results(self, test_type, 'fail') elif test.warned: self.benchmarker.report_verify_results(self, test_type, 'warn') elif test.passed: self.benchmarker.report_verify_results(self, test_type, 'pass') else: raise Exception("Unknown error - test did not pass,warn,or fail") verification.flush()
import pathlib import pandas as pd from app import app, server from utils import colors, fonts, header from pages import batsmenGraphs PATH = pathlib.Path(__file__) DATA_PATH = PATH.joinpath("../data").resolve() selected = ['V Kohli', 'SPD Smith'] df_main = pd.read_csv(DATA_PATH.joinpath("innings.csv")) available_players = df_main['Name'].unique() app.layout = html.Div([ dcc.Location(id='url', refresh=False), html.Div(header()), html.Div(id='batsmenGraphs-head', children=[ html.H3( ['Batsmen Compared'], style={ 'textAlign': 'center', 'color': colors['title'], 'fontFamily': fonts['title'], 'position': 'relative', 'top': '-100px', 'left': '20%' }), html.Div( [ html.Div(