def _get_db_log_path(conf): logs = [] if conf.db_logs_dir: log_file_path = lambda f : path(conf.db_logs_dir) + f collect_logs = conf.db_logs_files if conf.db_logs_files else ls(conf.db_logs_dir) for file in collect_logs: logs.append(log_file_path(file)) return logs
def _get_db_log_path(conf): logs = [] if conf.db_logs_dir: log_file_path = lambda f: path(conf.db_logs_dir) + f collect_logs = conf.db_logs_files if conf.db_logs_files else ls( conf.db_logs_dir) for file in collect_logs: logs.append(log_file_path(file)) return logs
if os.path.isfile(firstpost_name): # Record time of firstpost file creation start_time = os.path.getmtime(firstpost_name) # Makedir if necessary util.mkdir(os.path.join(score_dir, dataset_name, algo_name)) # Load info about this dataset info_file = os.path.join(data_dir, dataset_name, '%s_public.info' % dataset_name) info = libscores.get_info(info_file) # FIXME HACK info['metric'] = 'auc_metric' # END FIXME HACK # Load solution for this dataset solution_file = os.path.join(data_dir, dataset_name, '%s_%s.solution' % (dataset_name, TEST)) solution = libscores.read_array(solution_file) # For each set of predictions prediction_files = util.ls(os.path.join(pred_dir, '%s_%s_*.predict' % (dataset_name, TEST))) for prediction_file in prediction_files: # Time of file creation since algorithm start file_time = os.path.getmtime(prediction_file) - start_time # Open predictions prediction = libscores.read_array(prediction_file) # Check predictions match shape of solution if solution.shape != prediction.shape: raise ValueError("Mismatched prediction shape {} vs. {}".format(prediction.shape, solution.shape)) # Score if info['metric'] == 'r2_metric' or info['metric'] == 'a_metric': # Remove NaN and Inf for regression solution = libscores.sanitize_array(solution) prediction = libscores.sanitize_array(prediction) # TODO - remove eval score = eval(info['metric'] + '(solution, prediction, "' + info['task'] + '")')
def test_delete_smrlog_after_scaleout(self): util.print_frame() # start load generator util.log("start load_generator") for i in range(self.max_load_generator): ip, port = util.get_rand_gateway(self.cluster) self.load_gen_thrd_list[i] = load_generator.LoadGenerator(i, ip, port) self.load_gen_thrd_list[i].start() time.sleep(5) # generate load for 5 sec util.log("started load_generator") # servers for scale out servers = [config.server4, config.server5, config.server6] leader_cm = self.cluster['servers'][0] # Scale out cluster = config.clusters[0] ret = util.pg_add(cluster, servers, leader_cm) self.assertEqual(True, ret, 'Scale out fail. util.pg_add returns false') time.sleep(5) # pg0 -> pg1 cluster = config.clusters[1] ret = util.migration(cluster, 0, 1, 8000, 8191, 40000) self.assertEqual(True, ret, 'Migration Fail 0 -> 1') # get log file old_logs = {} for s in config.clusters[0]['servers']: parent_dir, log_dir = util.smr_log_dir(s['id']) path = '%s/%s' % (parent_dir, log_dir) old_logs[s['id']] = util.ls(path) # bgsave in order to make smrlogs deleted. for s in config.clusters[0]['servers']: bgsave_ret = util.bgsave(s) self.assertTrue(bgsave_ret, 'failed to bgsave. pgs%d' % s['id']) util.log('bgsave pgs%d is done.') # check consistency ok = True for j in range(len(self.load_gen_thrd_list)): self.assertTrue(self.load_gen_thrd_list[j].isConsistent(), 'Inconsistent after migration') # is smr-replicator delete smrlogs? i = 0 while i < 20: i += 1 # get current log files cur_logs = {} for s in config.clusters[0]['servers']: parent_dir, log_dir = util.smr_log_dir(s['id']) path = '%s/%s' % (parent_dir, log_dir) cur_logs[s['id']] = util.ls(path) # compare old and new temp_old_logs = copy.deepcopy(old_logs) for id, nl in cur_logs.items(): ol = temp_old_logs.get(id) self.assertNotEqual(ol, None, "failed to check logfiles. old logs for smr-replicator '%d' is not exist." % id) for log in nl: if log in ol: ol.remove(log) ok = True for id, ol in temp_old_logs.items(): if len(ol) == 0: ok = False util.log('Loop %d ---------------------------------------------------------' % i) util.log('deleted smrlog files: %s' % util.json_to_str(temp_old_logs)) if ok: break time.sleep(10) self.assertTrue(ok, 'smr-replicator does not delete smrlogs.') util.log('smr-replicator deletes smrlogs.') # check consistency of load_generator for i in range(len(self.load_gen_thrd_list)): self.load_gen_thrd_list[i].quit() for i in range(len(self.load_gen_thrd_list)): self.load_gen_thrd_list[i].join() self.assertTrue(self.load_gen_thrd_list[i].isConsistent(), 'Inconsistent after migration')
#!/usr/bin/python from util import rand, ls, load, promptint, promptstr print("XKCD Password Generator.") dicts = ls("/usr/share/dict") print("{:d} files found in /usr/share/dict".format(len(dicts))) print("Choose which one to use 0-{:d}".format(len(dicts)-1)) for i in range(len(dicts)): print(str(i) + ": "+dicts[i].split("/")[-1]) d = load("/usr/share/dict/"+dicts[promptint("Choice (0-{:d}): ".format(len(dicts)-1))], promptint("Minimum word length: "), promptint("Maximum word length: ")) pw = "" for i in range(promptint("Number of words: ")): pw = pw + d[rand(len(d))][:-1] + " " print(pw)
def test_delete_smrlog_after_scaleout(self): util.print_frame() # start load generator util.log("start load_generator") for i in range(self.max_load_generator): ip, port = util.get_rand_gateway(self.cluster) self.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, ip, port) self.load_gen_thrd_list[i].start() time.sleep(5) # generate load for 5 sec util.log("started load_generator") # servers for scale out servers = [config.server4, config.server5, config.server6] leader_cm = self.cluster['servers'][0] # Scale out cluster = config.clusters[0] ret = util.pg_add(cluster, servers, leader_cm) self.assertEqual(True, ret, 'Scale out fail. util.pg_add returns false') time.sleep(5) # pg0 -> pg1 cluster = config.clusters[1] ret = util.migration(cluster, 0, 1, 8000, 8191, 40000) self.assertEqual(True, ret, 'Migration Fail 0 -> 1') # get log file old_logs = {} for s in config.clusters[0]['servers']: parent_dir, log_dir = util.smr_log_dir(s['id']) path = '%s/%s' % (parent_dir, log_dir) old_logs[s['id']] = util.ls(path) # bgsave in order to make smrlogs deleted. for s in config.clusters[0]['servers']: bgsave_ret = util.bgsave(s) self.assertTrue(bgsave_ret, 'failed to bgsave. pgs%d' % s['id']) util.log('bgsave pgs%d is done.') # check consistency ok = True for j in range(len(self.load_gen_thrd_list)): self.assertTrue(self.load_gen_thrd_list[j].isConsistent(), 'Inconsistent after migration') # is smr-replicator delete smrlogs? i = 0 while i < 20: i += 1 # get current log files cur_logs = {} for s in config.clusters[0]['servers']: parent_dir, log_dir = util.smr_log_dir(s['id']) path = '%s/%s' % (parent_dir, log_dir) cur_logs[s['id']] = util.ls(path) # compare old and new temp_old_logs = copy.deepcopy(old_logs) for id, nl in cur_logs.items(): ol = temp_old_logs.get(id) self.assertNotEqual( ol, None, "failed to check logfiles. old logs for smr-replicator '%d' is not exist." % id) for log in nl: if log in ol: ol.remove(log) ok = True for id, ol in temp_old_logs.items(): if len(ol) == 0: ok = False util.log( 'Loop %d ---------------------------------------------------------' % i) util.log('deleted smrlog files: %s' % util.json_to_str(temp_old_logs)) if ok: break time.sleep(10) self.assertTrue(ok, 'smr-replicator does not delete smrlogs.') util.log('smr-replicator deletes smrlogs.') # check consistency of load_generator for i in range(len(self.load_gen_thrd_list)): self.load_gen_thrd_list[i].quit() for i in range(len(self.load_gen_thrd_list)): self.load_gen_thrd_list[i].join() self.assertTrue(self.load_gen_thrd_list[i].isConsistent(), 'Inconsistent after migration')
import util mods = util.ls("./mods-enabled") for mod in mods: __import__(mod) print "imported mod %s" % mod
# Record time of firstpost file creation start_time = os.path.getmtime(firstpost_name) # Makedir if necessary util.mkdir(os.path.join(score_dir, dataset_name, algo_name)) # Load info about this dataset info_file = os.path.join(data_dir, dataset_name, '%s_public.info' % dataset_name) info = libscores.get_info(info_file) # Load solution for this dataset solution_file = os.path.join( data_dir, dataset_name, '%s_%s.solution' % (dataset_name, TEST)) solution = libscores.read_array(solution_file) # For each set of predictions prediction_files = util.ls( os.path.join(pred_dir, '%s_%s_*.predict' % (dataset_name, TEST))) for prediction_file in prediction_files: # Time of file creation since algorithm start file_time = os.path.getmtime(prediction_file) - start_time # Open predictions prediction = libscores.read_array(prediction_file) # Check predictions match shape of solution if solution.shape != prediction.shape: raise ValueError( "Mismatched prediction shape {} vs. {}".format( prediction.shape, solution.shape)) # Score if info['metric'] == 'r2_metric' or info[ 'metric'] == 'a_metric': # Remove NaN and Inf for regression
def _build_video_paths(self, video_dir): video_paths = ls(video_dir) self.video_paths = [os.path.join(video_dir, f) for f in video_paths]
def _count_images(self): filenames = ls(self.image_dir) return len(filenames) // 3