def parse_simulation_file(simulated_vectors_file): constr_dict = Odict() states_dict = Odict() ids_vector = [] opti_vector = [] fitness = "" with open(simulated_vectors_file) as file: for line in file: if "CONSTRAINTS:" in line: constr_dict_raw = eval(line.split('CONSTRAINTS:')[-1]) for k, dd in constr_dict_raw.items(): # check if the experiments where fold or abs if len(k) == 2: constr_dict['/'.join(k)] = dd elif len(k) == 1: constr_dict[''.join(k)] = dd elif "VARIABLE PAR IDS:" in line: ids_vector = eval(line[line.index("["):]) elif "FITNESS:" in line: fitness = eval(line[line.index(":") + 1:]) elif "VECTOR:" in line: opti_vector = eval(line[line.index(":") + 1:]) elif "STATES:" in line: key = line[line.index(":") + 1:line.rindex(":")] value = eval(line[line.rindex(":") + 1:]) states_dict[key] = value return constr_dict, states_dict, fitness, ids_vector, opti_vector
def tests_notitle(self): """Testing: clean data - notitle as path name.""" self.importer.data = [Odict([('password', 'UuQHzvv6IHRIJGjwKru7')])] data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('path', 'notitle')]) ] self.importer.clean(clean=False, convert=False) self.assertEqual(self.importer.data, data_expected)
def tests_empty(self): """Testing: clean data - empty title and clean enabled.""" self.importer.data = [Odict([('password', 'UuQHzvv6IHRIJGjwKru7')])] data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('path', 'notitle')]) ] self.importer.clean(clean=True, convert=False) self.assertEqual(self.importer.data, data_expected)
def tests_login(self): """Testing: clean data - login as path name.""" self.importer.data = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm')]) ] data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('path', 'lnqYm3ZWtm')]) ] self.importer.clean(clean=False, convert=False) self.assertEqual(self.importer.data, data_expected)
def test_satanize_duplicate_paths(self): """Testing: satanize duplicate paths.""" self.importer.data = [ Odict([('title', 'ovh.com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm')]), Odict([('title', 'ovh.com'), ('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'lnqYm3ZWtm')]), Odict([('title', 'ovh.com'), ('password', '[Q&$\fd]!`vKA&b'), ('login', 'fmmhpvity')]), Odict([('title', 'ovh.com'), ('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'), ('login', 'ptfzlnvmj')]) ] data_expected = [ Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh.com')]), Odict([('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh.com0')]), Odict([('password', '[Q&$\fd]!`vKA&b'), ('login', 'fmmhpvity'), ('path', 'ovh.com1')]), Odict([('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'), ('login', 'ptfzlnvmj'), ('path', 'ovh.com2')]) ] self.importer.satanize(clean=False) self.assertTrue(self.importer.data == data_expected)
def test_paths(self): """Testing: duplicate paths.""" self.importer.data = [ Odict([('title', 'ovh.com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm')]), Odict([('title', 'ovh.com'), ('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'lnqYm3ZWtm')]), Odict([('title', 'ovh.com'), ('password', '[Q&$\fd]!`vKA&b'), ('login', 'fmmhpvity')]), Odict([('title', 'ovh.com'), ('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'), ('login', 'ptfzlnvmj')]) ] data_expected = [ Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh.com/lnqYm3ZWtm')]), Odict([('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh.com/lnqYm3ZWtm-1')]), Odict([('password', '[Q&$\fd]!`vKA&b'), ('login', 'fmmhpvity'), ('path', 'ovh.com/fmmhpvity')]), Odict([('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'), ('login', 'ptfzlnvmj'), ('path', 'ovh.com/ptfzlnvmj')]) ] self.importer.clean(clean=False, convert=False) self.assertEqual(self.importer.data, data_expected)
def main(cost_function_file, output_name, n_tests, cv=True, shuffle_one=False): output_name = cost_function_file.split('/')[:-1] + '/' + output_name n_tests = int(n_tests) controls = set() # conc_dict_samples = Odict() cost_func_dict = Odict() f = open(cost_function_file) for line in f: if line.startswith("ID"): cost_id = line.split("\t")[0].replace("ID:","") cost_val = line.split("\t")[1].strip() cost_func_dict[cost_id+":"+cost_val] = Odict() else: sample = line.split("\t")[0] value = "\t".join(line.split("\t")[1:]) if "/" in sample: drug = sample.split("/")[0] # pdb.set_trace() # control = getSampleName(sample.split("/")[1]) control = sample.split("/")[1] elif "CONTROL" in sample: control = getSampleName(sample) elif not '_' in sample: control = getSampleName(sample) controls.add(control) cost_func_dict[cost_id+":"+cost_val][sample] = value f.close() print len(controls) controls_list = list(controls) bucket_size = len(controls) if cv: testing_size = int(round(bucket_size/n_tests)) if not shuffle_one else 1 ids = range(0, n_tests*testing_size, testing_size) for i, start in enumerate(ids): end = len(controls) if i+1>=len(ids) else ids[i+1] testing_list = controls_list[start:end] training_list = list(controls-set(testing_list)) # print results to file saveData(output_name, cost_func_dict, i, training_list, testing_list) else: training_size = int(round(bucket_size*80/100)) if not shuffle_one else bucket_size-1 for i in xrange(n_tests): training_list = random.sample(controls, training_size) testing_list = list(controls-set(training_list)) # print results to file saveData(output_name, cost_func_dict, i, training_list, testing_list) return
def test_satanize_duplicate_paths(self): """ Testing: satanize duplicate paths """ self.importer.data = [ Odict([('title', 'ovh.com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm')]), Odict([('title', 'ovh.com'), ('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'lnqYm3ZWtm')]) ] data_expected = [ Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh.com')]), Odict([('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh.com0')]) ] self.importer.satanize(clean=False) self.assertTrue(self.importer.data == data_expected)
def test_convert(self): """Testing: convert password path.""" self.importer.data = [ Odict([('title', 'ovh>com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm')]), Odict([('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'fm/mhpv*ity')]) ] data_expected = [ Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'), ('login', 'lnqYm3ZWtm'), ('path', 'ovh-com')]), Odict([('password', 'VRiplZSniSBlHNnQvc9e'), ('login', 'fm/mhpv*ity'), ('path', 'fm-mhpv-ity')]) ] self.importer.clean(clean=False, convert=True) self.assertEqual(self.importer.data, data_expected)
def test_get_data(self): """Testing: convert dict to password entry.""" entry = Odict([('password', 'EaP:bCmLZliqa|]WR/#HZP-aa'), ('login', 'roddhjav'), ('comments', 'This is a comment')]) entry_expected = "EaP:bCmLZliqa|]WR/#HZP-aa\nlogin: roddhjav\ncomments: This is a comment\n" self.assertTrue(self.importer.get(entry) == entry_expected)
def read_conf_dir(dir_path): """ Read all config files in spcified folder, in alphabetical order. So, if files conain identical variants, the ones, read early would be replaced by the ones, read later. :param dir_path: path to folder :return: nested dict, with merged backup variants from all files """ if (not os.path.exists(dir_path)): Log.error( "Configuration folder '{}' doesn't exists".format(dir_path)) filenames = [ file for file in os.listdir(dir_path) if re.search("\.(py3?|ya?ml|json)$", file) ] files = [dir_path + "/" + file for file in sorted(filenames)] if len(files) == 0: Log.error( "There are no config files in folder '{}'".format(dir_path)) conf = Odict() Log.info("Found config files: " + ", ".join(files)) for file in files: variants = Conf.read_conf_file(file) if variants is not None: conf.update(variants) return conf
def setUp(self): self.importer = self.passimport.PasswordManager() self.importer.data = [ Odict([('title', 'twitter.com'), ('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'), ('comments', ''), ('group', 'Social'), ('address', ''), ('sex', ''), ('website', 'https://pujol.io'), ('uuid', '44jle5q3fdvrprmaahozexy2pi')]) ] self.data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'), ('website', 'https://pujol.io'), ('uuid', '44jle5q3fdvrprmaahozexy2pi'), ('path', 'Social/twitter.com')]) ]
def main(model_path_1, model_path_2): ''' modify model_1 progress_trace (ids and vectors) to appropriate size and format so they can be used on model_2 The size of the progress trace should be same as the vector_ids of progress_trace.txt of model_2. Results are saved at model_1 path ''' model_output_1 = model_path_1 + "/outputs/" cost_1_file = model_path_1 + "/inputs/cost_func.txt" prog_1_file = model_output_1 + "progress_trace.txt" cost_2_file = model_path_2 + "/inputs/cost_func.txt" prog_2_file = model_path_2 + "/outputs/progress_trace.txt" cost_1 = get_formula(cost_1_file) cost_1_dict = get_costDict(cost_1) ids_vect_1, opti_vect = get_progTrace(prog_1_file) cost_2 = get_formula(cost_2_file) cost_2_dict = get_costDict(cost_2) ids_vect_2, _ = get_progTrace(prog_2_file) # dict in form {'species_id' : [kCDmodel1, kCDmodel2]} # if not kCDmodel1 for species_id then x # if not kCDmodel2 for species_id then ommited, not present merged_dict = Odict() for k in cost_2_dict.keys(): merged_dict[k] = [] if k in cost_1_dict: merged_dict[k].append(cost_1_dict[k]) else: merged_dict[k].append('x') merged_dict[k].append(cost_2_dict[k]) res_vect = [] for vect in opti_vect: tail = vect[len(cost_1_dict.keys()):] # fill kCDs with 1.0 values with size of resulting vector head = [1.0 for i in range(0, len(cost_2_dict))] for k, v in merged_dict.items(): if not v[0] == 'x': # pdb.set_trace() head[ids_vect_2.index(v[1])] = vect[ids_vect_1.index(v[0])] res_vect.append(head + tail) res_ids = ids_vect_2[:len(cost_2_dict)] + ids_vect_1[len(cost_1_dict):] # pdb.set_trace() mod_prog_file = model_output_1 + "swaped_progress_trace.txt" mod_prog = open(mod_prog_file, "w") mod_prog.write("VARIABLE PAR IDS: {}\n\n".format(res_ids)) for vect in res_vect: mod_prog.write("VECTOR: {}\n\n".format(vect)) mod_prog.close()
def test_data(self): """Testing: clean data.""" self.importer.data = [ Odict([('title', 'twitter.com'), ('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'), ('comments', ''), ('group', 'Social'), ('address', ''), ('sex', ''), ('website', 'https://pujol.io'), ('uuid', '44jle5q3fdvrprmaahozexy2pi')]) ] self.data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'), ('website', 'https://pujol.io'), ('uuid', '44jle5q3fdvrprmaahozexy2pi'), ('path', 'Social/twitter.com')]) ] self.importer.clean(clean=False, convert=False) self.assertEqual(self.importer.data, self.data_expected)
def get_costDict(cost_str): param_dict = Odict() for nominator in cost_str.split("/"): for pair in nominator.split("+"): if pair.split("*")[1] in param_dict.keys(): raise ValueError( "Error! Same species present in cost multiple times!") # pdb.set_trace() param_dict[pair.split("*")[1]] = pair.split("*")[0] return param_dict
def test_satanize_path(self): """Testing: satanize data & generate password path.""" # Test url as path name self.importer.data = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('url', 'twitter.com')]) ] data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('url', 'twitter.com'), ('path', 'twitter.com')]) ] self.importer.satanize(clean=False) self.assertTrue(self.importer.data == data_expected) # Test login as path name self.importer.data = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm')]) ] data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('login', 'lnqYm3ZWtm'), ('path', 'lnqYm3ZWtm')]) ] self.importer.satanize(clean=False) self.assertTrue(self.importer.data == data_expected) # Test notitle as path name self.importer.data = [Odict([('password', 'UuQHzvv6IHRIJGjwKru7')])] data_expected = [ Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('path', 'notitle')]) ] self.importer.satanize(clean=False) self.assertTrue(self.importer.data == data_expected)
def test_numbers(self): """Testing: duplicate with numbers.""" self.importer.data = [ Odict([('title', 'ovh.com')]), Odict([('title', 'ovh.com')]), Odict([('title', 'ovh.com')]), Odict([('title', 'ovh.com')]) ] data_expected = [ Odict([('path', 'ovh.com/notitle')]), Odict([('path', 'ovh.com/notitle-1')]), Odict([('path', 'ovh.com/notitle-2')]), Odict([('path', 'ovh.com/notitle-3')]) ] self.importer.clean(clean=False, convert=False) self.assertEqual(self.importer.data, data_expected)
def create_sub_paths(paths, classes=None, image=None): paths_at_position = Odict() for p_id in xrange(0, len(paths)): for pos in list(paths[p_id]): if tuple(pos) in paths_at_position.keys(): if classes is not None: paths_at_position[tuple(pos)].append( (p_id, classes[p_id])) elif image is not None: paths_at_position[tuple( list(pos) + [image[tuple(pos)]])].append( (p_id, )) else: paths_at_position[tuple(pos)].append((p_id, )) else: if classes is not None: paths_at_position[tuple(pos)] = [(p_id, classes[p_id])] elif image is not None: paths_at_position[tuple( list(pos) + [image[tuple(pos)]])] = [(p_id, )] else: paths_at_position[tuple(pos)] = [(p_id, )] sub_paths = Odict() for key, p_ids in paths_at_position.iteritems(): if tuple(p_ids) in sub_paths.keys(): sub_paths[tuple(p_ids)].append(key) else: sub_paths[tuple(p_ids)] = [key] return sub_paths
def main(input_vectors_file, exp_table_file): rand = True f = open(input_vectors_file) data = f.readlines() par_ids = eval([l for l in data if l.startswith('PAR_IDS')][0].split('=')[-1]) var_ids = eval([l for l in data if l.startswith('VAR_PAR_IDS')][0].split("=")[-1]) vector = eval([l for l in data if l.startswith('RND_VECTOR')][0].split('=')[-1]) var_idx = eval([l for l in data if l.startswith('VAR_PAR_IDX')][0].split('=')[-1]) rnd_vect = [j for i, j in enumerate(vector) if i in var_idx] # pdb.set_trace() assert len(var_ids) == len(rnd_vect), "Something wrong with input data!" exp_table = pd.read_table(exp_table_file, index_col=0, low_memory=False) nparam = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2028, 3000] for i in list(reversed(nparam)): tmp_dict = Odict() print i if rand: var = random.sample(var_ids, i) fix = [k for k in var_ids if k not in var] else: var = var_ids[0:i] fix = var_ids[i:] for j in fix: if not j in tmp_dict: tmp_dict[j] = [rnd_vect[var_ids.index(j)] \ for m in xrange(0, len(exp_table.columns))] # pdb.set_trace() tmp_df = pd.DataFrame.from_dict(tmp_dict, orient='index', dtype='float64') tmp_df.columns = exp_table.columns res = pd.concat([exp_table, tmp_df]) res.index.rename('ID', inplace=True) print len(tmp_dict), exp_table.shape, tmp_df.shape, res.shape name = ('.').join( exp_table_file.split('.')[:-1]) + "_5_" + str(i) + ".csv" print name res.to_csv(name, sep='\t')
def make_otmword(valsi): try: entry = otm.Entry(valsi["@word"], int(valsi["definitionid"])) except: print(valsi["@word"]) selmaho = ": " + valsi["selmaho"] if "selmaho" in valsi.keys() else "" translation = otm.translation(valsi["@type"]+selmaho, [valsi["definition"]]) translations = [translation] tags = [] if "@unofficial" in valsi.keys(): tags.append("unofficial") contents = make_contents(valsi) variations = [] relations = [] return Odict([("entry", entry), ("translations", translations), ("tags", tags), ("contents", contents), ("variations", variations), ("relations", relations)])
def parse_cost_file_formulas(file, diff_species): # import pdb; pdb.set_trace() cost_ids = Odict() with open(file) as f: for line in f: if line.startswith('ID:'): cost_function = line.split('\t')[1].strip() cost_id = line.split('\t')[0].split(':')[1].strip() if 'kCD' in cost_function: cost_function_noK = re.sub('kCD(\d+)\*', '', cost_function) else: cost_function_noK = re.sub('k(\d+)\*', '', cost_function) pybios_ids = re.findall(r"\[\d+\]", cost_function_noK) # import pdb; pdb.set_trace() cost_ids[(cost_id, cost_function, cost_function_noK)] = \ {k: j[0] for k, j in diff_species.iteritems()\ if k in pybios_ids} return cost_ids
def make_otmjson(rawdict, filename, lang, *args): _vlaste = [] for valsi in rawdict: _vla = make_otmword(valsi) if lang == "ja": _vla = delete_emptynotes(sortcontents(integrate_gloss(goodnotes(_vla)))) if '--nodollar' in args: _vla = delete_dollar(_vla) _vlaste.append(_vla) if '--addrelations' in args: from time import time import concurrent.futures print('add relations...') start = time() entry_list = [word["entry"] for word in _vlaste] letters = ".abcdefgijklmnoprstuvwxyz" letters += letters[1:].upper() entry_dict = {letter: [e for e in entry_list if e["form"][0] == letter] for letter in letters} executor = concurrent.futures.ProcessPoolExecutor(max_workers=8) max_n = len(_vlaste) split_n = 8 list_size = max_n // split_n tasks_list = [_vlaste[x:x + list_size] for x in range(0, max_n, list_size)] futures = [executor.submit(add_relations_for_multi, tasks, entry_dict) for tasks in tasks_list] done_task = 0 new_vlaste = [] for future in concurrent.futures.as_completed(futures): done_task += len(future.result()) new_vlaste.extend(future.result()) sys.stdout.write("\r{}/{} words done.".format(done_task, max_n)) sys.stdout.flush() end = time() print(" ({:.1f} sec.)".format(end-start)) _vlaste = new_vlaste _langdata = {"from":"jbo", "to": lang} _date = str(datetime.date.today()) _j = Odict([("words", _vlaste), ("zpdic", {"alphabetOrder":".'aAbBcCdDeEfFgGiIjJkKlLmMnNoOpPrRsStTuUvVxXyYzZ"}), ("meta", {"lang": _langdata, "generated_date": _date})]) otmjson = json.dumps(_j, indent=2, ensure_ascii=False) with open(filename, 'w', encoding='utf-8') as f: f.write(otmjson) print("Written to {}.".format(filename))
def test_subfolder(self): """Testing: duplicate to subfolder.""" self.importer.data = [ Odict([('title', 'google.com'), ('login', '*****@*****.**'), ('group', 'Emails')]), Odict([('title', 'google.com'), ('login', '*****@*****.**'), ('group', 'Emails')]), Odict([('title', 'google.com'), ('login', '*****@*****.**'), ('group', 'Emails')]), Odict([('title', 'google.com'), ('login', '*****@*****.**'), ('group', 'Emails')]) ] data_expected = [ Odict([('login', '*****@*****.**'), ('path', 'Emails/google.com/[email protected]')]), Odict([('login', '*****@*****.**'), ('path', 'Emails/google.com/[email protected]')]), Odict([('login', '*****@*****.**'), ('path', 'Emails/google.com/[email protected]')]), Odict([('login', '*****@*****.**'), ('path', 'Emails/google.com/[email protected]')]) ] self.importer.clean(clean=False, convert=False) self.assertEqual(self.importer.data, data_expected)
def setUp(self): self.npix = 1500 self.wave1 = 3800. self.wave2 = 5200. self.w = np.linspace(self.wave1, self.wave2, self.npix) self.f = np.ones_like(self.w) self.f += 0.6 * np.exp(-((self.w - 4600.) / 30.)**2 / 2.) self.cfrat = 1000. * np.ones_like(self.w) counts = self.f * self.cfrat errors = np.sqrt(counts + 100) self.fe = errors / self.cfrat # Create header with all supported types self.head = Odict() self.head['RJD'] = 2451234.56789 self.head['Object'] = 'Fake spectrum' self.head['Expose'] = np.float32(120.) self.head['Record'] = 12345 # make a spectrum self.spec = molly.Molly.fromdata(self.w, self.f, self.fe, self.cfrat, self.head)
def main(ramax=58, ramin=56, decmin=-32, decmax=-31, t0=59215, tm=59945): res = ObsMetaData.getObservationMetaData(boundLength=2, boundType='circle', fieldRA=(ramin-3, ramax+3), fieldDec=(decmin-3, decmax+3), expMJD=(t0, tm)) parsed = [Odict(obsmd.summary['OpsimMetaData']) for obsmd in res \ if obsmd.bandpass in ("g", "r", "i", "z", "y")] df = pd.DataFrame(parsed) X = df[['obsHistID', 'filter', 'FWHMeff', 'descDitheredRA', 'descDitheredDec', 'airmass', 'fiveSigmaDepth', 'expMJD']].copy() X.descDitheredRA = np.degrees(X.descDitheredRA) X.descDitheredDec = np.degrees(X.descDitheredDec) X['d1'] = angularSeparation(ramin, decmax, X.descDitheredRA.values, X.descDitheredDec.values) X['d2'] = angularSeparation(ramin, decmin, X.descDitheredRA.values, X.descDitheredDec.values) X['d3'] = angularSeparation(ramax, decmax, X.descDitheredRA.values, X.descDitheredDec.values) X['d4'] = angularSeparation(ramax, decmin, X.descDitheredRA.values, X.descDitheredDec.values) Y = X.query('d1 < 1.75 | d2 < 1.75 | d3 < 1.75 |d4 < 1.75')#.obsHistID.size for afilter in np.unique(Y['filter']): sub = Y[Y['filter']==afilter] print(afilter, sub[['airmass', 'FWHMeff', 'fiveSigmaDepth']].describe()) print('\n') Y.to_csv('./catalogs+tables/visits_from_minion.csv')
def test_get_empty(self): """Testing: convert empty dict to password entry.""" entry = Odict() entry_expected = '\n' self.assertTrue(self.importer.get(entry) == entry_expected)
def run_simulation_test(model, sim_time_vector): from model import PybiosEqs import modeltools process_times = Odict() real_times = Odict() try: if (sys.argv[1] == "True" or sys.argv[1] == "true"): print "Running simulations with steady state as a start" flag_next = False else: print "Running normal simulations" flag_next = True except IndexError: print "Running normal simulations" flag_next = True flag_first = True for sim_time in sim_time_vector: try: model_raw = PybiosEqs(0.000000, sim_time, [model], modParPath=[model], filePrefix=model) except TypeError: # for older models model_raw = PybiosEqs(0.000000, sim_time, model) mp = modeltools.ModelProcessor() S = model_raw.S F = model_raw.F K = model_raw.K step = sim_time / 100 cctime = np.arange(0, float(sim_time) + 1, step, dtype=np.float) if flag_first: start_time = time.time() start_ptime = time.clock() not_in_steady_state = False try: profiler_start("simulate_normal_" + model[model[:model.rfind("/")].rfind("/") + 1:-1] + ".log") res_ccompile = mp.simulate_time_course(S, cctime, F, K) S_time = res_ccompile.timeCourse[-1] profiler_stop() except MemoryError: print "Memory Error" process_times[sim_time] = time.clock() - start_ptime real_times[sim_time] = time.time() - start_time print "T({}) Process time normal: {}".format( sim_time, process_times[sim_time]) print "T({}) Real time normal: {}".format(sim_time, real_times[sim_time]) flag_first = flag_next # change this if running with or without steady state (true normal, false ss) S_time = res_ccompile.timeCourse[-1] not_in_steady_state = check_steady_state(res_ccompile, cctime, not_in_steady_state) if not_in_steady_state: print "SPECIES NOT IN STEADY STATE" else: start_ptime = time.clock() start_time = time.time() try: profiler_start("simulate_short_" + model[model[:model.rfind("/")].rfind("/") + 1:-1] + ".log") res_ccompile = mp.simulate_time_course(S_time, cctime, F, K) profiler_stop() except MemoryError: print "Memory Error" process_times[sim_time] = time.clock() - start_ptime real_times[sim_time] = time.time() - start_time print "T({}) Process time short: {}".format( sim_time, process_times[sim_time]) print "T({}) Real time short: {}".format(sim_time, real_times[sim_time]) S_time = res_ccompile.timeCourse[-1] not_in_steady_state = check_steady_state(res_ccompile, cctime, not_in_steady_state) if not_in_steady_state: print "SPECIES NOT IN STEADY STATE RUNNING AGAIN WITH NORMAL SIMULATIONS" res_ccompile = mp.simulate_time_course(S, cctime, F, K) return res_ccompile, process_times, real_times, len(S)
****TODO: Handling of input parameters**** ''' mapping_ids_file = '/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/Speedy_v3_r403445.ID_mapping_parameters.txt' identifiers_file = '/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/identifiers.py' file_h5_opti = "/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/model-Speedy_v3_r403445_v2-srv23ib.712174.0-multistarts.h5" file_h5_names = "/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/Speedy_v3_r403445_v1_2.h5" taketop10 = False progress_file = '/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/hdf5_progress_trace.txt' # evaluate the indentifiers.py file with open(identifiers_file, 'r') as f: org = Odict() execfile(identifiers_file, Odict(), org) par_ids = org['par_ids'] par_ids = sorted(par_ids.items(), key=operator.itemgetter(1)) par_ids = Odict(par_ids) # read data from the hdf5 files hdf5_opti = h5py.File(file_h5_opti, 'r') # get cost values cost = np.array(hdf5_opti["/finalCost"][0]) cost[cost == 0.0] = np.nan
def main(input_file, id_file_org): transl = True assert os.path.isfile(input_file), \ "Input file" + input_file + " doesn't exists!" assert os.path.isfile(id_file_org), \ "Input file" + id_file_org + " doesn't exists!" if transl: progress_file = input_file[:-4] + '_progress_trace_wTrl.txt' else: progress_file = input_file[:-4] + '_progress_trace.txt' # read in the data from Fabian with open(input_file, 'r') as f: data = [l.strip().split('\t') for l in f] # read in the identifiers.py file with open(id_file_org, 'r') as f: org = Odict() execfile(id_file_org, Odict(), org) par_ids = org['par_ids'] # handle the optimized parameters data n = sum([1 for i in data[0] if 'kCD' in i]) k_ids = data[0][0:n] model_ids = data[0][n:] vectors = [[float(i) for i in j] for j in data[1:]] #!!!!! # check if kCDs where used if 'kCD0' in k_ids: kCDs = ['kCD' + str(i) for i in xrange(n)] else: kCDs = ['kCD' + str(i + 1) for i in xrange(n)] pybios_ids = [] for k in model_ids: pybios_ids.append(\ par_ids.keys()[zip(*par_ids.values())[0].index(int(k[1:])-1)]) # pdb.set_trace() index = kCDs + pybios_ids if not transl: index = kCDs + [i for i in pybios_ids if "DrugTranslocation" not in i] # create empty dataframe with correct sorted parameter ids columns = [i for i in xrange(len(vectors))] matrix = np.empty(( len(index), len(columns), )) matrix.fill(np.nan) df_res = pd.DataFrame(matrix, index=index, columns=columns) # create dataframe with mapped optimized values to pybios_ids items = [] keys = k_ids + pybios_ids assert len(keys) == len(vectors[0]), "Problems with param_ids vector!" for i, k in enumerate(keys): if not transl and "DrugTranslocation" in k: continue else: vals = [i for i in zip(*vectors)[i]] items.append((k, vals)) df_in = pd.DataFrame.from_items(items, columns=columns, orient='index') # fill in the empty df df_res.update(df_in) # pdb.set_trace() if df_res.isnull().values.any(): print "NaN values exists!" pdb.set_trace() with open(progress_file, "w") as prog_file: prog_file.write("VARIABLE PAR IDS:\t{}\n\n".format( df_res.index.tolist())) for column in df_res: prog_file.write("VECTOR:\t{}\n\n".format(df_res[column].tolist())) # with open(progress_file, "w") as prog_file: # prog_file.write("VARIABLE PAR IDS:\t{}\n\n".format(index)) # for vect in vectors: # prog_file.write("VECTOR:\t{}\n\n".format(vect)) return
("JUNK", JUNK), ("BAD_FLAT", BAD_FLAT), ("BAD_COLUMN", BAD_COLUMN), ("BAD_TIME", BAD_TIME), ("OUTLIER", OUTLIER), ) # messages if various bitflags are set FLAG_MESSAGES = Odict(( (NO_FWHM, "no FWHM could be measured"), (NO_SKY, "zero sky pixels"), (SKY_AT_EDGE, "sky aperture overlapped the edge of the data window"), (TARGET_AT_EDGE, "target aperture overlapped the edge of the data window"), (TARGET_SATURATED, "target aperture had saturated pixels"), (TARGET_NONLINEAR, "target aperture had non-linear pixels"), (NO_EXTRACTION, "no extraction was possible"), (NO_DATA, "there were no valid pixels in the target aperture"), (CLOUDS, "marked as affected by clouds"), (JUNK, "junk data of unspecified nature"), (BAD_FLAT, "bad flat field feature in target aperture"), (BAD_COLUMN, "bad column in in target aperture"), (BAD_TIME, "the time was flagged as bad"), (OUTLIER, "identified as an outlier") )) def version(): """Returns version number of installed HiPERCAM pipeline""" import pkg_resources return pkg_resources.require("hipercam")[0].version