def parse_simulation_file(simulated_vectors_file):
constr_dict = Odict()
states_dict = Odict()
ids_vector = []
opti_vector = []
fitness = ""
with open(simulated_vectors_file) as file:
for line in file:
if "CONSTRAINTS:" in line:
constr_dict_raw = eval(line.split('CONSTRAINTS:')[-1])
for k, dd in constr_dict_raw.items():
# check if the experiments where fold or abs
if len(k) == 2:
constr_dict['/'.join(k)] = dd
elif len(k) == 1:
constr_dict[''.join(k)] = dd
elif "VARIABLE PAR IDS:" in line:
ids_vector = eval(line[line.index("["):])
elif "FITNESS:" in line:
fitness = eval(line[line.index(":") + 1:])
elif "VECTOR:" in line:
opti_vector = eval(line[line.index(":") + 1:])
elif "STATES:" in line:
key = line[line.index(":") + 1:line.rindex(":")]
value = eval(line[line.rindex(":") + 1:])
states_dict[key] = value
return constr_dict, states_dict, fitness, ids_vector, opti_vector
def tests_notitle(self):
"""Testing: clean data - notitle as path name."""
self.importer.data = [Odict([('password', 'UuQHzvv6IHRIJGjwKru7')])]
data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('path', 'notitle')])
]
self.importer.clean(clean=False, convert=False)
self.assertEqual(self.importer.data, data_expected)
def tests_empty(self):
"""Testing: clean data - empty title and clean enabled."""
self.importer.data = [Odict([('password', 'UuQHzvv6IHRIJGjwKru7')])]
data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('path', 'notitle')])
]
self.importer.clean(clean=True, convert=False)
self.assertEqual(self.importer.data, data_expected)
def tests_login(self):
"""Testing: clean data - login as path name."""
self.importer.data = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm')])
]
data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('path', 'lnqYm3ZWtm')])
]
self.importer.clean(clean=False, convert=False)
self.assertEqual(self.importer.data, data_expected)
def test_satanize_duplicate_paths(self):
"""Testing: satanize duplicate paths."""
self.importer.data = [
Odict([('title', 'ovh.com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm')]),
Odict([('title', 'ovh.com'), ('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'lnqYm3ZWtm')]),
Odict([('title', 'ovh.com'), ('password', '[Q&$\fd]!`vKA&b'),
('login', 'fmmhpvity')]),
Odict([('title', 'ovh.com'),
('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'),
('login', 'ptfzlnvmj')])
]
data_expected = [
Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh.com')]),
Odict([('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh.com0')]),
Odict([('password', '[Q&$\fd]!`vKA&b'), ('login', 'fmmhpvity'),
('path', 'ovh.com1')]),
Odict([('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'),
('login', 'ptfzlnvmj'), ('path', 'ovh.com2')])
]
self.importer.satanize(clean=False)
self.assertTrue(self.importer.data == data_expected)
def test_paths(self):
"""Testing: duplicate paths."""
self.importer.data = [
Odict([('title', 'ovh.com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm')]),
Odict([('title', 'ovh.com'), ('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'lnqYm3ZWtm')]),
Odict([('title', 'ovh.com'), ('password', '[Q&$\fd]!`vKA&b'),
('login', 'fmmhpvity')]),
Odict([('title', 'ovh.com'),
('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'),
('login', 'ptfzlnvmj')])
]
data_expected = [
Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh.com/lnqYm3ZWtm')]),
Odict([('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh.com/lnqYm3ZWtm-1')]),
Odict([('password', '[Q&$\fd]!`vKA&b'), ('login', 'fmmhpvity'),
('path', 'ovh.com/fmmhpvity')]),
Odict([('password', 'DQm_Y+a(sDC)[1|U-S<8Dq!A'),
('login', 'ptfzlnvmj'), ('path', 'ovh.com/ptfzlnvmj')])
]
self.importer.clean(clean=False, convert=False)
self.assertEqual(self.importer.data, data_expected)
def main(cost_function_file, output_name, n_tests, cv=True, shuffle_one=False):
output_name = cost_function_file.split('/')[:-1] + '/' + output_name
n_tests = int(n_tests)
controls = set()
# conc_dict_samples = Odict()
cost_func_dict = Odict()
f = open(cost_function_file)
for line in f:
if line.startswith("ID"):
cost_id = line.split("\t")[0].replace("ID:","")
cost_val = line.split("\t")[1].strip()
cost_func_dict[cost_id+":"+cost_val] = Odict()
else:
sample = line.split("\t")[0]
value = "\t".join(line.split("\t")[1:])
if "/" in sample:
drug = sample.split("/")[0]
# pdb.set_trace()
# control = getSampleName(sample.split("/")[1])
control = sample.split("/")[1]
elif "CONTROL" in sample:
control = getSampleName(sample)
elif not '_' in sample:
control = getSampleName(sample)
controls.add(control)
cost_func_dict[cost_id+":"+cost_val][sample] = value
f.close()
print len(controls)
controls_list = list(controls)
bucket_size = len(controls)
if cv:
testing_size = int(round(bucket_size/n_tests)) if not shuffle_one else 1
ids = range(0, n_tests*testing_size, testing_size)
for i, start in enumerate(ids):
end = len(controls) if i+1>=len(ids) else ids[i+1]
testing_list = controls_list[start:end]
training_list = list(controls-set(testing_list))
# print results to file
saveData(output_name, cost_func_dict, i, training_list, testing_list)
else:
training_size = int(round(bucket_size*80/100)) if not shuffle_one else bucket_size-1
for i in xrange(n_tests):
training_list = random.sample(controls, training_size)
testing_list = list(controls-set(training_list))
# print results to file
saveData(output_name, cost_func_dict, i, training_list, testing_list)
return
def test_satanize_duplicate_paths(self):
""" Testing: satanize duplicate paths """
self.importer.data = [
Odict([('title', 'ovh.com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm')]),
Odict([('title', 'ovh.com'), ('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'lnqYm3ZWtm')])
]
data_expected = [
Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh.com')]),
Odict([('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh.com0')])
]
self.importer.satanize(clean=False)
self.assertTrue(self.importer.data == data_expected)
def test_convert(self):
"""Testing: convert password path."""
self.importer.data = [
Odict([('title', 'ovh>com'), ('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm')]),
Odict([('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'fm/mhpv*ity')])
]
data_expected = [
Odict([('password', 'AGJjkMPsRUqDXyUdLbC4'),
('login', 'lnqYm3ZWtm'), ('path', 'ovh-com')]),
Odict([('password', 'VRiplZSniSBlHNnQvc9e'),
('login', 'fm/mhpv*ity'), ('path', 'fm-mhpv-ity')])
]
self.importer.clean(clean=False, convert=True)
self.assertEqual(self.importer.data, data_expected)
def test_get_data(self):
"""Testing: convert dict to password entry."""
entry = Odict([('password', 'EaP:bCmLZliqa|]WR/#HZP-aa'),
('login', 'roddhjav'),
('comments', 'This is a comment')])
entry_expected = "EaP:bCmLZliqa|]WR/#HZP-aa\nlogin: roddhjav\ncomments: This is a comment\n"
self.assertTrue(self.importer.get(entry) == entry_expected)
def read_conf_dir(dir_path):
"""
Read all config files in spcified folder, in alphabetical order. So, if files conain identical variants,
the ones, read early would be replaced by the ones, read later.
:param dir_path: path to folder
:return: nested dict, with merged backup variants from all files
"""
if (not os.path.exists(dir_path)):
Log.error(
"Configuration folder '{}' doesn't exists".format(dir_path))
filenames = [
file for file in os.listdir(dir_path)
if re.search("\.(py3?|ya?ml|json)$", file)
]
files = [dir_path + "/" + file for file in sorted(filenames)]
if len(files) == 0:
Log.error(
"There are no config files in folder '{}'".format(dir_path))
conf = Odict()
Log.info("Found config files: " + ", ".join(files))
for file in files:
variants = Conf.read_conf_file(file)
if variants is not None:
conf.update(variants)
return conf
def setUp(self):
self.importer = self.passimport.PasswordManager()
self.importer.data = [
Odict([('title', 'twitter.com'),
('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'),
('comments', ''), ('group', 'Social'), ('address', ''),
('sex', ''), ('website', 'https://pujol.io'),
('uuid', '44jle5q3fdvrprmaahozexy2pi')])
]
self.data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'),
('website', 'https://pujol.io'),
('uuid', '44jle5q3fdvrprmaahozexy2pi'),
('path', 'Social/twitter.com')])
]
def main(model_path_1, model_path_2):
'''
modify model_1 progress_trace (ids and vectors) to appropriate
size and format so they can be used on model_2
The size of the progress trace should be same as the vector_ids
of progress_trace.txt of model_2. Results are saved at model_1 path
'''
model_output_1 = model_path_1 + "/outputs/"
cost_1_file = model_path_1 + "/inputs/cost_func.txt"
prog_1_file = model_output_1 + "progress_trace.txt"
cost_2_file = model_path_2 + "/inputs/cost_func.txt"
prog_2_file = model_path_2 + "/outputs/progress_trace.txt"
cost_1 = get_formula(cost_1_file)
cost_1_dict = get_costDict(cost_1)
ids_vect_1, opti_vect = get_progTrace(prog_1_file)
cost_2 = get_formula(cost_2_file)
cost_2_dict = get_costDict(cost_2)
ids_vect_2, _ = get_progTrace(prog_2_file)
# dict in form {'species_id' : [kCDmodel1, kCDmodel2]}
# if not kCDmodel1 for species_id then x
# if not kCDmodel2 for species_id then ommited, not present
merged_dict = Odict()
for k in cost_2_dict.keys():
merged_dict[k] = []
if k in cost_1_dict:
merged_dict[k].append(cost_1_dict[k])
else:
merged_dict[k].append('x')
merged_dict[k].append(cost_2_dict[k])
res_vect = []
for vect in opti_vect:
tail = vect[len(cost_1_dict.keys()):]
# fill kCDs with 1.0 values with size of resulting vector
head = [1.0 for i in range(0, len(cost_2_dict))]
for k, v in merged_dict.items():
if not v[0] == 'x':
# pdb.set_trace()
head[ids_vect_2.index(v[1])] = vect[ids_vect_1.index(v[0])]
res_vect.append(head + tail)
res_ids = ids_vect_2[:len(cost_2_dict)] + ids_vect_1[len(cost_1_dict):]
# pdb.set_trace()
mod_prog_file = model_output_1 + "swaped_progress_trace.txt"
mod_prog = open(mod_prog_file, "w")
mod_prog.write("VARIABLE PAR IDS: {}\n\n".format(res_ids))
for vect in res_vect:
mod_prog.write("VECTOR: {}\n\n".format(vect))
mod_prog.close()
def test_data(self):
"""Testing: clean data."""
self.importer.data = [
Odict([('title', 'twitter.com'),
('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'),
('comments', ''), ('group', 'Social'), ('address', ''),
('sex', ''), ('website', 'https://pujol.io'),
('uuid', '44jle5q3fdvrprmaahozexy2pi')])
]
self.data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('url', 'https://twitter.com'),
('website', 'https://pujol.io'),
('uuid', '44jle5q3fdvrprmaahozexy2pi'),
('path', 'Social/twitter.com')])
]
self.importer.clean(clean=False, convert=False)
self.assertEqual(self.importer.data, self.data_expected)
def get_costDict(cost_str):
param_dict = Odict()
for nominator in cost_str.split("/"):
for pair in nominator.split("+"):
if pair.split("*")[1] in param_dict.keys():
raise ValueError(
"Error! Same species present in cost multiple times!")
# pdb.set_trace()
param_dict[pair.split("*")[1]] = pair.split("*")[0]
return param_dict
def test_satanize_path(self):
"""Testing: satanize data & generate password path."""
# Test url as path name
self.importer.data = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('url', 'twitter.com')])
]
data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('url', 'twitter.com'),
('path', 'twitter.com')])
]
self.importer.satanize(clean=False)
self.assertTrue(self.importer.data == data_expected)
# Test login as path name
self.importer.data = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm')])
]
data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'),
('login', 'lnqYm3ZWtm'), ('path', 'lnqYm3ZWtm')])
]
self.importer.satanize(clean=False)
self.assertTrue(self.importer.data == data_expected)
# Test notitle as path name
self.importer.data = [Odict([('password', 'UuQHzvv6IHRIJGjwKru7')])]
data_expected = [
Odict([('password', 'UuQHzvv6IHRIJGjwKru7'), ('path', 'notitle')])
]
self.importer.satanize(clean=False)
self.assertTrue(self.importer.data == data_expected)
def test_numbers(self):
"""Testing: duplicate with numbers."""
self.importer.data = [
Odict([('title', 'ovh.com')]),
Odict([('title', 'ovh.com')]),
Odict([('title', 'ovh.com')]),
Odict([('title', 'ovh.com')])
]
data_expected = [
Odict([('path', 'ovh.com/notitle')]),
Odict([('path', 'ovh.com/notitle-1')]),
Odict([('path', 'ovh.com/notitle-2')]),
Odict([('path', 'ovh.com/notitle-3')])
]
self.importer.clean(clean=False, convert=False)
self.assertEqual(self.importer.data, data_expected)
def create_sub_paths(paths, classes=None, image=None):
paths_at_position = Odict()
for p_id in xrange(0, len(paths)):
for pos in list(paths[p_id]):
if tuple(pos) in paths_at_position.keys():
if classes is not None:
paths_at_position[tuple(pos)].append(
(p_id, classes[p_id]))
elif image is not None:
paths_at_position[tuple(
list(pos) + [image[tuple(pos)]])].append(
(p_id, ))
else:
paths_at_position[tuple(pos)].append((p_id, ))
else:
if classes is not None:
paths_at_position[tuple(pos)] = [(p_id,
classes[p_id])]
elif image is not None:
paths_at_position[tuple(
list(pos) + [image[tuple(pos)]])] = [(p_id, )]
else:
paths_at_position[tuple(pos)] = [(p_id, )]
sub_paths = Odict()
for key, p_ids in paths_at_position.iteritems():
if tuple(p_ids) in sub_paths.keys():
sub_paths[tuple(p_ids)].append(key)
else:
sub_paths[tuple(p_ids)] = [key]
return sub_paths
def main(input_vectors_file, exp_table_file):
rand = True
f = open(input_vectors_file)
data = f.readlines()
par_ids = eval([l for l in data
if l.startswith('PAR_IDS')][0].split('=')[-1])
var_ids = eval([l for l in data
if l.startswith('VAR_PAR_IDS')][0].split("=")[-1])
vector = eval([l for l in data
if l.startswith('RND_VECTOR')][0].split('=')[-1])
var_idx = eval([l for l in data
if l.startswith('VAR_PAR_IDX')][0].split('=')[-1])
rnd_vect = [j for i, j in enumerate(vector) if i in var_idx]
# pdb.set_trace()
assert len(var_ids) == len(rnd_vect), "Something wrong with input data!"
exp_table = pd.read_table(exp_table_file, index_col=0, low_memory=False)
nparam = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2028, 3000]
for i in list(reversed(nparam)):
tmp_dict = Odict()
print i
if rand:
var = random.sample(var_ids, i)
fix = [k for k in var_ids if k not in var]
else:
var = var_ids[0:i]
fix = var_ids[i:]
for j in fix:
if not j in tmp_dict:
tmp_dict[j] = [rnd_vect[var_ids.index(j)] \
for m in xrange(0, len(exp_table.columns))]
# pdb.set_trace()
tmp_df = pd.DataFrame.from_dict(tmp_dict,
orient='index',
dtype='float64')
tmp_df.columns = exp_table.columns
res = pd.concat([exp_table, tmp_df])
res.index.rename('ID', inplace=True)
print len(tmp_dict), exp_table.shape, tmp_df.shape, res.shape
name = ('.').join(
exp_table_file.split('.')[:-1]) + "_5_" + str(i) + ".csv"
print name
res.to_csv(name, sep='\t')
def make_otmword(valsi):
try:
entry = otm.Entry(valsi["@word"], int(valsi["definitionid"]))
except:
print(valsi["@word"])
selmaho = ": " + valsi["selmaho"] if "selmaho" in valsi.keys() else ""
translation = otm.translation(valsi["@type"]+selmaho, [valsi["definition"]])
translations = [translation]
tags = []
if "@unofficial" in valsi.keys():
tags.append("unofficial")
contents = make_contents(valsi)
variations = []
relations = []
return Odict([("entry", entry), ("translations", translations), ("tags", tags),
("contents", contents), ("variations", variations), ("relations", relations)])
def parse_cost_file_formulas(file, diff_species):
# import pdb; pdb.set_trace()
cost_ids = Odict()
with open(file) as f:
for line in f:
if line.startswith('ID:'):
cost_function = line.split('\t')[1].strip()
cost_id = line.split('\t')[0].split(':')[1].strip()
if 'kCD' in cost_function:
cost_function_noK = re.sub('kCD(\d+)\*', '', cost_function)
else:
cost_function_noK = re.sub('k(\d+)\*', '', cost_function)
pybios_ids = re.findall(r"\[\d+\]", cost_function_noK)
# import pdb; pdb.set_trace()
cost_ids[(cost_id, cost_function, cost_function_noK)] = \
{k: j[0] for k, j in diff_species.iteritems()\
if k in pybios_ids}
return cost_ids
def make_otmjson(rawdict, filename, lang, *args):
_vlaste = []
for valsi in rawdict:
_vla = make_otmword(valsi)
if lang == "ja":
_vla = delete_emptynotes(sortcontents(integrate_gloss(goodnotes(_vla))))
if '--nodollar' in args:
_vla = delete_dollar(_vla)
_vlaste.append(_vla)
if '--addrelations' in args:
from time import time
import concurrent.futures
print('add relations...')
start = time()
entry_list = [word["entry"] for word in _vlaste]
letters = ".abcdefgijklmnoprstuvwxyz"
letters += letters[1:].upper()
entry_dict = {letter: [e for e in entry_list if e["form"][0] == letter] for letter in letters}
executor = concurrent.futures.ProcessPoolExecutor(max_workers=8)
max_n = len(_vlaste)
split_n = 8
list_size = max_n // split_n
tasks_list = [_vlaste[x:x + list_size] for x in range(0, max_n, list_size)]
futures = [executor.submit(add_relations_for_multi, tasks, entry_dict) for tasks in tasks_list]
done_task = 0
new_vlaste = []
for future in concurrent.futures.as_completed(futures):
done_task += len(future.result())
new_vlaste.extend(future.result())
sys.stdout.write("\r{}/{} words done.".format(done_task, max_n))
sys.stdout.flush()
end = time()
print(" ({:.1f} sec.)".format(end-start))
_vlaste = new_vlaste
_langdata = {"from":"jbo", "to": lang}
_date = str(datetime.date.today())
_j = Odict([("words", _vlaste), ("zpdic", {"alphabetOrder":".'aAbBcCdDeEfFgGiIjJkKlLmMnNoOpPrRsStTuUvVxXyYzZ"}),
("meta", {"lang": _langdata, "generated_date": _date})])
otmjson = json.dumps(_j, indent=2, ensure_ascii=False)
with open(filename, 'w', encoding='utf-8') as f:
f.write(otmjson)
print("Written to {}.".format(filename))
def test_subfolder(self):
"""Testing: duplicate to subfolder."""
self.importer.data = [
Odict([('title', 'google.com'), ('login', '*****@*****.**'),
('group', 'Emails')]),
Odict([('title', 'google.com'), ('login', '*****@*****.**'),
('group', 'Emails')]),
Odict([('title', 'google.com'), ('login', '*****@*****.**'),
('group', 'Emails')]),
Odict([('title', 'google.com'), ('login', '*****@*****.**'),
('group', 'Emails')])
]
data_expected = [
Odict([('login', '*****@*****.**'),
('path', 'Emails/google.com/[email protected]')]),
Odict([('login', '*****@*****.**'),
('path', 'Emails/google.com/[email protected]')]),
Odict([('login', '*****@*****.**'),
('path', 'Emails/google.com/[email protected]')]),
Odict([('login', '*****@*****.**'),
('path', 'Emails/google.com/[email protected]')])
]
self.importer.clean(clean=False, convert=False)
self.assertEqual(self.importer.data, data_expected)
def setUp(self):
self.npix = 1500
self.wave1 = 3800.
self.wave2 = 5200.
self.w = np.linspace(self.wave1, self.wave2, self.npix)
self.f = np.ones_like(self.w)
self.f += 0.6 * np.exp(-((self.w - 4600.) / 30.)**2 / 2.)
self.cfrat = 1000. * np.ones_like(self.w)
counts = self.f * self.cfrat
errors = np.sqrt(counts + 100)
self.fe = errors / self.cfrat
# Create header with all supported types
self.head = Odict()
self.head['RJD'] = 2451234.56789
self.head['Object'] = 'Fake spectrum'
self.head['Expose'] = np.float32(120.)
self.head['Record'] = 12345
# make a spectrum
self.spec = molly.Molly.fromdata(self.w, self.f, self.fe, self.cfrat,
self.head)
def main(ramax=58, ramin=56, decmin=-32, decmax=-31, t0=59215, tm=59945):
res = ObsMetaData.getObservationMetaData(boundLength=2, boundType='circle',
fieldRA=(ramin-3, ramax+3),
fieldDec=(decmin-3, decmax+3),
expMJD=(t0, tm))
parsed = [Odict(obsmd.summary['OpsimMetaData']) for obsmd in res \
if obsmd.bandpass in ("g", "r", "i", "z", "y")]
df = pd.DataFrame(parsed)
X = df[['obsHistID', 'filter', 'FWHMeff', 'descDitheredRA',
'descDitheredDec', 'airmass', 'fiveSigmaDepth', 'expMJD']].copy()
X.descDitheredRA = np.degrees(X.descDitheredRA)
X.descDitheredDec = np.degrees(X.descDitheredDec)
X['d1'] = angularSeparation(ramin, decmax, X.descDitheredRA.values, X.descDitheredDec.values)
X['d2'] = angularSeparation(ramin, decmin, X.descDitheredRA.values, X.descDitheredDec.values)
X['d3'] = angularSeparation(ramax, decmax, X.descDitheredRA.values, X.descDitheredDec.values)
X['d4'] = angularSeparation(ramax, decmin, X.descDitheredRA.values, X.descDitheredDec.values)
Y = X.query('d1 < 1.75 | d2 < 1.75 | d3 < 1.75 |d4 < 1.75')#.obsHistID.size
for afilter in np.unique(Y['filter']):
sub = Y[Y['filter']==afilter]
print(afilter, sub[['airmass', 'FWHMeff', 'fiveSigmaDepth']].describe())
print('\n')
Y.to_csv('./catalogs+tables/visits_from_minion.csv')
def test_get_empty(self):
"""Testing: convert empty dict to password entry."""
entry = Odict()
entry_expected = '\n'
self.assertTrue(self.importer.get(entry) == entry_expected)
def run_simulation_test(model, sim_time_vector):
from model import PybiosEqs
import modeltools
process_times = Odict()
real_times = Odict()
try:
if (sys.argv[1] == "True" or sys.argv[1] == "true"):
print "Running simulations with steady state as a start"
flag_next = False
else:
print "Running normal simulations"
flag_next = True
except IndexError:
print "Running normal simulations"
flag_next = True
flag_first = True
for sim_time in sim_time_vector:
try:
model_raw = PybiosEqs(0.000000,
sim_time, [model],
modParPath=[model],
filePrefix=model)
except TypeError: # for older models
model_raw = PybiosEqs(0.000000, sim_time, model)
mp = modeltools.ModelProcessor()
S = model_raw.S
F = model_raw.F
K = model_raw.K
step = sim_time / 100
cctime = np.arange(0, float(sim_time) + 1, step, dtype=np.float)
if flag_first:
start_time = time.time()
start_ptime = time.clock()
not_in_steady_state = False
try:
profiler_start("simulate_normal_" +
model[model[:model.rfind("/")].rfind("/") +
1:-1] + ".log")
res_ccompile = mp.simulate_time_course(S, cctime, F, K)
S_time = res_ccompile.timeCourse[-1]
profiler_stop()
except MemoryError:
print "Memory Error"
process_times[sim_time] = time.clock() - start_ptime
real_times[sim_time] = time.time() - start_time
print "T({}) Process time normal: {}".format(
sim_time, process_times[sim_time])
print "T({}) Real time normal: {}".format(sim_time,
real_times[sim_time])
flag_first = flag_next # change this if running with or without steady state (true normal, false ss)
S_time = res_ccompile.timeCourse[-1]
not_in_steady_state = check_steady_state(res_ccompile, cctime,
not_in_steady_state)
if not_in_steady_state:
print "SPECIES NOT IN STEADY STATE"
else:
start_ptime = time.clock()
start_time = time.time()
try:
profiler_start("simulate_short_" +
model[model[:model.rfind("/")].rfind("/") +
1:-1] + ".log")
res_ccompile = mp.simulate_time_course(S_time, cctime, F, K)
profiler_stop()
except MemoryError:
print "Memory Error"
process_times[sim_time] = time.clock() - start_ptime
real_times[sim_time] = time.time() - start_time
print "T({}) Process time short: {}".format(
sim_time, process_times[sim_time])
print "T({}) Real time short: {}".format(sim_time,
real_times[sim_time])
S_time = res_ccompile.timeCourse[-1]
not_in_steady_state = check_steady_state(res_ccompile, cctime,
not_in_steady_state)
if not_in_steady_state:
print "SPECIES NOT IN STEADY STATE RUNNING AGAIN WITH NORMAL SIMULATIONS"
res_ccompile = mp.simulate_time_course(S, cctime, F, K)
return res_ccompile, process_times, real_times, len(S)
****TODO: Handling of input parameters**** ''' mapping_ids_file = '/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/Speedy_v3_r403445.ID_mapping_parameters.txt' identifiers_file = '/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/identifiers.py' file_h5_opti = "/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/model-Speedy_v3_r403445_v2-srv23ib.712174.0-multistarts.h5" file_h5_names = "/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/Speedy_v3_r403445_v1_2.h5" taketop10 = False progress_file = '/home/H0001-1/a.kovachev/simtools/data/models/Speedy_v3_r403445/hdf5_progress_trace.txt' # evaluate the indentifiers.py file with open(identifiers_file, 'r') as f: org = Odict() execfile(identifiers_file, Odict(), org) par_ids = org['par_ids'] par_ids = sorted(par_ids.items(), key=operator.itemgetter(1)) par_ids = Odict(par_ids) # read data from the hdf5 files hdf5_opti = h5py.File(file_h5_opti, 'r') # get cost values cost = np.array(hdf5_opti["/finalCost"][0]) cost[cost == 0.0] = np.nan
def main(input_file, id_file_org):
transl = True
assert os.path.isfile(input_file), \
"Input file" + input_file + " doesn't exists!"
assert os.path.isfile(id_file_org), \
"Input file" + id_file_org + " doesn't exists!"
if transl:
progress_file = input_file[:-4] + '_progress_trace_wTrl.txt'
else:
progress_file = input_file[:-4] + '_progress_trace.txt'
# read in the data from Fabian
with open(input_file, 'r') as f:
data = [l.strip().split('\t') for l in f]
# read in the identifiers.py file
with open(id_file_org, 'r') as f:
org = Odict()
execfile(id_file_org, Odict(), org)
par_ids = org['par_ids']
# handle the optimized parameters data
n = sum([1 for i in data[0] if 'kCD' in i])
k_ids = data[0][0:n]
model_ids = data[0][n:]
vectors = [[float(i) for i in j] for j in data[1:]] #!!!!!
# check if kCDs where used
if 'kCD0' in k_ids:
kCDs = ['kCD' + str(i) for i in xrange(n)]
else:
kCDs = ['kCD' + str(i + 1) for i in xrange(n)]
pybios_ids = []
for k in model_ids:
pybios_ids.append(\
par_ids.keys()[zip(*par_ids.values())[0].index(int(k[1:])-1)])
# pdb.set_trace()
index = kCDs + pybios_ids
if not transl:
index = kCDs + [i for i in pybios_ids if "DrugTranslocation" not in i]
# create empty dataframe with correct sorted parameter ids
columns = [i for i in xrange(len(vectors))]
matrix = np.empty((
len(index),
len(columns),
))
matrix.fill(np.nan)
df_res = pd.DataFrame(matrix, index=index, columns=columns)
# create dataframe with mapped optimized values to pybios_ids
items = []
keys = k_ids + pybios_ids
assert len(keys) == len(vectors[0]), "Problems with param_ids vector!"
for i, k in enumerate(keys):
if not transl and "DrugTranslocation" in k:
continue
else:
vals = [i for i in zip(*vectors)[i]]
items.append((k, vals))
df_in = pd.DataFrame.from_items(items, columns=columns, orient='index')
# fill in the empty df
df_res.update(df_in)
# pdb.set_trace()
if df_res.isnull().values.any():
print "NaN values exists!"
pdb.set_trace()
with open(progress_file, "w") as prog_file:
prog_file.write("VARIABLE PAR IDS:\t{}\n\n".format(
df_res.index.tolist()))
for column in df_res:
prog_file.write("VECTOR:\t{}\n\n".format(df_res[column].tolist()))
# with open(progress_file, "w") as prog_file:
# prog_file.write("VARIABLE PAR IDS:\t{}\n\n".format(index))
# for vect in vectors:
# prog_file.write("VECTOR:\t{}\n\n".format(vect))
return
("JUNK", JUNK),
("BAD_FLAT", BAD_FLAT),
("BAD_COLUMN", BAD_COLUMN),
("BAD_TIME", BAD_TIME),
("OUTLIER", OUTLIER),
)
# messages if various bitflags are set
FLAG_MESSAGES = Odict((
(NO_FWHM, "no FWHM could be measured"),
(NO_SKY, "zero sky pixels"),
(SKY_AT_EDGE, "sky aperture overlapped the edge of the data window"),
(TARGET_AT_EDGE, "target aperture overlapped the edge of the data window"),
(TARGET_SATURATED, "target aperture had saturated pixels"),
(TARGET_NONLINEAR, "target aperture had non-linear pixels"),
(NO_EXTRACTION, "no extraction was possible"),
(NO_DATA, "there were no valid pixels in the target aperture"),
(CLOUDS, "marked as affected by clouds"),
(JUNK, "junk data of unspecified nature"),
(BAD_FLAT, "bad flat field feature in target aperture"),
(BAD_COLUMN, "bad column in in target aperture"),
(BAD_TIME, "the time was flagged as bad"),
(OUTLIER, "identified as an outlier")
))
def version():
"""Returns version number of installed HiPERCAM pipeline"""
import pkg_resources
return pkg_resources.require("hipercam")[0].version
