def main():
# Parse and print arguments
args = make_args_parser()
print_args(args)
# Load input file as well as the timestamps
X, ts = util.load_file(args.input)
# Load and compile model
model = util.load_and_compile_model(MODEL_PATH)
# Print model summary
print("------------------------- Model Summary -------------------------")
model.summary()
# Predict model output
y_pred = model.predict(X, batch_size=32)
# Load actual values
y_true, _ = util.load_file(ACTUAL_PATH)
# Compute MAE, MAPE and MSE
mae = util.compute_mae(X, y_true, model)
mape = util.compute_mape(y_true, y_pred, model)
mse = util.compute_mse(X, y_true, model)
# Check if file already exists. If so delete it
if os.path.exists(PREDICTED_PATH):
os.remove(PREDICTED_PATH)
# Open file an write from scratch
with open(PREDICTED_PATH, 'a') as file:
file.write('MAE: {:.4f}\tMAPE: {:.4f}\tMSE: {:.4f}\n'.format(
mae, mape, mse))
df = pd.concat([ts, pd.DataFrame(y_pred)], axis=1)
df.to_csv(file, index=False, header=False, sep='\t', encoding='utf-8')
def save_output_file(path_name, file_name):
"""Reads all of the data from a single run and saves to a .mat file."""
converted_data = load_file('%s%s.p'%(path_name, file_name))
filtered_data = load_file('%s%s_filtered.p'%(path_name, file_name))
force_data = load_file('%s%s_forces.p'%(path_name, file_name))
# Save all the data in a useful way.
if force_data['dim'] == 2:
converted_data['z'] = []
output_data = {'config': filtered_data['q'],
'velocity': filtered_data['v'],
'acceleration': filtered_data['a'],
'reference_config': filtered_data['ref'],
'dimension': force_data['dim'],
'time': force_data['t'],
'link_length': filtered_data['link_length'],
'contact_point': filtered_data['cp'],
'reaction_forces': {'static': force_data['static'],
'dynamic': force_data['dyn'],
'discrete': force_data['discrete']},
'sampled_points': {'x': converted_data['x'],
'y': converted_data['y'],
'z': converted_data['z']}}
sio.savemat('%s%s_output.mat' %(path_name, file_name), output_data)
def test_alm_merge_basic(info):
if info.verbose:
print("") # fixes display for later...
info.run_merge()
exp = util.load_file(info.expected)
res = util.load_file(info.result)
util.file_compare(exp, res)
info.cleanup()
def test_calfactors_basic(info):
# if not stopped:
# import pdb; pdb.set_trace()
# global stopped
# stopped = True
info.run_merge()
exp = util.load_file(info.expected)
res = util.load_file(info.result)
compare(exp, res)
info.cleanup()
def kdw_reset_model_and_deck(col):
util.remove_model_and_deck(col, KDW_MODEL, KDW_DECK, log)
deck_id = col.decks.id(KDW_DECK)
model = col.models.new(KDW_MODEL)
model['did'] = deck_id
model['css'] = util.load_file('kdw.css', log) or model['css']
col.models.add(model)
col.models.addField(model, col.models.newField('Kanji'))
col.models.addField(model, col.models.newField('Furigana'))
col.models.addField(model, col.models.newField('Meaning'))
col.models.addField(model, col.models.newField('Examples'))
tmpl_read = col.models.newTemplate('Read')
tmpl_read['qfmt'] = util.load_template_file('kdw', 'read', 'front',
log) or ''
tmpl_read['afmt'] = util.load_template_file('kdw', 'read', 'back',
log) or ''
tmpl_read['bqfmt'] = tmpl_read['qfmt']
tmpl_read['bafmt'] = tmpl_read['afmt']
col.models.addTemplate(model, tmpl_read)
tmpl_meaning = col.models.newTemplate('Meaning')
tmpl_meaning['qfmt'] = util.load_template_file('kdw', 'meaning', 'front',
log) or ''
tmpl_meaning['afmt'] = util.load_template_file('kdw', 'meaning', 'back',
log) or ''
tmpl_meaning['bqfmt'] = tmpl_meaning['qfmt']
tmpl_meaning['bafmt'] = tmpl_meaning['afmt']
col.models.addTemplate(model, tmpl_meaning)
col.save()
return model, col.decks.get(deck_id)
def on_btnGetCols_clicked(self,widget):
print ''' sql 을 이용하여 conlumn정보 가져오기'''
self.tbColEng.set_text('')
self.tbColHan.set_text('')
bounds = self.tbSql.get_selection_bounds();
if( len(bounds) < 2):
print "no text is selected"
self.log("no text is selected\n")
tb = self.tbSql
sql = self.tbSql.get_text(tb.get_start_iter(), tb.get_end_iter())
else:
sql = self.tbSql.get_text(bounds[0],bounds[1]);
sql = self.rmSqlComment(sql)
self.log('SQL : ' + sql + '\n');
print sql
try:
dbCols = DbCols()
dbCols.con_str = self._get_con_str()
cols = dbCols.get_cols(sql)
tb = self.tbColEng
assert isinstance(tb , gtk.TextBuffer)
tb.set_text(cols)
mydict = util.load_file("dict.txt")
names = cols.split(',')
hannames = util.eng_to_han(mydict, names)
self.tbColHan.set_text(hannames)
except :
tb = traceback.format_exc().decode('euckr')
self.log("********* ERROR ********* \n")
self.log(tb)
raise
def plot_all_loss(folders):
loss = util.load_file(folder, 'loss')
for folder in folders:
with open('{}/loss'.format(folder), 'rb') as f:
loss = pickle.load(f)
plt.legend()
plt.show()
def plot_loss(folders):
for (dir_name, file_name) in folders:
loss = util.load_file(dir_name, file_name)
plt.plot(loss, label=labels[dir_name])
plt.xlabel('Number of epochs')
plt.ylabel('Training Loss')
#plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.legend()
plt.show()
def plot_accuracy(folders):
for (dir_name, file_name) in folders:
accs = util.load_file(dir_name, file_name)
plt.plot(accs, label=labels[dir_name])
plt.xlabel('Number of epochs')
plt.ylabel('SVHN Test Accuracy')
#plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.legend(loc=4)
#plt.show()
plt.savefig('fc_accs.png')
def info(path):
n_sample, sign, density, double_occ, sweep, n_sweep = \
util.load_file(path, "meas_eqlt/n_sample", "meas_eqlt/sign",
"meas_eqlt/density", "meas_eqlt/double_occ",
"state/sweep", "params/n_sweep")
print(f"n_sample={n_sample}, sweep={sweep}/{n_sweep}")
if n_sample > 0:
print(f"<sign>={(sign/n_sample)}")
print(f"<n>={(density/sign)}")
print(f"<m_z^2>={((density-2*double_occ)/sign)}")
def __init__(self, projdir):
for file in os.listdir(projdir):
if file.endswith('.all'):
df = util.load_file(projdir, file)
errors = self.get_errors(df)
# write file
output = file[:-4] + '_errors.csv'
util.write_file(errors, projdir, output,
cols=TError.get_columns())
def get_vofields(self,sql):
cols = self._get_cols(sql)
mydict = util.load_file("dict.txt")
s = ''
for col in cols:
(name, t,precision,scale) = col[0],col[1], col[4],col[5]
tstr = self.get_type_str(t,precision,scale)
camelstr = util.underscore_to_camel(name) + ' ;'
hanname = util.eng_to_han_one(mydict, name)
s += "private %-12s %-30s // %s\n" % (tstr, camelstr, hanname)
return s;
def on_btnGenSql_clicked(self,widget):
''' SQL생성.'''
tb = self.tbColEng
mydict = util.load_file("dict.txt")
assert isinstance(tb , gtk.TextBuffer)
s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
cols = util.split(s)
#self.log(crud.mk_insert(cols))
#self.log('\n\n')
self.log(crud.mk_insert2(mydict, cols))
self.log('\n\n')
self.log(crud.mk_update(mydict, cols))
self.log('\n\n')
self.log(crud.mk_select(mydict, cols))
self.log('\n\n')
def on_btnGenChkCode_clicked(self,widget):
''' 체크코드 생성 버튼 처리.'''
mydict = util.load_file("dict.txt")
tb = self.tbColHan
assert isinstance(tb , gtk.TextBuffer)
s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
o = bindItemToVar.BindItemVar(s,mydict)
self.log(o.getVarStr())
self.log('\n\n')
self.log(o.getChkStr())
self.log('\n\n')
self.log(o.getChkStr2())
self.log('\n\n')
self.log(o.getChkStr3())
self.log('\n\n')
def __init__(self, date, news_limit=5, net_limit=50):
self.section = util.load_file("section.txt")
self.date = date
self.news_limit = news_limit
self.net_limit = net_limit
self.refer = 0
self.mecab = Mecab()
self.exp = re.compile("NN|XR|VA|VV|MAG|VX")
self.temp_net = {}
self.temp_list = {}
self.word_net = [] # relative word and its frequency
self.word_list = [] # total word and its frequency (using for PMI)
self.news = [] # top # of news
self.sentiment = [0, 0] # [neg, pos]
self.counter = [0 for i in range(16)]
def __init__( self, date, news_limit = 5, net_limit = 50 ):
self.section = util.load_file("section.txt")
self.date = date
self.news_limit = news_limit
self.net_limit = net_limit
self.refer = 0
self.mecab = Mecab()
self.exp = re.compile("NN|XR|VA|VV|MAG|VX")
self.temp_net = {}
self.temp_list = {}
self.word_net = [] # relative word and its frequency
self.word_list = [] # total word and its frequency (using for PMI)
self.news = [] # top # of news
self.sentiment = [0, 0] # [neg, pos]
self.counter = [ 0 for i in range(16) ]
def make_cfg():
print ' * * * TShockweb configurator * * * '
print ''
print 'TShockweb requires a configuration file called "%s" to function. This file was not found, but we can make a configuration file for you. I just need some information about your environment, and I will generate a %s file. If you want to run this configurator again later, simply remove the %s file from the tshockweb folder.' % (props_file_name, props_file_name, props_file_name)
for q in configs:
print '\n\n************************************'
print q.token
print 'Description: %s \n' % (q.desc)
ans = raw_input('Enter a value, or leave blank to accept the default of "%s" >>> ' % (q.default))
ans = ans.replace('\\', '/').strip()
if ans is not None and len(ans) > 0:
q.value = ans
print '\n\nGenerating tshockweb.properties...'
default_config = util.load_file(default_config_file)
tokens = make_cfg_dict(configs)
modified_config = Template(default_config).safe_substitute(tokens)
print modified_config
print 'Done!'
def plot_all_kshot_for_exp(folder, kshots, subfolder, exp_type, num_epochs=None):
plt.clf()
acc = util.load_file(folder, 'inter_accuracy44')
finals = list()
for k in kshots:
x_acc = [a[k] for a in acc]
chunked = chunk(x_acc, SMOOTHING)
finals.append(chunked)
step_size = 25 * SMOOTHING # Validation accuracy calculated every STEP_SIZE epochs.
x = np.arange(1, len(finals[0]) * step_size, step_size)
lsubfolder = subfolder.lower()
if num_epochs:
comb = list(filter(lambda y: y[0] <= num_epochs, list(zip(x, acc))))
x = [c[0] for c in comb]
acc = [c[1] for c in comb]
assert len(x) == len(acc)
for (i, f) in enumerate(finals):
plt.plot(x, f, label=labels[kshots[i] + 1], linewidth=2, marker='.', \
markersize=6)
#plt.plot(x, [baselines[lsubfolder][0]] * len(acc), label='Baseline 1st Instance' ,\
# linewidth=2)
#plt.plot(x, [baselines[subfolder][1]] * len(acc), label='Baseline 2nd Instance')
#plt.plot(x, [baselines[subfolder][4]] * len(acc), label='Baseline 5th Instance')
#plt.plot(x, [baselines[lsubfolder][6]] * len(acc), label='Baseline 7th Instance', \
# linewidth=2)
plt.ylim(0, 1.1)
plt.xlabel('Number of epochs')
plt.ylabel('% Accurate Test Set')
"""
if subfolder == 'vgg':
plt.title('Pre-trained VGG19 Controller')
else:
plt.title('Default Controller (No encoder)')
"""
plt.title(subfolder)
plt.legend(loc=2)
#plt.show()
plt.savefig('plots/{}_{}.png'.format(exp_type, subfolder))
def make_cfg():
print ' * * * TShockweb configurator * * * '
print ''
print 'TShockweb requires a configuration file called "%s" to function. This file was not found, but we can make a configuration file for you. I just need some information about your environment, and I will generate a %s file. If you want to run this configurator again later, simply remove the %s file from the tshockweb folder.' % (
props_file_name, props_file_name, props_file_name)
for q in configs:
print '\n\n************************************'
print q.token
print 'Description: %s \n' % (q.desc)
ans = raw_input(
'Enter a value, or leave blank to accept the default of "%s" >>> '
% (q.default))
ans = ans.replace('\\', '/').strip()
if ans is not None and len(ans) > 0:
q.value = ans
print '\n\nGenerating tshockweb.properties...'
default_config = util.load_file(default_config_file)
tokens = make_cfg_dict(configs)
modified_config = Template(default_config).safe_substitute(tokens)
print modified_config
print 'Done!'
def main():
# parse and print arguments
args = make_args_parser()
print_args(args)
# load input file as well as the timestamps
X, ts = util.load_file(args.input)
# load and compile model
model = util.load_and_compile_model(MODEL_PATH)
# print model summary
print("------------------------- Model Summary -------------------------")
model.summary()
# Get new model having as output the first hidden of our pretrained model
new_model = util.get_intermediate_layer_model(model, 'dense_1')
# print model summary
print("--------------- Intermediate Layer Model Summary ----------------")
new_model.summary()
# predict new model output
y_pred = new_model.predict(X, batch_size=32)
# Open file an write from scratch
with open(OUTPUT_PATH, 'w') as file:
df = pd.concat([ts, pd.DataFrame(y_pred)], axis=1)
df.to_csv(file, index=False, header=False, sep='\t', encoding='utf-8')
def on_btnDefMap_clicked(self,widget):
mydict = util.load_file("dict.txt")
tb = self.tbColHan
assert isinstance(tb , gtk.TextBuffer)
s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
a = s.splitlines()
if( len(a) <2 ):
a = util.split(s)
msg = 'SQL의 SELECT구문을 이용하기 바랍니다\n\n'
msg += 'BufMap defValMap = new BufMap();\n'
msg += 'defValMap\n'
for l in a:
l = re.sub('/\*.*\*/','',l)
l = l.strip()
t = self.get_token(l)
engname = t
t = '"' + t + '"'
hanname = util.eng_to_han_one(mydict,engname )
msg += '\t\t.put(%-30s,"0")\t\t// %s\n' %(t,hanname)
msg += '\t\t;\n'
self.log("\n\n")
self.log(msg)
self.log("\n\n")
def plot_single_kshot_for_exps(kshot, folders, subfolders, exp_type, num_epochs=None):
plt.clf()
step_size = 25
for (subfolder, folder) in zip(subfolders, folders):
print(kshot, subfolder)
acc = util.load_file(folder, 'inter_accuracy44')
x = np.arange(1, len(acc) * step_size, step_size * SMOOTHING)
if subfolder=='VGG19' and kshot == 9:
k_acc = [a[6] for a in acc]
elif subfolder=='VGG19' and kshot == 7:
k_acc = [a[5] for a in acc]
else:
k_acc = [a[kshot] for a in acc]
chunked = chunk(k_acc, SMOOTHING)
if num_epochs:
comb = list(filter(lambda y: y[0] <= num_epochs, list(zip(x, chunked))))
x = [c[0] for c in comb]
chunked = [c[1] for c in comb]
assert len(x) == len(chunked)
if subfolder in labels:
label = labels[subfolder]
else:
label = subfolder
plt.plot(x, chunked, label=label, linewidth=2, marker='.', markersize=6)
if exp_type == 'difficulty':
plt.plot(x, [baselines[subfolder][kshot]] * len(chunked), label='Baseline')
else:
x = np.arange(1, num_epochs, step_size * SMOOTHING)
# set_trace()
plt.plot(x, [baselines['all'][kshot]] * len(chunked), label='Baseline')
plt.ylim(0.0, 0.8)
plt.xlabel('Number of epochs')
plt.ylabel('% Accurate Test Set')
plt.title("{}-Shot Accuracy".format(kshot))
plt.legend(loc=2)
#plt.show()
plt.savefig('plots/{}_{}.png'.format(exp_type, kshot))
def on_btnSqlComment_clicked(self,widget):
'''코멘트를 이용하여 SQL을 생성함.'''
tb = self.tbSql
mydict = util.load_file("dict.txt")
assert isinstance(tb , gtk.TextBuffer)
s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
s = s.strip().upper()
prefix = None
if s.find('.') >= 0:
(prefix, s ) = s.split('.')
constr = self._get_con_str()
self.log('연결문자열:' + constr + '\n')
self.log('테이블명 :' + s + '\n')
m = utildb.get_comment_info(constr, s.upper())
#self.log(crud.mk_insert(cols))
#self.log('\n\n')
self.log(crud2.mk_insert2(mydict,m['dic'],m['cols']))
self.log('\n\n')
self.log(crud2.mk_update(mydict, m['dic'],m['cols']))
self.log('\n\n')
self.log(crud2.mk_select(mydict, m['dic'],m['cols'],prefix))
self.log('\n\n')
self.log(crud2.mk_select_insert(mydict, m['dic'],m['cols']))
self.log('\n\n')
cleaned_text = re.sub(remove, '', cleaned_text)
tokens = cleaned_text.split()
tokens = [word.lower() for word in tokens if word != '' and word.isalpha()]
return tokens
def sequence_tokens(tokens, sequence_length=50):
length = sequence_length + 1
sequences = []
token_length = len(tokens)
for i in range(length, token_length):
sequence = tokens[i - length:i]
sequences.append(' '.join(sequence))
return sequences
def save_sequences(sequences, filename):
data = '\n'.join(sequences)
with open(filename, 'w') as file:
file.write(data)
if __name__ == "__main__":
filename = 'plato.txt'
text = load_file(filename)
save_sequences(sequence_tokens(clean_tokenize_text(text)), 'sequences.txt')
np.random.seed(231)
################################################################################
# Load CV Data
################################################################################
print("\nReading data ...")
DATA_DIR = 'data/CVSplits/'
num_folds = 5
data_cv = []
label_cv = []
max_col_prior = 0
# First get what is the maximum number of features across all folds
for i in range(num_folds):
_, _, max_col_prior = util.load_file(
DATA_DIR + 'training0' + str(i) + '.data', max_col_prior)
#print(max_col_prior)
for i in range(num_folds):
data_fold, label_fold, max_col_prior = util.load_file(
DATA_DIR + 'training0' + str(i) + '.data', max_col_prior)
data_cv.append(data_fold)
label_cv.append(label_fold)
################################################################################
# Load Train and Test Data
################################################################################
DATA_DIR = 'data/'
data_tr, label_tr, max_col_prior = util.load_file(DATA_DIR + 'train.liblinear',
max_col_prior)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--src_path', type=str, required=True)
parser.add_argument('--output_path', type=str, required=True)
parser.add_argument('--conceptnet_path', type=str, required=True)
args = parser.parse_args()
src_path = args.src_path
output_path = args.output_path
conceptnet_path = args.conceptnet_path
print("Loading src and output files from ", src_path, output_path)
src = load_file(src_path)
output = load_file(output_path)
assert len(src) == len(output)
# load conceptnet
print("loading conceptnet from {0}...".format(conceptnet_path))
CN = load_pickle(conceptnet_path)
CN_concepts = set(get_all_entities(CN))
CN_concept_pairs = set([(h, t) for h, r, t, w in CN])
print("number of conceptnet triplets: {0}".format(len(CN)))
print("number of conceptnet entities: {0}".format(len(CN_concepts)))
# load stopwords
stopwords = load_pickle("./data/KB/stopwords.pkl")
print()
# emotion_lexicon = load_pickle(emotion_lexicon_file)
CN = load_pickle(conceptnet_path) # the augmented conceptnet
associated_concepts = defaultdict(list)
for h,r,t,w in tqdm(CN):
associated_concepts[h].append((r,t,w))
# vocab_embedding = torch.load(dataset_vocab_embedding_path) # (vocab, emb_dim)
# load vocab
print("Loading dataset vocab from ", dataset_vocab_path)
vocab_ckpt = torch.load(dataset_vocab_path)
word2id = vocab_ckpt["src"].base_field.vocab.stoi
id2word = vocab_ckpt["src"].base_field.vocab.itos
print("dataset vocab size: ", len(word2id))
all_words = list(word2id.keys())
src_concepts = load_file(src_concept_file)
new_word_ids = []
new_word_scores = []
new_word_VAD_scores = []
unemotional_words_count = []
for sent_word_ids, sent_word_scores, sent_word_VAD_scores, concepts, emotion in \
tqdm(zip(concept_words[0], concept_words[1], concept_words[2], src_concepts, emotions), total=len(concept_words[0])):
qualified = []
emotions_to_exclude = [e for e in id2emotion if e != id2emotion[emotion]]
neighbor_hood_size = 1
while len(set([t for r,t,w in qualified])) < num_unemotional_words:
if neighbor_hood_size == 4:
# print("only {0} concepts".format(len(set([t for r,t,w in qualified]))))
break
print("loading conceptnet from {0}...".format(conceptnet_path))
CN = load_pickle(conceptnet_path)
CN_concepts = set(get_all_entities(CN))
print("number of conceptnet triplets: {0}".format(len(CN)))
print("number of conceptnet entities: {0}".format(len(CN_concepts)))
# load stopwords
stopwords = load_pickle("./data/KB/stopwords.pkl")
# load data
for split in ["train", "valid", "test"]:
if dataset == "reddit":
src_path = "./data/Reddit/{0}-src{1}.txt".format(
split, "-smaller" if smaller else "")
elif dataset == "twitter":
src_path = "./data/Twitter/{0}-src.txt".format(split)
print("Loading data from {0}".format(src_path))
src = load_file(src_path)
print("extracting concepts from dataset...")
src_concepts = [
get_concepts(line, CN_concepts, stopwords) for line in tqdm(src)
]
# save augmented conceptnet
src_concepts_path = src_path.replace(".txt", "-concepts.txt")
print("saving src concepts to {0}...".format(src_concepts_path))
with open(src_concepts_path, "w") as f:
for concepts in src_concepts:
f.write(", ".join(concepts) + "\n")
def analyze_perturbations():
baseline = util.load_file('baseline_fc_plots', 'baseline_aps')
ladder = util.load_file('ladder_fc_plots', 'ladder_aps')
for (i, c) in enumerate(['src', 'target']):
for l in range(7):
ladder[c][l] = 1.0 / (ladder[c][l]**2)
baseline[c][l] = 1.0 / (baseline[c][l]**2)
ladder[c][7] = i
baseline[c][7] = i
for l in range(7):
ladder_denom = ladder['src'][l] + ladder['target'][l]
baseline_denom = baseline['src'][l] + baseline['target'][l]
ladder['src'][l] = ladder['src'][l] / ladder_denom
ladder['target'][l] = ladder['target'][l] / ladder_denom
baseline['src'][l] = baseline['src'][l] / baseline_denom
baseline['target'][l] = baseline['target'][l] / baseline_denom
data = 5 * np.ones((16, 16))
set_trace()
eprint(ladder)
for (i, c) in enumerate(['src', 'target']):
for l in range(1, 8):
x_start = 2 * (8 - l) + 1
y_start = 9 * i + 2
data[x_start, y_start:y_start + 3] = ladder[c][l] * np.ones((1, 3))
light_color = [sns.light_palette((210, 90, 60), input="husl").as_hex()[0]]
cmap = ListedColormap(
sns.color_palette("Blues", 10).as_hex() + ['#000000'])
clipped_cmap = ListedColormap(sns.color_palette("Blues", 10).as_hex())
cbar_kws = {'ticks': np.arange(0, 1.2, 0.2)}
ax = sns.heatmap(data,
vmin=0,
vmax=1.2,
cmap=cmap,
xticklabels=False,
yticklabels=False,
cbar_kws=cbar_kws)
plt.xlabel('Source Target')
plt.savefig('ladder_aps.png')
plt.clf()
data = 5 * np.ones((16, 16))
eprint(baseline)
for (i, c) in enumerate(['src', 'target']):
for l in range(1, 8):
x_start = 2 * (8 - l) + 1
y_start = 9 * i + 2
data[x_start, y_start:y_start + 3] = baseline[c][l] * np.ones(
(1, 3))
light_color = [sns.light_palette((210, 90, 60), input="husl").as_hex()[0]]
cmap = ListedColormap(
sns.color_palette("Blues", 10).as_hex() + ['#000000'])
clipped_cmap = ListedColormap(sns.color_palette("Blues", 10).as_hex())
cbar_kws = {'ticks': np.arange(0, 1.2, 0.2)}
ax = sns.heatmap(data,
vmin=0,
vmax=1.2,
cmap=cmap,
xticklabels=False,
yticklabels=False,
cbar_kws=cbar_kws)
plt.xlabel('Source Target')
plt.savefig('baseline_aps.png')
pre_process = False
if "--pre" in sys.argv:
pre_process = True
is_train = False
if "--train" in sys.argv:
is_train = True
elif "--test" in sys.argv:
is_train = False
if pre_process:
util.pre_process(is_train)
print('pre-preocess done!!')
images, labels = util.load_file(is_train)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
xs = tf.placeholder(tf.float32, shape = [util.BATCH_SIZE, util.IMAGE_HEIGHT, util.IMAGE_WIDTH, 3])
ys = tf.placeholder(tf.float32, shape = [util.BATCH_SIZE, My_Network.OUTPUT_SIZE])
my_net = My_Network.Mynetwork()
prediction = my_net.model(xs)
loss = my_net.loss_func(prediction, ys)
correct = my_net.count_correct(prediction, ys)
training = my_net.train(LEARNING_RATE, loss)
# run training
with tf.Session() as sess:
"""
Classes and functions for uploading to the Small Family archive
"""
import json
import boto3
from util import load_file, handle_datetime
UPLOAD_POLICY = load_file('archive_upload_policy.json')
SESSION_DURATION = 900
class TokenRetriever(object):
def __init__(self, stsClient):
self.stsClient = stsClient
def getToken(self, name):
"""
Gets a federation token as described at
https://boto3.readthedocs.org/en/latest/reference/services/sts.html#STS.Client.get_federation_token.
The result of a successful API call is returned as JSON.
"""
return json.dumps(self.stsClient.get_federation_token(Name=name,
Policy=UPLOAD_POLICY,
DurationSeconds=SESSION_DURATION),
default=handle_datetime)
concept_pairs_path = args.concept_pairs_path
two_step_PPMI = True
# load data
if dataset == "reddit":
src_path = "./data/Reddit/train-src{0}.txt".format("-smaller" if smaller else "")
tgt_path = "./data/Reddit/train-tgt{0}.txt".format("-smaller" if smaller else "")
tgt_emotion_path = "./data/Reddit/train-tgt{0}-emotions.txt".format("-smaller" if smaller else "")
elif dataset == "twitter":
src_path = "./data/Twitter/train-src.txt"
tgt_path = "./data/Twitter/train-tgt.txt"
tgt_emotion_path = "./data/Twitter/train-tgt-emotions.txt"
print("Loading data from {0}".format([src_path, tgt_path, tgt_emotion_path]))
src = load_file(src_path)
tgt = load_file(tgt_path)
tgt_emotions = load_file(tgt_emotion_path)
tgt_emotions = [id2emotion[int(emo[0])] for emo in tgt_emotions]
assert len(src) == len(tgt) and len(src) == len(tgt_emotions)
print("number of conversation pairs: {0}".format(len(src)))
# load conceptnet
print("loading conceptnet from {0}...".format(conceptnet_path))
CN = load_pickle(conceptnet_path)
CN_concepts = set(get_all_entities(CN))
print("number of conceptnet triplets: {0}".format(len(CN)))
print("number of conceptnet entities: {0}".format(len(CN_concepts)))
# load stopwords
stopwords = load_pickle("./data/KB/stopwords.pkl")
def load_dataset(self, date):
return load_file(date.year, date.month)
def main():
"""
Runs the entire process of computing reaction forces from raw input to
final output. The file name containing the raw data should be given
as the only argument. The various options are read from the configuration
file. Since the dimension might be changed often, you can override the
dimension on the command line.
"""
print 20*'+'+'TRACKING FORCES'+21*'+'
# Get the input file(s) with the raw data.
parser = argparse.ArgumentParser(description='Compute reaction forces from\
whisker tracking images.')
parser.add_argument('data_file', help=".mat file with tracked image data")
parser.add_argument('--dim', type=int, default=-1)
args = parser.parse_args()
if os.path.splitext(args.data_file)[1]:
data_file = os.path.splitext(args.data_file)[0]
else:
data_file = args.data_file
# Read the options from the configuration file. If no configuration file,
# use default.
file_name = None
for files in os.listdir('.'):
if files.endswith('.cfg'):
file_name = files
if file_name is None:
file_name = resource_filename(Requirement.parse("tracking_forces"),
'tracking_forces/tracking_forces.cfg')
config = ConfigParser.SafeConfigParser()
config.read([file_name])
# Create a folder to save the outputs.
output_dir = './%s/' %data_file
if not os.path.exists(output_dir):
os.makedirs(output_dir)
dim = args.dim
if dim < 0:
dim = config.getint('general', 'dimension')
mm_per_pixel = config.getfloat('general', 'mm_per_pixel')
scale = mm_per_pixel/1000
dt = config.getfloat('general', 'dt')
# Load the raw data, sample the images based on the number of links and
# convert to configurations in terms of angles.
if dim == 2:
try:
variable_names = {'x': config.get('convert', 'mat_xname_2d'),
'y': config.get('convert', 'mat_yname_2d'),
'z': config.get('convert', 'mat_zname_2d'),
'cp': config.get('convert', 'mat_cpname_2d')}
except:
variable_names = {'x': 'xw', 'y': 'yw', 'z': None, 'cp': 'CP'}
else:
try:
variable_names = {'x': config.get('convert', 'mat_xname_3d'),
'y': config.get('convert', 'mat_yname_3d'),
'z': config.get('convert', 'mat_zname_3d'),
'cp': config.get('convert', 'mat_cpname_3d')}
except:
variable_names = {'x': 'xw3d', 'y': 'yw3d', 'z': 'zw3d', 'cp': 'CP'}
conversion_args = (data_file+'.mat', scale, dim,
config.getint('convert', 'N'),
config.getint('convert','start'),
config.getint('convert','stop'),
config.getint('convert','convert_k'),
config.getfloat('convert','convert_s'),
config.get('convert', 'rx0_method'),
True, output_dir, variable_names)
# Perform a new conversion.
if config.getboolean('convert', 'new_conversion'):
converted_data = convert_frames(*conversion_args)
# Load previously converted data.
else:
try:
converted_data = load_file('%s%s.p'%(output_dir,data_file))
except:
print "WARNING: Converted data not found. Performing new conversion."
converted_data = convert_frames(*conversion_args)
# Get the reference shape (index of frame where whisker is undeformed).
ref = config.getint('filter', 'ref_index')
if ref < 0:
ref = get_reference_frame(converted_data['CP'])
# Filter the trajectories.
filter_args = (data_file+'.p', output_dir, dt, dim, ref,
config.get('filter', 'filter_type'),
config._sections['filter'])
filtered_data = filter_data(*filter_args)
# Build the whisker system in trep.
whisker = make_whisker(dim, filtered_data['ref'],
converted_data['link_length'],
config.getfloat('whisker', 'rbase'),
config.getfloat('whisker', 'taper'),
config.getfloat('whisker', 'damping_ratio'),
config.getfloat('whisker', 'rho'),
config.getfloat('whisker', 'E'))
# Compute the reaction forces and moments.
forces_args = (whisker, filtered_data['q'],
filtered_data['v'],
filtered_data['a'],
filtered_data['cp'], dt,
data_file, output_dir,
config.getboolean('forces','overwrite'))
force_data = calc_forces(dim, *forces_args)
# Plot the results.
if config.getboolean('plot', 'run_plot'):
plot_args = ('%s_forces.p' %(data_file), output_dir, './%s.mat' %data_file,
config.getboolean('plot', 'show_plots'))
plot_and_save(dim, *plot_args)
# Animate the whisker's motion.
if config.getboolean('animate', 'run_animation'):
animate_args = (data_file, output_dir, dt,
config.getboolean('animate', 'show_animation'),
config.getboolean('animate', 'save_animation'),
config.getboolean('animate', 'debug_conversion'))
animate_whisker(dim, *animate_args)
save_output_file(output_dir, data_file)
def load_sim_file(self):
self.simi_data = util.load_file(save_file)
pre_process = False
if "--pre" in sys.argv:
pre_process = True
is_train = False
if "--train" in sys.argv:
is_train = True
elif "--test" in sys.argv:
is_train = False
if pre_process:
image_folder_list, label_folder_list = util.produce_path(is_train)
util.pre_process(image_folder_list, label_folder_list)
print('pre-preocess done!!')
images, labels = util.load_file()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
my_net = My_Network.Mynetwork()
training = my_net.train(images, labels)
# run training
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
if is_train:
for i in range(ITERATION):
<BindItem id="item11" compid="Edit08" propid="value" datasetid="ds_detail" columnid="FXNUM"/>
<BindItem id="item12" compid="edt_acnutNo" propid="value" datasetid="ds_detail" columnid="ACNUT_NO"/>
<BindItem id="item13" compid="edt_acnutOwnerNm" propid="value" datasetid="ds_detail" columnid="ACNUT_OWNER_NM"/>
<BindItem id="item14" compid="cbo_bank" propid="value" datasetid="ds_detail" columnid="DELNG_BANK_CODE"/>
<BindItem id="item15" compid="Edit06" propid="value" datasetid="ds_detail" columnid="TLPHON_NO"/>
<BindItem id="item16" compid="Edit07" propid="value" datasetid="ds_detail" columnid="MOBLPHON_NO"/>
<BindItem id="item17" compid="Edit03" propid="value" datasetid="ds_detail" columnid="ADRES"/>
</Bind>
'''
s = '''
<Cell col="1" text="bind:BUF_GRAD_CODE_NM" editlimit="-1"/>
<Cell col="2" displaytype="number" edittype="masknumber" text="bind:MCT_POCTPAYM__RATE" mask="99%" editlimit="2" editlimitbymask="both"/>
<Cell col="3" displaytype="number" edittype="masknumber" text="bind:MCT_DRYGRASS_SILAGE_WGHTVAL" mask="9.9" editlimit="3" editlimitbymask="both" suppress="0"/>
<Cell col="4" displaytype="number" edittype="masknumber" text="bind:MCT_SPORT_TON_AMOUNT" mask="99,999" editlimit="5" editlimitbymask="both" combodisplayrowcount="5"/>
</Band>
'''
print ''
mydict = util.load_file("dict.txt")
o = BindItemVar(s, mydict)
print o.getVarStr();
print ''
print ''
print o.getChkStr()
print ''
print o.getChkStr2()
print ''
print o.getChkStr3()
selection: """
# list of readLines when we first loaded the file. This way if we need to start the decipher over we can without
# reading the file
file_contents = []
# list to be manipulated
cipher_text_list = []
# keep a history of changes for further analysis later
history = History()
substitution = Substitution()
poly_alphabetic = PolyAlphabetic()
selection = ""
while selection != "x":
selection = input(menu)
if selection == "1":
file_contents = util.load_file(history)
cipher_text_list = file_contents
print("Done")
if selection == "2":
# reset everything but history
cipher_text_list = file_contents
history.append("resetting cipher to", str(cipher_text_list))
substitution = Substitution()
print("Done")
if selection == "3":
history.display()
if selection == "4":
history.save(input("output file to save to: "))
if selection == "5":
util.find_pattern(cipher_text_list, history)
if selection == "6":
def test_reportsettings_merge_basic(info):
info.run_merge()
exp = util.load_file(info.expected)
res = util.load_file(info.result)
util.file_compare(exp, res)
info.cleanup()
import util
from keyword_anaylze import keyword_anaylze
date = "2015-12-02"
topic_loc = "../Resources/topic/"
result_loc = "../Resources/result/"
if __name__ == "__main__":
global topic_loc, result_loc
global date
topic = util.load_file(topic_loc + date)
for tp in topic:
print(tp)
ka = keyword_anaylze(date)
senti, news, network, counter = ka.anaylze(tp)
f = open(result_loc + tp, "w")
f.write("{0:.1f}% (p:{1}, n:{2})\n\n".format(
senti[1] / (senti[1] + senti[0]) * 100, senti[1], senti[0]))
# write counter of each community
f.write("positive\n")
for i in range(8):
f.write("{0}. {1}\n".format(i, counter[2 * i + 1]))
f.write("negative\n")
for i in range(8):
f.write("{0}. {1}\n".format(i, counter[2 * i]))
def make_stereogram(depthmap, pattern, color):
dm_img = util.load_file(depthmap)
dm_img = util.redistribute_grays(dm_img, FORCE_DEPTH)
pattern_width = int((dm_img.size[0] / PATTERN_FRACTION))
canvas_img = im.new(mode="RGB",
size=(dm_img.size[0] + pattern_width, dm_img.size[1]),
color=color)
pattern_strip_img = im.new(mode="RGB",
size=(pattern_width, dm_img.size[1]),
color=(0, 0, 0))
pattern_raw_img = util.load_file(pattern)
p_w = pattern_raw_img.size[0]
p_h = pattern_raw_img.size[1]
pattern_raw_img = pattern_raw_img.resize(
(pattern_width, (int)((pattern_width * 1.0 / p_w) * p_h)), im.LANCZOS)
region = pattern_raw_img.crop(
(0, 0, pattern_raw_img.size[0], pattern_raw_img.size[1]))
y = 0
while y < pattern_strip_img.size[1]:
pattern_strip_img.paste(
region,
(0, y, pattern_raw_img.size[0], y + pattern_raw_img.size[1]))
y += pattern_raw_img.size[1]
# Oversample. Smoother results.
dm_img = dm_img.resize(((int)(dm_img.size[0] * OVERSAMPLE),
(int)(dm_img.size[1] * OVERSAMPLE)))
canvas_img = canvas_img.resize(((int)(canvas_img.size[0] * OVERSAMPLE),
(int)(canvas_img.size[1] * OVERSAMPLE)))
pattern_strip_img = pattern_strip_img.resize(
((int)(pattern_strip_img.size[0] * OVERSAMPLE),
(int)(pattern_strip_img.size[1] * OVERSAMPLE)))
pattern_width = pattern_strip_img.size[0]
def shift_pixels(dm_start_x, depthmap_image_object, canvas_image_object,
direction):
depth_factor = pattern_width * SHIFT_RATIO
cv_pixels = canvas_image_object.load()
while 0 <= dm_start_x < dm_img.size[0]:
for dm_y in range(depthmap_image_object.size[1]):
constrained_end = max(
0,
min(dm_img.size[0] - 1,
dm_start_x + direction * pattern_width))
for dm_x in range(int(dm_start_x), int(constrained_end),
direction):
dm_pix = dm_img.getpixel((dm_x, dm_y))
px_shift = int(dm_pix / 255.0 * depth_factor *
(1 if WALL else -1)) * direction
if direction == 1:
cv_pixels[dm_x + pattern_width,
dm_y] = canvas_img.getpixel(
(px_shift + dm_x, dm_y))
if direction == -1:
cv_pixels[dm_x, dm_y] = canvas_img.getpixel(
(dm_x + pattern_width + px_shift, dm_y))
dm_start_x += direction * pattern_strip_img.size[0]
dm_center_x = dm_img.size[0] / 2
canvas_img.paste(pattern_strip_img,
(int(dm_center_x), 0, int(dm_center_x + pattern_width),
canvas_img.size[1]))
if not WALL:
canvas_img.paste(pattern_strip_img,
(int(dm_center_x - pattern_width), 0,
int(dm_center_x), canvas_img.size[1]))
shift_pixels(dm_center_x, dm_img, canvas_img, 1)
shift_pixels(dm_center_x + pattern_width, dm_img, canvas_img, -1)
if pattern:
canvas_img = canvas_img.resize(
((int)(canvas_img.size[0] / OVERSAMPLE),
(int)(canvas_img.size[1] / OVERSAMPLE)),
im.LANCZOS) # NEAREST, BILINEAR, BICUBIC, LANCZOS
return canvas_img
import util
from keyword_anaylze import keyword_anaylze
date = "2015-12-02"
topic_loc = "../Resources/topic/"
result_loc = "../Resources/result/"
if __name__ == "__main__":
global topic_loc, result_loc
global date
topic = util.load_file(topic_loc+date)
for tp in topic:
print(tp)
ka = keyword_anaylze(date)
senti, news, network, counter = ka.anaylze(tp)
f = open(result_loc+tp, "w")
f.write("{0:.1f}% (p:{1}, n:{2})\n\n" .format(senti[1]/(senti[1]+senti[0])*100, senti[1], senti[0]))
# write counter of each community
f.write("positive\n")
for i in range(8):
f.write("{0}. {1}\n" .format(i, counter[2*i+1]))
f.write("negative\n")
for i in range(8):
f.write("{0}. {1}\n" .format(i, counter[2*i]))
for n in news:
# -*- coding: utf-8 -*-
import re
from util import load_random_substring, load_file
random_file_content = load_random_substring(load_file(), 5000)
lines = random_file_content.split('\n')[1:-1]
words = re.compile(r'\s').split(random_file_content)
print([(len(w), w) for w in words])
def test_sv_merge_basic(info):
info.run_merge()
exp = util.load_file(info.expected)
res = util.load_file(info.result)
util.file_compare(exp, res, sv_flattenify)
info.cleanup()
def test_lv_parse(ff):
exp = util.load_file(ff)
LV = fromfn(ff)
res = LV.toxml()
util.file_compare(exp, res)
limit_max = 80
np.random.seed(231)
################################################################################
# Load CV Data
################################################################################
DATA_DIR = 'dataset/CVSplits/'
num_folds = 5
data_cv = []
label_cv = []
max_col_prior = 0
# First get what is the maximum number of features across all folds
for i in range(num_folds):
_, _, max_col_prior = util.load_file(
DATA_DIR + 'training0' + str(i) + '.data', max_col_prior)
#print(max_col_prior)
for i in range(num_folds):
data_fold, label_fold, max_col_prior = util.load_file(
DATA_DIR + 'training0' + str(i) + '.data', max_col_prior)
data_cv.append(data_fold)
label_cv.append(label_fold)
################################################################################
# Load Train, Dev and Test Data
################################################################################
DATA_DIR = 'dataset/'
data_tr, label_tr, max_col_prior = util.load_file(DATA_DIR + 'diabetes.train',
max_col_prior)
if (n == m):
print(res)
else:
print(res[n - m:m - n])
def tester_acorr(dt, n, m):
A = dt_in[0:n]
res = auto_corr_opt(A, n, m)
print(res)
res = np.correlate(A, A, "full")
if (n == m):
print(res)
else:
print(res[n - m:m - n])
if __name__ == '__main__':
fn = "crosscorr.in"
dt_in = util.load_file(fn)
dt_in = [x + y * 1j for x, y in dt_in]
#tester_xcorr(dt_in, 5, 3)
#tester_xcorr(dt_in, 16, 16)
#tester_xcorr(dt_in, 32, 32)
#tester_xcorr(dt_in, 32, 16)
#tester_xcorr(dt_in, 39, 25)
tester_acorr(dt_in, 16, 16)
import argparse
from util import load_file
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--path',
required=True,
type=str,
help='The path to the output file')
args = parser.parse_args()
path = args.path
lines = load_file(path)
avg_length = len([w for l in lines for w in l]) / len(lines)
print("Average sentence length: {0}".format(avg_length))
