def addEntry(self):
print "New Entry..."
name = raw_input("Entry name : ")
if not name:
print "name cant be blank"
return
if self.entries.has_key(name):
print "An entry with the name ", name ," exists already"
return False
username = raw_input("Username : "******"Genrate password? (Y/n)")
if genPass == False:
password = raw_input("Password : "******"Password : "******"URL : ")
newEntry = entry.Entry(name, username, password, "internet", URL)
self.entries[name]= newEntry
self.dbChanged = True
copyToClipboard = utils.choice("Copy password to clipboard? (Y/n)")
print "\n"
return True;
def load_data(self):
print('loading data')
dataset = pkl.load(open('data/{}.pkl'.format(FLAGS.dataset), 'rb'))
self.train_data = dataset['train_data']
self.test_data = dataset['test_data']
FLAGS.threshold = self.train_data[:, 2].mean()
print('setting threshold =', FLAGS.threshold)
adj_mats = dataset['adj']
if FLAGS.kg:
support, self.entity_map = get_support(adj_mats, self.train_data,
self.test_data)
else:
support, self.entity_map = get_support(None, self.train_data,
self.test_data)
self.n_entity = len(self.entity_map)
adj_list, _ = support_to_adj_list(self.n_entity, support)
self.adj_list = []
for al in adj_list:
if len(al) >= FLAGS.max_degree:
self.adj_list.append(choice(al, FLAGS.max_degree))
else:
if len(al):
self.adj_list.append(
choice(al, FLAGS.max_degree, replace=True))
else:
self.adj_list.append([(0, 0)] * FLAGS.max_degree)
self.adj_list = np.array(self.adj_list, dtype=np.int32)
def editEntry(self, name):
if not self.entries.has_key(name):
print 'No entry with name ', name
return
print "Edit Entry ", name
print "Press enter to no edit field"
username = raw_input('Username : '******'Password : '******'URL : ')
if URL == "":
URL = self.entries[name].URL
tag = "internet"
change = entry.Entry(name, username, password,tag, URL)
change.display(True)
modify = utils.choice("complete edit ? (Y/n)")
if modify:
self.entries[name].edit(change)
print "Editted entry ", name
self.dbChanged = True;
def __init__(self, fact_obj, airport_list, flight_number,
ref_departure_date, ref_return_date):
# 1. origin and destination
self.origin = random.randint(0, len(airport_list) - 1)
self.dest = random.randint(0, len(airport_list) - 1)
if self.dest == self.origin:
self.dest = (self.dest + 1) % len(airport_list)
self.dest = airport_list[self.dest]
self.origin = airport_list[self.origin]
# 2. date and time
d1 = np.random.normal(ref_departure_date, fact_obj.time_deviation,
1)[0]
d2 = np.random.normal(ref_return_date, fact_obj.time_deviation, 1)[0]
# makes ure that departure date comes first
if d1 < d2:
self.departure_date = d1
self.return_date = d2
else:
self.departure_date = d2
self.return_date = d1
# assert self.departure_date <= self.return_date
# assert self.return_date - self.departure_date <= 3600*24*365/2
# 3. class
self.flight_class = utils.choice(fact_obj.class_member,
1,
p=fact_obj.class_prior)[0]
# 4. connections
self.connection = utils.choice(fact_obj.connection_member,
1,
p=fact_obj.connection_prior)[0]
# 5. price
mean_base_price = fact_obj.flight_price_mean[self.flight_class]
base = mean_base_price * fact_obj.low_cost_mean_fraction[
self.flight_class]
self.price = int(
np.random.normal(
base, base * fact_obj.flight_price_norm_std[self.connection]))
self.price = utils.discrete_price(self.price)
# 6. flight number
# self.flight_number = random.randint(1000, 9999)
self.flight_number = flight_number
# 7. airline
airline_ind = random.randint(0, len(fact_obj.airline_list) - 1)
self.airline = fact_obj.airline_list.keys()[airline_ind]
# post process
self.departure_month, self.departure_day = utils.get_month_and_day(
fact_obj, self.departure_date)
self.return_month, self.return_day = utils.get_month_and_day(
fact_obj, self.return_date)
self.departure_time_num = utils.get_hour(self.departure_date)
self.return_time_num = utils.get_hour(self.return_date)
def getVal_text(self, mode):
if mode == 'right' and self.cloak_only:
print('Right mode can not be used in cloak only mode')
return
if not self.__chars_count:
with open(self.__column_path, "r") as read_file:
self.__chars_count = json.load(read_file)
self.__email = self.is_email()
options = ["synthetic", "fromCloak"]
prob = [
self.__chars_count["hidden"] / self.__chars_count["total_count"],
1 -
(self.__chars_count["hidden"] / self.__chars_count["total_count"])
]
option = utils.choice(options, p=prob)
if option == "synthetic":
generated_str = self.generateString(mode)
while self.__email and not utils.is_valid_address(generated_str):
generated_str = self.generateString(mode)
if self.__email:
generated_str = generated_str.replace('.....', '.')
generated_str = generated_str.replace('....', '.')
generated_str = generated_str.replace('...', '.')
generated_str = generated_str.replace('..', '.')
generated_str = generated_str.replace('.@', '@')
generated_str = generated_str.replace('@.', '@')
if generated_str[-1] == ".":
generated_str = generated_str[:-1]
else:
generated_str = utils.getRandomString(
self.__chars_count["strings"], self.__chars_count["counts"])
return generated_str
def main_menu(name):
while True:
choice = utils.choice('Main Menu', [
'Games', 'Harry Potter Games', 'Programs',
'Run Your Own Python Code'
], 'What do you want to do?')
if choice == 0:
games(name)
elif choice == 1:
hpgames(name)
elif choice == 2:
programs(name)
elif choice == 3:
print('===== Python Runner =====')
print('Enter your Python code below. Leave blank to end.\n')
code = ''
while True:
line = input('> ')
if line == '':
break
else:
code += line + '\n'
print('\nRunning This:')
print(code)
print('-----')
exec(code)
def programs(name):
rooms = ['this']
choice = utils.choice('Games', ['[Back]', 'Zen of Python'],
'What do you want to do?')
if choice == 0: return
exec('import ' + rooms[choice - 1])
def hpgames(name):
rooms = [
'this', 'hpgames.selection', 'hpgames.hp_game', 'hpgames.mm',
'hpgames.story', 'hpgames.hpapi', 'hpgames.elflife', 'hpgames.pf'
]
choice = utils.choice('Games', [
'[Back]', 'Zen of Python', 'House Selection', 'Journey of Options',
'Muggle Madness', 'Madlibs Story', 'HP Trivia Game', 'HP Elf Life',
'Patronus Finder'
], 'What do you want to do?')
if choice == 0: return
exec('import ' + rooms[choice - 1])
def getVal_numeric(self, low, high):
if not self.__length_frequency:
with open(self.__column_path, "r") as read_file:
self.__length_frequency = json.load(read_file)
self.__length_frequency.pop('cloak_only', None)
lengths = list(self.__length_frequency.keys())
counts = list(self.__length_frequency.values())
lengths = np.array(lengths, dtype=np.int64)
counts = np.array(counts, dtype=np.float64)
counts /= counts.sum()
values = np.linspace(low, high)
generated = []
for x in range(len(values)):
length = utils.choice(lengths, p=counts)
value = np.round(utils.getRandomVal(values),
length) if length > 0 else int(
utils.getRandomVal(values))
generated.append(value)
return utils.getRandomVal(generated)
def sample_word(self, to_N, sampling_error_samples):
# empty targets...
if to_N == -1:
return self.word
# print("word: {}".format(self.word))
if to_N in self.target_samples:
possibilities = self.target_samples[to_N]
if sampling_error_samples == 'uniform':
i = random.randint(0, len(possibilities) - 1)
return possibilities[i]
# sample word by probability ....
wrds = []
weights = []
for poss in possibilities:
w, p = poss
wrds.append(w)
weights.append(1 / p)
# if self.word not in w.split():
# narrower.append((w, p))
# print(poss)
weights = [float(i) / max(weights) for i in weights]
selected = choice(wrds, weights)
return selected
# this won't happen is N i selected by Node function select_N
else:
self.logger.warning(
"for id:{} word:'{}' target_meta to_N[{}] not available".
format(self.sampling_id, self.word, to_N))
self.logger.warning("{}".format(self.target_samples))
# Just select randomly some other [available] key...
N_list = list(self.target_samples.keys())
i = random.randint(0, len(N_list) - 1)
N = N_list[i]
return self.sample_word(N, sampling_error_samples)
def select_N(self, sampling_N):
# select N
# we do it weighted.... the higher the N, the lower the probability
if len(self.target_samples) == 0:
# self.logger.warning("for id:{} word:'{}' empty targets, returning -1".format(self.sampling_id, self.word))
return -1
N_list = list(self.target_samples.keys()) # 1 2 4 ... 7
if sampling_N == 'uniform':
i = random.randint(0, len(N_list) - 1)
N = N_list[i]
else:
_sum = sum(N_list)
weights = [_sum - v + 1 for v in N_list]
N = choice(N_list, weights)
return N
def games(name):
rooms = [
'this', 'planet.main', 'games.swr1r', 'games.sw2e', 'games.sw2eJD',
'games.spacetrails', 'games.jd', 'games.sugar1', 'games.sugar2',
'games.dog2', 'games.trivia', 'games.rhyme', 'games.walker',
'games.cookie', 'games.murder', 'games.translator', 'games.orc',
'games.cataclone1', 'games.hangman'
]
choice = utils.choice('Games', [
'[Back]', 'Zen of Python', 'Planet Selector',
'Space Wars 1 - Retirement', 'Space Wars 2 - Escape (MAXs Version)',
'Space Wars 2 - Escape (JDs Version)', 'Space Trails',
'Star Wars Quiz', 'Sugar Bank 1', 'Sugar Bank 2', 'Dog Adventure 2',
'Trivia', 'Rhyme Adventure', 'Time Travel', 'Fortune Cookie',
'Murder Mystery', 'Translator', 'Orc', 'Cataclone', 'Hangman',
'Guess the Number'
], 'What do you want to do?')
if choice == 0: return
exec('import ' + rooms[choice - 1])
def run(self):
# For all sentences in the dataset ...
for s in input_sentence_generator:
source_doc_path, sentence_id, sentence = s
# We must add sentence to the graph
self.sentence_graph.set_sentence(sentence)
# Score of the original sentence (only for debug purposes ...)
if self.verbose:
if self.bert_lm is not None:
score = self.bert_lm.get_score(sentence)
self.logger.debug("LM[{:.2f}]{}".format(score, sentence))
else:
self.logger.debug("LM[ - ]{}".format(sentence))
# Now we generate multiple (cnt_sentence_samples) "noisified variants" from current sentence
avg_wer = 0
samples_list = []
# A. Crate samples
tries = -1
while True:
tries += 1
# all samples collected ( or we do not want to wait too long ... )
if len(
samples_list
) == self.cnt_sentence_samples or tries * 2 > self.cnt_sentence_samples:
break
debug, sample = self.sentence_graph.sample_sentence()
if self.bert_lm is not None:
score = self.bert_lm.get_score(sample)
else:
score = 1.
error = wer(sentence, sample)
if self.min_wer <= error <= self.max_wer:
avg_wer += error
samples_list.append((sample, score, error))
# DEBUG: print the all sentence the variants
if self.verbose:
for sam in samples_list:
sample, score, error = sam
if self.bert_lm is not None:
self.logger.debug(" LM[{:.2f}] WER[{:.2f}]{}".format(
score, error, sample))
else:
self.logger.debug(" LM[ - ] WER[{:.2f}]{}".format(
error, sample))
self.logger.debug("avg WER: {:.2f}".format(
avg_wer / self.cnt_sentence_samples))
# B. Finally we choose one sentence .....
if len(samples_list) == 0:
selected_sentence = sentence
else:
sentences = []
lm_weights = []
for s in samples_list: # (sentence, LM, WER)
sentences.append(s[0])
lm_weights.append(s[1])
selected_sentence = utils.choice(sentences, lm_weights)
# And we write it to file
source_doc = ntpath.basename(source_doc_path)
target_doc_path = os.path.join(self.base_target_dir, source_doc)
if os.path.isfile(target_doc_path):
newline = True
else:
newline = False
with open(target_doc_path, "a") as file:
if newline:
file.write("\n")
file.write(selected_sentence)
self.logger.info("All files successfully processed")
self.logger.info("Calculating WER on files...")
time.sleep(0.5)
return wer_over_files(self.input_source_dir, self.base_target_dir,
self.input_filename_list)
def sample_sentence(self):
"""
Example of how the final sentence is constructed:
0 1 2 3 4 5 6 7 8
original sentence: Hello THIS , is 123 welcome ??? there !
0 1 3 5 7
alpha sentence : Hello THIS is welcome there
0 1 3 5 7
synth sentence : [hellish ] [is this] [welcomere]
0 1 2 3 4 5 7 8
final sentecnce : [Hellish] , [is this] 123 [welcomere] !
we can see, that non alpha token '???' at position 6 in the original sentence have to be omitted
also the letter casing is forgotten
and finally, spacing between alpha and non-alpha characters (like punctuation symbols) will also be different
"""
current_node = self.start_node
debug_sentence = []
synth_sentence = []
while True:
if current_node.from_M_variant == -2: # ... '.'
# sentence.append(".")
break
if current_node.from_M_variant == -1:
pass # sentence.append("<eps> ")
elif current_node.from_M_variant == 0: # original word
debug_sentence.append(current_node.word + " ")
else: # sampled word ...
# sentence.append(">" + current_node.word + "<[{}] ".format(current_node.variant))
# we are sampling now ....
to_N_parts = current_node.select_N(self.sampling_N)
sampled_word = current_node.sample_word(
to_N_parts, self.sampling_error_samples)
synth_sentence.append(
(sampled_word, current_node.sentence_start_idx,
current_node.from_M_variant))
debug_sentence.append(
">" + sampled_word +
"<[{}->{}] ".format(current_node.from_M_variant,
to_N_parts)) # current_node.variant
# current_node.display()
if len(current_node.neighbours) == 1:
current_node = current_node.neighbours[0]
else:
current_node = choice(current_node.neighbours,
current_node.weights)
final_sentence = []
last_idx_in = -1
for synth_word in synth_sentence:
word, start_idx, cnt_words = synth_word
word = word.replace(".", "")
if start_idx == 0:
word = word.capitalize()
# fill in missing non-alpha words
while 1 + last_idx_in < start_idx:
last_idx_in += 1
final_sentence.append(self.original_words[last_idx_in])
# we fill current word
assert start_idx == 1 + last_idx_in, "safety check"
final_sentence.append(word)
# wrong... last_idx_in = last_idx_in + cnt_words # from 1 , from 2 ...
# because we can occasionally skip " this && is " => "thisis" ] and we need to check this ... here
# ( we actually moved 3 words forward, not 2 )
# we check it against the index in the alpha sentence
for final_idx, w_p_idx in enumerate(self.words_alpha_pos):
_, _, orig_idx = w_p_idx
if orig_idx == start_idx:
_, _, true_idx = self.words_alpha_pos[final_idx +
cnt_words - 1]
last_idx_in = true_idx
# synth word: welcomere(5)
# words alpha pos: welcome(5) there(7)
# last indx in : not 6
# but ... .... 7
break
# fill the end of the sentence
while last_idx_in + 1 < len(self.original_words):
last_idx_in += 1
final_sentence.append(self.original_words[last_idx_in])
final_sentence = " ".join(final_sentence)
# fix [ ( {
final_sentence = re.sub(
r'\s*([(\[{])\s*', r' \1',
final_sentence) # "is ( maybe ) good" -> "is (maybe ) good"
# fix } ) }
final_sentence = re.sub(
r'\s*([)\]\}])\s*', r'\1 ',
final_sentence) # "is (maybe ) good" -> "is (maybe) good"
# fix - @
final_sentence = re.sub(
r'\s*([@])\s*', r'\1',
final_sentence) # "hello - kitty" -> "hello-kitty"
# fix , . ; : ! ? % $
final_sentence = re.sub(
r'\s([?,.;!:%$](?:\s|$))', r'\1',
final_sentence) # " hello , Peter" -> hello, Peter
# fix ``
# final_sentence = re.sub(r'\s*(``)\s*', r' \1', final_sentence)
final_sentence = re.sub(r"(``)\s", r'"', final_sentence)
# fix ''
final_sentence = re.sub(r"\s(''(?:\s|$))", r'" ', final_sentence)
final_sentence = final_sentence.strip()
# def remove_s(final_sentence, sym):
# p1 = -1
# for i, s in enumerate(final_sentence):
# if s == sym:
# if p1 == -1:
# if i + 1 < len(final_sentence):
# if final_sentence[i + 1] == " ":
# if i == 0 or final_sentence[i - 1] == " ":
# p1 = i
# continue
# if p1 != -1:
# if final_sentence[i-1] == " " and i-2 != p1:
# final_sentence = final_sentence[0:p1+1] + final_sentence[p1+2: i-1] + final_sentence[i:]
# else:
# final_sentence = final_sentence[0:p1 + 1] + final_sentence[p1 + 2: ]
# break
# return final_sentence
#
# cnt = final_sentence.count('"')
# for i in range(math.floor(cnt/2)):
# final_sentence = remove_s(final_sentence, '"')
# 012345678
# A " B " C
# 2 6
# [0:3]
debug_sentence = "\t".join(debug_sentence)
return debug_sentence, final_sentence
def create_x_y(data, vocab, stats, test=False, verbose=False):
''' create input-output pairs for neural network training
the input is (#examples,
'''
data = filter_vocab(data, vocab, stats)
max_span = stats['max_span']
max_q = stats['max_q']
surround_size = stats['surround_size']
neg_samples = stats['neg_samples']
ivocab = create_idict(vocab)
X = []
verbose=0
def print_sentence(name, sen):
if verbose:
print(name, ' '.join([ivocab[v] for v in sen if v]))
def map_vocab(word):
if word in vocab:
return vocab[word]
else:
return vocab['<unk>']
try:
for paragraph in data:
context = paragraph['context.tokens']
all_spans = sum(paragraph['spans'], [])
for qa in paragraph['qas']:
# extract question.
q = np.zeros(max_q)
for (i, word) in enumerate(qa['question.tokens']):
if i >= len(q):
break
q[i] = map_vocab(word)
def extract(pos, span, is_answer=False):
if verbose:
print('is_answer', is_answer)
print_sentence('question', q)
# extract span.
s = np.zeros(max_span)
for (i, word) in enumerate(span):
if i >= len(s):
break
s[i] = map_vocab(word)
print_sentence('span', s)
# extract context left.
answer_start = pos
cl = np.zeros(surround_size)
cr = np.zeros(surround_size)
for i in range(surround_size):
ind = answer_start - 1 - i
if ind >= 0:
cl[i] = map_vocab(context[ind])
print_sentence('cl', cl)
for i in range(surround_size):
ind = answer_start + len(span) + i
if ind < len(context):
cr[i] = map_vocab(context[ind])
print_sentence('cr', cr)
if verbose:
print()
return (s, q, cl, cr)
if not test:
for answer in qa['answers']:
X.append(extract(answer['answer_start'],
answer['text.tokens'], is_answer=True) + (1.,))
spans = choice(all_spans, neg_samples, replace=True)
#spans = all_spans
if test:
spans = all_spans
for span in spans:
pos = locate(context, span)
X.append(extract(pos, span) + (0.,))
except Exception as e:
print(e.message)
traceback.print_exc()
import pdb; pdb.set_trace();
#raise e
if not test:
random.shuffle(X)
return X
import math as ma
import utils
if __name__ == "__main__":
# start the number at 1
my_num = 1
while True:
if utils.is_prime(my_num):
print "> Prime number found: %d" % my_num
if not utils.choice("Find another?", "y"):
break
# increment is prime not found or user wants to continue
my_num += 1
def __init__(self, facts_obj, book_window, airport_list):
# 1. origin and destination, airport_list guarantees to have unique locations
self.origin = random.randint(0, len(airport_list) - 1)
self.dest = random.randint(0, len(airport_list) - 1)
if self.dest == self.origin:
self.dest = (self.dest + 1) % len(airport_list)
self.dest = airport_list[self.dest]
self.origin = airport_list[self.origin]
# 2. date
base_time = facts_obj.base_departure_time_epoch
a_year_from_now = base_time + 3600 * 24 * 365
# randomly pick a date between base_time and a_year_from_now
self.departure_date = random.randint(base_time, a_year_from_now)
# return adte is book_window away from the departure date
self.return_date = self.departure_date + 3600 * 24 * book_window
# 4. passenger information
num_passengers = 1
self.passengers = []
len_first_name = len(facts_obj.first_name_list)
len_last_name = len(facts_obj.last_name_list)
# '_' will later be replaced in intent standalization
for _ in xrange(num_passengers):
self.passengers.append(
facts_obj.first_name_list[random.randint(
0, len_first_name - 1)] + '_' +
facts_obj.last_name_list[random.randint(0, len_last_name - 1)])
# non-required fields during initial query
# 3. time
self.departure_time = utils.choice(facts_obj.time_list,
1,
p=facts_obj.time_prior)[0]
self.return_time = utils.choice(facts_obj.time_list,
1,
p=facts_obj.time_prior)[0]
# 5. class limit and price limit
self.class_limit = utils.choice(facts_obj.class_list,
1,
p=facts_obj.class_list_prior)[0]
# 6. price limist
if self.class_limit == 'all':
self.price_limit = facts_obj.price_limit_list[random.randint(
0,
len(facts_obj.price_limit_list) - 1)]
elif self.class_limit == 'economy':
self.price_limit = facts_obj.price_limit_list[random.randint(
0,
len(facts_obj.price_limit_list) - 2)]
elif self.class_limit == 'business':
self.price_limit = facts_obj.price_limit_list[random.randint(
1,
len(facts_obj.price_limit_list) - 1)]
# 7. num of connections
self.max_connection = utils.choice(facts_obj.connection_member,
1,
p=facts_obj.connection_prior)[0]
# 8. airline preference
self.airline = utils.choice(facts_obj.airline_preference,
1,
p=facts_obj.airline_preference_prior)[0]
# 10 post process
self.departure_month, self.departure_day = utils.get_month_and_day(
facts_obj, self.departure_date)
self.return_month, self.return_day = utils.get_month_and_day(
facts_obj, self.return_date)
# 11 change reservation
self.goal = utils.choice([0, 1, 2], p=facts_obj.goal_probaility)