def process_data(string):
try:
return list(map(int, get_lines(string)))
except ValueError:
pass
finally:
return get_lines(string)
def main():
french = get_lines("data/french.txt")
english = get_lines("data/english.txt")
french_english_pairs = list(zip(french, english))
training_pairs, validation_pairs = create_data_splits(french_english_pairs)
training_input_sentences = [pair[0] for pair in training_pairs]
training_output_sentences = [pair[1] for pair in training_pairs]
validation_input_sentences = [pair[0] for pair in validation_pairs]
validation_output_sentences = [pair[1] for pair in validation_pairs]
with open("data/french_vocab.pkl", "rb") as f:
fv = pickle.load(f)
with open("data/french_reverse_vocab.pkl", "rb") as f:
frv = pickle.load(f)
with open("data/english_vocab.pkl", "rb") as f:
ev = pickle.load(f)
with open("data/english_reverse_vocab.pkl", "rb") as f:
erv = pickle.load(f)
t_loss, t_accuracy = train(training_input_sentences,
training_output_sentences, fv, ev, frv, erv,
hyperparameters, writer)
print("Training Accuracy = {:.1f}".format(100. * np.mean(t_accuracy)))
evaluate(validation_input_sentences, validation_output_sentences, fv, ev,
frv, erv, hyperparameters, writer)
def story():
global titleA
if request.method =="GET":
return render_template("stories.html",story=titleA,lines=utils.get_lines(titleA))
if request.method =="POST":
if(request.form['button'] == "Add Line"):
utils.add_to_story(titleA,str(request.form["newline"]))
return render_template("stories.html",story=titleA,lines=utils.get_lines(titleA))
def puzzle1():
max_id = 0
for bp in get_lines('day5'):
bp_id = BoardingPass(bp).get_id()
if bp_id > max_id:
max_id = bp_id
print(max_id)
def puzzle1():
lines = get_lines('day18')
total = 0
for expr in lines:
tokens = tokenize(expr)
total += evaluate(tokens)
print(total)
def puzzle2():
lines = get_lines('day18')
total = 0
for expr in lines:
tokens = tokenize(expr)
total += evaluate(tokens, flatten=add_mul)
print(total)
def main():
lines = get_lines('input/03.txt')
line_len = len(lines[0])
print(line_len)
# only counts 1 bits
bit_counter = [0] * line_len
line_c = 0
for l in lines:
for i in range(line_len):
if l[i] == '1':
bit_counter[i] += 1
line_c += 1
# most common,least common bit
gamma_rate, epsilon_rate = 0, 0
# most common bit calc
for i in range(line_len):
gamma_rate <<= 1
epsilon_rate <<= 1
if bit_counter[i] >= int(line_c / 2):
gamma_rate += 1
else:
epsilon_rate += 1
print(
f'gamma rate {gamma_rate} * epsilon rate {epsilon_rate} = {gamma_rate * epsilon_rate}'
)
def puzzle2():
code = get_lines('day8')
cpu = ConsoleCpu(code)
for mod_line in range(len(code)):
if cpu.mod_halt(mod_line):
break
print(cpu.accumulator)
def puzzle2():
lines = get_lines('day17')
# lines = ['.#.', '..#', '###']
grid = CubeGrid4d(lines)
for _ in range(6):
# grid.render()
grid.iterate()
print(len(grid))
def puzzle1():
code = get_lines('day8')
cpu = ConsoleCpu(code)
ip_seen = set()
while cpu.ip not in ip_seen:
ip_seen.add(cpu.ip)
cpu.step()
print(cpu.accumulator)
def puzzle2():
bp_ids = []
for bp in get_lines('day5'):
bp_ids.append(BoardingPass(bp).get_id())
bp_ids.sort()
for val in bp_ids:
if val + 1 not in bp_ids:
print(val + 1)
break
def puzzle2():
grid = TreeGrid(get_lines('day3'))
t_11 = grid.traverse(1, 1)
t_31 = grid.traverse(3, 1)
t_51 = grid.traverse(5, 1)
t_71 = grid.traverse(7, 1)
t_12 = grid.traverse(1, 2)
print(t_11 * t_31 * t_51 * t_71 * t_12)
def puzzle1():
grid = TreeGrid(get_lines('day3'))
count = 0
x_pos = 0
for row in range(grid.height):
if grid.tree_at(row, x_pos):
count += 1
x_pos += 3
print(count)
def puzzle1():
rules = get_lines('day7')
rp = RuleParser(rules)
count = 0
for bt in rp.bag_types():
if rp.can_contain(bt, 'shiny gold'):
count += 1
print(count)
def puzzle1():
lines = get_lines('day25')
card_pk, door_pk = (int(x) for x in lines)
card_loop_size = find_loop_size(card_pk)
door_loop_size = find_loop_size(door_pk)
assert card_pk == transform(7, card_loop_size)
assert door_pk == transform(7, door_loop_size)
enc_key = transform(door_pk, card_loop_size)
print(enc_key)
def create_keras_sequences(data_path='../text/cleaned/'):
"""Previous stuff before running model train."""
logging.info('Loading and wrangling data.')
lines = get_lines(data_path)
tokenizer = get_tokenizer(lines)
all_words_list = ordereddict_to_list(tokenizer.word_counts)
p = p_distribution(lines)
index = int(len(lines) * VAL_RATIO)
return (TextSequence(lines[:-index], tokenizer, all_words_list, BATCH_SIZE,
8, 20, p),
TextSequence(lines[-index:], tokenizer, all_words_list, BATCH_SIZE,
8, 20, p), len(tokenizer.word_index) + 1)
def create_random_set_single(src_lang, trg_lang, src_in_file, trg_in_file,
set_num, out_dir):
set_name = os.path.basename(src_in_file)
set_id = get_set_id(set_name)
s_lines = utils.get_lines(src_in_file)
t_lines = utils.get_lines(trg_in_file)
input_size = len(s_lines)
set_dir = os.path.join(out_dir, set_id)
if not os.path.exists(set_dir):
os.makedirs(set_dir)
for set_id in range(0, set_num):
fos = open(os.path.join(set_dir, "%d.%s" % (set_id, src_lang)), 'w')
fot = open(os.path.join(set_dir, "%d.%s" % (set_id, trg_lang)), 'w')
for i in range(0, input_size):
id = random.randint(0, input_size - 1)
fos.write(s_lines[id])
fot.write(t_lines[id])
fos.close()
fot.close()
def puzzle1():
lines = get_lines('day13')
earliest = int(lines[0])
bus_ids = [int(bus) for bus in lines[1].split(',') if bus != 'x']
min_bus = 0
min_delay = 1e12
for bus_id in bus_ids:
delay = bus_id - (earliest % bus_id)
if delay < min_delay:
min_delay = delay
min_bus = bus_id
print(min_bus * min_delay)
def get_known_words(self):
file_names = os.listdir(self.all_dicts_path)
known_words = []
for name in file_names:
if name.endswith("txt"):
dict_file = os.path.join(self.all_dicts_path, name)
words = get_lines(dict_file)
self.logger.info("file:{}, num of words:{}".format(
name, len(words)))
update_file(words, dict_file)
known_words.extend(words)
update_file(known_words, self.known_words_file)
self.logger.info("num of known words: {}".format(len(known_words)))
def get_def_var_names(probname):
lines = get_lines(nl_fname(probname))
def to_index_name(line):
if line[0] == 'V':
# "Vi j k #name" -> i as int
index = int(line[1:].split(None, 1)[0])
# "Vi j k #name" -> name
name = line.rsplit('#',
1)[1].strip() # Needs: option nl_comments 1;
return index, name
return None
return OrderedDict(i_name for i_name in imap(to_index_name, lines)
if i_name)
def puzzle2():
lines = get_lines('day13')
bus_ids = [int(bus) if bus != 'x' else None
for bus in lines[1].split(',')]
ts = 0
delta = bus_ids[0] # assume not None...
for idx, val in enumerate(bus_ids[1:], 1):
while True:
if val is None:
break
if (ts + idx) % val == 0:
delta = lcm(delta, val)
break
ts += delta
print(ts)
def __init__(self):
# paras
self.annotate_logger = "log/annotate.log"
self.entities_file = "all_dicts/entities.txt"
self.data_after_seg = "temp/data_after_seg.txt"
self.train_file = "data/train.txt"
self.dev_file = "data/dev.txt"
self.test_file = "data/test.txt"
# logging
self.logger = get_logger(self.annotate_logger)
# annotate
self.entities = get_lines(self.entities_file)
self.annotate()
def main():
# part 1
#def up(v: int):
# global vertical
# vertical -= v
#def down(v: int):
# global vertical
# vertical += v
#def forward(v: int):
# global horizontal
# horizontal += v
#
def up(v: int):
global aim
aim -= v
def down(v: int):
global aim
aim += v
def forward(v: int):
global horizontal
global vertical
global aim
horizontal += v
vertical += (aim * v)
operations = {
'up': up,
'down': down,
'forward': forward,
}
i = 0
for l in get_lines('input/02.txt'):
operation, value = l.split(' ')
operations[operation](int(value))
print(f'o {operation} val {value}\n\tv {vertical}, h {horizontal}')
if i >= 10:
continue
i += 1
print(f'h {horizontal} * v {vertical} = {horizontal * vertical}')
def evaluate_file(config):
config = load_config(config["config_file"])
logger = get_logger(config["log_file"])
entities_now = get_lines("all_dicts/entities.txt")
new_entities = set()
with open(config["map_file"], "rb") as f:
word_to_id, id_to_word, tag_to_id, id_to_tag = pickle.load(f)
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
model = Model(config)
logger.info("读取现有模型...")
ckpt = tf.train.get_checkpoint_state(
config["ckpt_path"]) # 从模型路径获取ckpt
model.saver.restore(sess, ckpt.model_checkpoint_path)
with open("temp/all_data_clean.txt", "r", encoding="utf-8") as f:
for index, line in enumerate(f):
if index % 1000 == 0:
print(index)
result = model.evaluate_line(sess,
input_from_line(line, word_to_id),
id_to_tag)
entities_pre = list(
set([item["word"] for item in result["entities"]]))
for en in entities_pre:
if en not in entities_now:
new_entities.add(en)
# save new words
with open("results/new_entities.txt", "w+", encoding="utf-8") as fout:
for en in new_entities:
fout.write(en + "\n")
print("num of new entities:{}".format(len(new_entities)))
def puzzle2():
items = get_lines('day21')
food = dict()
allergen_candidates = dict()
all_allergens = set()
all_ingredients = set()
for line in items:
ingredients, _, allergens = line.partition(' (contains ')
allergens = set(allergens.rstrip(')').split(', '))
ingredients = set(ingredients.split())
all_ingredients.update(ingredients)
all_allergens.update(allergens)
food[frozenset(ingredients)] = allergens
for allergen in allergens:
if allergen in allergen_candidates:
# remove all candidates which don't appear in
# current ingredient list
allergen_candidates[allergen] &= ingredients
else:
allergen_candidates[allergen] = ingredients.copy()
known_bad = {}
while True:
for allergen, cand in allergen_candidates.items():
unknown = cand - set(known_bad)
if len(unknown) == 1:
known_bad[next(iter(unknown))] = allergen
break
else:
# implies we made no changes; we're done.
break
# Canonical 'bad ingredients' list *SORTED BY ALLERGEN*
print(','.join(sorted(known_bad, key=lambda x: known_bad[x])))
def puzzle1():
items = get_lines('day21')
food = dict()
allergen_candidates = dict()
all_allergens = set()
all_ingredients = set()
for line in items:
ingredients, _, allergens = line.partition(' (contains ')
allergens = set(allergens.rstrip(')').split(', '))
ingredients = set(ingredients.split())
all_ingredients.update(ingredients)
all_allergens.update(allergens)
food[frozenset(ingredients)] = allergens
for allergen in allergens:
if allergen in allergen_candidates:
# remove all candidates which don't appear in
# current ingredient list
allergen_candidates[allergen] &= ingredients
else:
allergen_candidates[allergen] = ingredients.copy()
safe_ingredients = all_ingredients.copy()
for ing in all_ingredients:
for possible in allergen_candidates.values():
if ing in possible:
safe_ingredients.discard(ing)
count = 0
for recipe in food:
for ing in recipe:
if ing in safe_ingredients:
count += 1
print(count)
def fields_count(f):
left_names = []
for b in f:
if len(b) > 1:
left_names += b
for name in list(set(left_names)):
if left_names.count(name) == 1:
return name
return None
if __name__ == "__main__":
data = get_lines("ticket.txt")
# Collect ranges
end = data.index('')
fields = collect_fields(data[:end])
print(len(fields.keys()))
# Collect my ticket
data = data[end + 2:]
my_ticket = collect_tickets(data[:1])[0]
# Collect nearby tickets
data = data[3:]
nearby_tickets = collect_tickets(data)
# Remove invalid tickets
from utils import get_lines
def find_bus_id(timestamp):
while True:
for i in ids:
if timestamp % i == 0:
return timestamp, i
timestamp += 1
if __name__ == "__main__":
ts, ids = get_lines("shuttle.txt")
ts = int(ts)
ids = ids.split(",")
ids = [int(a) for a in ids if a != "x"]
arrival, bus = find_bus_id(ts)
print((arrival - ts) * bus)
###"""
else:
# Current file filepath
thisfile = current_file(__file__)
# AOC day number
DAY_NO: int = day_number(thisfile.stem)
raw_data = read_data(f"day-{DAY_NO}-input.txt")
# raw_data = read_data(f"day-17-input.txt")
# Split data into lines for further processing. Skip any missing or blank lines.
try:
data = list(map(int, get_lines(raw_data)))
except ValueError:
pass
finally:
data = get_lines(raw_data)
grid = [list(i) for i in data]
# -- objects and functions --- #
def create_cubes(matrix):
cube_set = set()
for r in range(len(matrix)):
for c in range(len(matrix[0])):
if matrix[r][c] == "#":
import re
from utils import get_lines
data = get_lines('bags.txt')
print(data)
bags = dict()
# Collecting the colors
pattern = r'\w* \w* bag'
for line in data:
bag = re.findall(pattern, line)
bag = [a[:len(a) - 4] for a in bag if a != 'no other bag']
bags[bag[0]] = bag[1:]
# ### PART 1 ###
opened = ['shiny gold']
final = []
while True:
new = []
for bag in opened:
new += [x for x in bags.keys() if bag in bags[x]]
if len(new) == 0:
break
opened = list(set(new))
final += opened
print(len(set(final)))
def get_unknown_words(self):
known_words = get_lines(self.known_words_file)
with open(self.unknown_words_file, "w+", encoding="utf-8") as fout:
for word, fre in self.word_fre:
if word not in known_words:
fout.write("{}\t{}\n".format(word, fre))
def show_log(old):
content = logviewer.get_content(old, utils.get_inverted(), utils.get_lines(), True)
logviewer.window(utils.ADDON_NAME, content, default=utils.is_default_window())
def get_graph(request):
data = {'nodes': utils.get_nodes(),
'lines': utils.get_lines()}
return HttpResponse(content=json.dumps(data), content_type='application/json')
def tree_process(self):
my_log = logging.getLogger('train:tree_processing')
if self.config.settings["tree_file"] is None or self.config.settings["tree_file"] == "":
# create clade list (own method?)
descandents = {}
for organism in self.organisms:
for rank in self.config.settings["taxonomy_ranks"]:
parent = self.sqlite_taxonomy.parent_at_rank(organism, rank)
print('!!! Call: parent = self.sqlite_taxonomy.parent_at_rank(organism, rank) !!!')
if parent in descandents:
descandents[parent] += 1
else:
descandents[parent] = 1
for parent in descandents.keys():
if parent is None or parent == "":
continue
if descandents[parent] >= self.config.settings["n_min_genomes_generic"]:
self.nodes.append(parent)
else:
# read the tree_string
nodes_tmp = utils.get_lines(self.config.settings["tree_file"])
for n in nodes_tmp:
if n != "" and n is not None:
self.nodes.append(n.rstrip("\n"))
tree_string = self.nodes[0]
# check if the tree is a newick string or a node list
t_file = os.path.join(self.config.settings["project_dir"], "tree.newick")
if ";" not in self.nodes[0] and len(self.nodes) >= 1:
my_log.debug("Generating tree from the clades list ({} clades)....".format(str(len(self.nodes))))
clades_file = os.path.join(self.config.settings["project_dir"], "clades.txt")
utils.list_to_file(self.nodes, clades_file)
if self.stat is not None:
self.stat.add_written_file(clades_file)
self.stat.succesfully_written(clades_file)
# run script to create tree
obj = ncbi2newick.Ncbi2Newick(self.config.ncbi_tax_db, logged=True)
obj.tree_from_nodes(clades_file)
obj.tree_to_file(t_file)
obj.close()
else:
my_log.debug("Copying tree to the project directory...")
fw = open(t_file, "w")
if self.stat is not None:
self.stat.add_written_file(t_file)
fw.write(tree_string)
if self.stat is not None:
self.succesfully_written(t_file)
fw.close()
# change tree_file to the tree in the project directory
# get back the tree_string and nodes, this is necessary for further processing
self.tree_file = os.path.join(self.config.settings["project_dir"], "tree.newick")
fr = open(self.tree_file, "r")
tree_string = fr.readline().rstrip()
fr.close()
if tree_string == "":
my_log.critical("First line is empty in the newick file: {}".format(self.tree_file))
sys.exit(1)
self.nodes = ncbi2newick.get_nodes_from_newick(self.tree_file)
