def get(self):
args = self.parser.parse_args()
result = {"status": "success", "msg": "读取文件成功"}
ext = get_splitext(args["path"])
result["ext"] = ext[1]
path = self.app.config["AUTO_HOME"] + "/workspace/%s%s" % (session["username"], args["path"])
data = read_file(path)
if not data["status"]:
result["status"] = "fail"
result["msg"] = "读取文件失败"
result["data"] = data["data"]
return result, 201
def reset_next_build_numb(output):
next_build_number = output + "/nextBuildNumber"
index = 1
data = "%d" % (index + 1)
if not exists_path(next_build_number):
make_nod(next_build_number)
else:
index = int(read_file(next_build_number)["data"])
data = "%d" % (index + 1)
write_file(next_build_number, data)
out = output + "/%d" % index
if not exists_path(output):
mk_dirs(output)
return (out, index)
def __init__(self, filename):
self.filename = filename
self.data = read_file(self.filename)
self.table_tag = BeautifulSoup(self.data, 'html.parser')
# copy attribute from tag that specifies alignment
# to the td tag. This is needed since we will
# remove all tags other than table, tr, td, and th.
self.add_style_attr_to_td()
self.shorten()
self.rows = self.add_rows()
self.df = self.create_dataframe()
self.cell_data()
self.value_locations()
self.table = self.build_table()
def test_set_max_token_len():
print('Getting filenames ...', end=' ')
base_path = tables_extracted_split_tables_dir()
fns = list(
get_filenames(
[os.path.join(base_path, '*', '10-k', '*', '*.table-extracted')]))
print('done.')
bar = ChargingBar('Processing files', max=len(fns))
max_token_len = 0
for fn in fns:
token_len = len(read_file(fn).split())
if token_len > max_token_len:
max_token_len = token_len
bar.next()
bar.finish()
with open(os.path.join(base_path, 'max_token_len'), 'w') as f:
f.write(f'max_token_len: {max_token_len}')
def get_case_filename(execute_case_config_path=r".\config\testcase.ini",
case_file_path=r".\test_cases"):
"""
获取有效用例文件名
:param execute_case_config_path: 用例执行范围的配置文件
:param case_file_path: 读取用例的根目录
:return: {"py_case_filename": [py文件名称,], "api_case_filename": [非py文件名称,]}
"""
if not os.path.exists(execute_case_config_path):
raise ValueError(
f"用例配置文件 {execute_case_config_path} 在目录{os.getcwd()}中不存在，请确认！")
if not os.path.exists(case_file_path):
raise ValueError(f"用例根目录 {case_file_path} 在目录{os.getcwd()}中不存在，请确认！")
case_files = file.read_file(execute_case_config_path).splitlines()
# 分离配置文件中的py文件和非py文件
py_case_filename = []
api_case_filename = []
not_exist_file = []
for case_file in case_files:
if case_file.startswith(";") or case_file.startswith(
"#") or case_file == "":
continue
file_path = file.find_file(case_file_path, case_file)
if not file_path:
not_exist_file.append(case_file)
if case_file.endswith("py"):
py_case_filename.append(case_file)
else:
api_case_filename.append(case_file)
# 如果存在无效用例名，则抛错
if not_exist_file:
raise ValueError(f"这些用例文件{not_exist_file}不存在，请核实！")
return {
"py_case_filename": py_case_filename,
"api_case_filename": api_case_filename
}
def train_set_max_token_len():
print('Getting filenames ...', end=' ')
base_path = os.path.join(generated_data_dir())
fns = list(get_filenames([os.path.join(base_path, 'html', '*.unescaped')]))
fns.extend(list(get_filenames([os.path.join(base_path, 'expected_json',
'*.expected_json')])))
print('done')
bar = ChargingBar('Processing files', max=len(fns))
max_token_len = 0
for fn in fns:
token_len = len(read_file(fn).split())
if token_len > max_token_len:
max_token_len = token_len
bar.next()
bar.finish()
with open(os.path.join(base_path, 'max_token_len'), 'w') as f:
f.write(f'max_token_len: {max_token_len}')
def check_hand_created_samples():
result = True
for samples_dir, input_name in \
zip([text_samples_dir(), html_samples_dir()],
['text_input', 'html_input']):
data_filenames = get_filenames(samples_dir, input_name, '*')
json_filenames = get_filenames(samples_dir, 'json_input', '*')
data_filenames = sorted(data_filenames)
json_filenames = sorted(json_filenames)
for d_fn, j_fn in zip(data_filenames, json_filenames):
print(f'Checking:\n {d_fn}\n {j_fn}\n')
input_data = read_file(d_fn)
json_input_data = get_json_from_file(j_fn)
if data_contains_all_elements(input_data,
json_input_data) is False:
print(f'Errors found in:\n input: {d_fn}\n'
f' json_input: {j_fn}')
result = False
return result
def find_unprocessed_tag_names():
unprocessed_tags = set()
unprocessed_tags_exist = False
for filename in get_filenames(extracted_tables_dir(),
'*', '10-k', '*', '*', '*'):
table_tag = BeautifulSoup(read_file(filename), 'html.parser')
descendant_tag_names = find_descendant_tag_names(table_tag.descendants)
diff = descendant_tag_names - set(tag_actions.keys())
unprocessed_tags.update(diff)
if len(diff) > 0:
unprocessed_tags_exist = True
print(f'filename: {filename}')
print(f'unprocessed_tags: {unprocessed_tags}')
if unprocessed_tags_exist:
print(f'unprocessed_tags: {unprocessed_tags}')
else:
print('No unprocessed tags found')
def get(self):
args = self.parser.parse_args()
key = args["key"].replace(
"--", "/") if args["key"] else args["path"].replace("--", "/")
self.log.debug("Get args:{}".format(args))
result = {"status": "success", "msg": "读取文件成功."}
ext = get_splitext(key)
result["ext"] = ext[1]
#path = self.app.config["AUTO_HOME"] + "/workspace/%s%s" % (session["username"], args["path"])
#path = args["key"]
data = read_file(key)
if not data["status"]:
result["status"] = "fail"
result["msg"] = "读取文件失败."
result["data"] = data["data"]
return result, 201
def editor(key):
rpkey = key.replace("--", "/")
t = get_splitext(rpkey)
log.info("*RPKEY:" + rpkey)
log.info("*KEY:" + key)
log.info("*File ext:" + t[1])
default = "default.html"
if t[1] in (".html", ".htm"):
if os.path.exists(rpkey):
default = rpkey
return send_file(default)
if t[1] in (".txt", ".robot", ".resource", ".py", ".js", ".yaml", ".conf",
".ini", ".sh", ".md", ".tplt"):
default = "editor.html"
if t[1] == ".yaml":
mode = 'yaml'
elif t[1] == '.py':
mode = 'python'
elif t[1] == '.md':
mode = 'textile'
else:
mode = 'python'
return render_template(default, key=rpkey, mode=mode)
if t[1] in (".bmp", ".jpg", ".jpeg", ".png", ".gif"):
return send_file(rpkey)
if t[1] in (".tmd"):
res = read_file(rpkey)
return render_template("test_design.html",
key=rpkey,
value=res["data"])
return render_template(default)
def __fetch(self, data_source_path):
if data_source_path.startswith(
"http://") or data_source_path.startswith("https://"):
import requests
if not self.paging:
print("Fetching from URL {0}".format(data_source_path))
response = requests.get(data_source_path, headers=self.headers)
if response.ok:
self.source = response.text
else:
print("Failed to fetch from {0}".format(data_source_path))
print("Error response: {0} {1}".format(
response.status_code, response.text))
raise Exception(
"Failed to fetch from {0}".format(data_source_path))
else:
self.source = []
for page_number in range(self.paging["min"],
self.paging["max"] + 1):
paging_path = "{0}&{1}={2}".format(data_source_path,
self.paging["iterator"],
page_number)
print("Fetching from URL {0}".format(paging_path))
response = requests.get(paging_path, headers=self.headers)
if response.ok:
self.source.append(response.text)
else:
print("Failed to fetch from {0}".format(
data_source_path))
print("Error response: {0} {1}".format(
response.status_code, response.text))
raise Exception("Failed to fetch from {0}".format(
data_source_path))
else:
self.source = read_file(data_source_path)
acc, jump = ops[op](arg, acc)
pointer += jump
return pointer == len(program), acc
def star_a(program):
return run(program, operations)[1]
def star_b(program, corrupt_instructions={'nop': 'jmp', 'jmp': 'nop'}):
for instruction in filter(
lambda i: program[i]['op'] in corrupt_instructions.keys(),
range(len(program))):
patch = {
'line_number': instruction,
'op': corrupt_instructions[program[instruction]['op']]
}
success, result = run(program, operations, patch)
if success:
return result
code = read_file('day08/1.in').splitlines()
operations = {
'nop': lambda arg, acc: (acc, 1),
'jmp': lambda arg, acc: (acc, arg),
'acc': lambda arg, acc: (acc + arg, 1)
}
print(star_a(compile_source(code)), star_b(compile_source(code)))
def create_wire(file):
wire = dict()
for i, val in enumerate(read_file("day03/{}".format(file)).splitlines()):
wire[i] = create_wire_from_string(val)
return wire
from utils.file import read_file
from itertools import product
def star_a(i):
return [a * b for a, b in zip(i, [2020 - j for j in i]) if b in i][0]
def star_b(i):
return [
a * b * c for a, b, c in [(a, b, 2020 - (a + b))
for a, b in product(i, i)] if c in i
][0]
i = list(map(int, read_file('day01/1.in').splitlines()))
print(star_a(i), star_b(i))
def test_read_file():
assert read_file("utils/test/foo.txt") == "bar"
from functools import reduce
from utils.file import read_file
def a(i, dx, dy):
def xg(mx=len(i[0]), x=0):
while True:
x = (x + dx) % mx
yield x
return reduce(
lambda c, v: c + v,
map(lambda c: c, [
1 if i[y][x] == '#' else 0
for y, x in zip(range(dy, len(i), dy), xg())
]))
def b(i, slopes):
return reduce(lambda x, y: x * y, map(lambda slope: a(i, *slope), slopes))
i = read_file('day03/1.in').splitlines()
print(a(i, *[3, 1]), b(i, [[1, 1], [3, 1], [5, 1], [7, 1], [1, 2]]))
for cube in active_cubes:
neighbors = get_neighbors(cube, dimension)
if len(active_cubes.intersection(neighbors)) in [2, 3]:
answer.add(cube)
inactive_neigbor_cubes |= (neighbors - active_cubes)
for inactive_cube in inactive_neigbor_cubes:
if len(
active_cubes.intersection(
get_neighbors(inactive_cube, dimension))) == 3:
answer.add(inactive_cube)
return answer
def solve(initial_state, dimension, cycles=6):
zeros = [0] * (dimension - 2)
active_cubes = set()
for y, row in enumerate(initial_state.splitlines()):
for x in [x for x, c in enumerate(list(row)) if c == '#']:
active_cubes.add((x, y, *zeros))
for _ in range(cycles):
active_cubes = cycle(active_cubes, dimension)
return len(active_cubes)
data = read_file('day17/1.in')
print(solve(data, 3), solve(data, 4))
def generate_input(input_fn, fn_type, json_input_fn):
def separate_values_strings(top_tag):
filter_str = lambda x: True if x[0] == YIELDED_STR else False
map_str = lambda x: (x[1], x[2])
filter_value = lambda x: True if x[0] == YIELDED_NUM else False
map_value = lambda x: (x[1], x[2])
def extract_tag_str(html_strings_tags_values, filter_fn, map_fn):
temp = filter(filter_fn, html_strings_tags_values)
return list(map(map_fn, temp))
html_strings_tags_values = list(strings_and_values_in_html(top_tag))
html_tags_strings = extract_tag_str(html_strings_tags_values,
filter_str, map_str)
html_tags_values = extract_tag_str(html_strings_tags_values,
filter_value, map_value)
return html_tags_strings, html_tags_values
if fn_type != 'unescaped':
raise ValueError(f'We are only supporting HTML at this point - '
f'not {fn_type}')
input_data = read_file(input_fn)
top_tag = BeautifulSoup(input_data, 'html.parser')
remove_comments(top_tag)
html_tags_strings, html_tags_values = separate_values_strings(top_tag)
json_input = get_json_from_file(json_input_fn)
json_names = top_level_names(json_input)
json_names.extend(get_names(json_input))
mappings = {} # original string to new string
for json_name in json_names:
mappings[json_name] = randomize_string(json_name)
json_values = list(get_values(json_input))
json_values = \
list(filter(lambda x: True if is_number(x) else False, json_values))
replace_names(json_names, html_tags_strings, mappings)
# print(f'mappings: {mappings}\n\n')
value_mappings = \
{value: randomize_string(value)
for value in json_values}
mappings.update(value_mappings)
mappings['-'] = '999999999'
# print(f'mappings: {mappings}')
# check_missing_tokens(html_tags_values, json_values)
number_tokens = list(numeric_tokens(html_tags_values))
for token in number_tokens:
mappings[token] = randomize_string(token)
replace_values(json_values, html_tags_values, mappings)
json_expected = update_expected_strings(json_input, mappings)
return str(top_tag), json_expected
from utils.file import read_file
def star_a(numbers, target):
all_spoken = {}
for turn, number in enumerate(numbers, 1):
all_spoken[number] = turn
most_recently_spoken = numbers[-1]
for turn in range(len(all_spoken), target):
if most_recently_spoken not in all_spoken.keys():
all_spoken[most_recently_spoken], most_recently_spoken = turn, 0
else:
all_spoken[most_recently_spoken], most_recently_spoken = turn, turn - all_spoken[most_recently_spoken]
return most_recently_spoken
data = list(map(int, read_file('day15/1.in').split(',')))
print(star_a(data, 2020), star_a(data, 30000000))
answer[bag] = [] if c[0] == 'no other bags' else [
to_val(*re.match('([0-9]+).(.*bag)', x).groups()) for x in c
]
return answer
def can_hold_gold_bag(b, bags):
return reduce(
lambda a, v: True
if v[1] == 'shiny gold bags' else a | can_hold_gold_bag(v[1], bags),
bags[b], False)
def count_bags_in_bag(bag, bags):
return reduce(lambda a, v: a + (v[0] * count_bags_in_bag(v[1], bags)),
bags[bag], 1)
def star_a(bags):
return sum(
[1 if can_hold_gold_bag(bag, bags) else 0 for bag in bags.keys()])
def star_b(bags):
return count_bags_in_bag('shiny gold bags', bags) - 1
data = read_file('day07/1.in').replace('.', '').splitlines()
print(star_a(parse(data)), star_b(parse(data)))
def all_encodings(filenames, base_dirname, tokens_path):
# Since we're writing tokens to a file for each company,
# and later merging these tokens, the token number
# must always keep incrementing. This way, our dictionary with
# (token_num: token_value) will not miss any tokens.
out_dirname_json = \
os.path.join(os.sep.join(tokens_path.split(os.sep)[:-1]),
'expected_json',
'encoded')
out_dirname_html = \
os.path.join(os.sep.join(tokens_path.split(os.sep)[:-1]),
'html',
'encoded')
create_dirs([out_dirname_json, out_dirname_html])
current_company_dir = ''
token_num = Number.START_WORD_NUM.value
tokens = set()
tokens_filename = ''
# num_dirs_to_process = 3
for filename in filenames:
# filename = '/Volumes/datadrive/tags-cleaned/0000707605_AMERISERV_FINANCIAL_INC__PA_/10-k/2018-01-01_2018-12-31_10-K/tables-extracted/162.table-extracted'
print(f'filename: {filename}')
text = read_file(filename)
company_dir_idx = len(base_dirname)
if base_dirname == generated_data_dir():
company_dir = ''
else:
company_dir = filename[company_dir_idx + 1:].split(os.sep)[0]
if current_company_dir != company_dir:
if len(tokens) > 0:
write_tokens_file(tokens, tokens_filename, token_num)
token_num += len(tokens)
del tokens
tokens = set()
current_company_dir = company_dir
# num_dirs_to_process -= 1
# if num_dirs_to_process <= 0:
# break
else:
# We have to create this variable, and assign to it.
# This way, we have access to the last filename
# in the else clause of this for statement.
tokens_filename = get_tokens_filename(filename, company_dir_idx,
company_dir, "tokens")
if filename.endswith('unescaped') or filename.endswith('html') \
or filename.endswith('table-extracted'):
find_html_table_encodings(out_dirname_html, filename, text, tokens)
elif filename.endswith('json'):
find_json_encodings(out_dirname_json, filename, text, tokens)
else:
write_tokens_file(tokens, tokens_filename, token_num)
all_tokens_filename = os.path.join(base_dirname, 'tokens')
all_tokens = set()
for filename in get_filenames([tokens_path]):
tokens = read_tokens_file(filename)
all_tokens.update(get_token_values(tokens))
print(f'len(all_tokens): {len(all_tokens)}')
# We need to give the offset as the last value in this function call.
# This allows us to interpret the value of 1 as the start of a
# number sequence, and not confuse it with an entry in the tokens
# file that has key = 1.
write_tokens_file(all_tokens, all_tokens_filename,
Number.START_WORD_NUM.value)
def star_a(navigation_instructions, ship_dir, ship_pos=0+0j):
for cmd, arg in navigation_instructions:
if cmd in dirs.keys():
ship_pos += dirs[cmd] * arg
elif cmd in rotations.keys():
ship_dir *= rotations[cmd] ** int(arg / 90)
else:
ship_pos += ship_dir * arg
return mhd(ship_pos)
def star_b(navigation_instructions, waypoint, ship_pos=0+0j):
for cmd, arg in navigation_instructions:
if cmd in dirs.keys():
waypoint += dirs[cmd] * arg
elif cmd in rotations.keys():
waypoint *= rotations[cmd] ** int(arg / 90)
else:
ship_pos += waypoint * arg
return mhd(ship_pos)
data = list(map(lambda m: (m.group(1), int(m.group(2))),
[re.search('([NSEWLRF])([0-9]+)', i) for i in read_file('day12/1.in').splitlines()]))
dirs = {'N': 0 - 1j, 'E': 1 + 0j, 'S': 0 + 1j, 'W': -1 + 0j}
rotations = {'L': 0 - 1j, 'R': 0 + 1j}
print(star_a(data, dirs['E']), star_b(data, 10 - 1j))
matches_per_rule = set()
for rule_idx in [
i for i, rule in enumerate(rules)
if reduce(lambda a, v: a & rule(v), values_for_a_col, True)
]:
matches_per_rule.add(rule_idx)
rules_matching_all_entries_per_col.append(matches_per_rule)
mapping = {}
while len(mapping) < len(rules):
for i, val in [
(i, s.pop())
for i, s in enumerate(rules_matching_all_entries_per_col)
if len(s) == 1
]:
mapping[val] = i
remove_value_from_sets(rules_matching_all_entries_per_col, val)
answer = 1
for col in [
i for i, row in enumerate(notes[0].splitlines())
if row.startswith('departure')
]:
answer *= my_ticket[mapping[col]]
return answer
data = read_file('day16/1.in').split('\n\n')
print(star_a(data), star_b(data))
def add_zero_and_max_then_sort(adapters):
adapters.extend([0, max(adapters) + 3])
return sorted(adapters)
def star_a(adapters):
jolts = Counter([adapters[i + 1] - adapters[i] for i in range(len(adapters) - 1)])
return reduce(lambda c, v: c * v, [jolts[i] for i in [1, 3]])
def star_b(adapters):
p = [0] * len(adapters)
p[0] = 1
for i, a, b, c in [(i, i - 1, i - 2, i - 3) for i in range(1, len(adapters))]:
p[i] = p[a]
if adapters[i] - adapters[b] <= 2:
p[i] += p[b]
if adapters[i] - adapters[c] <= 3:
p[i] += p[c]
return p[-1]
data = add_zero_and_max_then_sort(list(map(int, read_file('day10/1.in').splitlines())))
start = time.time()
print(star_a(data), star_b(data))
end = time.time()
print(end - start)
def main():
i = read_file("day01/input.txt")
print(a(i))
print(b(i))
result[p] = f'{replacement[i]}'
answer.append(int(''.join(result), 2))
return answer
def apply_mask_b(mask, value):
answer = list(get_as_binary(0))
for bit in range(len(value)):
answer[bit] = mask[bit] if mask[bit] in ['X', '1'] else value[bit]
return answer
def star_b(program):
mem = {}
mask = None
for line in program:
if re.search('mask', line):
mask = re.search('mask = (.*)', line).group(1)
else:
m = re.search('mem\\[([0-9]+)] = (.*)', line)
address, value = int(m.group(1)), int(m.group(2))
result = apply_mask_b(mask, get_as_binary(address))
for address in get_addresses(result):
mem[address] = value
return sum(mem.values())
data = read_file('day14/1.in').splitlines()
print(star_a(data), star_b(data))
[adapters[i + 1] - adapters[i] for i in range(len(adapters) - 1)])
return reduce(lambda c, v: c * v, [jolts[i] for i in [1, 3]])
def star_b(adapters):
answer = 1
diff = [x - y for y, x in zip(adapters[:-1], adapters[1:])]
i = 0
while i < len(diff):
perm, jmp = 1, 1
if diff[i:i + 4] == [1, 1, 1, 1]:
perm, jmp = 7, 3
elif diff[i:i + 3] == [1, 1, 1]:
perm, jmp = 4, 2
elif diff[i:i + 2] == [1, 1]:
perm = 2
answer *= perm
i += jmp
return answer
data = add_zero_and_max_then_sort(
list(map(int,
read_file('day10/1.in').splitlines())))
start = time.time()
print(star_a(data), star_b(data))
end = time.time()
print(end - start)
