def test_topostfix():
case = (
("8-(3+2*6)/5+4", "8 3 2 6 * + 5 / - 4 +"),
("((5+7)/3*7-(3*2))+(7-3)*3+2*5+4*5+1*6+1*5", "5 7 + 3 / 7 * 3 2 * - 7 3 - 3 * + 2 5 * + 4 5 * + 1 6 * + 1 5 * +"),
("1*((2+3) /2) ^5", "1 2 3 + 2 / 5 ** *"),
("-1+((2+3)*4/5)", "1 - 2 3 + 4 * 5 / +"),
("-1+-5", "1 - 5 - +"),
("pow(3,2)+ sin(45)", "3 2 , pow 45 sin +"),
("-$a+$b*(-$c+$d)+$e", "_a - _b _c - _d + * + _e +"),
("pOw(1*(2+3), 6+5)", "1 2 3 + * 6 5 + , pow"),
("-1+5*-6", "1 - 5 6 - * +"),
("Sum([1+1,5+6*7,POW (3,2)])", "1 1 + 5 6 7 * + , 3 2 , pow , sum"),
("5/ 4mod sin(1)+SUm([1,2])^3.5* 2-4", "5 4 1 sin % / 1 2 , sum 3.5 ** 2 * + 4 -"),
("4", "4"),
("-sin(45)", "45 sin -"),
("-+-+sin(45)", "45 sin + - + -"),
("$var", "_var"),
("-(+5+6)", "5 + 6 + -"),
("$vAr", "_var"),
)
for c, r in case:
exp = Convertor.preprocessing(c)
gtk = Convertor.tokenize(exp)
rr = Convertor.topostfix(gtk)
assert " ".join(map(str, rr)) == r
def select(self):
self.convertor = Convertor(
filedialog.askopenfilename(initialdir="/",
title="Select file",
filetypes=(("jpeg", "*.jpg"), ("png",
"*.png"),
("all files", "*.*"))))
if self.target:
self.target.destroy()
self.target = tk.Label(self.app,
text='TExt',
image=self.convertor.tkImg)
self.target.grid(row=1, column=0)
self.filemenu.entryconfigure(1, state=tk.NORMAL)
self.filemenu.entryconfigure(2, state=tk.NORMAL)
self.funcmenu.entryconfigure(0, state=tk.NORMAL)
self.funcmenu.entryconfigure(1, state=tk.NORMAL)
self.funcmenu.entryconfigure(2, state=tk.NORMAL)
self.filtermenu.entryconfigure(0, state=tk.NORMAL)
self.filtermenu.entryconfigure(1, state=tk.NORMAL)
self.filtermenu.entryconfigure(2, state=tk.NORMAL)
self.filtermenu.entryconfigure(3, state=tk.NORMAL)
self.filtermenu.entryconfigure(4, state=tk.NORMAL)
self.filtermenu.entryconfigure(5, state=tk.NORMAL)
def convert_fields(self):
converter = Convertor()
self.op = converter.binary_to_decimal(self.op)
self.rs = converter.binary_to_decimal(self.rs)
self.rt = converter.binary_to_decimal(self.rt)
self.rd = converter.binary_to_decimal(self.rd)
self.shamt = converter.binary_to_decimal(self.shamt)
self.funct = converter.binary_to_decimal(self.funct)
def check(wdbconn, dbsrc, dbdst, conf, count):
'''
检测转换后的数据的正确性
'''
check_cond = conf.get('check')
if check_cond is None:
sys.stderr.write(u'无法校验数据，需要在配置文件中指定 check 字段')
return
tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to']
where = conf.get('where', '1')
update_by = conf.get('update_by')
from convertor import Convertor
c = Convertor(conf['map'], conf.get('const'), wdbconn)
progress = make_progress(wdbconn, tbfrom, where, count)
check_where = conf.get('check_where', '1')
check_src_use_where = conf.get('check_src_use_where', False)
if not check_src_use_where:
where = '1'
rand_sql = make_check_rand_sql(tbto, check_where)
limit = 0
ncomm = 0
while limit < count:
# 从目标数据随机取出一条数据
with wdbconn.cursor() as wcsr:
wcsr.execute(rand_sql)
dst_data = wcsr.fetchone()
if dst_data is None:
sys.stderr.write(u'数据不足，无法检测')
break
# 获取对应的源数据
src_sql = make_check_src_sql(c.keys, tbfrom, where, dst_data,
check_cond)
wcsr.execute(src_sql)
src_data = wcsr.fetchall()
for sd in src_data:
src_dst_data = c.process(sd)
# 比较双方的数据
if compare(src_dst_data, dst_data):
ncomm += 1
break
progress.update(1)
limit += 1
progress.close()
ndiff = count - ncomm
print(u'''随机检测条数 %d
相同 %d
不同 %d
正确率 %.2f
错误率 %.2f''' % (count, ncomm, ndiff, ncomm * 100.0 / count,
ndiff * 100.0 / count))
def __test__(dp, model, fbid):
"Using for quick test a crawled account"
# Create a convertor and convert files in a account to vector
convertor = Convertor("data")
profile = convertor.read_profile(fbid)
profile = pd.DataFrame([profile])
# Load datapreprocessing object and nomalizing vector
datapreprocessing = load(dp)
profile = datapreprocessing.convert(profile)
# Load model and predict result
randomforest = load(model)
result = randomforest.predict_proba(profile)[0]
print(result)
def test_preprocessing():
case = (
("1+1", "1+1"),
("1 + 1 ", "1 + 1"),
(" 1 +1", "1 +1"),
("1 pOw $e", "1 pow _e"),
)
for c, r in case:
assert Convertor.preprocessing(c) == r
def __init__(self, folder="data"):
self.folder = folder
self.filename = input("Enter EPUB Filename: ")
self.lang_src = "en"
self.lang_tgt = "fr"
path_without_ext = os.path.join(self.folder, self.filename).lower()
path_with_ext = path_without_ext + ".epub"
if os.path.exists(path_with_ext):
c = Convertor(path_without_ext)
c.to_txt()
##t = Translator(path_without_ext, self.lang_src, self.lang_tgt)
#t.google_translate()
#r = Reader(t.filename, self.lang_tgt)
#r.play()
else:
print("File {} is Missing".format(path))
def save_func(self, func, exp):
"""保存函数"""
if func.startswith(Configuration.FuncPrefix):
if exp.find(Configuration.UserDeclarator) != -1:
raise Exception("invalid syntax")
if exp.find("=") != -1:
raise Exception("invalid syntax")
func = Convertor.format_usrname(func)
lmd = Supporter.args2list(func_lambda(exp, self))#包装自定义函数
self._handler.add_api(func, lmd)#加入到虚拟空间中
else:
raise Exception("Function should starts with #")
def convert(wdbconn, rdbconn, dbsrc, dbdst, conf):
limit = max(conf.get('limit', 0), 0)
tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to']
where = conf.get('where', '1')
update_by = conf.get('update_by')
from convertor import Convertor
c = Convertor(conf['map'], conf.get('const'), wdbconn)
progress = make_progress(wdbconn, tbfrom, where, limit)
rcsr = rdbconn.cursor()
srcsql = make_src_sql(c.keys, tbfrom, where)
rcsr.execute(srcsql)
# 获取数据表
offset, step = 0, 200
while limit == 0 or offset < limit:
with wdbconn.cursor() as wcsr:
srcdata = rcsr.fetchmany(step)
# 因为每条 sql 可能都不一样，所以不能使用 executemany
for sd in srcdata:
dstdata = c.process(sd)
dstsql = make_dst_sql(dstdata, tbto, update_by)
# print(dstsql)
if dstsql:
wcsr.execute(dstsql, tuple(dstdata.values()))
wdbconn.commit()
progress.update(step)
if len(srcdata) < step:
break
offset += step
rcsr.close()
progress.close()
def get_elements(self):
self.elements = Convertor.convert(self.entry.get('1.0', END))
if self.elements == False:
showerror(
title='Помилка!',
message=
f"Введіть правильну кількість символів '{self.size**2}' та переконайтеся, що ви все правильно ввели!"
)
elif len(self.elements) != self.size**2:
showerror(
title='Помилка!',
message=
f"Введіть правильну кількість символів '{self.size**2}' та переконайтеся, що ви все правильно ввели!"
)
else:
self.matr = self.elements
self.convert_to_matr()
def test_grapetoken():
exp = "sum([-1+1*2,4+5/2^2,3mod mod(3,2)])"
res = (
"sum",
"(",
"(",
"-",
"1",
"+",
"1",
"*",
"2",
",",
"4",
"+",
"5",
"/",
"2",
"**",
"2",
",",
"3",
"%",
"mod",
"(",
"3",
",",
"2",
")",
")",
")",
)
rr = Convertor.tokenize(exp)
for c,r in zip(rr, res):
assert str(c.value) == r
assert rr[3].type == ElementTypeEnum.UOP
assert isinstance(rr[0].value, operators.UnaryOperator)
assert rr[5].type == ElementTypeEnum.BOP
assert isinstance(rr[5].value, operators.BinaryOperator)
from conf import Cfg
from finder import Finder
from convertor import Convertor
from storage import Storage
import glob
import os
from parser import Parser
import scholarly
cfg = Cfg()
db = Storage(cfg)
db.load()
finder = Finder(cfg)
convertor = Convertor(cfg)
def process_file(f):
checksum = Storage.file_checksum(f)
if checksum in db.data:
print("file {} already processed ({})".format(f, checksum))
return
db.store(checksum, {"pdf": f})
convertor.convert(f)
for f in finder.find_all():
process_file(f)
db.load()
class Main(tk.Frame):
def __init__(self, frame):
super().__init__(root)
self.frame = frame
self.app = tk.Frame(frame, bg='Gray')
self.app.pack()
self.target = False
self.img = False
self.convertor = False
self.menu = tk.Menu(frame)
frame.config(menu=self.menu)
self.filemenu = tk.Menu(self.menu, tearoff=0)
self.funcmenu = tk.Menu(self.menu, tearoff=0)
self.filtermenu = tk.Menu(self.menu, tearoff=0)
self.menu.add_cascade(label='File', menu=self.filemenu)
self.menu.add_cascade(label='Functions', menu=self.funcmenu)
self.menu.add_cascade(label='Filters', menu=self.filtermenu)
self.menu.add_cascade(label='Face detection', command=self.detection)
self.menu.add_cascade(label='To original', command=self.original)
self.filemenu.add_cascade(label='Select', command=self.select)
self.filemenu.add_cascade(label='Save', command=self.save)
self.filemenu.add_cascade(label='Save as', command=self.saveAs)
self.filemenu.entryconfigure(1, state=tk.DISABLED)
self.filemenu.entryconfigure(2, state=tk.DISABLED)
self.filemenu.add_separator()
self.filemenu.add_cascade(label='Exit', command=self.close)
self.funcmenu.add_cascade(label='Resize', command=self.resize)
self.funcmenu.add_cascade(label='Blur', command=self.blur)
self.funcmenu.add_cascade(label='Contour', command=self.contour)
self.funcmenu.entryconfigure(0, state=tk.DISABLED)
self.funcmenu.entryconfigure(1, state=tk.DISABLED)
self.funcmenu.entryconfigure(2, state=tk.DISABLED)
self.filtermenu.add_cascade(label='Brightness',
command=self.brightness)
self.filtermenu.add_cascade(label='Negative', command=self.negative)
self.filtermenu.add_cascade(label='White-Black',
command=self.white_black)
self.filtermenu.add_cascade(label='Grayscale', command=self.grayscale)
self.filtermenu.add_cascade(label='Sepia', command=self.sepia)
self.filtermenu.add_cascade(label='Contrast', command=self.contrast)
self.filtermenu.entryconfigure(0, state=tk.DISABLED)
self.filtermenu.entryconfigure(1, state=tk.DISABLED)
self.filtermenu.entryconfigure(2, state=tk.DISABLED)
self.filtermenu.entryconfigure(3, state=tk.DISABLED)
self.filtermenu.entryconfigure(4, state=tk.DISABLED)
self.filtermenu.entryconfigure(5, state=tk.DISABLED)
tk.Frame(self.app, width=700, height=0).grid(row=0, column=0)
tk.Canvas(self.app, width=700, height=400, bg='Gray',
bd=0).grid(row=1, column=0)
def close(self):
self.frame.destroy()
def select(self):
self.convertor = Convertor(
filedialog.askopenfilename(initialdir="/",
title="Select file",
filetypes=(("jpeg", "*.jpg"), ("png",
"*.png"),
("all files", "*.*"))))
if self.target:
self.target.destroy()
self.target = tk.Label(self.app,
text='TExt',
image=self.convertor.tkImg)
self.target.grid(row=1, column=0)
self.filemenu.entryconfigure(1, state=tk.NORMAL)
self.filemenu.entryconfigure(2, state=tk.NORMAL)
self.funcmenu.entryconfigure(0, state=tk.NORMAL)
self.funcmenu.entryconfigure(1, state=tk.NORMAL)
self.funcmenu.entryconfigure(2, state=tk.NORMAL)
self.filtermenu.entryconfigure(0, state=tk.NORMAL)
self.filtermenu.entryconfigure(1, state=tk.NORMAL)
self.filtermenu.entryconfigure(2, state=tk.NORMAL)
self.filtermenu.entryconfigure(3, state=tk.NORMAL)
self.filtermenu.entryconfigure(4, state=tk.NORMAL)
self.filtermenu.entryconfigure(5, state=tk.NORMAL)
def saveAs(self):
self.convertor.saveAs(
filedialog.asksaveasfilename(defaultextension='.png',
initialdir="/",
title="Select file",
filetypes=(("jpeg files", "*.jpg"),
("png files", "*.png"))))
def save(self):
self.convertor.save()
def resize(self):
Resize(self.convertor.resize, self.updateImg)
def blur(self):
Blur(self.convertor.blur, self.updateImg)
def updateImg(self):
if self.target:
self.target.destroy()
self.target = tk.Label(self.app,
text='TExt',
image=self.convertor.tkImg)
self.target.grid(row=1, column=0)
def detection(self):
self.convertor.detection()
self.updateImg()
def contour(self):
self.convertor.contour()
self.updateImg()
def brightness(self):
Brightness(self.convertor.brightness, self.updateImg)
def negative(self):
self.convertor.negative()
self.updateImg()
def white_black(self):
WhiteBlack(self.convertor.white_black, self.updateImg)
def grayscale(self):
self.convertor.grayscale()
self.updateImg()
def sepia(self):
self.convertor.sepia()
self.updateImg()
def contrast(self):
Contrast(self.convertor.contrast, self.updateImg)
def original(self):
self.convertor.original()
self.updateImg()
def __solver__(conn, addr, data, **kwargs):
"Resolve a package 'data' come from client"
# Change data from string to json
data = json.loads(data)
# Get parameter server
if "server" not in kwargs:
raise Exception("Expected server parameter")
server = kwargs["server"]
if "uid" in data:
uid = data["uid"]
else:
uid = None
# Solve data have key fb_id
if "fb_id" in data:
fb_id = scrape_utils.__create_original_link__("https://",
data["fb_id"])
while True:
# Get email and password from server object
email = server.__email__[server.__current_account__]
password = server.__password__[server.__current_account__]
# Create scraper and start scraping facebook account
try:
scraper = Scraper2(email, password, verbose="file", sender=uid)
bSuccess = scraper(fb_id)
except Exception as e:
print(str(e))
scraper.__driver__.close()
conn.close()
return
# Not success if the crawler account is banned
if bSuccess is not False:
break
content = json.dumps({
"kind": "notify",
"data": "Error in crawling, restart crawling...",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
# Switch account
server.__current_account__ = (server.__current_account__ +
1) % len(server.__email__)
content = json.dumps({
"kind": "notify",
"data": "Converting crawled data to vector......",
"level": 0,
"end": ""
})
__print__(content, verbose=server.__verbose__, file=uid)
# Create convertor and convert crawled data to vector
convertor = Convertor("data")
profile = convertor.read_profile(fb_id.split("/")[-1])
profile = pd.DataFrame([profile])
content = json.dumps({
"kind": "notify",
"data": "Done",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
content = json.dumps({
"kind": "notify",
"data": "Preprocessing data......",
"level": 0,
"end": ""
})
__print__(content, verbose=server.__verbose__, file=uid)
# Load datapreprocessing object and normalizing vector
datapreprocessing = load("pkg/DataPreprocessingremove.dp")
profile = datapreprocessing.convert(profile)
content = json.dumps({
"kind": "notify",
"data": "Done",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
content = json.dumps({
"kind": "notify",
"data": "Predicting using Random forest......",
"level": 0,
"end": ""
})
__print__(content, verbose=server.__verbose__, file=uid)
# Load model and predict result
randomforest = load("pkg/overRandomForestremove.model")
result = randomforest.predict_proba(profile)[0][0] > 0.6
content = json.dumps({
"kind": "notify",
"data": "Done",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
result = "real" if result == True else "fake"
folder = os.path.join(os.getcwd(), "data")
target_dir = os.path.join(folder, fb_id.split("/")[-1])
filename = os.path.join(target_dir, "result.txt")
with open(filename, mode="w") as f:
f.write(result)
content = json.dumps({
"kind": "result",
"data": result,
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
conn.close()
import sys
from convertor import Convertor
input_file_path = sys.argv[1]
if not input_file_path:
print('please provide input file path.')
sys.exit()
convertor = Convertor(input_file_path)
convertor.encode()
# convertor.decode()
def convert_fields(self):
converter = Convertor()
self.op = converter.binary_to_decimal(self.op)
self.rs = converter.binary_to_decimal(self.rs)
self.rt = converter.binary_to_decimal(self.rt)
self.address = converter.binary_to_decimal(self.address)
def __solver__(conn, addr, data, **kwargs):
"Resolve a package 'data' come from client"
# Change data from string to json
data = json.loads(data)
# Get parameter server
if "server" not in kwargs:
raise Exception("Expected server parameter")
server = kwargs["server"]
# Solve data have key fb_id
if "fb_id" in data:
while True:
# Get email and password from server object
email = server.__email__[server.__current_account__]
password = server.__password__[server.__current_account__]
# Create scraper and start scraping facebook account
try:
scraper = Scraper2(email, password, verbose= "send", sender= conn)
bSuccess = scraper(data["fb_id"])
except Exception as e:
print(str(e))
scraper.__driver__.close()
conn.close()
return
# Not success if the crawler account is banned
if bSuccess:
break
content = json.dumps({
"kind": "notify",
"data": "Error in crawling, restart crawling...",
"level": None,
"end": "\n"
})
conn.send(content.encode())
# Switch account
server.__current_account__ = (server.__current_account__ + 1) % len(server.__email__)
content = json.dumps({"kind": "notify", "data": "Converting crawled data to vector......", "level": 0, "end": ""})
conn.send(content.encode())
# Create convertor and convert crawled data to vector
convertor = Convertor("data")
profile = convertor.read_profile(data["fb_id"].split("/")[-1])
profile = pd.DataFrame([profile])
content = json.dumps({"kind": "notify", "data": "Done", "level": None, "end": "\n"})
conn.send(content.encode())
content = json.dumps({"kind": "notify", "data": "Preprocessing data......", "level": 0, "end": ""})
conn.send(content.encode())
# Load datapreprocessing object and normalizing vector
datapreprocessing = load("pkg/DataPreprocessingremove.dp")
profile = datapreprocessing.convert(profile)
content = json.dumps({"kind": "notify", "data": "Done", "level": None, "end": "\n"})
conn.send(content.encode())
content = json.dumps({"kind": "notify", "data": "Predicting using Random forest......", "level": 0, "end": ""})
conn.send(content.encode())
# Load model and predict result
randomforest = load("pkg/overRandomForestremove.model")
result = randomforest.predict_proba(profile)[0][0] > 0.6
content = json.dumps({"kind": "notify", "data": "Done", "level": None, "end": "\n"})
conn.send(content.encode())
result = "real" if result == True else "fake"
content = json.dumps({"kind": "result", "data": result, "level": None, "end": "\n"})
conn.send(content.encode())
conn.close()
def get_obj(self, name):
"""获取对象"""
name = Convertor.format_usrname(name)
return self._handler.Environment.get(name)
def del_objs(self, vars):
"""删除对象"""
vars = ",".join(Convertor.format_usrname(v) for v in vars)
self._handler.exec_code("del %s" % vars)
def save_vars(self, vars, vals):
"""保存变量"""
for n, v in zip(vars, vals):
n = Convertor.format_usrname(n)
#实际成为了YCPY虚拟环境中全局变量
self._handler.add_api(n, v)
def format_exp(self, exp):
"""格式化表达式"""
return Convertor.format(exp)#获得等价的Python表达式
from convertor import Convertor
convertor = Convertor('head_rules.txt')
with open('ctb.bracketed', 'r') as f_in, open('ctb.conll', 'w') as f_out:
for line in f_in:
if line.startswith('#'):
f_out.write(line)
else:
deps = convertor.convert(line)
f_out.write(deps + '\n')
def get_hex_input():
try_again = True
while try_again:
hex = input("Enter an instruction: ")
parameter_one = is_hex(hex)
parameter_two = is_8_values_long(hex)
if (parameter_one == True and parameter_two == True):
try_again = False
else:
print("Enter a hexadecimal instruction with 8 characters")
return hex
registers = Registers()
memory = Memory(50)
convertor = Convertor()
print_registers_and_memory(registers, memory)
on = True
while on:
instruction = get_hex_input()
'''
00430825 - or $1,$2,$3
00430820 - add $1,$2,$3
00430822 - sub $1,$2,$3
00430829 - and $1,$2,$3
00430826 - xor $1,$2,$3
20410003 - addi $1,$2,3
8C410003 - lw $1,3($2)
AC410003 - sw $1,3($2)
Args:
- None BUT need to change the source code according the where the input/output are
- TO USE:
- change the orignal DTAL path
e.g. path = "/home/alta/BLTSpeaking/ged-kmk/dtal/data/GEM4/indtsv"
- change the A1, A2, B1, B2, C output paths
e.g. a1 = "/home/alta/BLTSpeaking/ged-pm574/dtal/data/GEM4-grade-dependent/BLXXXeval3_annotator5_A1.tsv"
Output:
- .ged.tsv split by grades
'''
import os
from convertor import Convertor
convertor = Convertor()
convertor.read_map(convertor.mapping_file1)
convertor.read_grade(convertor.grading_file1)
def main():
path = "/home/alta/BLTSpeaking/ged-kmk/dtal/data/GEM4/indtsv"
print('-------------------------------------------------------')
print(path)
print('-------------------------------------------------------')
files = [os.path.join(path, f) for f in os.listdir(path) if (os.path.isfile(os.path.join(path, f))) and '.tsv' in f]
files_a1 = []
files_a2 = []
files_b1 = []
files_b2 = []
files_c = []
for file in files:
