def select(self):
self.convertor = Convertor(
filedialog.askopenfilename(initialdir="/",
title="Select file",
filetypes=(("jpeg", "*.jpg"), ("png",
"*.png"),
("all files", "*.*"))))
if self.target:
self.target.destroy()
self.target = tk.Label(self.app,
text='TExt',
image=self.convertor.tkImg)
self.target.grid(row=1, column=0)
self.filemenu.entryconfigure(1, state=tk.NORMAL)
self.filemenu.entryconfigure(2, state=tk.NORMAL)
self.funcmenu.entryconfigure(0, state=tk.NORMAL)
self.funcmenu.entryconfigure(1, state=tk.NORMAL)
self.funcmenu.entryconfigure(2, state=tk.NORMAL)
self.filtermenu.entryconfigure(0, state=tk.NORMAL)
self.filtermenu.entryconfigure(1, state=tk.NORMAL)
self.filtermenu.entryconfigure(2, state=tk.NORMAL)
self.filtermenu.entryconfigure(3, state=tk.NORMAL)
self.filtermenu.entryconfigure(4, state=tk.NORMAL)
self.filtermenu.entryconfigure(5, state=tk.NORMAL)
def convert_fields(self):
converter = Convertor()
self.op = converter.binary_to_decimal(self.op)
self.rs = converter.binary_to_decimal(self.rs)
self.rt = converter.binary_to_decimal(self.rt)
self.rd = converter.binary_to_decimal(self.rd)
self.shamt = converter.binary_to_decimal(self.shamt)
self.funct = converter.binary_to_decimal(self.funct)
def check(wdbconn, dbsrc, dbdst, conf, count):
'''
检测转换后的数据的正确性
'''
check_cond = conf.get('check')
if check_cond is None:
sys.stderr.write(u'无法校验数据，需要在配置文件中指定 check 字段')
return
tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to']
where = conf.get('where', '1')
update_by = conf.get('update_by')
from convertor import Convertor
c = Convertor(conf['map'], conf.get('const'), wdbconn)
progress = make_progress(wdbconn, tbfrom, where, count)
check_where = conf.get('check_where', '1')
check_src_use_where = conf.get('check_src_use_where', False)
if not check_src_use_where:
where = '1'
rand_sql = make_check_rand_sql(tbto, check_where)
limit = 0
ncomm = 0
while limit < count:
# 从目标数据随机取出一条数据
with wdbconn.cursor() as wcsr:
wcsr.execute(rand_sql)
dst_data = wcsr.fetchone()
if dst_data is None:
sys.stderr.write(u'数据不足，无法检测')
break
# 获取对应的源数据
src_sql = make_check_src_sql(c.keys, tbfrom, where, dst_data,
check_cond)
wcsr.execute(src_sql)
src_data = wcsr.fetchall()
for sd in src_data:
src_dst_data = c.process(sd)
# 比较双方的数据
if compare(src_dst_data, dst_data):
ncomm += 1
break
progress.update(1)
limit += 1
progress.close()
ndiff = count - ncomm
print(u'''随机检测条数 %d
相同 %d
不同 %d
正确率 %.2f
错误率 %.2f''' % (count, ncomm, ndiff, ncomm * 100.0 / count,
ndiff * 100.0 / count))
def __test__(dp, model, fbid):
"Using for quick test a crawled account"
# Create a convertor and convert files in a account to vector
convertor = Convertor("data")
profile = convertor.read_profile(fbid)
profile = pd.DataFrame([profile])
# Load datapreprocessing object and nomalizing vector
datapreprocessing = load(dp)
profile = datapreprocessing.convert(profile)
# Load model and predict result
randomforest = load(model)
result = randomforest.predict_proba(profile)[0]
print(result)
def __init__(self, folder="data"):
self.folder = folder
self.filename = input("Enter EPUB Filename: ")
self.lang_src = "en"
self.lang_tgt = "fr"
path_without_ext = os.path.join(self.folder, self.filename).lower()
path_with_ext = path_without_ext + ".epub"
if os.path.exists(path_with_ext):
c = Convertor(path_without_ext)
c.to_txt()
##t = Translator(path_without_ext, self.lang_src, self.lang_tgt)
#t.google_translate()
#r = Reader(t.filename, self.lang_tgt)
#r.play()
else:
print("File {} is Missing".format(path))
def convert(wdbconn, rdbconn, dbsrc, dbdst, conf):
limit = max(conf.get('limit', 0), 0)
tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to']
where = conf.get('where', '1')
update_by = conf.get('update_by')
from convertor import Convertor
c = Convertor(conf['map'], conf.get('const'), wdbconn)
progress = make_progress(wdbconn, tbfrom, where, limit)
rcsr = rdbconn.cursor()
srcsql = make_src_sql(c.keys, tbfrom, where)
rcsr.execute(srcsql)
# 获取数据表
offset, step = 0, 200
while limit == 0 or offset < limit:
with wdbconn.cursor() as wcsr:
srcdata = rcsr.fetchmany(step)
# 因为每条 sql 可能都不一样，所以不能使用 executemany
for sd in srcdata:
dstdata = c.process(sd)
dstsql = make_dst_sql(dstdata, tbto, update_by)
# print(dstsql)
if dstsql:
wcsr.execute(dstsql, tuple(dstdata.values()))
wdbconn.commit()
progress.update(step)
if len(srcdata) < step:
break
offset += step
rcsr.close()
progress.close()
def __solver__(conn, addr, data, **kwargs):
"Resolve a package 'data' come from client"
# Change data from string to json
data = json.loads(data)
# Get parameter server
if "server" not in kwargs:
raise Exception("Expected server parameter")
server = kwargs["server"]
if "uid" in data:
uid = data["uid"]
else:
uid = None
# Solve data have key fb_id
if "fb_id" in data:
fb_id = scrape_utils.__create_original_link__("https://",
data["fb_id"])
while True:
# Get email and password from server object
email = server.__email__[server.__current_account__]
password = server.__password__[server.__current_account__]
# Create scraper and start scraping facebook account
try:
scraper = Scraper2(email, password, verbose="file", sender=uid)
bSuccess = scraper(fb_id)
except Exception as e:
print(str(e))
scraper.__driver__.close()
conn.close()
return
# Not success if the crawler account is banned
if bSuccess is not False:
break
content = json.dumps({
"kind": "notify",
"data": "Error in crawling, restart crawling...",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
# Switch account
server.__current_account__ = (server.__current_account__ +
1) % len(server.__email__)
content = json.dumps({
"kind": "notify",
"data": "Converting crawled data to vector......",
"level": 0,
"end": ""
})
__print__(content, verbose=server.__verbose__, file=uid)
# Create convertor and convert crawled data to vector
convertor = Convertor("data")
profile = convertor.read_profile(fb_id.split("/")[-1])
profile = pd.DataFrame([profile])
content = json.dumps({
"kind": "notify",
"data": "Done",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
content = json.dumps({
"kind": "notify",
"data": "Preprocessing data......",
"level": 0,
"end": ""
})
__print__(content, verbose=server.__verbose__, file=uid)
# Load datapreprocessing object and normalizing vector
datapreprocessing = load("pkg/DataPreprocessingremove.dp")
profile = datapreprocessing.convert(profile)
content = json.dumps({
"kind": "notify",
"data": "Done",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
content = json.dumps({
"kind": "notify",
"data": "Predicting using Random forest......",
"level": 0,
"end": ""
})
__print__(content, verbose=server.__verbose__, file=uid)
# Load model and predict result
randomforest = load("pkg/overRandomForestremove.model")
result = randomforest.predict_proba(profile)[0][0] > 0.6
content = json.dumps({
"kind": "notify",
"data": "Done",
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
result = "real" if result == True else "fake"
folder = os.path.join(os.getcwd(), "data")
target_dir = os.path.join(folder, fb_id.split("/")[-1])
filename = os.path.join(target_dir, "result.txt")
with open(filename, mode="w") as f:
f.write(result)
content = json.dumps({
"kind": "result",
"data": result,
"level": None,
"end": "\n"
})
__print__(content, verbose=server.__verbose__, file=uid)
conn.close()
import sys
from convertor import Convertor
input_file_path = sys.argv[1]
if not input_file_path:
print('please provide input file path.')
sys.exit()
convertor = Convertor(input_file_path)
convertor.encode()
# convertor.decode()
def __solver__(conn, addr, data, **kwargs):
"Resolve a package 'data' come from client"
# Change data from string to json
data = json.loads(data)
# Get parameter server
if "server" not in kwargs:
raise Exception("Expected server parameter")
server = kwargs["server"]
# Solve data have key fb_id
if "fb_id" in data:
while True:
# Get email and password from server object
email = server.__email__[server.__current_account__]
password = server.__password__[server.__current_account__]
# Create scraper and start scraping facebook account
try:
scraper = Scraper2(email, password, verbose= "send", sender= conn)
bSuccess = scraper(data["fb_id"])
except Exception as e:
print(str(e))
scraper.__driver__.close()
conn.close()
return
# Not success if the crawler account is banned
if bSuccess:
break
content = json.dumps({
"kind": "notify",
"data": "Error in crawling, restart crawling...",
"level": None,
"end": "\n"
})
conn.send(content.encode())
# Switch account
server.__current_account__ = (server.__current_account__ + 1) % len(server.__email__)
content = json.dumps({"kind": "notify", "data": "Converting crawled data to vector......", "level": 0, "end": ""})
conn.send(content.encode())
# Create convertor and convert crawled data to vector
convertor = Convertor("data")
profile = convertor.read_profile(data["fb_id"].split("/")[-1])
profile = pd.DataFrame([profile])
content = json.dumps({"kind": "notify", "data": "Done", "level": None, "end": "\n"})
conn.send(content.encode())
content = json.dumps({"kind": "notify", "data": "Preprocessing data......", "level": 0, "end": ""})
conn.send(content.encode())
# Load datapreprocessing object and normalizing vector
datapreprocessing = load("pkg/DataPreprocessingremove.dp")
profile = datapreprocessing.convert(profile)
content = json.dumps({"kind": "notify", "data": "Done", "level": None, "end": "\n"})
conn.send(content.encode())
content = json.dumps({"kind": "notify", "data": "Predicting using Random forest......", "level": 0, "end": ""})
conn.send(content.encode())
# Load model and predict result
randomforest = load("pkg/overRandomForestremove.model")
result = randomforest.predict_proba(profile)[0][0] > 0.6
content = json.dumps({"kind": "notify", "data": "Done", "level": None, "end": "\n"})
conn.send(content.encode())
result = "real" if result == True else "fake"
content = json.dumps({"kind": "result", "data": result, "level": None, "end": "\n"})
conn.send(content.encode())
conn.close()
from conf import Cfg
from finder import Finder
from convertor import Convertor
from storage import Storage
import glob
import os
from parser import Parser
import scholarly
cfg = Cfg()
db = Storage(cfg)
db.load()
finder = Finder(cfg)
convertor = Convertor(cfg)
def process_file(f):
checksum = Storage.file_checksum(f)
if checksum in db.data:
print("file {} already processed ({})".format(f, checksum))
return
db.store(checksum, {"pdf": f})
convertor.convert(f)
for f in finder.find_all():
process_file(f)
db.load()
def get_hex_input():
try_again = True
while try_again:
hex = input("Enter an instruction: ")
parameter_one = is_hex(hex)
parameter_two = is_8_values_long(hex)
if (parameter_one == True and parameter_two == True):
try_again = False
else:
print("Enter a hexadecimal instruction with 8 characters")
return hex
registers = Registers()
memory = Memory(50)
convertor = Convertor()
print_registers_and_memory(registers, memory)
on = True
while on:
instruction = get_hex_input()
'''
00430825 - or $1,$2,$3
00430820 - add $1,$2,$3
00430822 - sub $1,$2,$3
00430829 - and $1,$2,$3
00430826 - xor $1,$2,$3
20410003 - addi $1,$2,3
8C410003 - lw $1,3($2)
AC410003 - sw $1,3($2)
def convert_fields(self):
converter = Convertor()
self.op = converter.binary_to_decimal(self.op)
self.rs = converter.binary_to_decimal(self.rs)
self.rt = converter.binary_to_decimal(self.rt)
self.address = converter.binary_to_decimal(self.address)
from convertor import Convertor
convertor = Convertor('head_rules.txt')
with open('ctb.bracketed', 'r') as f_in, open('ctb.conll', 'w') as f_out:
for line in f_in:
if line.startswith('#'):
f_out.write(line)
else:
deps = convertor.convert(line)
f_out.write(deps + '\n')
