class DBManager():
def __init__(self, store):
self._db = SqliteDict(store, autocommit=True)
self.commands_cache = {}
def add(self, command):
if self.get_command(command.get_name()) != None:
return False
self._db[command.get_name()] = command
return True
def update(self, command):
self.add(command)
def remove(self, command):
del self._db[command.get_name()]
def empty(self):
self.commands_cache = {}
self._db.clear()
def count(self):
return len(self._db)
def get_command(self, name):
if name not in self.commands_cache:
try:
self.commands_cache[name] = self._db[name]
except KeyError:
return None
return self.commands_cache[name]
def get_all_commands(self):
list = []
for key, val in self._db.iteritems():
list.append(key)
return list
class DBManager():
def __init__(self):
self._db = SqliteDict('./matches.sqlite', autocommit=True)
self.matches_cache = {}
def add(self, match):
self._db[match.get_code()] = match
def update(self, match):
self.add(match)
def remove(self, match):
del self._db[match.get_code()]
def empty(self):
self._db.clear()
def count(self):
return len(self._db)
def get_match(self, code):
if code not in self.matches_cache:
try:
self.matches_cache[code] = self._db[code]
except KeyError:
return None
return self.matches_cache[code]
def get_all_matches(self):
list = []
for key, val in self._db.iteritems():
list.append(key)
return list
class db(object):
def __init__(self, user):
self.user = str(user)
self.db = SqliteDict(self.getCfgPath(), autocommit=True)
def get(self, key=''):
return self.db.get(key) if key else self.db.iteritems()
def set(self, key='', data=''):
if not key:
key = self.user
if data:
self.db[key] = data
else:
self.db.__delitem__(key)
def getCfgPath(self):
if os.path.isdir('hoshino'):
if not os.path.isdir('hoshino/modules/ASF_Plus/config'):
os.mkdir('hoshino/modules/ASF_Plus/config')
return os.path.join(
os.path.abspath('hoshino/modules/ASF_Plus/config'),
f'{self.user}.sqlite')
else:
if not os.path.isdir('../config'):
os.mkdir('../config')
return os.path.join(os.path.abspath('../config'),
f'{self.user}.sqlite')
def modify(self):
mydict = SqliteDict('./my_db.sqlite', autocommit=True)
if self.index == 2:
self.ttlog.log("快速收录重置-->开始")
elif self.index == 1:
self.ttlog.log("普通收录重置-->开始")
tree_len = len(self.tree.get_children())
time.sleep(10)
for key, value in sorted(mydict.iteritems()):
if value[self.index] == "已提交":
# 是否修改列表，看是否加载列表
value[self.index] = "未提交"
if tree_len != 0:
self.tree.item(value[4], value=value[:4])
self.ttlog.log("重置状态：" + key + "")
# 修改数据库
mydict[key] = value
if self.index == 2:
self.ttlog.log("快速收录重置-->完成")
self.content.config(text="快速收录重置-->完成")
elif self.index == 1:
self.ttlog.log("普通收录重置-->完成")
self.content.config(text="普通收录重置-->完成")
self.ttlog.stop_log()
def get_url(self):
url_list = []
mydict = SqliteDict('./my_db.sqlite', autocommit=True)
for key, value in sorted(mydict.iteritems()):
if value[2] == "未提交":
url_list.append(value)
self.ttlog.log("快速收录-->共有没推送的网页链接数 ：{} 条!".format(len(url_list)))
print("共有没快速收录推送的网页链接数 ：{} 条!".format(len(url_list)))
return url_list
def criaTeste():
mydict = SqliteDict('db/teste.sqlite', "mails", autocommit=True)
mydict['1'] = "batata"
mydict['2'] = "banana"
mydict['3'] = "oi"
mydict['4'] = "teste"
for key, value in mydict.iteritems():
print(key, value)
print(len(mydict)) # etc... all dict functions work
mydict.close()
class Scribe(object):
def __init__(self, location, table_name, exp_name):
filename = "{}/scribe.sqlite".format(location)
self.book = SqliteDict(filename, autocommit=True, tablename=table_name)
unique_id = datetime.now().strftime("date_%m.%d_time_%H.%M")
self.exp_name = exp_name + "_" + unique_id
self.observation_index = 0
def record(self, value, type="general"):
key = "{}; {}; {}".format(self.exp_name, self.observation_index, type)
self.book[key] = value
self.observation_index += 1
observe = record #sometimes i forget which
def lookup(self,
type=None,
exp_name=None,
ret_sorted=False,
strip_keys=False):
type_func = lambda *args: True
name_func = lambda *args: True
if type:
type_func = lambda x: x[2] == type
if exp_name:
name_func = lambda x: exp_name in x[0]
key_func = lambda x: type_func(x) and name_func(x)
unpack = lambda x: [
f(x.strip()) for f, x in zip([str, int, str], x.split(";"))
]
items = {k: v for k, v in self.book.iteritems() if key_func(unpack(k))}
if ret_sorted:
return self.sort_results(items, strip_keys)
return items
def sort_results(self, result_dict, only_val_return=False):
unpack = lambda x: [
f(x.strip()) for f, x in zip([str, int, str], x.split(";"))
]
ranker = lambda x: unpack(x[0])[1]
sorted_items = sorted(result_dict.items(), key=ranker)
if only_val_return:
return [v for k, v in sorted_items]
return sorted_items
def close(self):
self.book.close()
def list_db(dbname, options):
# No context manager (and no close()) as this is read-only and
# close() can block for a long time for no apparent reason.
db = SqliteDict(dbname, flag='r', autocommit=False)
if not options.keys:
for k, v in db.iteritems():
output(k, v.rstrip('\n'), options)
else:
for k in options.keys:
try:
v = db[k]
except KeyError as e:
error('no such key: "{}"'.format(k))
else:
output(k, v.rstrip('\n'), options)
def list_annotations(dbname, options):
# No context manager: close() can block and this is read-only
doc_count, ann_count = 0, 0
db = SqliteDict(dbname, flag='r', autocommit=False)
for k, v in db.iteritems():
root, ext = os.path.splitext(os.path.basename(k))
if ext != options.suffix:
continue
for line in v.splitlines():
if options.random is not None and random() > options.random:
continue
print('{}\t{}'.format(root, line))
ann_count += 1
doc_count += 1
print('Done, listed {} annotations in {} docs from {}'.format(
ann_count, doc_count, dbname),
file=sys.stderr)
class Scribe(object):
def __init__(self, location, table_name, exp_name):
filename = "{}/scribe.sqlite".format(location)
self.book = SqliteDict(filename, autocommit=True, tablename=table_name)
unique_id = datetime.now().strftime("date_%m.%d_time_%H.%M")
self.exp_name = exp_name+"_"+unique_id
self.observation_index = 0
def record(self, value, type="general"):
key = "{}; {}; {}".format(self.exp_name, self.observation_index, type)
self.book[key] = value
self.observation_index += 1
observe = record #sometimes i forget which
def lookup(self, type=None, exp_name=None, ret_sorted=False, strip_keys=False):
type_func = lambda *args: True
name_func = lambda *args: True
if type:
type_func = lambda x: x[2] == type
if exp_name:
name_func = lambda x: exp_name in x[0]
key_func = lambda x: type_func(x) and name_func(x)
unpack = lambda x: [f(x.strip()) for f,x in zip([str,int,str],x.split(";"))]
items = {k:v for k,v in self.book.iteritems() if key_func(unpack(k))}
if ret_sorted:
return self.sort_results(items, strip_keys)
return items
def sort_results(self, result_dict, only_val_return=False):
unpack = lambda x: [f(x.strip()) for f,x in zip([str,int,str],x.split(";"))]
ranker = lambda x: unpack(x[0])[1]
sorted_items = sorted(result_dict.items(), key=ranker)
if only_val_return:
return [v for k,v in sorted_items]
return sorted_items
def close(self):
self.book.close()
else:
flag = 'c'
mergehll = SqliteDict(outputfile, flag=flag, journal_mode='MEMORY',
encode=hll_encode, decode=hll_decode)
mergekeys = set()
for k in mergehll.keys():
mergekeys.add(k)
for f in files:
if not os.path.exists(f):
print("{}: File not found".format(f))
sys.exit()
filehll = SqliteDict(f, encode=hll_encode, decode=hll_decode)
# merge into outputfile
for tld, tldhll in filehll.iteritems():
if tld in mergekeys: # key already in outputfile -> merge
hll = mergehll[tld]
if not hll.__eq__(tldhll): # merge only if HLL is different
hll.update(tldhll)
mergehll[tld] = hll
else: # new key in outfile -> copy
mergehll[tld] = filehll[tld]
mergekeys.add(tld)
filehll.close()
print("Total TLDs {}".format(len(mergehll)))
print("Total IPs {}".format(len(mergehll['TOTAL'])))
mergehll.commit()
mergehll.close()
class Update_window(object):
"""sitemap更新窗体"""
def __init__(self, tree, eblog, sitemap):
self.newroot = tk.Toplevel()
self.newroot.title('下载文件中')
self.newroot.iconbitmap("favicon.ico")
self.newroot.wm_attributes('-topmost', 1)
win_width = self.newroot.winfo_screenwidth()
win_higth = self.newroot.winfo_screenheight()
width_adjust = (win_width - 800) / 2
higth_adjust = (win_higth - 250) / 2
self.newroot.geometry("%dx%d+%d+%d" %
(800, 250, width_adjust, higth_adjust))
# 进度条
self.bar = ttk.Progressbar(self.newroot,
length=740,
mode="indeterminate",
orient=tk.HORIZONTAL)
self.bar.place(
x=30,
y=150,
)
self.bar.start(10)
# 提示内容
self.content = tk.Label(self.newroot, text="正在下载Sitemap.xml文件...")
self.content.place(
x=30,
y=30,
)
self.content2 = tk.Label(self.newroot,
text="下载速度和文件大小以及服务器带宽有关，请耐心等待......",
wraplength=740,
justify="left")
self.content2.place(
x=30,
y=60,
)
self.eblog = eblog
self.sitemap = sitemap
self.tree = tree
self.mydict = SqliteDict('./my_db.sqlite', autocommit=True)
# 开启处理线程
self.p = Thread(target=self.update)
self.p.setDaemon(True)
self.p.start()
self.eblog.log("Sitemap线程：开启sitemap线程,下载Sitemap.xml中...")
# 关闭右上角
self.newroot.protocol("WM_DELETE_WINDOW", self.close)
# 列表添加item,返回iid
def append_item(self, item_list):
# 加最后/加前面都可以，因为要是前面iid全是1
item = self.tree.insert("",
0,
values=(item_list[0], item_list[1],
item_list[2], item_list[3]))
return item
# 处理函数
def update(self):
try:
with open("sitemap.xml", "wb") as f:
f.write(requests.get(self.sitemap).content)
with open("sitemap.xml", 'r', encoding='utf-8') as f:
xml_data = f.read()
self.content.configure(text="Sitemap文件下载完成，正在对比分析....")
urls = re.findall(r'<loc>(.+?)</loc>', xml_data, re.S)
self.eblog.log("Sitemap线程-->下载Sitemap.xml完成,正在解析xml文件...")
tuple_list = list(self.mydict.iteritems())
tree_urls = [i[0] for i in tuple_list]
# 求交集
c = list(set(urls).intersection(set(tree_urls)))
# tree中多余的
tree_urls_ = list(set(tree_urls).difference(set(c)))
# 交集不动,tree把tree中把不是交集的删掉,把siemaop中不是交集的增添上.
for key, value in sorted(tuple_list):
self.content2.config(text="当前处理-->检查" + key)
if key not in c:
# 是否删除,看有没有列表
if len(self.tree.get_children()) != 0:
self.tree.delete(value[4])
self.mydict.__delitem__(key=key)
self.eblog.log("Sitemap线程-->本地删除" + str(tree_urls_))
# sitemap中新添加的
urls_ = list(set(urls).difference(set(c)))
for url in sorted(urls_):
cur_time = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
iid = self.append_item([url, "未提交", "未提交", cur_time])
self.mydict[url] = [url, "未提交", "未提交", cur_time, iid]
self.content2.config(text="当前处理-->正在添加" + url)
self.eblog.log("Sitemap线程-->本地添加" + str(urls_))
self.eblog.log("Sitemap线程-->关闭sitemap线程,更新完成。")
self.close()
except:
self.eblog.log(traceback.format_exc())
self.eblog.log("Sitemap线程-->更新失败")
def close(self):
self.newroot.destroy()
from sqlitedict import SqliteDict
import pickle
tall = []
c = 0
c_tall = 0
mydict = SqliteDict(
"/../../../../../scratch/manoelribeiro/helpers/authors_dict.sqlite",
tablename="authors",
flag="r")
for _, value in mydict.iteritems():
if (c % 100000) == 0:
print(f"c = {c}, c_all = {c_tall}")
c += 1
if c >= 11400000:
break
diff_comm = set()
for comm in value:
cat = comm["category"]
if cat == "Alt-right" or cat == "Alt-lite" or cat == "Intellectual Dark Web":
diff_comm.add(cat)
else:
diff_comm.add("control")
if len(diff_comm) > 1:
continue
tall.append(value)
c_tall += 1
class Admin(commands.Cog):
MY_ID = 192520503443849217
def __init__(self, bot):
self.bot = bot
self.description = "Admin/debug commands."
self.gamble_db = SqliteDict('./gamble.sqlite', autocommit=True)
self.remind_db = SqliteDict('./remind.sqlite', autocommit=True)
self.prefix_db = SqliteDict('./prefix.sqlite', autocommit=True)
@commands.command(help="[admin] reset points for this server")
async def perish(self, ctx):
server = str(ctx.guild.id)
if ctx.author.id == self.MY_ID:
try:
# del db['gamble'][server]
del self.gamble_db[server]
await ctx.send("donezo")
except KeyError:
await ctx.send("no-op, table is already gone")
else:
await ctx.send("Perish this command is not for you")
@commands.command(help="[admin] reset all points for all servers")
async def thanos(self, ctx):
if (ctx.author.id == self.MY_ID):
for key in db:
del db[key]
try:
for key in db['gamble']:
del db['gamble'][key]
await ctx.send("*thanos snaps*")
except KeyError:
await ctx.send("no-op, table is already gone")
else:
await ctx.send("Perish this command is not for you")
@commands.command(help="[admin] drop all tables")
async def drop(self, ctx):
if (ctx.author.id == self.MY_ID):
for key in db:
del db[key]
await ctx.send("monkaSteer")
else:
await ctx.send("Perish this command is not for you")
@commands.command(help="[admin] drop raid table")
async def rdrop(self, ctx):
if (ctx.author.id == self.MY_ID):
if db['reminders']:
for key in db['reminders']:
del db['reminders'][key]
del db['reminders']
await ctx.send("all done")
@commands.command(help="[admin] log all tables")
async def tables(self, ctx):
for key, value in self.gamble_db.iteritems():
print(f"key: {key} value: {value}")
for key, value in self.prefix_db.iteritems():
print(f"key: {key} value: {value}")
if (ctx.author.id == self.MY_ID):
print('KEYS')
print(db.keys())
print('SUBTABLES')
for key in db:
print(key)
for k in db[key]:
print(f'KEY: {k} // VALUE: {db[key][k]}')
else:
await ctx.send("Perish this command is not for you")
ICON_FONT_FILE = 'pifile.ttf'
ICON_FONT_JSON = 'config.json'
ICONS = icon(ICON_FONT_JSON, ICON_FONT_FILE)
##################################
# SCREEN BANNER VARIABLES
###################################
LOADING_MESSEGES = []
TREKNOBABBLE = 'treknobabble.tbdb'
_TREKNOBABBLE = os.path.join('resources/', TREKNOBABBLE)
tb_db = SqliteDict(_TREKNOBABBLE)
# for item in dict_db.iteritems():
# pprint(item)
for key, value in tb_db.iteritems():
LOADING_MESSEGES.append(value)
# return_event = pygame.event.Event(RETURN_EVENT)
# Dict of events that are accessible to screens
piscreenevents = {
"button": TFTBUTTONCLICK,
"update": UPDATESCREEN,
"nextscreen": NEXTSCREEN,
"toggle_fps": TOGGLE_FPS_EVENT,
}
myset = []
for link in soup.find_all('h1'):
tmp = link.get_text()
string = tmp[5:-3]
myset.append(string)
if myset[0] not in dict_db.iterkeys():
dict_db[myset[0]] = myset[1]
unique += 1
else:
print "Duplicate"
duplicates += 1
treknobabble.append(failure)
# get_tb()
for item in dict_db.iteritems():
pprint(item)
print len(dict_db)
dict_db.close()
# print "Dups: " + str(duplicates)
# f = open('treknobable.txt', 'w')
# f.write(x.get_string(sortby="Timestamp"))
# for thing in treknobabble:
# pprint(thing)
# for key in thing:
# f.write('\n' + '----------\n')
# f.write(key + '\n')
# f.write(thing[key])
class Tree_control(object):
"""列表加载窗体"""
def __init__(self, tree, eblog):
self.newroot = tk.Toplevel()
self.newroot.title('加载列表')
self.newroot.iconbitmap("favicon.ico")
self.newroot.wm_attributes('-topmost', 1)
win_width = self.newroot.winfo_screenwidth()
win_higth = self.newroot.winfo_screenheight()
width_adjust = (win_width - 400) / 2
higth_adjust = (win_higth - 250) / 2
self.newroot.geometry("%dx%d+%d+%d" %
(400, 250, width_adjust, higth_adjust))
# 进度条
self.__showFlag = True
self.__width = 300
self.__heigth = 20
self.__sleep = 0
self.bar = ttk.Progressbar(self.newroot,
length=self.__width,
mode="indeterminate",
orient=tk.HORIZONTAL)
self.bar.pack(expand=True)
self.bar.start(10)
# 提示内容
self.content2 = tk.Label(self.newroot,
text="正在加载列表中,请不要中断操作，请耐心等待......")
self.content2.place(
x=50,
y=30,
)
self.content = tk.Label(self.newroot, text="")
self.content.place(
x=50,
y=60,
)
self.eblog = eblog
self.tree = tree
self.mydict = SqliteDict('./my_db.sqlite', autocommit=True)
# 开启处理线程
self.p = Thread(target=self.add_item)
self.p.setDaemon(True)
self.p.start()
# 点击关闭右上角
self.newroot.protocol("WM_DELETE_WINDOW", self.close)
# 加载item
def add_item(self):
len_items = len(sorted(self.mydict.iteritems()))
i = 0
for key, value in sorted(self.mydict.iteritems()):
i = i + 1
self.content.config(text="当前正处理：第" + str(i) + "个，共有" +
str(len_items) + "个链接")
self.tree.insert("",
0,
iid=value[4],
values=(value[0], value[1], value[2], value[3]))
self.close()
return 1
def close(self):
self.newroot.destroy()
class tiny_db():
def __init__(self):
self.debug = True
self.dict_db = SqliteDict('./racks.sqlite', autocommit=True)
self.days_to_keep = 5
self._define_mem_db()
# SET UP DATABASE VARIABLES
self.row_height = DATABASE_SETTINGS['rows']
self.column_width = DATABASE_SETTINGS['columns']
self.get_last_filed()
self.rack_day = self._today()
self.next = {}
self._today()
self.next_location()
self._db_info()
if self.last_filed is None:
try:
self._convert_to_sqlitedb()
except:
print "Database is Empty"
# @profile
def _define_mem_db(self):
self.mem_db = []
for item in self.dict_db.iteritems():
self.mem_db.append(item[1])
# @profile
def _db_info(self):
if len(self.mem_db) > 0:
x = PrettyTable([" ", "size"])
x.add_row(["DB Size", len(self.dict_db)])
x = PrettyTable(["stats", "accn", "Date", "Timestamp"])
# pprint(self.last_filed)
readable = self._conv_timestamp(self.last_filed['time'])
x.add_row(["Last",
self.last_filed['accn'],
readable,
self.last_filed['time']])
first_filed = self.get_first_filed()
other_readable = self._conv_timestamp(first_filed['time'])
x.add_row(["First",
first_filed['accn'],
other_readable,
first_filed['time']])
# print x
return x
def _list_size(self):
size = {'memory': len(self.mem_db), 'disk': len(self.dict_db)}
return size
def _print_database(self):
if len(self.mem_db) < 2:
return none
x = PrettyTable(["Accn", "Rack", "Position", "Time", "Timestamp"])
# x.padding_width = 1
for item in self.mem_db:
x.add_row(self._make_entry_row(item))
# print x.get_string(sortby="Timestamp")
f = open('_database_info.txt', 'w')
f.write(x.get_string(sortby="Timestamp"))
f.write(self._db_info().get_string())
f.close()
print x.get_string(fields=["Accn", "Rack", "Position"])
def _make_entry_row(self, item):
readable = self._conv_timestamp(item['time'])
# print item['row']
x = [item['accn'],
item['rackDay'] + ' ' + str(item['rack']),
ROWS[str(item['row'])] + ' ' + str(item['column']),
readable,
item['time']]
return x
def _conv_timestamp(self, ts):
dt = datetime.datetime.fromtimestamp(float(ts))
return dt.strftime("%H:%M - %m.%d.%Y")
def _today(self):
self.purge_date = int(
mktime(
(datetime.date.today() -
datetime.timedelta(
self.days_to_keep)).timetuple()))
return strftime('%a', localtime(time()))
# @profile
def file_accn(self, accn):
insert = {
'accn': accn,
'rack': self.next['rack'],
'rackDay': self.next['rackDay'],
'column': self.next['column'],
'row': self.next['row'],
'time': str(time())
}
self.mem_db.append(insert)
self.last_filed = insert
self.dict_db[insert['time']] = insert
self.next_location()
# @profile
def get_last_filed(self):
_last_filed_id = None
self.last_filed = None
for item in self.mem_db:
if _last_filed_id < item['time']:
_last_filed_id = item['time']
self.last_filed = item
return self.last_filed
# @profile
def get_first_filed(self):
# if len(self.mem_db) == 1:
# return self.last_filed
# else:
# print self.last_filed
if self.last_filed is None:
return self.last_filed
else:
_smallest_id = self.last_filed['time']
first_filed = None
for item in self.mem_db:
if item:
# print item
if _smallest_id > item['time']:
# print "smaller"
_smallest_id = item['time']
first_filed = item
# print first_filed
return first_filed
# @profile
def new_day(self):
self.next['column'] = 1
self.next['rack'] = 1
self.next['row'] = 1
self.next['rackDay'] = self._today()
# @profile
def next_location(self):
if self.last_filed is None:
self.new_day()
return
if self.last_filed['rackDay'] != self._today():
self.new_day()
elif self.last_filed['column'] == self.column_width:
if self.last_filed['row'] == self.row_height:
self.next['column'] = 1
self.next['row'] = 1
self.next['rack'] = self.last_filed['rack'] + 1
else:
self.next['column'] = 1
self.next['row'] = self.last_filed['row'] + 1
self.next['rack'] = self.last_filed['rack']
else:
self.next['column'] = self.last_filed['column'] + 1
self.next['rack'] = self.last_filed['rack']
self.next['row'] = self.last_filed['row']
self.next['rackDay'] = self._today()
# @profile
def find_accn(self, accn):
result = []
for item in self.mem_db:
if item['accn'] == accn:
result.append(item)
elif accn in item['accn']:
result.append(item)
return result
# @profile
def _convert_to_sqlitedb(self):
old_db = TinyDB(DATABASE_SETTINGS['database'])
self.mem_db = []
total = 0
for table in old_db.tables():
for item in old_db.table(table).all():
total += 1
print "MIGRATING DATABASE"
print "--OLD DATABASE: " + str(total)
i = 0
for table in old_db.tables():
for item in old_db.table(table).all():
if len(item['accn']) < 15:
if int(item['time']) > self.purge_date:
self.mem_db.append(item)
print " Converted: " + str(i) + " of " + str(total)
i += 1
self.dict_db[str(item['time'])] = item
else:
print "REMOVING OLD THING"
print "DATABASE MIGRATION COMPLETE"
print "COMMITING CHANGES"
self.get_last_filed()
# @profile
def clean(self):
print "HARD STORAGE: " + str(len(self.dict_db))
print "Mem: " +str(len(self.mem_db))
self.mem_db = []
for item in self.dict_db.iteritems():
# print int(item[1]['time'][:-3])
# print self.purge_date
if float(item[1]['time']) < self.purge_date:
print "---"
print float(item[1]['time'])
# print "---"
print self.purge_date
del self.dict_db[item[0]]
self._define_mem_db()
print self.purge_date
print "HARD STORAGE: " + str(len(self.dict_db))
print "Mem: " +str(len(self.mem_db))
class Baidu_ordinary_windows(object):
"""百度普通收录窗体"""
def __init__(self, tree, site, token):
# 展示等待窗体
self.newroot = tk.Toplevel()
self.newroot.title('普通收录')
self.newroot.iconbitmap("favicon.ico")
win_width = self.newroot.winfo_screenwidth()
win_higth = self.newroot.winfo_screenheight()
width_adjust = (win_width - 800) / 2
higth_adjust = (win_higth - 250) / 2
self.newroot.geometry("%dx%d+%d+%d" %
(800, 250, width_adjust, higth_adjust))
# 提示内容
self.content = tk.Label(self.newroot,
text="正在普通收录中,请不要中断操作，请耐心等待......")
self.content.place(
x=10,
y=30,
)
self.content2 = tk.Label(self.newroot, text="")
self.content2.place(
x=10,
y=60,
)
# 窗体日志
self.ttlog = ttlog(master=self.newroot)
self.ttlog.place(x=10, y=70, width=780, height=150)
self.tree = tree
self.site = site
self.token = token
self.mydict = SqliteDict('./my_db.sqlite', autocommit=True)
# 开始处理线程
self.p = Thread(target=self.main)
self.p.setDaemon(True)
self.p.start()
self.ttlog.log("普通收录-->开启普通收录线程.....")
# 点击关闭右上角
self.newroot.protocol("WM_DELETE_WINDOW", self.close)
def close(self):
self.ttlog.stop_log()
self.newroot.destroy()
# 获取未提交的urls
def get_url(self):
url_list = []
for key, value in sorted(self.mydict.iteritems()):
if value[1] == "未提交":
url_list.append(value)
self.ttlog.log("普通收录-->共有没推送的网页链接数 ：{} 条!".format(len(url_list)))
print("共有没普通收录推送的网页链接数 ：{} 条!".format(len(url_list)))
return url_list
# 查询剩余次数
def get_remain(self):
post_url = "http://data.zz.baidu.com/urls?site={}&token={}".format(
self.site, self.token)
headers = {
'User-Agent': 'curl/7.12.1',
'Host': 'data.zz.baidu.com',
'Content-Type': 'text/plain',
'Content-Length': '83',
}
response = requests.post(post_url, headers=headers, data=self.site)
req = response.text
if "success" in req:
req_json = json.loads(req)
if req_json["remain"] == 0:
self.ttlog.log(
"普通收录-->查询剩余次数,今天普通收录推送任务已经完成,\n当天剩余的可推送url条数: " +
req_json["remain"] + "条。")
else:
self.ttlog.log(
"普通收录-->查询剩余次数,推送成功：" + self.site +
'\n当天剩余的可推送url条数: {}条'.format(req_json["remain"]))
return req_json["remain"]
else:
return 0
# 提交urls
def api(self, url):
post_url = "http://data.zz.baidu.com/urls?site={}&token={}".format(
self.site, self.token)
headers = {
'User-Agent': 'curl/7.12.1',
'Host': 'data.zz.baidu.com',
'Content-Type': 'text/plain',
'Content-Length': '83',
}
response = requests.post(post_url, headers=headers, data=url[0])
req = response.text
if "success" in req:
req_json = json.loads(req)
if req_json["remain"] == 0:
self.ttlog.log("普通收录-->今天普通收录推送任务已经完成,当天剩余的可推送url条数: 0条。")
else:
# 是否修改列表，看是否加载列表
tree_len = len(self.tree.get_children())
if tree_len != 0:
print("普通收录-->修改列表")
self.tree.item(url[4],
value=(url[0], "已提交", url[2], url[3]))
# 修改数据库
self.mydict[url[0]] = [url[0], "已提交", url[2], url[3], url[4]]
self.ttlog.log(
"普通收录-->推送成功：" + url[0] +
'\n当天剩余的可推送url条数: {}条'.format(req_json["remain"]))
else:
req_json = json.loads(req)
self.ttlog.log(r"普通收录-->推送失败:" + req_json["message"] +
"，当天可剩余推送数量为0条。")
return None
# 处理函数
def main(self):
# 获取未提交的urls
urls = self.get_url()
# 查询剩余次数
num = self.get_remain()
# 确定执行的urls
post_urls = urls[:num]
# 是否开始处理
flag = tk.messagebox.askquestion(
"提交", "本地共有没推送的网页数 ：{} 条!\n"
"当前剩余主动推送的次数 ：{} 条!\n"
"选“是”开始提交，选“否”取消提交".format(len(urls), num))
if flag == "yes":
try:
# 窗体置顶
self.newroot.wm_attributes('-topmost', 1)
cpu_num = multiprocessing.cpu_count()
self.ttlog.log("CPU核心数：" + str(cpu_num))
self.ttlog.log("开启线程池，能一定程度加速")
pool = ThreadPool(cpu_num)
results = pool.map(self.api, post_urls)
pool.close()
pool.join()
self.ttlog.stop_log()
self.ttlog.log("普通收录-->今日的推送任务完成！")
self.content.config(text="普通收录-->今日的推送任务完成！")
except Exception as e:
self.ttlog.log('错误代码：{}'.format(e))
self.ttlog.log("Error: unable to start thread")
else:
self.ttlog.log("你选择了否，没有推送网页链接")
def testeSelectSQLDICT():
mails = SqliteDict('db/teste.sqlite', "mails", autocommit=True)
for key, value in sorted(mails.iteritems(), reverse=True):
print(key, value)
class Cache(object):
def __init__(self, **kwargs):
self.name = kwargs.get('name', DEFAULT_DATABASE_NAME)
self._db = SqliteDict('./%s.sqlite' % self.name,
encode=cache_encode,
decode=cache_decode,
autocommit=True)
logger.info('name=%s size=%s', self.name, len(self._db))
def close(self):
self._db.close()
def __del__(self):
pass # self.close()
def __iter__(self):
return self._db.__iter__()
def iteritems(self):
return self._db.iteritems()
def iterkeys(self):
return self._db.iterkeys()
def itervalues(self):
return self._db.itervalues()
def items(self):
return self.iteritems()
def __getitem__(self, key):
return self._db.__getitem__(key)
def get(self, key, default=None):
try:
return self.__getitem__(key)
except KeyError:
return default
def __setitem__(self, key, value):
return self._db.__setitem__(key, value)
def set(self, key, value):
return self.__setitem__(key, value)
def __len__(self):
return len(self._db)
def __contains__(self, key):
return key in self._db
def __delitem__(self, key):
if key in self._db:
del self._db[key]
def list(self, limit=None):
logger.info('list')
keys = []
for key in self:
keys.append(key)
if limit is not None and len(keys) == limit: break
return keys
def index():
# Main pages
import logging
logging.basicConfig(level=logging.DEBUG)
def load_graph(model_file):
graph = tf.Graph()
graph_def = tf.GraphDef()
logging.debug("############ Graph ##################")
with open(model_file, "rb") as f:
graph_def.ParseFromString(f.read())
with graph.as_default():
tf.import_graph_def(graph_def)
return graph
def read_tensor_from_image_file(file_name, input_height=299, input_width=299,
input_mean=0, input_std=255):
input_name = "file_reader"
output_name = "normalized"
file_reader = tf.read_file(file_name, input_name)
if file_name.endswith(".png"):
image_reader = tf.image.decode_png(file_reader, channels = 3,
name='png_reader')
elif file_name.endswith(".gif"):
image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
name='gif_reader'))
elif file_name.endswith(".bmp"):
image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
else:
image_reader = tf.image.decode_jpeg(file_reader, channels = 3,
name='jpeg_reader')
float_caster = tf.cast(image_reader, tf.float32)
dims_expander = tf.expand_dims(float_caster, 0);
resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
sess = tf.Session()
result = sess.run(normalized)
return result
def load_labels(label_file):
label = []
proto_as_ascii_lines = tf.gfile.GFile(label_file).readlines()
for l in proto_as_ascii_lines:
label.append(l.rstrip())
return label
while True:
import datetime
now = datetime.datetime.now()
if now.hour in [22,23,24,0,1,2,3,4,5,6,7]:
pass
else:
resource = urllib.request.urlopen(target)
filename = gen_filename()
file_name = r'/Users/antigen/dev/RateMyView/uploads' + '/' + filename
output = open(file_name,"wb")
output.write(resource.read())
output.close()
GRAPH = load_graph(model_file)
t = read_tensor_from_image_file(file_name,
input_height=input_height,
input_width=input_width,
input_mean=input_mean,
input_std=input_std)
logging.debug("############ Reading File ##################")
input_name = "import/" + input_layer
output_name = "import/" + output_layer
input_operation = GRAPH.get_operation_by_name(input_name)
output_operation = GRAPH.get_operation_by_name(output_name)
with tf.Session(graph=GRAPH) as sess:
start = time.time()
results = sess.run(output_operation.outputs[0],
{input_operation.outputs[0]: t})
end=time.time()
results = np.squeeze(results)
top_k = results.argsort()[-5:][::-1]
labels = load_labels(label_file)
mydict = SqliteDict(sql_dict_store, autocommit=True)
mydict[filename] = top_k[0]+1
for key, value in mydict.iteritems():
logging.debug((key, str(value)))
mydict.close()
logging.debug(top_k[0]+1)
score = str(top_k[0]+1)
response = score + get_fortune(compliments, moar)
time.sleep(30*60)
return render_template('view.html', score=response)
class AnnotationInterface(object):
"""Access to stored annotations.
"""
def __init__(self, source_data, db):
"""
"""
# process current records in db
self.db = SqliteDict(db, autocommit=True)
# process csv file
with smart_open(source_data) as h:
self.data = []
fields = h.readline().strip().split(',')
required_fields = [
'task_id', 'model_id', 'topic_no', 'task', 'answer'
]
if fields != required_fields:
raise ValueError('invalid csv file: %s does not match %s' %
(fields, required_fields))
for line in h.readlines():
record = dict(zip(fields, line.strip().split(',')))
self.data.append(record)
@staticmethod
def key(record):
return '-'.join([
record['task_id'], record['model_id'], record['topic_no'],
record['user']
])
def agreement(self):
stats = defaultdict(list)
users_agree = defaultdict(int)
users_disagree = defaultdict(int)
users_skip = defaultdict(int)
for key, value in self.db.iteritems():
task_id, model_id, topic_no, user = key.split('-')
stats[(task_id, model_id, topic_no)].append(
(user, value['annotation']))
for recs in stats.values():
c = Counter([a for u, a in recs if '?' not in a and '!' not in a])
for user, answer in recs:
freq = c.get(answer, 0)
if freq == 0:
users_skip[user] += 1
elif freq == 1:
users_disagree[user] += 1
else:
users_agree[user] += 1
all_users = set(users_agree.keys()).union(users_disagree.keys()).union(
users_skip.keys())
results = {}
for user in all_users:
results[user] = (users_agree.get(user,
0), users_disagree.get(user, 0),
users_skip.get(user, 0))
return results
def score(self):
stats = defaultdict(list)
for key, value in self.db.iteritems():
task_id, model_id, topic_no, user = key.split('-')
stats[(task_id, model_id,
topic_no)].append(value['answer'] == value['annotation'])
ok, fail = 0, 0
for recs in stats.values():
if Counter(recs).get(True, 0) > 1:
ok += 1
else:
fail += 1
return ok, fail
def export(self):
stats = defaultdict(list)
for key, value in self.db.iteritems():
task_id, model_id, topic_no, user = key.split('-')
stats[(task_id, model_id, topic_no)].append(value)
results = []
for recs in stats.values():
head = recs[0]
print(head)
results.append(
(head['task_id'], head['model_id'], head['topic_no'],
head['task'], head['answer'], [r['annotation']
for r in recs]))
return results
def save(self, data, user_name):
no_underscore = [key for key in data if not key.startswith('_')][0]
record = {
'task_id':
data['_task_id'],
'model_id':
data['_model_id'],
'topic_no':
data['_topic_no'],
'answer':
data['_answer'],
'user':
user_name,
'task':
data['_task'],
'annotation':
no_underscore.replace('!', 'There is no answer!').replace(
'?', 'I don\'t know?')
}
self.db[self.key(record)] = record
logger.info('saved %s', record)
def get_stats(self, user_name):
impossible, skipped, done = 0, 0, 0
for key, value in self.db.iteritems():
task_id, model_id, topic_no, user = key.split('-')
if user == user_name:
if '?' in value['annotation']:
skipped += 1
elif '!' in value['annotation']:
impossible += 1
else:
done += 1
return {'skipped': skipped, 'done': impossible + done}
def get(self, user_name):
user_stats = defaultdict(list)
answer_stats = defaultdict(list)
for key, value in self.db.iteritems():
task_id, model_id, topic_no, user = key.split('-')
user_stats[(task_id, model_id, topic_no)].append(user)
answer_stats[(task_id, model_id,
topic_no)].append(value['annotation'])
selected = None
for data in self.data:
users = user_stats.get(
(data['task_id'], data['model_id'], data['topic_no']), [])
done = answer_stats.get(
(data['task_id'], data['model_id'], data['topic_no']), [])
if user_name not in users and len(done) == len(set(done)):
selected = data
break
return selected
