class DBManager(): def __init__(self, store): self._db = SqliteDict(store, autocommit=True) self.commands_cache = {} def add(self, command): if self.get_command(command.get_name()) != None: return False self._db[command.get_name()] = command return True def update(self, command): self.add(command) def remove(self, command): del self._db[command.get_name()] def empty(self): self.commands_cache = {} self._db.clear() def count(self): return len(self._db) def get_command(self, name): if name not in self.commands_cache: try: self.commands_cache[name] = self._db[name] except KeyError: return None return self.commands_cache[name] def get_all_commands(self): list = [] for key, val in self._db.iteritems(): list.append(key) return list
class DBManager(): def __init__(self): self._db = SqliteDict('./matches.sqlite', autocommit=True) self.matches_cache = {} def add(self, match): self._db[match.get_code()] = match def update(self, match): self.add(match) def remove(self, match): del self._db[match.get_code()] def empty(self): self._db.clear() def count(self): return len(self._db) def get_match(self, code): if code not in self.matches_cache: try: self.matches_cache[code] = self._db[code] except KeyError: return None return self.matches_cache[code] def get_all_matches(self): list = [] for key, val in self._db.iteritems(): list.append(key) return list
class db(object): def __init__(self, user): self.user = str(user) self.db = SqliteDict(self.getCfgPath(), autocommit=True) def get(self, key=''): return self.db.get(key) if key else self.db.iteritems() def set(self, key='', data=''): if not key: key = self.user if data: self.db[key] = data else: self.db.__delitem__(key) def getCfgPath(self): if os.path.isdir('hoshino'): if not os.path.isdir('hoshino/modules/ASF_Plus/config'): os.mkdir('hoshino/modules/ASF_Plus/config') return os.path.join( os.path.abspath('hoshino/modules/ASF_Plus/config'), f'{self.user}.sqlite') else: if not os.path.isdir('../config'): os.mkdir('../config') return os.path.join(os.path.abspath('../config'), f'{self.user}.sqlite')
def modify(self): mydict = SqliteDict('./my_db.sqlite', autocommit=True) if self.index == 2: self.ttlog.log("快速收录重置-->开始") elif self.index == 1: self.ttlog.log("普通收录重置-->开始") tree_len = len(self.tree.get_children()) time.sleep(10) for key, value in sorted(mydict.iteritems()): if value[self.index] == "已提交": # 是否修改列表,看是否加载列表 value[self.index] = "未提交" if tree_len != 0: self.tree.item(value[4], value=value[:4]) self.ttlog.log("重置状态:" + key + "") # 修改数据库 mydict[key] = value if self.index == 2: self.ttlog.log("快速收录重置-->完成") self.content.config(text="快速收录重置-->完成") elif self.index == 1: self.ttlog.log("普通收录重置-->完成") self.content.config(text="普通收录重置-->完成") self.ttlog.stop_log()
def get_url(self): url_list = [] mydict = SqliteDict('./my_db.sqlite', autocommit=True) for key, value in sorted(mydict.iteritems()): if value[2] == "未提交": url_list.append(value) self.ttlog.log("快速收录-->共有没推送的网页链接数 :{} 条!".format(len(url_list))) print("共有没快速收录推送的网页链接数 :{} 条!".format(len(url_list))) return url_list
def criaTeste(): mydict = SqliteDict('db/teste.sqlite', "mails", autocommit=True) mydict['1'] = "batata" mydict['2'] = "banana" mydict['3'] = "oi" mydict['4'] = "teste" for key, value in mydict.iteritems(): print(key, value) print(len(mydict)) # etc... all dict functions work mydict.close()
class Scribe(object): def __init__(self, location, table_name, exp_name): filename = "{}/scribe.sqlite".format(location) self.book = SqliteDict(filename, autocommit=True, tablename=table_name) unique_id = datetime.now().strftime("date_%m.%d_time_%H.%M") self.exp_name = exp_name + "_" + unique_id self.observation_index = 0 def record(self, value, type="general"): key = "{}; {}; {}".format(self.exp_name, self.observation_index, type) self.book[key] = value self.observation_index += 1 observe = record #sometimes i forget which def lookup(self, type=None, exp_name=None, ret_sorted=False, strip_keys=False): type_func = lambda *args: True name_func = lambda *args: True if type: type_func = lambda x: x[2] == type if exp_name: name_func = lambda x: exp_name in x[0] key_func = lambda x: type_func(x) and name_func(x) unpack = lambda x: [ f(x.strip()) for f, x in zip([str, int, str], x.split(";")) ] items = {k: v for k, v in self.book.iteritems() if key_func(unpack(k))} if ret_sorted: return self.sort_results(items, strip_keys) return items def sort_results(self, result_dict, only_val_return=False): unpack = lambda x: [ f(x.strip()) for f, x in zip([str, int, str], x.split(";")) ] ranker = lambda x: unpack(x[0])[1] sorted_items = sorted(result_dict.items(), key=ranker) if only_val_return: return [v for k, v in sorted_items] return sorted_items def close(self): self.book.close()
def list_db(dbname, options): # No context manager (and no close()) as this is read-only and # close() can block for a long time for no apparent reason. db = SqliteDict(dbname, flag='r', autocommit=False) if not options.keys: for k, v in db.iteritems(): output(k, v.rstrip('\n'), options) else: for k in options.keys: try: v = db[k] except KeyError as e: error('no such key: "{}"'.format(k)) else: output(k, v.rstrip('\n'), options)
def list_annotations(dbname, options): # No context manager: close() can block and this is read-only doc_count, ann_count = 0, 0 db = SqliteDict(dbname, flag='r', autocommit=False) for k, v in db.iteritems(): root, ext = os.path.splitext(os.path.basename(k)) if ext != options.suffix: continue for line in v.splitlines(): if options.random is not None and random() > options.random: continue print('{}\t{}'.format(root, line)) ann_count += 1 doc_count += 1 print('Done, listed {} annotations in {} docs from {}'.format( ann_count, doc_count, dbname), file=sys.stderr)
class Scribe(object): def __init__(self, location, table_name, exp_name): filename = "{}/scribe.sqlite".format(location) self.book = SqliteDict(filename, autocommit=True, tablename=table_name) unique_id = datetime.now().strftime("date_%m.%d_time_%H.%M") self.exp_name = exp_name+"_"+unique_id self.observation_index = 0 def record(self, value, type="general"): key = "{}; {}; {}".format(self.exp_name, self.observation_index, type) self.book[key] = value self.observation_index += 1 observe = record #sometimes i forget which def lookup(self, type=None, exp_name=None, ret_sorted=False, strip_keys=False): type_func = lambda *args: True name_func = lambda *args: True if type: type_func = lambda x: x[2] == type if exp_name: name_func = lambda x: exp_name in x[0] key_func = lambda x: type_func(x) and name_func(x) unpack = lambda x: [f(x.strip()) for f,x in zip([str,int,str],x.split(";"))] items = {k:v for k,v in self.book.iteritems() if key_func(unpack(k))} if ret_sorted: return self.sort_results(items, strip_keys) return items def sort_results(self, result_dict, only_val_return=False): unpack = lambda x: [f(x.strip()) for f,x in zip([str,int,str],x.split(";"))] ranker = lambda x: unpack(x[0])[1] sorted_items = sorted(result_dict.items(), key=ranker) if only_val_return: return [v for k,v in sorted_items] return sorted_items def close(self): self.book.close()
else: flag = 'c' mergehll = SqliteDict(outputfile, flag=flag, journal_mode='MEMORY', encode=hll_encode, decode=hll_decode) mergekeys = set() for k in mergehll.keys(): mergekeys.add(k) for f in files: if not os.path.exists(f): print("{}: File not found".format(f)) sys.exit() filehll = SqliteDict(f, encode=hll_encode, decode=hll_decode) # merge into outputfile for tld, tldhll in filehll.iteritems(): if tld in mergekeys: # key already in outputfile -> merge hll = mergehll[tld] if not hll.__eq__(tldhll): # merge only if HLL is different hll.update(tldhll) mergehll[tld] = hll else: # new key in outfile -> copy mergehll[tld] = filehll[tld] mergekeys.add(tld) filehll.close() print("Total TLDs {}".format(len(mergehll))) print("Total IPs {}".format(len(mergehll['TOTAL']))) mergehll.commit() mergehll.close()
class Update_window(object): """sitemap更新窗体""" def __init__(self, tree, eblog, sitemap): self.newroot = tk.Toplevel() self.newroot.title('下载文件中') self.newroot.iconbitmap("favicon.ico") self.newroot.wm_attributes('-topmost', 1) win_width = self.newroot.winfo_screenwidth() win_higth = self.newroot.winfo_screenheight() width_adjust = (win_width - 800) / 2 higth_adjust = (win_higth - 250) / 2 self.newroot.geometry("%dx%d+%d+%d" % (800, 250, width_adjust, higth_adjust)) # 进度条 self.bar = ttk.Progressbar(self.newroot, length=740, mode="indeterminate", orient=tk.HORIZONTAL) self.bar.place( x=30, y=150, ) self.bar.start(10) # 提示内容 self.content = tk.Label(self.newroot, text="正在下载Sitemap.xml文件...") self.content.place( x=30, y=30, ) self.content2 = tk.Label(self.newroot, text="下载速度和文件大小以及服务器带宽有关,请耐心等待......", wraplength=740, justify="left") self.content2.place( x=30, y=60, ) self.eblog = eblog self.sitemap = sitemap self.tree = tree self.mydict = SqliteDict('./my_db.sqlite', autocommit=True) # 开启处理线程 self.p = Thread(target=self.update) self.p.setDaemon(True) self.p.start() self.eblog.log("Sitemap线程:开启sitemap线程,下载Sitemap.xml中...") # 关闭右上角 self.newroot.protocol("WM_DELETE_WINDOW", self.close) # 列表添加item,返回iid def append_item(self, item_list): # 加最后/加前面都可以,因为要是前面iid全是1 item = self.tree.insert("", 0, values=(item_list[0], item_list[1], item_list[2], item_list[3])) return item # 处理函数 def update(self): try: with open("sitemap.xml", "wb") as f: f.write(requests.get(self.sitemap).content) with open("sitemap.xml", 'r', encoding='utf-8') as f: xml_data = f.read() self.content.configure(text="Sitemap文件下载完成,正在对比分析....") urls = re.findall(r'<loc>(.+?)</loc>', xml_data, re.S) self.eblog.log("Sitemap线程-->下载Sitemap.xml完成,正在解析xml文件...") tuple_list = list(self.mydict.iteritems()) tree_urls = [i[0] for i in tuple_list] # 求交集 c = list(set(urls).intersection(set(tree_urls))) # tree中多余的 tree_urls_ = list(set(tree_urls).difference(set(c))) # 交集不动,tree把tree中把不是交集的删掉,把siemaop中不是交集的增添上. for key, value in sorted(tuple_list): self.content2.config(text="当前处理-->检查" + key) if key not in c: # 是否删除,看有没有列表 if len(self.tree.get_children()) != 0: self.tree.delete(value[4]) self.mydict.__delitem__(key=key) self.eblog.log("Sitemap线程-->本地删除" + str(tree_urls_)) # sitemap中新添加的 urls_ = list(set(urls).difference(set(c))) for url in sorted(urls_): cur_time = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') iid = self.append_item([url, "未提交", "未提交", cur_time]) self.mydict[url] = [url, "未提交", "未提交", cur_time, iid] self.content2.config(text="当前处理-->正在添加" + url) self.eblog.log("Sitemap线程-->本地添加" + str(urls_)) self.eblog.log("Sitemap线程-->关闭sitemap线程,更新完成。") self.close() except: self.eblog.log(traceback.format_exc()) self.eblog.log("Sitemap线程-->更新失败") def close(self): self.newroot.destroy()
from sqlitedict import SqliteDict import pickle tall = [] c = 0 c_tall = 0 mydict = SqliteDict( "/../../../../../scratch/manoelribeiro/helpers/authors_dict.sqlite", tablename="authors", flag="r") for _, value in mydict.iteritems(): if (c % 100000) == 0: print(f"c = {c}, c_all = {c_tall}") c += 1 if c >= 11400000: break diff_comm = set() for comm in value: cat = comm["category"] if cat == "Alt-right" or cat == "Alt-lite" or cat == "Intellectual Dark Web": diff_comm.add(cat) else: diff_comm.add("control") if len(diff_comm) > 1: continue tall.append(value) c_tall += 1
class Admin(commands.Cog): MY_ID = 192520503443849217 def __init__(self, bot): self.bot = bot self.description = "Admin/debug commands." self.gamble_db = SqliteDict('./gamble.sqlite', autocommit=True) self.remind_db = SqliteDict('./remind.sqlite', autocommit=True) self.prefix_db = SqliteDict('./prefix.sqlite', autocommit=True) @commands.command(help="[admin] reset points for this server") async def perish(self, ctx): server = str(ctx.guild.id) if ctx.author.id == self.MY_ID: try: # del db['gamble'][server] del self.gamble_db[server] await ctx.send("donezo") except KeyError: await ctx.send("no-op, table is already gone") else: await ctx.send("Perish this command is not for you") @commands.command(help="[admin] reset all points for all servers") async def thanos(self, ctx): if (ctx.author.id == self.MY_ID): for key in db: del db[key] try: for key in db['gamble']: del db['gamble'][key] await ctx.send("*thanos snaps*") except KeyError: await ctx.send("no-op, table is already gone") else: await ctx.send("Perish this command is not for you") @commands.command(help="[admin] drop all tables") async def drop(self, ctx): if (ctx.author.id == self.MY_ID): for key in db: del db[key] await ctx.send("monkaSteer") else: await ctx.send("Perish this command is not for you") @commands.command(help="[admin] drop raid table") async def rdrop(self, ctx): if (ctx.author.id == self.MY_ID): if db['reminders']: for key in db['reminders']: del db['reminders'][key] del db['reminders'] await ctx.send("all done") @commands.command(help="[admin] log all tables") async def tables(self, ctx): for key, value in self.gamble_db.iteritems(): print(f"key: {key} value: {value}") for key, value in self.prefix_db.iteritems(): print(f"key: {key} value: {value}") if (ctx.author.id == self.MY_ID): print('KEYS') print(db.keys()) print('SUBTABLES') for key in db: print(key) for k in db[key]: print(f'KEY: {k} // VALUE: {db[key][k]}') else: await ctx.send("Perish this command is not for you")
ICON_FONT_FILE = 'pifile.ttf' ICON_FONT_JSON = 'config.json' ICONS = icon(ICON_FONT_JSON, ICON_FONT_FILE) ################################## # SCREEN BANNER VARIABLES ################################### LOADING_MESSEGES = [] TREKNOBABBLE = 'treknobabble.tbdb' _TREKNOBABBLE = os.path.join('resources/', TREKNOBABBLE) tb_db = SqliteDict(_TREKNOBABBLE) # for item in dict_db.iteritems(): # pprint(item) for key, value in tb_db.iteritems(): LOADING_MESSEGES.append(value) # return_event = pygame.event.Event(RETURN_EVENT) # Dict of events that are accessible to screens piscreenevents = { "button": TFTBUTTONCLICK, "update": UPDATESCREEN, "nextscreen": NEXTSCREEN, "toggle_fps": TOGGLE_FPS_EVENT, }
myset = [] for link in soup.find_all('h1'): tmp = link.get_text() string = tmp[5:-3] myset.append(string) if myset[0] not in dict_db.iterkeys(): dict_db[myset[0]] = myset[1] unique += 1 else: print "Duplicate" duplicates += 1 treknobabble.append(failure) # get_tb() for item in dict_db.iteritems(): pprint(item) print len(dict_db) dict_db.close() # print "Dups: " + str(duplicates) # f = open('treknobable.txt', 'w') # f.write(x.get_string(sortby="Timestamp")) # for thing in treknobabble: # pprint(thing) # for key in thing: # f.write('\n' + '----------\n') # f.write(key + '\n') # f.write(thing[key])
class Tree_control(object): """列表加载窗体""" def __init__(self, tree, eblog): self.newroot = tk.Toplevel() self.newroot.title('加载列表') self.newroot.iconbitmap("favicon.ico") self.newroot.wm_attributes('-topmost', 1) win_width = self.newroot.winfo_screenwidth() win_higth = self.newroot.winfo_screenheight() width_adjust = (win_width - 400) / 2 higth_adjust = (win_higth - 250) / 2 self.newroot.geometry("%dx%d+%d+%d" % (400, 250, width_adjust, higth_adjust)) # 进度条 self.__showFlag = True self.__width = 300 self.__heigth = 20 self.__sleep = 0 self.bar = ttk.Progressbar(self.newroot, length=self.__width, mode="indeterminate", orient=tk.HORIZONTAL) self.bar.pack(expand=True) self.bar.start(10) # 提示内容 self.content2 = tk.Label(self.newroot, text="正在加载列表中,请不要中断操作,请耐心等待......") self.content2.place( x=50, y=30, ) self.content = tk.Label(self.newroot, text="") self.content.place( x=50, y=60, ) self.eblog = eblog self.tree = tree self.mydict = SqliteDict('./my_db.sqlite', autocommit=True) # 开启处理线程 self.p = Thread(target=self.add_item) self.p.setDaemon(True) self.p.start() # 点击关闭右上角 self.newroot.protocol("WM_DELETE_WINDOW", self.close) # 加载item def add_item(self): len_items = len(sorted(self.mydict.iteritems())) i = 0 for key, value in sorted(self.mydict.iteritems()): i = i + 1 self.content.config(text="当前正处理:第" + str(i) + "个,共有" + str(len_items) + "个链接") self.tree.insert("", 0, iid=value[4], values=(value[0], value[1], value[2], value[3])) self.close() return 1 def close(self): self.newroot.destroy()
class tiny_db(): def __init__(self): self.debug = True self.dict_db = SqliteDict('./racks.sqlite', autocommit=True) self.days_to_keep = 5 self._define_mem_db() # SET UP DATABASE VARIABLES self.row_height = DATABASE_SETTINGS['rows'] self.column_width = DATABASE_SETTINGS['columns'] self.get_last_filed() self.rack_day = self._today() self.next = {} self._today() self.next_location() self._db_info() if self.last_filed is None: try: self._convert_to_sqlitedb() except: print "Database is Empty" # @profile def _define_mem_db(self): self.mem_db = [] for item in self.dict_db.iteritems(): self.mem_db.append(item[1]) # @profile def _db_info(self): if len(self.mem_db) > 0: x = PrettyTable([" ", "size"]) x.add_row(["DB Size", len(self.dict_db)]) x = PrettyTable(["stats", "accn", "Date", "Timestamp"]) # pprint(self.last_filed) readable = self._conv_timestamp(self.last_filed['time']) x.add_row(["Last", self.last_filed['accn'], readable, self.last_filed['time']]) first_filed = self.get_first_filed() other_readable = self._conv_timestamp(first_filed['time']) x.add_row(["First", first_filed['accn'], other_readable, first_filed['time']]) # print x return x def _list_size(self): size = {'memory': len(self.mem_db), 'disk': len(self.dict_db)} return size def _print_database(self): if len(self.mem_db) < 2: return none x = PrettyTable(["Accn", "Rack", "Position", "Time", "Timestamp"]) # x.padding_width = 1 for item in self.mem_db: x.add_row(self._make_entry_row(item)) # print x.get_string(sortby="Timestamp") f = open('_database_info.txt', 'w') f.write(x.get_string(sortby="Timestamp")) f.write(self._db_info().get_string()) f.close() print x.get_string(fields=["Accn", "Rack", "Position"]) def _make_entry_row(self, item): readable = self._conv_timestamp(item['time']) # print item['row'] x = [item['accn'], item['rackDay'] + ' ' + str(item['rack']), ROWS[str(item['row'])] + ' ' + str(item['column']), readable, item['time']] return x def _conv_timestamp(self, ts): dt = datetime.datetime.fromtimestamp(float(ts)) return dt.strftime("%H:%M - %m.%d.%Y") def _today(self): self.purge_date = int( mktime( (datetime.date.today() - datetime.timedelta( self.days_to_keep)).timetuple())) return strftime('%a', localtime(time())) # @profile def file_accn(self, accn): insert = { 'accn': accn, 'rack': self.next['rack'], 'rackDay': self.next['rackDay'], 'column': self.next['column'], 'row': self.next['row'], 'time': str(time()) } self.mem_db.append(insert) self.last_filed = insert self.dict_db[insert['time']] = insert self.next_location() # @profile def get_last_filed(self): _last_filed_id = None self.last_filed = None for item in self.mem_db: if _last_filed_id < item['time']: _last_filed_id = item['time'] self.last_filed = item return self.last_filed # @profile def get_first_filed(self): # if len(self.mem_db) == 1: # return self.last_filed # else: # print self.last_filed if self.last_filed is None: return self.last_filed else: _smallest_id = self.last_filed['time'] first_filed = None for item in self.mem_db: if item: # print item if _smallest_id > item['time']: # print "smaller" _smallest_id = item['time'] first_filed = item # print first_filed return first_filed # @profile def new_day(self): self.next['column'] = 1 self.next['rack'] = 1 self.next['row'] = 1 self.next['rackDay'] = self._today() # @profile def next_location(self): if self.last_filed is None: self.new_day() return if self.last_filed['rackDay'] != self._today(): self.new_day() elif self.last_filed['column'] == self.column_width: if self.last_filed['row'] == self.row_height: self.next['column'] = 1 self.next['row'] = 1 self.next['rack'] = self.last_filed['rack'] + 1 else: self.next['column'] = 1 self.next['row'] = self.last_filed['row'] + 1 self.next['rack'] = self.last_filed['rack'] else: self.next['column'] = self.last_filed['column'] + 1 self.next['rack'] = self.last_filed['rack'] self.next['row'] = self.last_filed['row'] self.next['rackDay'] = self._today() # @profile def find_accn(self, accn): result = [] for item in self.mem_db: if item['accn'] == accn: result.append(item) elif accn in item['accn']: result.append(item) return result # @profile def _convert_to_sqlitedb(self): old_db = TinyDB(DATABASE_SETTINGS['database']) self.mem_db = [] total = 0 for table in old_db.tables(): for item in old_db.table(table).all(): total += 1 print "MIGRATING DATABASE" print "--OLD DATABASE: " + str(total) i = 0 for table in old_db.tables(): for item in old_db.table(table).all(): if len(item['accn']) < 15: if int(item['time']) > self.purge_date: self.mem_db.append(item) print " Converted: " + str(i) + " of " + str(total) i += 1 self.dict_db[str(item['time'])] = item else: print "REMOVING OLD THING" print "DATABASE MIGRATION COMPLETE" print "COMMITING CHANGES" self.get_last_filed() # @profile def clean(self): print "HARD STORAGE: " + str(len(self.dict_db)) print "Mem: " +str(len(self.mem_db)) self.mem_db = [] for item in self.dict_db.iteritems(): # print int(item[1]['time'][:-3]) # print self.purge_date if float(item[1]['time']) < self.purge_date: print "---" print float(item[1]['time']) # print "---" print self.purge_date del self.dict_db[item[0]] self._define_mem_db() print self.purge_date print "HARD STORAGE: " + str(len(self.dict_db)) print "Mem: " +str(len(self.mem_db))
class Baidu_ordinary_windows(object): """百度普通收录窗体""" def __init__(self, tree, site, token): # 展示等待窗体 self.newroot = tk.Toplevel() self.newroot.title('普通收录') self.newroot.iconbitmap("favicon.ico") win_width = self.newroot.winfo_screenwidth() win_higth = self.newroot.winfo_screenheight() width_adjust = (win_width - 800) / 2 higth_adjust = (win_higth - 250) / 2 self.newroot.geometry("%dx%d+%d+%d" % (800, 250, width_adjust, higth_adjust)) # 提示内容 self.content = tk.Label(self.newroot, text="正在普通收录中,请不要中断操作,请耐心等待......") self.content.place( x=10, y=30, ) self.content2 = tk.Label(self.newroot, text="") self.content2.place( x=10, y=60, ) # 窗体日志 self.ttlog = ttlog(master=self.newroot) self.ttlog.place(x=10, y=70, width=780, height=150) self.tree = tree self.site = site self.token = token self.mydict = SqliteDict('./my_db.sqlite', autocommit=True) # 开始处理线程 self.p = Thread(target=self.main) self.p.setDaemon(True) self.p.start() self.ttlog.log("普通收录-->开启普通收录线程.....") # 点击关闭右上角 self.newroot.protocol("WM_DELETE_WINDOW", self.close) def close(self): self.ttlog.stop_log() self.newroot.destroy() # 获取未提交的urls def get_url(self): url_list = [] for key, value in sorted(self.mydict.iteritems()): if value[1] == "未提交": url_list.append(value) self.ttlog.log("普通收录-->共有没推送的网页链接数 :{} 条!".format(len(url_list))) print("共有没普通收录推送的网页链接数 :{} 条!".format(len(url_list))) return url_list # 查询剩余次数 def get_remain(self): post_url = "http://data.zz.baidu.com/urls?site={}&token={}".format( self.site, self.token) headers = { 'User-Agent': 'curl/7.12.1', 'Host': 'data.zz.baidu.com', 'Content-Type': 'text/plain', 'Content-Length': '83', } response = requests.post(post_url, headers=headers, data=self.site) req = response.text if "success" in req: req_json = json.loads(req) if req_json["remain"] == 0: self.ttlog.log( "普通收录-->查询剩余次数,今天普通收录推送任务已经完成,\n当天剩余的可推送url条数: " + req_json["remain"] + "条。") else: self.ttlog.log( "普通收录-->查询剩余次数,推送成功:" + self.site + '\n当天剩余的可推送url条数: {}条'.format(req_json["remain"])) return req_json["remain"] else: return 0 # 提交urls def api(self, url): post_url = "http://data.zz.baidu.com/urls?site={}&token={}".format( self.site, self.token) headers = { 'User-Agent': 'curl/7.12.1', 'Host': 'data.zz.baidu.com', 'Content-Type': 'text/plain', 'Content-Length': '83', } response = requests.post(post_url, headers=headers, data=url[0]) req = response.text if "success" in req: req_json = json.loads(req) if req_json["remain"] == 0: self.ttlog.log("普通收录-->今天普通收录推送任务已经完成,当天剩余的可推送url条数: 0条。") else: # 是否修改列表,看是否加载列表 tree_len = len(self.tree.get_children()) if tree_len != 0: print("普通收录-->修改列表") self.tree.item(url[4], value=(url[0], "已提交", url[2], url[3])) # 修改数据库 self.mydict[url[0]] = [url[0], "已提交", url[2], url[3], url[4]] self.ttlog.log( "普通收录-->推送成功:" + url[0] + '\n当天剩余的可推送url条数: {}条'.format(req_json["remain"])) else: req_json = json.loads(req) self.ttlog.log(r"普通收录-->推送失败:" + req_json["message"] + ",当天可剩余推送数量为0条。") return None # 处理函数 def main(self): # 获取未提交的urls urls = self.get_url() # 查询剩余次数 num = self.get_remain() # 确定执行的urls post_urls = urls[:num] # 是否开始处理 flag = tk.messagebox.askquestion( "提交", "本地共有没推送的网页数 :{} 条!\n" "当前剩余主动推送的次数 :{} 条!\n" "选“是”开始提交,选“否”取消提交".format(len(urls), num)) if flag == "yes": try: # 窗体置顶 self.newroot.wm_attributes('-topmost', 1) cpu_num = multiprocessing.cpu_count() self.ttlog.log("CPU核心数:" + str(cpu_num)) self.ttlog.log("开启线程池,能一定程度加速") pool = ThreadPool(cpu_num) results = pool.map(self.api, post_urls) pool.close() pool.join() self.ttlog.stop_log() self.ttlog.log("普通收录-->今日的推送任务完成!") self.content.config(text="普通收录-->今日的推送任务完成!") except Exception as e: self.ttlog.log('错误代码:{}'.format(e)) self.ttlog.log("Error: unable to start thread") else: self.ttlog.log("你选择了否,没有推送网页链接")
def testeSelectSQLDICT(): mails = SqliteDict('db/teste.sqlite', "mails", autocommit=True) for key, value in sorted(mails.iteritems(), reverse=True): print(key, value)
class Cache(object): def __init__(self, **kwargs): self.name = kwargs.get('name', DEFAULT_DATABASE_NAME) self._db = SqliteDict('./%s.sqlite' % self.name, encode=cache_encode, decode=cache_decode, autocommit=True) logger.info('name=%s size=%s', self.name, len(self._db)) def close(self): self._db.close() def __del__(self): pass # self.close() def __iter__(self): return self._db.__iter__() def iteritems(self): return self._db.iteritems() def iterkeys(self): return self._db.iterkeys() def itervalues(self): return self._db.itervalues() def items(self): return self.iteritems() def __getitem__(self, key): return self._db.__getitem__(key) def get(self, key, default=None): try: return self.__getitem__(key) except KeyError: return default def __setitem__(self, key, value): return self._db.__setitem__(key, value) def set(self, key, value): return self.__setitem__(key, value) def __len__(self): return len(self._db) def __contains__(self, key): return key in self._db def __delitem__(self, key): if key in self._db: del self._db[key] def list(self, limit=None): logger.info('list') keys = [] for key in self: keys.append(key) if limit is not None and len(keys) == limit: break return keys
def index(): # Main pages import logging logging.basicConfig(level=logging.DEBUG) def load_graph(model_file): graph = tf.Graph() graph_def = tf.GraphDef() logging.debug("############ Graph ##################") with open(model_file, "rb") as f: graph_def.ParseFromString(f.read()) with graph.as_default(): tf.import_graph_def(graph_def) return graph def read_tensor_from_image_file(file_name, input_height=299, input_width=299, input_mean=0, input_std=255): input_name = "file_reader" output_name = "normalized" file_reader = tf.read_file(file_name, input_name) if file_name.endswith(".png"): image_reader = tf.image.decode_png(file_reader, channels = 3, name='png_reader') elif file_name.endswith(".gif"): image_reader = tf.squeeze(tf.image.decode_gif(file_reader, name='gif_reader')) elif file_name.endswith(".bmp"): image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader') else: image_reader = tf.image.decode_jpeg(file_reader, channels = 3, name='jpeg_reader') float_caster = tf.cast(image_reader, tf.float32) dims_expander = tf.expand_dims(float_caster, 0); resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width]) normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std]) sess = tf.Session() result = sess.run(normalized) return result def load_labels(label_file): label = [] proto_as_ascii_lines = tf.gfile.GFile(label_file).readlines() for l in proto_as_ascii_lines: label.append(l.rstrip()) return label while True: import datetime now = datetime.datetime.now() if now.hour in [22,23,24,0,1,2,3,4,5,6,7]: pass else: resource = urllib.request.urlopen(target) filename = gen_filename() file_name = r'/Users/antigen/dev/RateMyView/uploads' + '/' + filename output = open(file_name,"wb") output.write(resource.read()) output.close() GRAPH = load_graph(model_file) t = read_tensor_from_image_file(file_name, input_height=input_height, input_width=input_width, input_mean=input_mean, input_std=input_std) logging.debug("############ Reading File ##################") input_name = "import/" + input_layer output_name = "import/" + output_layer input_operation = GRAPH.get_operation_by_name(input_name) output_operation = GRAPH.get_operation_by_name(output_name) with tf.Session(graph=GRAPH) as sess: start = time.time() results = sess.run(output_operation.outputs[0], {input_operation.outputs[0]: t}) end=time.time() results = np.squeeze(results) top_k = results.argsort()[-5:][::-1] labels = load_labels(label_file) mydict = SqliteDict(sql_dict_store, autocommit=True) mydict[filename] = top_k[0]+1 for key, value in mydict.iteritems(): logging.debug((key, str(value))) mydict.close() logging.debug(top_k[0]+1) score = str(top_k[0]+1) response = score + get_fortune(compliments, moar) time.sleep(30*60) return render_template('view.html', score=response)
class AnnotationInterface(object): """Access to stored annotations. """ def __init__(self, source_data, db): """ """ # process current records in db self.db = SqliteDict(db, autocommit=True) # process csv file with smart_open(source_data) as h: self.data = [] fields = h.readline().strip().split(',') required_fields = [ 'task_id', 'model_id', 'topic_no', 'task', 'answer' ] if fields != required_fields: raise ValueError('invalid csv file: %s does not match %s' % (fields, required_fields)) for line in h.readlines(): record = dict(zip(fields, line.strip().split(','))) self.data.append(record) @staticmethod def key(record): return '-'.join([ record['task_id'], record['model_id'], record['topic_no'], record['user'] ]) def agreement(self): stats = defaultdict(list) users_agree = defaultdict(int) users_disagree = defaultdict(int) users_skip = defaultdict(int) for key, value in self.db.iteritems(): task_id, model_id, topic_no, user = key.split('-') stats[(task_id, model_id, topic_no)].append( (user, value['annotation'])) for recs in stats.values(): c = Counter([a for u, a in recs if '?' not in a and '!' not in a]) for user, answer in recs: freq = c.get(answer, 0) if freq == 0: users_skip[user] += 1 elif freq == 1: users_disagree[user] += 1 else: users_agree[user] += 1 all_users = set(users_agree.keys()).union(users_disagree.keys()).union( users_skip.keys()) results = {} for user in all_users: results[user] = (users_agree.get(user, 0), users_disagree.get(user, 0), users_skip.get(user, 0)) return results def score(self): stats = defaultdict(list) for key, value in self.db.iteritems(): task_id, model_id, topic_no, user = key.split('-') stats[(task_id, model_id, topic_no)].append(value['answer'] == value['annotation']) ok, fail = 0, 0 for recs in stats.values(): if Counter(recs).get(True, 0) > 1: ok += 1 else: fail += 1 return ok, fail def export(self): stats = defaultdict(list) for key, value in self.db.iteritems(): task_id, model_id, topic_no, user = key.split('-') stats[(task_id, model_id, topic_no)].append(value) results = [] for recs in stats.values(): head = recs[0] print(head) results.append( (head['task_id'], head['model_id'], head['topic_no'], head['task'], head['answer'], [r['annotation'] for r in recs])) return results def save(self, data, user_name): no_underscore = [key for key in data if not key.startswith('_')][0] record = { 'task_id': data['_task_id'], 'model_id': data['_model_id'], 'topic_no': data['_topic_no'], 'answer': data['_answer'], 'user': user_name, 'task': data['_task'], 'annotation': no_underscore.replace('!', 'There is no answer!').replace( '?', 'I don\'t know?') } self.db[self.key(record)] = record logger.info('saved %s', record) def get_stats(self, user_name): impossible, skipped, done = 0, 0, 0 for key, value in self.db.iteritems(): task_id, model_id, topic_no, user = key.split('-') if user == user_name: if '?' in value['annotation']: skipped += 1 elif '!' in value['annotation']: impossible += 1 else: done += 1 return {'skipped': skipped, 'done': impossible + done} def get(self, user_name): user_stats = defaultdict(list) answer_stats = defaultdict(list) for key, value in self.db.iteritems(): task_id, model_id, topic_no, user = key.split('-') user_stats[(task_id, model_id, topic_no)].append(user) answer_stats[(task_id, model_id, topic_no)].append(value['annotation']) selected = None for data in self.data: users = user_stats.get( (data['task_id'], data['model_id'], data['topic_no']), []) done = answer_stats.get( (data['task_id'], data['model_id'], data['topic_no']), []) if user_name not in users and len(done) == len(set(done)): selected = data break return selected