def load_into_sqlite(): # Update data by id by_id = data_by_id() with _LogTime("loading data into SQLite"): d = sqlitedict.SqliteDict(SQLITE_DB, table=SQLITE_BY_ID_TABLE, init=by_id) with _LogTime("computing prefixes"): pd = {} td = {} n = 0 for record in by_id.itervalues(): (ps, ts) = prefixes_and_tokens(record) v = record["id"] for p in ps: if not pd.has_key(p): pd[p] = set() pd[p].add(v) for t in ts: if not td.has_key(t): td[t] = set() td[t].add(v) n += 1 if not n % 10000: print n with _LogTime("writing prefixes to SQLite"): pds = sqlitedict.SqliteDict(SQLITE_DB, table="prefixes", init=pd) tds = sqlitedict.SqliteDict(SQLITE_DB, table="tokens", init=td)
def test_readonly(self): fname = norm_file('tests/db/sqlitedict-override-test.sqlite') orig_db = sqlitedict.SqliteDict(filename=fname) orig_db['key'] = 'value' orig_db['key_two'] = 2 orig_db.commit() orig_db.close() readonly_db = sqlitedict.SqliteDict(filename=fname, flag='r') self.assertTrue(readonly_db['key'] == 'value') self.assertTrue(readonly_db['key_two'] == 2) def attempt_write(): readonly_db['key'] = ['new_value'] def attempt_update(): readonly_db.update(key='value2', key_two=2.1) def attempt_delete(): del readonly_db['key'] def attempt_clear(): readonly_db.clear() def attempt_terminate(): readonly_db.terminate() attempt_funcs = [ attempt_write, attempt_update, attempt_delete, attempt_clear, attempt_terminate ] for func in attempt_funcs: with self.assertRaises(RuntimeError): func()
def user_page_crawl(): res = [] pool = Pool(8) idxd = sqlitedict.SqliteDict('./idx_db.db') upgd = sqlitedict.SqliteDict('./upg_db.db', autocommit=True) def get_upg(uid, upgd): jo = get_uid_page(uid) upgd[source] = jo print dump_json(len(jo['data']), i=2) for k, idx_js in idxd.items(): rj = idx_js source = rj['source'] print k, dump_json(source) # print dump_json(rj,i=2) # print rj.keys() try: uid = rj['media_url'].split('/')[-2] except: continue pool.spawn(get_upg, uid, upgd) pool.join()
def test_default_reuse_existing_flag_c(self): """Re-opening of a database does not destroy it.""" # given, fname = norm_file('tests/db/sqlitedict-override-test.sqlite') orig_db = sqlitedict.SqliteDict(filename=fname) orig_db['key'] = 'value' orig_db.commit() orig_db.close() next_db = sqlitedict.SqliteDict(filename=fname) self.assertIn('key', next_db.keys()) self.assertEqual(next_db['key'], 'value')
def match(q, limit=None): results = [] bp = sqlitedict.SqliteDict(SQLITE_DB, table="prefixes") bi = sqlitedict.SqliteDict(SQLITE_DB, table="shareable_medication_by_id") bt = sqlitedict.SqliteDict(SQLITE_DB, table="tokens") #bp = _bp() #bi = _bi() tokens = tokenize(q) for t in tokens[:-1]: # Skip tokens of length 1 because the list is too long # TODO: In the future, have a globally popular list of # ~200 things for each letter to return instead of the # overwhelmingly huge list that would be returned now if len(t) < 2: continue # Take the ones where the second part of the tuple is True # which means that the string matches a complete token, i.e. # "viagra" not "via" (which would just be the beginning of a string) results.append(bt[t]) # For the last token, we take both partial tokens and complete tokens, # i.e. "via" and "viagra" both match something that has "viagra" in it # TODO: In the future, make it so its not always the last token that # is partial but wherever the cursor is if len(tokens[-1]) >= 2: results.append(bp[tokens[-1]]) if not results: results.append(set()) rr1 = set.intersection(*results) # If the matching we've done doesn't have any hits, then # fallback to looking at the first things you type if not limit or len(rr1) < limit: r2 = [] for t in tokens: if len(t) >= 2: r2.append(bp[t]) if not r2: r2.append(set()) rr2 = set.intersection(*r2) print "rr2=%r" % rr2 rr2.difference_update(rr1) else: rr2 = set() rr = [bi[x] for x in rr1] + [bi[x] for x in rr2] if limit is None: return rr else: return rr[:limit]
def test_overwrite_using_flag_n(self): """Re-opening of a database with flag='c' destroys it all.""" # given, fname = norm_file('tests/db/sqlitedict-override-test.sqlite') orig_db = sqlitedict.SqliteDict(filename=fname, tablename='sometable') orig_db['key'] = 'value' orig_db.commit() orig_db.close() # verify, next_db = sqlitedict.SqliteDict(filename=fname, tablename='sometable', flag='n') self.assertNotIn('key', next_db.keys())
def stat_feed(): from collections import Counter cnt = Counter() idxd = sqlitedict.SqliteDict('./idx_db.db') upgd = sqlitedict.SqliteDict('./upg_db.db') for k, idx_js in upgd.items(): print dump_json(k.decode('utf8')), idx_js.keys() for k, idx_js in idxd.items(): uj = idx_js # cnt[dump_json([uj['source'],uj['api_meta'][0]])]+=1 cnt[dump_json([uj['api_meta'][0]])] += 1 # cnt[dump_json([uj['api_meta']])]+=1 for k, v in cnt.most_common(3): print k, v print 'category_cnt', len(cnt), 'item_cnt', len(idxd)
def test_readonly(self): fname = norm_file('tests/db/sqlitedict-override-test.sqlite') orig_db = sqlitedict.SqliteDict(filename=fname) orig_db['key'] = 'value' orig_db.commit() orig_db.close() readonly_db = sqlitedict.SqliteDict(filename=fname, flag='r') self.assertTrue(readonly_db['key'] == 'value') def attempt_write(): readonly_db['key'] = ['new_value'] with self.assertRaises(RuntimeError): attempt_write()
def check_length(self): # Load the current sqlitedict db = sqlitedict.SqliteDict(self.db_name, 'iterations') # Get the number of current iterations # Minus one because OpenMDAO uses 1-indexing self.num_iters = int(db.keys()[-1].split('|')[-1])
def cache_topics(rss): """Add topics to cache file.""" path = os.path.join(os.path.dirname(os.path.realpath(__file__)), CACHE_FILE) with sqlitedict.SqliteDict(path) as db: feed_link = rss['args'].source dates = db.get(feed_link, {}) for topic in rss['topics']: date = dateutil.parser.parse(topic['date']) date = date.strftime('%Y%m%d') news = dates.get(date, {}) link = topic['link'] if link in news: logging.info('Topic with link {} already cached'.format(link)) continue logging.info('Cache topic with link {} from {}'.format( link, feed_link)) news[link] = topic dates[date] = news db[feed_link] = dates db.commit()
def __init__(self): super().__init__() self.setupUi() self.dictionarydb = sqlitedict.SqliteDict("DBS2.db", autocommit=True) self.Database = self.dictionarydb.get('Data', []) "self.dictionarydb.clear() #database clear"
def openfile(cls, fpath, tablename='vntree0', flag='c'): """Class method that opens (load) a vn4 (sqlite) file. :param fpath: the file path for the vn4 file. :type fpath: str :returns: root node of tree or `False` if failure. :rtype: Node or bool """ if not os.path.isfile(fpath): logger.error("%s.openfile: arg `fpath`=«%s» not valid." % (cls.__name__, fpath)) return False try: #with sqlitedict.SqliteDict(fpath, encode=sqlitedict_encode, decode=sqlitedict_decode) as _vndict: with sqlitedict.SqliteDict(fpath, tablename=tablename, flag=flag) as _vndict: print(f"_vndict={_vndict}") pkldata = _vndict["_vntree"] #rootnode = _vndict["_vntree"] #print(f"type(pkldata)={type(pkldata)}") #print(pkldata) rootnode = cls(treedict=pkldata) rootnode._vntree_fpath = os.path.abspath(fpath) except Exception as err: logger.error("%s.openfile: data in file «%s» not valid: %s" % (cls.__name__, fpath, err)) return False for _n in rootnode: _n.load_data() return rootnode
def main(quiet=True): try: get_ipython except NameError: nested = 0 cfg = load_default_config() cfg.TerminalInteractiveShell.prompts_class = CustPrompt else: print 'Running nested copies of IPython. Augmenting configuration...' cfg = load_default_config() nested = 1 from IPython.terminal.embed import InteractiveShellEmbed cfg.TerminalInteractiveShell.confirm_exit = False cfg.TerminalInteractiveShell.debug = True ipshell = InteractiveShellEmbed.instance( config=cfg, banner1='Welcome to the sqlenv IPython Shell...\n') # Setup sqlitedict as `sqdb` sqdb_path = os.path.join(os.path.abspath('.'), '.local_db.sqlite') sqdb = sqlitedict.SqliteDict(sqdb_path) ipshell()
def read_sqlite_dict_file(filepath, *, with_dill=False, **kwargs): if with_dill: sqlitedict = tricks.module_sqlitedict_with_dill(dill_detect_trace=True) else: import sqlitedict with sqlitedict.SqliteDict(filepath, **kwargs) as sd: return dict(sd)
def test_irregular_tablenames(self): """Irregular table names need to be quoted""" db = sqlitedict.SqliteDict(':memory:', tablename='9nine') db['key'] = 'value' db.commit() self.assertEqual(db['key'], 'value') db.close() db = sqlitedict.SqliteDict(':memory:', tablename='outer space') db['key'] = 'value' db.commit() self.assertEqual(db['key'], 'value') db.close() with self.assertRaisesRegexp(ValueError, r'^Invalid tablename '): sqlitedict.SqliteDict(':memory:', '"')
def test_commit_nonblocking(self): """Coverage for non-blocking commit.""" # given, with sqlitedict.SqliteDict(autocommit=True) as d: # exercise: the implicit commit is nonblocking d['key'] = 'value' d.commit(blocking=False)
def test_with_statement(self): """Verify using sqlitedict as a contextmanager . """ with sqlitedict.SqliteDict() as d: self.assertTrue(isinstance(d, sqlitedict.SqliteDict)) self.assertEqual(dict(d), {}) self.assertEqual(list(d), []) self.assertEqual(len(d), 0)
def set_plan_dict(self, plan_dict): id_curr = max([int(k) for k in plan_dict]) with sqlitedict.SqliteDict(filename=self.filename) as sqldict: sqldict['plan_dict'] = plan_dict sqldict['id_curr'] = id_curr + 1 sqldict.commit() self.plan_dict_cache = None
def index_crawl(): pool = Pool(8) cols = 'news_finance,news_entertainment,news_tech,news_game,news_sports,news_travel,news_car,news_hot,news_military,news_fashion,news_history,news_world,news_discovery,news_regime,news_baby,news_essay'.split( ',') car_cols = 'car_new_arrival,SUV,car_guide,car_usage' idxd = sqlitedict.SqliteDict('./idx_db.db', autocommit=True) def fetch_one_col(col, i, idxd): try: maxhot = str(ts2unix(get_date()) - i * 2000) jo = get_index_page(col, maxhot) ilist = extract_index_user_list(jo) for art in jo['data']: key = 'idx_%s' % (art['item_id']) art['api_meta'] = [col, maxhot] idxd[key] = art except Exception as e: print e.message, col for col in cols[:]: for i in range(0, 90): pool.spawn(fetch_one_col, col, i, idxd) pool.join() idxd.close()
def getDB(self): ''' returns and initialize sigleton instance of db ''' if not self.__created : self.__db=sql.SqliteDict(self.__dbpath,autocommit=True) self.__created=True return self.__db
def save(self, id, value): plan_dict = self.plan_dict if id in plan_dict: with sqlitedict.SqliteDict(filename=self.filename) as sqldict: sqldict[id] = {'id': id, 'x': plan_dict[id], 'value': value} sqldict.commit() else: raise AttributeError(f'Такого ключа для сохранения нет {id}')
def test_directory_notfound(self): """Verify RuntimeError: directory does not exist.""" # given: a non-existent directory, folder = tempfile.mkdtemp(prefix='sqlitedict-test') os.rmdir(folder) # exercise, with self.assertRaises(RuntimeError): sqlitedict.SqliteDict(filename=os.path.join(folder, 'nonexistent'))
def check_length(self): # Load the current sqlitedict db = sqlitedict.SqliteDict(self.db_name, 'iterations') cr = self.case_reader = SqliteCaseReader(self.db_name) # Get the number of current iterations # Minus one because OpenMDAO uses 1-indexing self.num_iters = int(cr.driver_cases.list_cases()[-1].split('|')[-1])
def process(dbname, path, options): with sqlitedict.SqliteDict(dbname, autocommit=True) as db: if is_tar_gzip(path): return process_tgz(db, path, options) elif os.path.isfile(path): return process_file(db, path, options) elif os.path.isdir(path): raise NotImplementedError()
def populate_db(self, ids: np.array, sents: np.array): db_file = p.abspath(self.sub_dir / f'{self.seed_name}.sqlite') id_to_sent = sqld.SqliteDict(db_file, autocommit=True) for i in range(ids.shape[0]): id_to_sent[str(ids[i])] = str(sents[i]) id_to_sent.close()
def __init__(self, path, *, compressed=True): self.compressed = compressed tablename = 'responses_gzip' if compressed else 'responses' self.db = sqlitedict.SqliteDict(path, tablename=tablename, autocommit=True, encode=self.encode, decode=self.decode)
def __enter__(self): try: import sqlitedict except ImportError: error('failed to import sqlitedict; try `pip3 install sqlitedict`') raise self.db = sqlitedict.SqliteDict(self.dbname, autocommit=True) return self
def test_with_statement(self): ''' test_with_statement ''' with sqlitedict.SqliteDict() as d: self.assertTrue(isinstance(d, sqlitedict.SqliteDict)) self.assertEqual(dict(d), {}) self.assertEqual(list(d), []) self.assertEqual(len(d), 0)
def _get_linkdb(self, autocommit=False): return sqlitedict.SqliteDict( filename=str(self._conf_root / 'glink.db'), tablename='links', journal_mode='WAL', autocommit=autocommit, encode=partial(json.dumps, ensure_ascii=False), decode=json.loads )
def test_as_str(self): """Verify SqliteDict.__str__().""" # given, db = sqlitedict.SqliteDict() # exercise db.__str__() # test when db closed db.close() db.__str__()