def process_block(height: int, blockhash: str): print('processing block: height=%d hash=%s' % (height, blockhash)) height_bin = b'HGHT' + struct.pack('>I', height) blockhash_bin = binascii.unhexlify(blockhash) h = db.get(height_bin) if h is not None and h == blockhash_bin and db.get(b'HASH' + blockhash_bin): # block already seen and processed print('* already seen and processed') return if h is not None and h != blockhash_bin: # seen another block with this height, delete old info print('* reorg') db.delete(b'HASH' + h) else: # this is new block print('* new block') db.put(height_bin, binascii.unhexlify(blockhash)) process_transactions(blockhash)
def update_matches_per_tournament_details(key, from_cache=True): tournament = db.get(key) url = tournament["url"] if not url: return matches = matches_list_per_tournament(url, from_cache=from_cache) for item in matches: item = dict( item, tournament_year=tournament["year"], tournament_slug=tournament["slug"], tournament_code=tournament["code"], ) match_key = build_match_key(item) merged_item = merge(db.get(match_key) or {}, item) log.track_lacked(key, url, merged_item) db.set(match_key, merged_item) player = merged_item["winner"] if player.get("url"): player_key = build_player_key(player["slug"], player["code"]) db.set(player_key, player) else: log.warning(f"Player without link, {url} {player}") player = merged_item["looser"] if player.get("url"): player_key = build_player_key(player["slug"], player["code"]) db.set(player_key, player) else: log.warning(f"Player without link, {url} {player}")
def votemeta(line, date): log(3, 'vote title is "%s"' % line) res={'rapporteur': []} m=docre.search(line) if m: doc=m.group(1).replace(' ', '') log(4,'setting doc to "%s"' % doc) res['doc']=doc reports=db.get("dossiers_by_doc", doc) if reports: res['epref']=[report['procedure']['reference'] for report in reports] if len(reports) > 1: log(3,"more than 1 dossier referencing document %s, %s" % (doc,[d['procedure']['reference'] for d in reports])) else: if doc in VOTE_DOX_RE: res['epref']=[VOTE_DOX_RE[doc]] elif doc not in ignoredox: log(2,'%s despite matching regex could not associate dossier with vote in "%s"' % (doc,line)) return res m=refre.search(line) if m and db.get('ep_dossiers',m.group(1)): res['epref']=[m.group(1)] return res for k,v in VOTE_DOX.items(): if k in line: res['epref']=[v] return res log(4,'no associated dossier for: "%s"' % line) return res
def test_successful_get(rethink_connect): global unique_id result = get(database, table, unique_id, rethink_connect) assert result is not None assert len(result) == 10 assert result["name"] == "The Dark Knight Rises" assert set(keys).difference(result) == set()
def get_code_list(self): code_list = [] sql = "select code, code_type from stock" result = db.get(sql) for row in result: code_list.append(row[1].lower() + row[0]) return code_list
def update_match_details(key, from_cache=True): match = db.get(key) or {} url = match["url"] if not url: return db.set(key, merge(match, match_detail(url)))
def get_value_for_server(self, guild_id): raw_dict = db.get(self.DB_KEY) if raw_dict is None: return None else: dictionary = get_dict(raw_dict) return dictionary.get(str(guild_id))
def do(self, hashparams): jQuery(SLOT).empty() jQuery(SLOT).append("<p>The address is {}</p>".format( db.get('ck').publicAddress)) jQuery(SLOT).append(_t1) jQuery("#smsg").submit(self.spost) pass
def __init__(self, username="******", password="", loader=False): addr = re.split('@', username) user = db.get("user:%s:login" % addr[0]) if not user == username and loader: return None self.username = username self.user = addr[0] self.password = password
def single(quote_id): quotes = [ db.get(quote_id) ] admin = authDB.isAuthenticated(request) if None in quotes: abort(404) if request.wants_json(): return json_nyi() return render_template('quotes.html', nav=navs, quotes=quotes, isAdmin=admin)
def remove(quote_id): quote = db.get(quote_id) if quote is None: abort(404) if request.provided_json(): return json_nyi() db.delete(quote) return "success"
def users(request): if request.method == 'POST': user = db.insert('users', request.data) return JSONResponse(user, status=HTTP_CREATED) data = db.get('users') return JSONResponse(data=data)
def build_tournaments_csv(year): headers = [ 'id', 'slug', 'code', 'name', 'city', 'country', 'year', 'week', 'url', 'type', 'surface', 'category', 'date_start', 'date_end' ] keys = get_tournament_keys(year) objects = (db.get(key) for key in keys) build_csv(objects, headers, f'export/tournaments_{year}.csv')
def remove_server(self, guild_id: int): raw_dict = db.get(self.DB_KEY) if raw_dict is not None: dictionary = get_dict(raw_dict) try: dictionary.pop(str(guild_id)) db.set(self.DB_KEY, json.dumps(dictionary)) except KeyError: pass
def set_value_for_server(self, guild_id, value): raw_dict = db.get(self.DB_KEY) if raw_dict is None: dictionary = {} else: dictionary = get_dict(raw_dict) dictionary[str(guild_id)] = value db.set(self.DB_KEY, json.dumps(dictionary))
def request_html(url, from_cache=True): html = db.get(url) if not html or not from_cache: log.debug("Request {}".format(url)) html = get_html(get_ajax_url(url)).get(blocking=True) db.set(url, html) return html
def update_player(key, from_cache=True): player = db.get(key) or {} url = player["url"] if not url: return player = merge(player, player_detail(url, from_cache=from_cache)) log.track_lacked(key, url, player) db.set(key, player)
def remove(quote_id): if not 'user' in session: abort(403) quote = db.get(quote_id) if quote is None: abort(404) if request.provided_json(): return json_nyi() db.delete(quote) return "success"
def fetch_votes(quote_id): quote = db.get(quote_id) if quote is None: abort(404) json = { 'links': [ build_link('/quotes/%s' %(quote_id), 'pyqdb/quote', Quote.json_mimetype), build_link('/quotes/%s/votes' %(quote_id), 'pyqdb/quote/cast-vote', Vote.json_mimetype, method='put') ], 'type': '', # up or down 'id': quote_id } return jsonify(json, Vote.json_mimetype)
def single(quote_id): quotes = [db.get(quote_id)] admin = authDB.isAuthenticated(request) if None in quotes: abort(404) if request.wants_json(): return json_nyi() return render_template('quotes.html', nav=navs, quotes=quotes, isAdmin=admin)
def update_tournament_details(key, from_cache=True): tournament = db.get(key) or {} url = tournament["url"] if not url: return db.set( key, merge(tournament, tournament_detail(tournament["url"], from_cache=from_cache)), )
def get_prefix_for_guild(guild_id: int): prefixes_raw_dict = db.get(PREFIXES_DB_KEY) if prefixes_raw_dict is not None: try: return get_dict(prefixes_raw_dict)[str(guild_id)] except KeyError: log_event( f"Failed trying to fetch prefix for server id {guild_id}", logging.CRITICAL) return DEFAULT_PREFIX log_event(f"Error Fetching prefixes DB", logging.CRITICAL) return DEFAULT_PREFIX
def __save_report(key, doc): """Saves report using data from specified document.""" date = __get_date_by_name(doc["event"]) rpt_id = "report/" + doc["location"] + "-" + str.join("-", map(lambda x: str(x), date)) rpt = db.get(rpt_id, {}) # get report or create new one rpt["_id"] = key rpt["location"] = doc["location"] rpt["event"] = doc["event"] rpt["date"] = date rpt["data"] = parse(doc["report"]) db[rpt_id] = rpt
def test_dump(): """ don't try this at home. it's an example, of how you can get a glimpse on some nosql collection""" from db import db, TABLES from utils.log import set_level set_level(0) print(_html_header()) for table in TABLES.keys(): #print(table, db.count(table, None), file = sys.stderr) vals = db.get(table, None).values() if len(vals) == 0: continue dump_schema(vals, ['changes'], title=table) print(_html_footer())
def on_request(self, request): token = request.headers.get('token') user = None if token: try: user = db.get('users', int(token)) except Exception: pass request.user = user or { 'name': 'Anonymous' } return request
def addchangednames(mep): mepid = mep['UserID'] m = db.get('ep_meps', mepid) if not m: return mep prevnames = [ c['data'][0] for changes in m.get('changes', {}).values() for c in changes if c['path'] == ['Name', 'full'] ] aliases = set(mep['Name']['aliases']) for name in prevnames: aliases |= set(mangleName(name, mepid)['aliases']) mep['Name']['aliases'] = sorted( [x for x in set(unws(n) for n in aliases) if x]) return mep
def cast_vote(quote_id): ip = request.remote_addr quote = db.get(quote_id) if quote is None: abort(404) if request.provided_json(): return json_nyi() else: type = request.form['type'] if type == "up": quote = db.up_vote(quote_id, ip) elif type == "down": quote = db.down_vote(quote_id, ip) else: abort(400) return jsonify(quote, Quote.json_mimetype)
def fetch_votes(quote_id): quote = db.get(quote_id) if quote is None: abort(404) json = { 'links': [ build_link('/quotes/%s' % (quote_id), 'pyqdb/quote', Quote.json_mimetype), build_link('/quotes/%s/votes' % (quote_id), 'pyqdb/quote/cast-vote', Vote.json_mimetype, method='put') ], 'type': '', # up or down 'id': quote_id } return jsonify(json, Vote.json_mimetype)
def getdocs(line): issue = {} m = instdocre.search(line) if m.group(1): issue[u'procedure'] = m.group(1) if m.group(2): issue[u'reading'] = m.group(2) if m.group(3): issue[u'epdoc'] = m.group(3) dossier = db.dossier(m.group(3)) if dossier: issue[u'docref'] = dossier['procedure']['reference'] if m.group(4): dossiers = db.get('dossiers_by_doc', m.group(4)) or [] dossier = None if not dossiers else dossiers[0] if dossier: issue[u'docref'] = dossier['procedure']['reference'] issue[u'comdoc'] = m.group(4) else: issue[u'comdoc'] = m.group(4) if m.group(5): issue[u'otherdoc'] = m.group(5) return issue
process_transactions(blockhash) SEGWIT_START = 481824 blockcount = rpc.getblockcount() for height in range(blockcount, SEGWIT_START - 1, -1): blockhash = rpc.getblockhash(height) process_block(height, blockhash) print() data = [] for height in range(blockcount, SEGWIT_START - 1, -1): height_bin = b'HGHT' + struct.pack('>I', height) h = db.get(height_bin) v = db.get(b'HASH' + h) strippedsize, size, weight, txtotal, txsegwit = struct.unpack('>IIIII', v) data.append({ 'height': height, # 'hash': binascii.hexlify(h).decode(), 'strippedsize': strippedsize, 'size': size, 'weight': weight, 'txtotal': txtotal, 'txsegwit': txsegwit, }) shutil.copyfile('data.json', 'data.json.last') json.dump(data, open('data.json', 'wt'), sort_keys=True)
def update_tournaments(tournaments): for tournament in tournaments: key = build_tournament_key(tournament["year"], tournament["slug"], tournament["code"]) db.set(key, merge(db.get(key) or {}, tournament))
def scrape(id, terms, mepname, **kwargs): activity_types = ( ('plenary-speeches', 'CRE'), ('reports', "REPORT"), ('reports-shadow', "REPORT-SHADOW"), ('opinions', "COMPARL"), ('opinions-shadow', "COMPARL-SHADOW"), ('motions-instit', "MOTION"), ('oral-questions', "OQ"), # other activities ('written-explanations', 'WEXP'), ('major-interpellations', 'MINT'), ('written-questions', "WQ"), ('motions-indiv', "IMOTION"), ('written-declarations', "WDECL"), ) activities = {} for type, TYPE in activity_types: for term in terms: page = 0 cnt = 20 url = "http://www.europarl.europa.eu/meps/en/%s/loadmore-activities/%s/%s/?page=%s&count=%s" % ( id, type, term, page, cnt) try: root = fetch(url) except: log(1, "failed to fetch {}".format(url)) raise ValueError #continue #print(url, file=sys.stderr) while (len(root.xpath('//div[@class="erpl_document"]')) > 0): for node in root.xpath('//div[@class="erpl_document"]'): if type == 'written-explanations': item = { 'title': unws(''.join( node.xpath( './div/h3/span[@class="t-item"]//text()')) ), 'date': datetime.strptime( node.xpath('./div[1]/div[1]/span[1]/text()') [0], u"%d-%m-%Y"), 'text': unws(''.join(node.xpath('./div[2]/div//text()'))) } elif type == 'written-declarations': if len(node.xpath('./div[1]/div')) != 3: log( 2, "written decl item has not 3 divs but %d %s" % (len(node.xpath('./div[1]/div')), url)) continue if len(node.xpath('./div[1]/div[1]/span')) != 3: log( 2, "written decl item has not 3 but %d spans in the 1st div at %s" % (len(node.xpath('./div[1]/div[1]/span')), url)) continue item = { 'title': unws(''.join( node.xpath( './div/h3/span[@class="t-item"]//text()')) ), 'date': datetime.strptime( node.xpath('./div[1]/div[1]/span[1]/text()') [0], u"%d-%m-%Y"), 'id': unws(''.join( node.xpath('./div[1]/div[1]/span[2]/text()') [0])), 'status': unws(''.join( node.xpath('./div[1]/div[1]/span[3]/text()') [0])), 'formats': [{ 'type': unws(fnode.xpath('./span/text()')[0]), 'url': str(fnode.xpath('./@href')[0]), 'size': unws(fnode.xpath('./span/span/text()')[0]) } for fnode in node.xpath( './div[1]/div[2]/div[@class="d-inline"]/a')], 'authors': [{ 'name': name.strip(), "mepid": db.mepid_by_name(name.strip()) } for name in node.xpath( './div[1]/div[3]/span/text()')], } for info in node.xpath('./div[2]/div'): label = unws(''.join(info.xpath('./text()')))[:-2] value = unws(''.join(info.xpath('./span/text()'))) if 'date' in label.lower(): value = datetime.strptime(value, u"%d-%m-%Y") if label == 'Number of signatories': number, date = value.split(' - ') value = int(number) item["No of sigs date"] = datetime.strptime( date, u"%d-%m-%Y") item[label] = value else: #from lxml.etree import tostring #print('\n'.join(tostring(e).decode() for e in node.xpath('./div/div[1]'))) # all other activities share the following scraper ref = unws(''.join( node.xpath('./div[1]/div[1]/span[2]/text()'))) if ref.startswith('- '): ref = ref[2:] if ref.endswith(' -'): ref = ref[:-2] item = { 'date': datetime.strptime( node.xpath('./div[1]/div[1]/span[1]/text()') [0], u"%d-%m-%Y"), 'reference': ref, } if type not in ['written-questions', 'oral-questions']: ref = unws(''.join( node.xpath('./div[1]/div[1]/span[3]/text()'))) if ref: if not pere.match(ref): log( 2, "pe, has not expected format: '%s'" % ref) else: item['pe'] = ref # opinions don't have title urls... why would they? refurl = node.xpath('./div[1]/h3/a/@href') if refurl: item['url'] = str(refurl[0]) item['title'] = unws(''.join( node.xpath( './div/h3//span[@class="t-item"]//text()'))) abbr = node.xpath( './div[1]/div[1]/span/span[contains(concat(" ",normalize-space(@class)," ")," erpl_badge-committee ")]/text()' ) if len(abbr): item['committee'] = [ a for a in [unws(c) for c in abbr] if a ] formats = [] for fnode in node.xpath( './div[1]/div[2]/div[@class="d-inline"]/a'): elem = { 'type': unws(fnode.xpath('./span/text()')[0]), 'url': str(fnode.xpath('./@href')[0]) } tmp = fnode.xpath('./span/span/text()') if len(tmp) > 0: elem['size'] = unws(tmp[0]) formats.append(elem) if formats: item['formats'] = formats authors = [{ 'name': name.strip(), "mepid": db.mepid_by_name(name.strip()) } for name in node.xpath('./div[1]/div[3]/span/text()') ] if authors: item['authors'] = authors if type in ['opinions-shadow', 'opinions']: for f in item['formats']: if f['type'] == 'PDF': ref = pdf2ref(f['url']) if ref is not None: item['dossiers'] = [ref] break else: # try to deduce dossier from document reference dossiers = db.get('dossiers_by_doc', item['reference']) or [] if len(dossiers) > 0: item['dossiers'] = [ d['procedure']['reference'] for d in dossiers ] elif not '+DOC+PDF+' in item['url']: # try to figure out the associated dossier by making an (expensive) http request to the ep log( 4, "fetching primary activity page %s" % item['url']) try: refroot = fetch(item['url']) except: refroot = None if refroot is not None: if '/doceo/' in item[ 'url']: # stupid new EP site removed the span with the procedure, bastards. fulla = refroot.xpath( '//table[@class="buttondocwin"]//a/img[@src="/doceo/data/img/navi_moredetails.gif"]/..' ) if fulla: fullurl = fulla[0].get('href') if fullurl.endswith('.html'): if fullurl[-7:-5] != 'EN': fullurl = fullurl[:-7] + 'EN.html' log( 4, 'loading activity full text page %s' % fullurl) if fullurl.startswith( '/doceo'): fullurl = 'https://www.europarl.europa.eu' + fullurl if fullurl != item['url']: refroot = fetch(fullurl) else: log( 4, 'no fulla for %s' % item['url']) anchor = refroot.xpath( '//span[@class="contents" and text()="Procedure : " and not(ancestor::div[@style="display:none"])]' ) if len(anchor) == 1: dossier = anchor[0].xpath( "./following-sibling::a/text()") if len(dossier) == 1: item['dossiers'] = [ unws(dossier[0]) ] elif len(dossier) > 1: log( 2, "more than one dossier in ep info page: %d %s" % (len(dossier), item['url'])) elif len(anchor) > 1: log( 2, "more than one anchor in ep info page: %d %s" % (len(anchor), item['url'])) item['term'] = term if TYPE not in activities: activities[TYPE] = [] activities[TYPE].append(item) if len(root.xpath('//div[@class="erpl_document"]')) < cnt: break page += 1 url = "http://www.europarl.europa.eu/meps/en/%s/loadmore-activities/%s/%s/?page=%s&count=%s" % ( id, type, term, page, cnt) try: root = fetch(url) except: log(1, "failed to fetch {}".format(url)) #raise ValueError break #print(url, file=sys.stderr) if TYPE in activities: activities[TYPE] = sorted(activities[TYPE], key=lambda x: x['date']) activities['mep_id'] = id if len(activities.keys()) > 1: process(activities, id, db.activities, 'ep_mep_activities', mepname, nodiff=True) return activities return {}
async def data_manager(websocket, path): logging.info(f"[INFO] Current path: {str(path)}") try: if path == "/": raise Exception(f"No operation defined on the current path." f"\nReceived path: {str(path)}") data = await websocket.recv() data = json.loads(data) identifier = data["id"] request_type = data["type"] payload = data["payload"] time = data["time"] logging.info(f"[INFO] Data received: {data}") if request_type != RDM_CALL: raise Exception(f"Type of service not as expected. Probably a call" f"for other service. Verify logs for further infor" f"mation!") if save_audit(identifier, request_type, payload, time): pass else: logging.error(f"[ERROR] Error while trying to save data to audit" f" database. Verify Rethink Data Manager logs for " f"furhter information!") try: connection = configure_database(ReDB_HOST, ReDB_PORT, ReDB_DEFAULT_DB, ReDB_USER, ReDB_PASS) if str(path) == op[1]: database = payload["database"] table = payload["table"] payload = payload["data"] logging.info(f"[INFO] Inserting data:\n\tDATABASE: " f"{database}\n\tTABLE: {table}\n\tDATA: " f"{payload}") result = insert(database, table, payload, connection) if result == {}: raise Exception(f"[ERROR] Error trying to insert data. " f"Verify the logs for the full traceback. " f"The data was not inserted!") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[2]: database = payload["database"] table = payload["table"] logging.info(f"[INFO] Getting all data:\n\tDATABASE: " f"{database}\n\tTABLE: {table}") result = get_all(database, table, connection) if result == {}: raise Exception(f"[ERROR] Error trying to getting data. " f"Verify the logs for the full traceback. " f"Neither the table is empty or no table" f" was found!") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[3]: database = payload["database"] table = payload["table"] identifier = payload["identifier"] logging.info(f"[INFO] Getting objetct data:\n\tDATABASE: " f"{database}\n\tTABLE: {table}\n\tIDENTIFIER: " f"{identifier}") result = get(database, table, identifier, connection) if result == {}: raise Exception(f"[ERROR] Error trying to getting data. " f"Verify the logs for the full traceback. " f"Neither the object doesn't exists or no" f" table was found!") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[4]: database = payload["database"] table = payload["table"] identifier = payload["identifier"] statment = payload["data"] logging.info(f"[INFO] Updating object:\n\tDATABASE: " f"{database}\n\tTABLE: {table}\n\tIDENTIFIER: " f"{identifier}\n\tUPDATE STATEMENT: {statment}") result = update(database, table, identifier, statment, connection) if result == {}: raise Exception(f"[ERROR] Error trying to update object. " f"Verify the logs for the full traceback. " f"Neither the object doesn't exists or no" f" table was found!") elif result["objects_updated"] == 0: raise Exception(f"[ERROR] Error trying to update object. " f"Verify the logs for the full traceback.") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[5]: database = payload["database"] table = payload["table"] identifier = payload["identifier"] logging.info(f"[INFO] Deleting object:\n\tDATABASE: " f"{database}\n\tTABLE: {table}\n\tIDENTIFIER: " f"{identifier}") result = delete(database, table, identifier, connection) if result == {}: raise Exception(f"[ERROR] Error trying to delete object. " f"Verify the logs for the full traceback. " f"Neither the object doesn't exists or no" f" table was found!") elif result["objects_deleted"] == 0: raise Exception(f"[ERROR] Error trying to delete object. " f"Verify the logs for the full traceback.") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[6]: database = payload["database"] table = payload["table"] logging.info(f"[INFO] Deleting all objects:\n\tDATABASE: " f"{database}\n\tTABLE: {table}") result = delete_all(database, table, connection) if result == {}: raise Exception(f"[ERROR] Error trying to delete objects. " f"Verify the logs for the full traceback. " f"Neither some error was found or no table" f" was found!") elif result["objects_deleted"] == 0: raise Exception(f"[ERROR] Error trying to delete object. " f"Verify the logs for the full traceback.") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[7]: database = payload["database"] table = payload["table"] logging.info(f"[INFO] Creating new table:\n\tDATABASE: " f"{database}\n\tTABLE: {table}") result = create_table(database, table, connection) if result == {}: raise Exception(f"[ERROR] Unknown error while trying to " f"create a new table. Verify the logs" f" for the the full traceback.") elif result["tables_created"] == 0: raise Exception(f"[ERROR] This table ({table})" f" already exists!") else: await websocket.send(json.dumps(success_msg(result))) elif str(path) == op[8]: database = payload["database"] table = payload["table"] statment = payload["filter"] statment = json.loads(statment) logging.info(f"[INFO] Getting objetct data:\n\tDATABASE: " f"{database}\n\tTABLE: {table}\n\tFILTER: " f"{statment}") result = get_where(database, table, statment, connection) if result == {}: raise Exception(f"[ERROR] Unknown error while trying to " f"create a new table. Verify the logs" f" for the the full traceback.") else: await websocket.send(json.dumps(success_msg(result))) else: raise Exception(f"Unknown operation on the current path." f"\nReceived path: {str(path)}") except Exception as err: if connection: disconnect_database(connection) raise err except KeyboardInterrupt as err: disconnect_database(connection) raise err except ConnectionClosed as err: disconnect_database(connection) raise err except CancelledError as err: disconnect_database(connection) raise err except Exception as err: logging.error(f"[ERROR] Error at services/data_manager." f"\nTraceback: {err}") await websocket.send(json.dumps(error_msg(path))) else: pass
def get_music_channel_id_for_guild(guild_id: int): music_channels_raw_dict = db.get(MUSIC_CH_DB_KEY) if music_channels_raw_dict is None: raise KeyError return get_dict(music_channels_raw_dict)[str(guild_id)]
def test_unsuccessful_get(rethink_connect): global unique_id result = get(database, empty_table, unique_id, rethink_connect) assert result is not None assert len(result) == 0 assert result == {}
def is_valid_login(self): pwd = db.get("user:%s:password" % self.user) if pwd == self.password: return True return False