def get_fastgraph_transactions(self, secret, query, queryType, raw=False, both=True, skip=None): from yadacoin.crypt import Crypt cipher = Crypt(secret) for transaction in self.mongo.db.fastgraph_transactions.find(query): if 'txn' in transaction: try: if transaction.get('id') in skip: continue if 'relationship' not in transaction: continue if not transaction['relationship']: continue res = self.mongo.db.fastgraph_transaction_cache.find_one({ 'txn.id': transaction.get('id'), }) if res: continue if not raw: decrypted = cipher.decrypt(transaction['relationship']) relationship = json.loads(decrypted.decode('latin1')) transaction['relationship'] = relationship self.mongo.db.fastgraph_transaction_cache.update( { 'txn': transaction, } , upsert=True) except: continue for x in self.mongo.db.fastgraph_transaction_cache.find({ 'txn': {'$exists': True} }): yield x['tnx']
def get_relationships(self, wif): # from block import Block # from transaction import Transaction from yadacoin.crypt import Crypt relationships = [] for block in self.config.BU.get_blocks(): for transaction in block.get('transactions'): try: cipher = Crypt(wif) decrypted = cipher.decrypt(transaction['relationship']) relationship = json.loads(decrypted.decode('latin1')) relationships.append(relationship) except: continue return relationships
def get_transaction_by_rid(self, selector, wif=None, bulletin_secret=None, rid=False, raw=False, theirs=False, my=False, public_key=None): # from block import Block # from transaction import Transaction from yadacoin.crypt import Crypt if not rid: ds = bulletin_secret selectors = [ TU.hash(ds + selector), TU.hash(selector + ds) ] else: if not isinstance(selector, list): selectors = [selector, ] else: selectors = selector def txn_gen(): res = self.mongo.db.blocks.find( {"transactions": {"$elemMatch": {"relationship": {"$ne": ""}, "rid": {"$in": selectors}}}}) for x in res: yield x res = self.mongo.db.fastgraph_transactions.find( {"txn": {"$elemMatch": {"relationship": {"$ne": ""}, "rid": {"$in": selectors}}}}) for x in res: yield x for block in txn_gen(): for transaction in block.get('transactions'): if theirs and public_key == transaction['public_key']: continue if my and public_key != transaction['public_key']: continue if not raw: try: cipher = Crypt(wif) decrypted = cipher.decrypt(transaction['relationship']) relationship = json.loads(decrypted.decode('latin1')) transaction['relationship'] = relationship except: continue if 'rid' in transaction and transaction['rid'] in selectors: return transaction
def get_posts(self, rids): from yadacoin.crypt import Crypt if not isinstance(rids, list): rids = [rids, ] posts_cache = self.mongo.db.posts_cache.find({ 'rid': {'$in': rids} }).sort([('height', -1)]) latest_block = self.config.BU.get_latest_block() if posts_cache.count(): posts_cache = posts_cache[0] block_height = posts_cache['height'] else: block_height = 0 transactions = self.mongo.db.blocks.aggregate([ { "$match": { "index": {'$gt': block_height} } }, { "$match": { "transactions": {"$elemMatch": {"relationship": {"$ne": ""}}}, "transactions.dh_public_key": '', "transactions.rid": '' } }, {"$unwind": "$transactions"}, { "$project": { "_id": 0, "txn": "$transactions", "height": "$index" } }, { "$match": { "txn.relationship": {"$ne": ""}, "txn.dh_public_key": '', "txn.rid": '' } }, { "$sort": {"height": 1} } ]) fastgraph_transactions = self.mongo.db.fastgraph_transactions.find({ "txn.relationship": {"$ne": ""}, "txn.dh_public_key": '', "txn.rid": '' }) transactions = [x for x in transactions] + [x for x in fastgraph_transactions] # transactions are all posts not yet cached by this rid # so we want to grab all bulletin secrets for this rid mutual_bulletin_secrets = self.get_mutual_bulletin_secrets(rids) friends = [] for friend in self.get_transactions_by_rid(rids, self.config.bulletin_secret, rid=True): if 'their_bulletin_secret' in friend['relationship']: friends.append(friend['relationship']['their_bulletin_secret']) friends = list(set(friends)) had_txns = False if friends: mutual_bulletin_secrets.extend(friends) for i, x in enumerate(transactions): res = self.mongo.db.posts_cache.find_one({ 'rid': {'$in': rids}, 'id': x['txn']['id'] }) if res: continue for bs in mutual_bulletin_secrets: try: crypt = Crypt(bs) decrypted = crypt.decrypt(x['txn']['relationship']) try: decrypted = base64.b64decode(decrypted) except: raise data = json.loads(decrypted.decode('utf-8')) x['txn']['relationship'] = data if 'postText' in data: had_txns = True self.app_log.debug('caching posts at height: {}'.format(x.get('height', 0))) for rid in rids: self.mongo.db.posts_cache.update({ 'rid': rid, 'height': x.get('height', 0), 'id': x['txn']['id'], 'bulletin_secret': bs }, { 'rid': rid, 'height': x.get('height', 0), 'id': x['txn']['id'], 'txn': x['txn'], 'bulletin_secret': bs, 'success': True }, upsert=True) except Exception as e: for rid in rids: self.mongo.db.posts_cache.update({ 'rid': rid, 'height': x.get('height', 0), 'id': x['txn']['id'], 'bulletin_secret': bs }, { 'rid': rid, 'height': x.get('height', 0), 'id': x['txn']['id'], 'txn': x['txn'], 'bulletin_secret': bs, 'success': False }, upsert=True) self.app_log.debug(e) if not had_txns: for rid in rids: self.mongo.db.posts_cache.insert({ 'rid': rid, 'height': latest_block['index'], 'success': False }) i = 1 for x in self.mongo.db.fastgraph_transaction_cache.find({ 'txn.dh_public_key': '', 'txn.relationship': {'$ne': ''}, 'txn.rid': '' }): if 'txn' in x: x['txn']['height'] = block_height + i yield x['txn'] i += 1 for x in self.mongo.db.posts_cache.find({'rid': {'$in': rids}, 'success': True}): if 'txn' in x: x['txn']['height'] = x['height'] x['txn']['bulletin_secret'] = x['bulletin_secret'] yield x['txn']
def get_transactions_by_rid(self, selector, bulletin_secret, wif=None, rid=False, raw=False, returnheight=True, lt_block_height=None, requested_rid=False): # selectors is old code before we got an RID by sorting the bulletin secrets # from block import Block # from transaction import Transaction from yadacoin.crypt import Crypt if not rid: ds = bulletin_secret selectors = [ TU.hash(ds + selector), TU.hash(selector + ds) ] else: if not isinstance(selector, list): selectors = [selector, ] else: selectors = selector transactions_by_rid_cache = self.mongo.db.transactions_by_rid_cache.find( { 'raw': raw, 'rid': rid, 'bulletin_secret': bulletin_secret, 'returnheight': returnheight, 'selector': {'$in': selectors}, 'requested_rid': requested_rid } ).sort([('height', -1)]) latest_block = self.config.BU.get_latest_block() transactions = [] if lt_block_height: query = {"transactions.rid": {"$in": selectors}, "transactions": {"$elemMatch": {"relationship": {"$ne": ""}}}, 'index': {'$lte': lt_block_height}} if requested_rid: query["transactions.requested_rid"] = {"$in": selectors} blocks = self.mongo.db.blocks.find(query) else: if transactions_by_rid_cache.count(): transactions_by_rid_cache = transactions_by_rid_cache[0] block_height = transactions_by_rid_cache['height'] else: block_height = 0 query = {"transactions.rid": {"$in": selectors}, "transactions": {"$elemMatch": {"relationship": {"$ne": ""}}}, 'index': {'$gt': block_height}} if requested_rid: query = { "$or": [ { "transactions.rid": { "$in": selectors } }, { "transactions.requested_rid": { "$in": selectors } } ], "transactions": { "$elemMatch": { "relationship": { "$ne": "" } } }, 'index': { '$gt': block_height } } else: query = { "transactions.rid": { "$in": selectors }, "transactions": { "$elemMatch": { "relationship": { "$ne": "" } } }, 'index': { '$gt': block_height } } blocks = self.mongo.db.blocks.find(query) cipher = Crypt(self.config.wif) for block in blocks: for transaction in block.get('transactions'): if 'relationship' in transaction and transaction['relationship']: if returnheight: transaction['height'] = block['index'] if not raw: try: decrypted = cipher.decrypt(transaction['relationship']) relationship = json.loads(decrypted.decode('latin1')) transaction['relationship'] = relationship except: continue for selector in selectors: self.app_log.debug('caching transactions_by_rid at height: {}'.format(block['index'])) self.mongo.db.transactions_by_rid_cache.insert( { 'raw': raw, 'rid': rid, 'bulletin_secret': bulletin_secret, 'returnheight': returnheight, 'selector': selector, 'txn': transaction, 'height': block['index'], 'requested_rid': requested_rid } ) transactions.append(transaction) if not transactions: for selector in selectors: self.mongo.db.transactions_by_rid_cache.insert( { 'raw': raw, 'rid': rid, 'bulletin_secret': bulletin_secret, 'returnheight': returnheight, 'selector': selector, 'height': latest_block['index'], 'requested_rid': requested_rid } ) for ftxn in self.mongo.db.fastgraph_transactions.find({'txn.rid': {'$in': selectors}}): if 'txn' in ftxn: yield ftxn['txn'] last_id = '' for x in self.mongo.db.transactions_by_rid_cache.find({ 'raw': raw, 'rid': rid, 'returnheight': returnheight, 'selector': {'$in': selectors}, 'requested_rid': requested_rid }).sort([('txn.id', 1)]): if 'txn' in x and x['txn']['id'] != last_id: last_id = x['txn']['id'] yield x['txn']
def get_transactions(self, wif, query, queryType, raw=False, both=True, skip=None): if not skip: skip = [] #from block import Block #from transaction import Transaction from yadacoin.crypt import Crypt get_transactions_cache = self.mongo.db.get_transactions_cache.find( { 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'queryType': queryType } ).sort([('height', -1)]) latest_block = self.get_latest_block() if get_transactions_cache.count(): get_transactions_cache = get_transactions_cache[0] block_height = get_transactions_cache['height'] else: block_height = 0 cipher = Crypt(wif) transactions = [] for block in self.mongo.db.blocks.find({"transactions": {"$elemMatch": {"relationship": {"$ne": ""}}}, 'index': {'$gt': block_height}}): for transaction in block.get('transactions'): try: if transaction.get('id') in skip: continue if 'relationship' not in transaction: continue if not transaction['relationship']: continue if not raw: decrypted = cipher.decrypt(transaction['relationship']) relationship = json.loads(decrypted.decode('latin1')) transaction['relationship'] = relationship transaction['height'] = block['index'] self.mongo.db.get_transactions_cache.update( { 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'height': latest_block['index'], 'queryType': queryType, 'id': transaction['id'] }, { 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'height': latest_block['index'], 'txn': transaction, 'queryType': queryType, 'id': transaction['id'] } , upsert=True) except: self.app_log.debug('failed decrypt. block: {}'.format(block['index'])) if both: transaction['height'] = block['index'] self.mongo.db.get_transactions_cache.update( { 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'height': latest_block['index'], 'queryType': queryType }, { 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'height': latest_block['index'], 'txn': transaction, 'queryType': queryType } , upsert=True) continue if not transactions: self.mongo.db.get_transactions_cache.insert({ 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'queryType': queryType, 'height': latest_block['index'] }) fastgraph_transactions = self.get_fastgraph_transactions(wif, query, queryType, raw=False, both=True, skip=None) for fastgraph_transaction in fastgraph_transactions: yield fastgraph_transaction search_query = { 'public_key': self.config.public_key, 'raw': raw, 'both': both, 'skip': skip, 'queryType': queryType, 'txn': {'$exists': True} } search_query.update(query) transactions = self.mongo.db.get_transactions_cache.find(search_query).sort([('height', -1)]) for transaction in transactions: yield transaction['txn']