def get_fastgraph_transactions(self, secret, query, queryType, raw=False, both=True, skip=None):
from yadacoin.crypt import Crypt
cipher = Crypt(secret)
for transaction in self.mongo.db.fastgraph_transactions.find(query):
if 'txn' in transaction:
try:
if transaction.get('id') in skip:
continue
if 'relationship' not in transaction:
continue
if not transaction['relationship']:
continue
res = self.mongo.db.fastgraph_transaction_cache.find_one({
'txn.id': transaction.get('id'),
})
if res:
continue
if not raw:
decrypted = cipher.decrypt(transaction['relationship'])
relationship = json.loads(decrypted.decode('latin1'))
transaction['relationship'] = relationship
self.mongo.db.fastgraph_transaction_cache.update(
{
'txn': transaction,
}
, upsert=True)
except:
continue
for x in self.mongo.db.fastgraph_transaction_cache.find({
'txn': {'$exists': True}
}):
yield x['tnx']
def get_relationships(self, wif):
# from block import Block
# from transaction import Transaction
from yadacoin.crypt import Crypt
relationships = []
for block in self.config.BU.get_blocks():
for transaction in block.get('transactions'):
try:
cipher = Crypt(wif)
decrypted = cipher.decrypt(transaction['relationship'])
relationship = json.loads(decrypted.decode('latin1'))
relationships.append(relationship)
except:
continue
return relationships
def get_transaction_by_rid(self, selector, wif=None, bulletin_secret=None, rid=False, raw=False,
theirs=False, my=False, public_key=None):
# from block import Block
# from transaction import Transaction
from yadacoin.crypt import Crypt
if not rid:
ds = bulletin_secret
selectors = [
TU.hash(ds + selector),
TU.hash(selector + ds)
]
else:
if not isinstance(selector, list):
selectors = [selector, ]
else:
selectors = selector
def txn_gen():
res = self.mongo.db.blocks.find(
{"transactions": {"$elemMatch": {"relationship": {"$ne": ""}, "rid": {"$in": selectors}}}})
for x in res:
yield x
res = self.mongo.db.fastgraph_transactions.find(
{"txn": {"$elemMatch": {"relationship": {"$ne": ""}, "rid": {"$in": selectors}}}})
for x in res:
yield x
for block in txn_gen():
for transaction in block.get('transactions'):
if theirs and public_key == transaction['public_key']:
continue
if my and public_key != transaction['public_key']:
continue
if not raw:
try:
cipher = Crypt(wif)
decrypted = cipher.decrypt(transaction['relationship'])
relationship = json.loads(decrypted.decode('latin1'))
transaction['relationship'] = relationship
except:
continue
if 'rid' in transaction and transaction['rid'] in selectors:
return transaction
def get_posts(self, rids):
from yadacoin.crypt import Crypt
if not isinstance(rids, list):
rids = [rids, ]
posts_cache = self.mongo.db.posts_cache.find({
'rid': {'$in': rids}
}).sort([('height', -1)])
latest_block = self.config.BU.get_latest_block()
if posts_cache.count():
posts_cache = posts_cache[0]
block_height = posts_cache['height']
else:
block_height = 0
transactions = self.mongo.db.blocks.aggregate([
{
"$match": {
"index": {'$gt': block_height}
}
},
{
"$match": {
"transactions": {"$elemMatch": {"relationship": {"$ne": ""}}},
"transactions.dh_public_key": '',
"transactions.rid": ''
}
},
{"$unwind": "$transactions"},
{
"$project": {
"_id": 0,
"txn": "$transactions",
"height": "$index"
}
},
{
"$match": {
"txn.relationship": {"$ne": ""},
"txn.dh_public_key": '',
"txn.rid": ''
}
},
{
"$sort": {"height": 1}
}
])
fastgraph_transactions = self.mongo.db.fastgraph_transactions.find({
"txn.relationship": {"$ne": ""},
"txn.dh_public_key": '',
"txn.rid": ''
})
transactions = [x for x in transactions] + [x for x in fastgraph_transactions]
# transactions are all posts not yet cached by this rid
# so we want to grab all bulletin secrets for this rid
mutual_bulletin_secrets = self.get_mutual_bulletin_secrets(rids)
friends = []
for friend in self.get_transactions_by_rid(rids, self.config.bulletin_secret, rid=True):
if 'their_bulletin_secret' in friend['relationship']:
friends.append(friend['relationship']['their_bulletin_secret'])
friends = list(set(friends))
had_txns = False
if friends:
mutual_bulletin_secrets.extend(friends)
for i, x in enumerate(transactions):
res = self.mongo.db.posts_cache.find_one({
'rid': {'$in': rids},
'id': x['txn']['id']
})
if res:
continue
for bs in mutual_bulletin_secrets:
try:
crypt = Crypt(bs)
decrypted = crypt.decrypt(x['txn']['relationship'])
try:
decrypted = base64.b64decode(decrypted)
except:
raise
data = json.loads(decrypted.decode('utf-8'))
x['txn']['relationship'] = data
if 'postText' in data:
had_txns = True
self.app_log.debug('caching posts at height: {}'.format(x.get('height', 0)))
for rid in rids:
self.mongo.db.posts_cache.update({
'rid': rid,
'height': x.get('height', 0),
'id': x['txn']['id'],
'bulletin_secret': bs
},
{
'rid': rid,
'height': x.get('height', 0),
'id': x['txn']['id'],
'txn': x['txn'],
'bulletin_secret': bs,
'success': True
},
upsert=True)
except Exception as e:
for rid in rids:
self.mongo.db.posts_cache.update({
'rid': rid,
'height': x.get('height', 0),
'id': x['txn']['id'],
'bulletin_secret': bs
},
{
'rid': rid,
'height': x.get('height', 0),
'id': x['txn']['id'],
'txn': x['txn'],
'bulletin_secret': bs,
'success': False
},
upsert=True)
self.app_log.debug(e)
if not had_txns:
for rid in rids:
self.mongo.db.posts_cache.insert({
'rid': rid,
'height': latest_block['index'],
'success': False
})
i = 1
for x in self.mongo.db.fastgraph_transaction_cache.find({
'txn.dh_public_key': '',
'txn.relationship': {'$ne': ''},
'txn.rid': ''
}):
if 'txn' in x:
x['txn']['height'] = block_height + i
yield x['txn']
i += 1
for x in self.mongo.db.posts_cache.find({'rid': {'$in': rids}, 'success': True}):
if 'txn' in x:
x['txn']['height'] = x['height']
x['txn']['bulletin_secret'] = x['bulletin_secret']
yield x['txn']
def get_transactions_by_rid(self, selector, bulletin_secret, wif=None, rid=False, raw=False,
returnheight=True, lt_block_height=None, requested_rid=False):
# selectors is old code before we got an RID by sorting the bulletin secrets
# from block import Block
# from transaction import Transaction
from yadacoin.crypt import Crypt
if not rid:
ds = bulletin_secret
selectors = [
TU.hash(ds + selector),
TU.hash(selector + ds)
]
else:
if not isinstance(selector, list):
selectors = [selector, ]
else:
selectors = selector
transactions_by_rid_cache = self.mongo.db.transactions_by_rid_cache.find(
{
'raw': raw,
'rid': rid,
'bulletin_secret': bulletin_secret,
'returnheight': returnheight,
'selector': {'$in': selectors},
'requested_rid': requested_rid
}
).sort([('height', -1)])
latest_block = self.config.BU.get_latest_block()
transactions = []
if lt_block_height:
query = {"transactions.rid": {"$in": selectors}, "transactions": {"$elemMatch": {"relationship": {"$ne": ""}}},
'index': {'$lte': lt_block_height}}
if requested_rid:
query["transactions.requested_rid"] = {"$in": selectors}
blocks = self.mongo.db.blocks.find(query)
else:
if transactions_by_rid_cache.count():
transactions_by_rid_cache = transactions_by_rid_cache[0]
block_height = transactions_by_rid_cache['height']
else:
block_height = 0
query = {"transactions.rid": {"$in": selectors}, "transactions": {"$elemMatch": {"relationship": {"$ne": ""}}},
'index': {'$gt': block_height}}
if requested_rid:
query = {
"$or": [
{
"transactions.rid": {
"$in": selectors
}
},
{
"transactions.requested_rid": {
"$in": selectors
}
}
],
"transactions": {
"$elemMatch": {
"relationship": {
"$ne": ""
}
}
},
'index': {
'$gt': block_height
}
}
else:
query = {
"transactions.rid": {
"$in": selectors
},
"transactions": {
"$elemMatch": {
"relationship": {
"$ne": ""
}
}
},
'index': {
'$gt': block_height
}
}
blocks = self.mongo.db.blocks.find(query)
cipher = Crypt(self.config.wif)
for block in blocks:
for transaction in block.get('transactions'):
if 'relationship' in transaction and transaction['relationship']:
if returnheight:
transaction['height'] = block['index']
if not raw:
try:
decrypted = cipher.decrypt(transaction['relationship'])
relationship = json.loads(decrypted.decode('latin1'))
transaction['relationship'] = relationship
except:
continue
for selector in selectors:
self.app_log.debug('caching transactions_by_rid at height: {}'.format(block['index']))
self.mongo.db.transactions_by_rid_cache.insert(
{
'raw': raw,
'rid': rid,
'bulletin_secret': bulletin_secret,
'returnheight': returnheight,
'selector': selector,
'txn': transaction,
'height': block['index'],
'requested_rid': requested_rid
}
)
transactions.append(transaction)
if not transactions:
for selector in selectors:
self.mongo.db.transactions_by_rid_cache.insert(
{
'raw': raw,
'rid': rid,
'bulletin_secret': bulletin_secret,
'returnheight': returnheight,
'selector': selector,
'height': latest_block['index'],
'requested_rid': requested_rid
}
)
for ftxn in self.mongo.db.fastgraph_transactions.find({'txn.rid': {'$in': selectors}}):
if 'txn' in ftxn:
yield ftxn['txn']
last_id = ''
for x in self.mongo.db.transactions_by_rid_cache.find({
'raw': raw,
'rid': rid,
'returnheight': returnheight,
'selector': {'$in': selectors},
'requested_rid': requested_rid
}).sort([('txn.id', 1)]):
if 'txn' in x and x['txn']['id'] != last_id:
last_id = x['txn']['id']
yield x['txn']
def get_transactions(self, wif, query, queryType, raw=False, both=True, skip=None):
if not skip:
skip = []
#from block import Block
#from transaction import Transaction
from yadacoin.crypt import Crypt
get_transactions_cache = self.mongo.db.get_transactions_cache.find(
{
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'queryType': queryType
}
).sort([('height', -1)])
latest_block = self.get_latest_block()
if get_transactions_cache.count():
get_transactions_cache = get_transactions_cache[0]
block_height = get_transactions_cache['height']
else:
block_height = 0
cipher = Crypt(wif)
transactions = []
for block in self.mongo.db.blocks.find({"transactions": {"$elemMatch": {"relationship": {"$ne": ""}}}, 'index': {'$gt': block_height}}):
for transaction in block.get('transactions'):
try:
if transaction.get('id') in skip:
continue
if 'relationship' not in transaction:
continue
if not transaction['relationship']:
continue
if not raw:
decrypted = cipher.decrypt(transaction['relationship'])
relationship = json.loads(decrypted.decode('latin1'))
transaction['relationship'] = relationship
transaction['height'] = block['index']
self.mongo.db.get_transactions_cache.update(
{
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'height': latest_block['index'],
'queryType': queryType,
'id': transaction['id']
},
{
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'height': latest_block['index'],
'txn': transaction,
'queryType': queryType,
'id': transaction['id']
}
, upsert=True)
except:
self.app_log.debug('failed decrypt. block: {}'.format(block['index']))
if both:
transaction['height'] = block['index']
self.mongo.db.get_transactions_cache.update(
{
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'height': latest_block['index'],
'queryType': queryType
},
{
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'height': latest_block['index'],
'txn': transaction,
'queryType': queryType
}
, upsert=True)
continue
if not transactions:
self.mongo.db.get_transactions_cache.insert({
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'queryType': queryType,
'height': latest_block['index']
})
fastgraph_transactions = self.get_fastgraph_transactions(wif, query, queryType, raw=False, both=True, skip=None)
for fastgraph_transaction in fastgraph_transactions:
yield fastgraph_transaction
search_query = {
'public_key': self.config.public_key,
'raw': raw,
'both': both,
'skip': skip,
'queryType': queryType,
'txn': {'$exists': True}
}
search_query.update(query)
transactions = self.mongo.db.get_transactions_cache.find(search_query).sort([('height', -1)])
for transaction in transactions:
yield transaction['txn']