def create_vesting(doc):
vesting = {
"vesting": {
"amount": convert_amount(doc["vesting_shares_value"], VESTING_PRECISION),
"decs": utils.UInt64(VESTING_PRECISION),
"sym": VESTING_SYMBOL
},
"delegate_vesting": {
"amount": convert_amount(doc["delegated_vesting_shares_value"], VESTING_PRECISION),
"decs": utils.UInt64(VESTING_PRECISION),
"sym": VESTING_SYMBOL
},
"received_vesting": {
"amount": convert_amount(doc["received_vesting_shares_value"], VESTING_PRECISION),
"decs": utils.UInt64(VESTING_PRECISION),
"sym": VESTING_SYMBOL
},
"unlocked_limit": {
"amount": utils.Int64(0),
"decs": utils.UInt64(VESTING_PRECISION),
"sym": VESTING_SYMBOL
},
"_SERVICE_" : {
"scope" : doc["name"],
"rev" : utils.Int64(1),
"payer" : doc["name"],
"size" : 64
}
}
cyberway.vesting.append(vesting)
def create_witness(doc, exist_accounts):
total_weight = 0
acc_list = []
for vote in golos.votes.table.find({"witness": doc["owner"]}):
if "removed" in vote.keys():
continue
if not vote["account"] in exist_accounts:
continue
if vote["account"] in acc_list:
continue
acc_list.append(vote["account"])
for acc in acc_list:
total_weight += int(
cyberway.vesting.table.find_one(
{"_SERVICE_.scope": acc})["vesting"]["amount"].to_decimal())
witness = {
"name": doc["owner"],
"key": doc["signing_key"],
"url": doc["url"],
"active": True,
"total_weight": utils.UInt64(total_weight),
"_SERVICE_": {
"scope": "gls.ctrl",
"rev": utils.Int64(1),
"payer": doc["owner"],
"size": 61
}
}
cyberway.witness.append(witness)
def create_balance(doc):
balance = {
"balance": {
"amount": convert_amount(doc["balance_value"], BALANCE_PRECISION),
"decs": utils.UInt64(BALANCE_PRECISION),
"sym": BALANCE_SYMBOL
},
"payments": {
"amount" : utils.Int64(0),
"decs" : utils.UInt64(BALANCE_PRECISION),
"sym" : BALANCE_SYMBOL
},
"_SERVICE_" : {
"scope" : doc["name"],
"rev" : utils.Int64(1),
"payer" : doc["name"],
"size" : 16
}
}
cyberway.balance.append(balance)
def update_childcount(self):
total = len(self.childcount_dict)
updated = 0
processed = 0
for key, data in self.childcount_dict.items():
processed += 1
if data[0] != data[1]:
update_filter = {
"$and":[{"id":utils.UInt64(key[1])}, {"_SERVICE_.scope":key[0]}]
}
update_obj = {
"$set":{"childcount":utils.UInt64(data[0])}
}
self.update_list.append(UpdateOne(update_filter, update_obj))
if len(self.update_list) == 10000:
updated += len(self.update_list)
self.write_childcount()
if processed % self.cache_period == 0:
utils.printProgressBar(processed, total, prefix = 'Progress:', suffix = 'Complete', length = 50)
if len(self.update_list):
updated += len(self.update_list)
self.write_childcount()
utils.printProgressBar(processed, total, prefix = 'Progress:', suffix = 'Complete', length = 50)
print("\nUpdate childcount: %d/%d = %.2f%%" % (updated, len(self.childcount_dict), 100.*updated/len(self.childcount_dict)))
def create_permission(doc, name, parent_id):
permusage_id = cyberway.permusage.nextId()
permusage = {
"id" : utils.UInt64(permusage_id),
"last_used" : doc["last_owner_update"],
"_SERVICE_" : {
"scope" : "",
"rev" : utils.Int64(1),
"payer" : "",
"size" : 16
}
}
cyberway.permusage.append(permusage)
permission_id = cyberway.permission.nextId()
permission = {
"id" : utils.UInt64(permission_id),
"usage_id" : utils.UInt64(permusage_id),
"parent" : utils.UInt64(parent_id),
"owner" : doc["account"],
"name" : name,
"last_updated" : doc["last_owner_update"],
"auth" : {
"threshold" : utils.UInt64(1),
"keys" : [
{
"key" : doc[name][0],
"weight" : utils.UInt64(1)
}
],
"accounts" : [ ],
"waits" : [ ]
},
"_SERVICE_" : {
"scope" : "",
"rev" : utils.Int64(1),
"payer" : "",
"size" : 91
}
}
cyberway.permission.append(permission)
return permission_id
def convert_posts(self, query={}):
print("convert_posts")
golos_gpo = self.golos_db['dynamic_global_property_object'].find()[0]
golos_posts = self.golos_db['comment_object']
reward_pools = self.publish_db['rewardpools']
pool = reward_pools.find()[0] #we have to create it before converting
pool["state"]["funds"]["amount"] = utils.Int64(
utils.get_golos_asset_amount(
golos_gpo["total_reward_fund_steem_value"]))
pool["state"]["funds"]["decs"] = utils.UInt64(BALANCE_PRECISION)
pool["state"]["funds"]["sym"] = BALANCE_SYMBOL
pool["state"]["msgs"] = utils.UInt64(0)
rshares_sum = 0
cursor = golos_posts.find(query)
length = cursor.count()
print("total messages = ", length)
utils.printProgressBar(0,
length,
prefix='Progress:',
suffix='Complete',
length=50)
added = 0
passed = -1
for doc in cursor:
passed += 1
try:
if doc["removed"] or (not (doc["author"] in self.exists_accs)):
continue
cur_mssg_id = self.get_post_id(doc["author"], doc["permlink"])
cur_rshares_raw = utils.get_fixp_raw(doc["net_rshares"])
messagestate = {
"netshares":
utils.Int64(cur_rshares_raw),
"voteshares":
utils.Int64(utils.get_fixp_raw(doc["vote_rshares"])),
"sumcuratorsw":
utils.Int64(self.mssgs_curatorsw[(doc["author"],
cur_mssg_id)])
}
date_close = datetime.strptime(
"2106-02-07T06:28:15", '%Y-%m-%dT%H:%M:%S').isoformat()
if (doc["cashout_time"].isoformat() != date_close
and doc["cashout_time"].isoformat() >
datetime.now().isoformat()):
rshares_sum += cur_rshares_raw
pool["state"]["msgs"] = utils.UInt64(
pool["state"]["msgs"].to_decimal() + 1)
orphan_comment = (len(doc["parent_author"]) > 0) and (
not (doc["parent_author"] in self.exists_accs))
message = {
"id":
utils.UInt64(cur_mssg_id),
"permlink":
doc["permlink"],
"date":
utils.UInt64(
int(doc["last_update"].timestamp()) * 1000000),
"parentacc":
"" if orphan_comment else doc["parent_author"],
"parent_id":
utils.UInt64(0 if (
orphan_comment or (len(doc["parent_permlink"]) == 0)
) else self.get_post_id(doc["parent_author"],
doc["parent_permlink"])),
"tokenprop":
utils.Int64(
utils.get_prop_raw(doc["percent_steem_dollars"] / 2)),
"beneficiaries": [],
"rewardweight":
utils.Int64(utils.get_prop_raw(doc["reward_weight"])),
"state":
messagestate,
"childcount":
utils.UInt64(doc["children"]),
"cashout_time":
utils.UInt64(
int(doc["cashout_time"].timestamp()) * 1000000),
"level":
utils.UInt64(
0 if orphan_comment else doc["depth"]
), # this value will be incorrect for comment to orphan comment
# but we only use level for comments nesting limits,
# so it seems that this is not a problem
"_SERVICE_": {
"scope": doc["author"],
"rev": utils.Int64(1),
"payer": doc["author"],
"size": 50
}
}
self.publish_tables.message.append(message)
tags = []
if (isinstance(doc["json_metadata"], dict)):
if ("tags" in doc["json_metadata"]):
tags = create_tags(doc["json_metadata"]["tags"])
if (isinstance(doc["json_metadata"], str)):
try:
if (doc["json_metadata"]):
json_str = doc["json_metadata"]
if ((json_str.find("\"") == 0) and
(json_str.rfind("\"") == len(json_str) - 1)):
json_str = json_str[1:len(json_str) - 1]
dict_metadata = json.loads(json_str)
if (dict_metadata["tags"]):
tags = create_tags(dict_metadata["tags"])
except Exception:
tags = []
content = {
"id": utils.UInt64(cur_mssg_id),
"headermssg": doc["title"],
"bodymssg": doc["body"],
"languagemssg": "",
"tags": tags,
"jsonmetadata": doc["json_metadata"],
"_SERVICE_": {
"scope": doc["author"],
"rev": utils.Int64(1),
"payer": doc["author"],
"size": 50
}
}
self.publish_tables.content.append(content)
if added % self.cache_period == 0:
self.publish_tables.writeCache()
print("messages converted -- ", added)
utils.printProgressBar(passed + 1,
length,
prefix='Progress:',
suffix='Complete',
length=50)
added += 1
except pymongo.errors.BulkWriteError as e:
print(doc)
print(e.args)
print(e.details)
print(traceback.format_exc())
break
try:
self.publish_tables.writeCache()
utils.printProgressBar(passed + 1,
length,
prefix='Progress:',
suffix='Complete',
length=50)
except Exception as e:
print(e.args)
print(traceback.format_exc())
pool["state"]["rshares"] = utils.Int128(rshares_sum)
pool["state"]["rsharesfn"] = utils.Int128(rshares_sum)
reward_pools.save(pool)
print("...done")
def convert_votes(self, query={}):
print("convert_votes")
golos_votes = self.golos_db['comment_vote_object']
cyberway_messages = self.publish_db['message']
cursor = golos_votes.find(query)
length = cursor.count()
print("total votes = ", length)
length = golos_votes.count()
utils.printProgressBar(0,
length,
prefix='Progress:',
suffix='Complete',
length=50)
added = 0
passed = -1
for doc in cursor:
passed += 1
try:
if (not (doc["author"] in self.exists_accs)) or (
not (doc["voter"] in self.exists_accs)):
continue
post_id = (doc["author"], doc["permlink"])
cur_mssg_id = self.get_post_id(doc["author"], doc["permlink"])
vote = {
"id":
utils.UInt64(self.publish_tables.vote.nextId()),
"message_id":
utils.UInt64(cur_mssg_id),
"voter":
doc["voter"],
"weight":
doc["vote_percent"],
"time":
utils.UInt64(
int(doc["last_update"].timestamp()) * 1000000),
"count":
utils.Int64(doc["num_changes"]),
"delegators": [],
"curatorsw":
utils.Int64(doc["weight"] / 2),
"rshares":
utils.Int64(utils.get_fixp_raw(doc["rshares"])),
"_SERVICE_": {
"scope": doc["author"],
"rev": utils.Int64(1),
"payer": doc["author"],
"size": 50
}
}
self.publish_tables.vote.append(vote)
self.mssgs_curatorsw[(doc["author"],
cur_mssg_id)] += vote["curatorsw"]
if added % self.cache_period == 0:
self.publish_tables.writeCache()
print("votes converted -- ", added)
utils.printProgressBar(passed + 1,
length,
prefix='Progress:',
suffix='Complete',
length=50)
added += 1
except Exception as e:
print(doc)
print(e.args)
print(traceback.format_exc())
break
try:
self.publish_tables.writeCache()
utils.printProgressBar(passed + 1,
length,
prefix='Progress:',
suffix='Complete',
length=50)
except Exception as e:
print(e.args)
print(traceback.format_exc())
print('...done')
def create_account(doc):
account_id = cyberway.account.nextId()
last_code_update = datetime.datetime.strptime("1970-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.000Z")
account = {
"id" : utils.UInt64(account_id),
"name" : doc["name"],
"vm_type" : utils.UInt64(0),
"vm_version" : utils.UInt64(0),
"privileged" : False,
"last_code_update" : last_code_update,
"code_version" : "0000000000000000000000000000000000000000000000000000000000000000",
"creation_date" : doc["created"],
"code" : "",
"abi" : "",
"_SERVICE_" : {
"scope" : "",
"rev" : utils.Int64(1),
"payer" : "",
"size" : 65
}
}
cyberway.account.append(account)
accountseq = {
"id" : utils.UInt64(account_id),
"name" : doc["name"],
"recv_sequence" : utils.UInt64(0),
"auth_sequence" : utils.UInt64(0),
"code_sequence" : utils.UInt64(0),
"abi_sequence" : utils.UInt64(0),
"_SERVICE_" : {
"scope" : "",
"rev" : utils.Int64(1),
"payer" : "",
"size" : 48
}
}
cyberway.accountseq.append(accountseq)
reslimit = {
"id" : utils.UInt64(account_id),
"owner" : doc["name"],
"pending" : False,
"net_weight" : utils.Int64(-1),
"cpu_weight" : utils.Int64(-1),
"ram_bytes" : utils.Int64(-1),
"_SERVICE_" : {
"scope" : "",
"rev" : utils.Int64(1),
"payer" : "",
"size" : 41
}
}
cyberway.reslimit.append(reslimit)
resusage = {
"id" : utils.UInt64(account_id),
"owner" : doc["name"],
"net_usage" : {
"last_ordinal" : utils.UInt64(0),
"value_ex" : utils.UInt64(0),
"consumed" : utils.UInt64(0)
},
"cpu_usage" : {
"last_ordinal" : utils.UInt64(0),
"value_ex" : utils.UInt64(0),
"consumed" : utils.UInt64(0)
},
"ram_usage" : utils.UInt64(2724),
"_SERVICE_" : {
"scope" : "",
"rev" : utils.Int64(1),
"payer" : "",
"size" : 64
}
}
cyberway.resusage.append(resusage)
def convert_amount(amount, precision):
return utils.UInt64(int(amount*(10**precision)))