class DbBarData(ModelBase):
"""
Candlestick bar data for database storage.
Index is defined unique with datetime, interval, symbol
"""
id = AutoField()
symbol: str = CharField()
exchange: str = CharField()
datetime: datetime = DateTimeField()
interval: str = CharField()
volume: float = FloatField()
open_interest: float = FloatField()
open_price: float = FloatField()
high_price: float = FloatField()
low_price: float = FloatField()
close_price: float = FloatField()
class Meta:
database = db
indexes = ((("symbol", "exchange", "interval", "datetime"),
True), )
@staticmethod
def from_bar(bar: BarData):
"""
Generate DbBarData object from BarData.
"""
db_bar = DbBarData()
db_bar.symbol = bar.symbol
db_bar.exchange = bar.exchange.value
db_bar.datetime = bar.datetime
db_bar.interval = bar.interval.value
db_bar.volume = bar.volume
db_bar.open_interest = bar.open_interest
db_bar.open_price = bar.open_price
db_bar.high_price = bar.high_price
db_bar.low_price = bar.low_price
db_bar.close_price = bar.close_price
return db_bar
def to_bar(self):
"""
Generate BarData object from DbBarData.
"""
bar = BarData(
symbol=self.symbol,
exchange=Exchange(self.exchange),
datetime=self.datetime,
interval=Interval(self.interval),
volume=self.volume,
open_price=self.open_price,
high_price=self.high_price,
open_interest=self.open_interest,
low_price=self.low_price,
close_price=self.close_price,
gateway_name="DB",
)
return bar
@staticmethod
def save_all(objs: List["DbBarData"], progress_bar_dict=None):
"""
save a list of objects, update if exists.
"""
dicts = [i.to_dict() for i in objs]
with db.atomic():
if driver is Driver.POSTGRESQL:
for bar in dicts:
DbBarData.insert(bar).on_conflict(
update=bar,
conflict_target=(
DbBarData.symbol,
DbBarData.exchange,
DbBarData.interval,
DbBarData.datetime,
),
).execute()
else:
total_sz = len(dicts)
loaded = 0
for c in chunked(dicts, 50):
DbBarData.insert_many(
c).on_conflict_replace().execute()
if 'save_progress_bar' in progress_bar_dict:
loaded += 50
percent_saved = min(
round(100 * loaded / total_sz, 2), 100)
QApplication.processEvents()
progress_bar_dict['save_progress_bar'].setValue(
percent_saved)
class YeFengWordTable(BaseModel):
id = IntegerField(primary_key=True)
word = CharField()
py = CharField()
priority = IntegerField()
updatedt = DateTimeField("%Y-%m-%d %H:%M:%S")
class TableVersion(BaseModel):
"""table to keep track of table versions"""
name = CharField(max_length=60, index=True)
version = IntegerField(default=0)
class TaskCategory(BaseModel):
id = PrimaryKeyField()
name = CharField()
description = CharField()
class Application(_BaseModel):
client_id = UUIDField(default=uuid.uuid4, primary_key=True)
client_secret = CharField()
app_name = CharField()
class DbTickData(Model):
"""
Tick data for database storage.
Index is defined unique with vt_symbol, interval and datetime.
"""
symbol = CharField()
exchange = CharField()
datetime = DateTimeField()
name = CharField()
volume = FloatField()
last_price = FloatField()
last_volume = FloatField()
limit_up = FloatField()
limit_down = FloatField()
open_price = FloatField()
high_price = FloatField()
low_price = FloatField()
pre_close = FloatField()
bid_price_1 = FloatField()
bid_price_2 = FloatField()
bid_price_3 = FloatField()
bid_price_4 = FloatField()
bid_price_5 = FloatField()
ask_price_1 = FloatField()
ask_price_2 = FloatField()
ask_price_3 = FloatField()
ask_price_4 = FloatField()
ask_price_5 = FloatField()
bid_volume_1 = FloatField()
bid_volume_2 = FloatField()
bid_volume_3 = FloatField()
bid_volume_4 = FloatField()
bid_volume_5 = FloatField()
ask_volume_1 = FloatField()
ask_volume_2 = FloatField()
ask_volume_3 = FloatField()
ask_volume_4 = FloatField()
ask_volume_5 = FloatField()
vt_symbol = CharField()
gateway_name = CharField()
class Meta:
database = DB
indexes = ((("vt_symbol", "datetime"), True),)
@staticmethod
def from_tick(tick: TickData):
"""
Generate DbTickData object from TickData.
"""
db_tick = DbTickData()
db_tick.symbol = tick.symbol
db_tick.exchange = tick.exchange.value
db_tick.datetime = tick.datetime
db_tick.name = tick.name
db_tick.volume = tick.volume
db_tick.last_price = tick.last_price
db_tick.last_volume = tick.last_volume
db_tick.limit_up = tick.limit_up
db_tick.limit_down = tick.limit_down
db_tick.open_price = tick.open_price
db_tick.high_price = tick.high_price
db_tick.low_price = tick.low_price
db_tick.pre_close = tick.pre_close
db_tick.bid_price_1 = tick.bid_price_1
db_tick.ask_price_1 = tick.ask_price_1
db_tick.bid_volume_1 = tick.bid_volume_1
db_tick.ask_volume_1 = tick.ask_volume_1
if tick.bid_price_2:
db_tick.bid_price_2 = tick.bid_price_2
db_tick.bid_price_3 = tick.bid_price_3
db_tick.bid_price_4 = tick.bid_price_4
db_tick.bid_price_5 = tick.bid_price_5
db_tick.ask_price_2 = tick.ask_price_2
db_tick.ask_price_3 = tick.ask_price_3
db_tick.ask_price_4 = tick.ask_price_4
db_tick.ask_price_5 = tick.ask_price_5
db_tick.bid_volume_2 = tick.bid_volume_2
db_tick.bid_volume_3 = tick.bid_volume_3
db_tick.bid_volume_4 = tick.bid_volume_4
db_tick.bid_volume_5 = tick.bid_volume_5
db_tick.ask_volume_2 = tick.ask_volume_2
db_tick.ask_volume_3 = tick.ask_volume_3
db_tick.ask_volume_4 = tick.ask_volume_4
db_tick.ask_volume_5 = tick.ask_volume_5
db_tick.vt_symbol = tick.vt_symbol
db_tick.gateway_name = "DB"
return tick
def to_tick(self):
"""
Generate TickData object from DbTickData.
"""
tick = TickData(
symbol=self.symbol,
exchange=Exchange(self.exchange),
datetime=self.datetime,
name=self.name,
volume=self.volume,
last_price=self.last_price,
last_volume=self.last_volume,
limit_up=self.limit_up,
limit_down=self.limit_down,
open_price=self.open_price,
high_price=self.high_price,
low_price=self.low_price,
pre_close=self.pre_close,
bid_price_1=self.bid_price_1,
ask_price_1=self.ask_price_1,
bid_volume_1=self.bid_volume_1,
ask_volume_1=self.ask_volume_1,
gateway_name=self.gateway_name,
)
if self.bid_price_2:
tick.bid_price_2 = self.bid_price_2
tick.bid_price_3 = self.bid_price_3
tick.bid_price_4 = self.bid_price_4
tick.bid_price_5 = self.bid_price_5
tick.ask_price_2 = self.ask_price_2
tick.ask_price_3 = self.ask_price_3
tick.ask_price_4 = self.ask_price_4
tick.ask_price_5 = self.ask_price_5
tick.bid_volume_2 = self.bid_volume_2
tick.bid_volume_3 = self.bid_volume_3
tick.bid_volume_4 = self.bid_volume_4
tick.bid_volume_5 = self.bid_volume_5
tick.ask_volume_2 = self.ask_volume_2
tick.ask_volume_3 = self.ask_volume_3
tick.ask_volume_4 = self.ask_volume_4
tick.ask_volume_5 = self.ask_volume_5
return tick
class weather(BaseModel):
area = CharField(primary_key=True, index=True, max_length=100)
updated = IntegerField(index=True, null=True)
description = CharField(null=True, index=True, max_length=50)
windspeed = CharField(null=True, index=True, max_length=50)
temperature = IntegerField(null=True, index=True)
class Users(BaseModel):
user_id = CharField(primary_key=True)
user_name = CharField()
profile_pic_url = TextField()
auth_token = TextField(null=True)
class Users(LocalPaste):
username = CharField(unique=True)
Password = CharField()
class User(BaseModel):
uid = IntegerField(primary_key=True)
name = CharField(max_length=250)
path = IntegerField()
step = IntegerField(default=1)
updated = DateTimeField(default=datetime.datetime.now)
class Company(BaseModel):
"""
Model class for the companies table.
:param company_name: Name of the companies
:param email: Email of the companies
:param address_1: address_1 of the companies
:param address_2: address_2 of the companies
:param city: city of the companies
:param province: province of the companies
:param zipcode: zipcode of the companies
:param country: country of the companies
:param phone: phone of the companies
:param password: Password of the companies
:param state: State of the companies. Should be an enum value of the above
:param created_at: Time of creation of the row
:param modified_at: Time of last modification of the row
"""
id = AutoField() # Primary key
company_name = CharField()
email = CharField(unique=True)
password = CharField()
address_1 = CharField(null=True)
address_2 = CharField(null=True)
city = CharField(null=True)
province = CharField(null=True)
zipcode = CharField(null=True)
country = CharField(null=True)
phone = CharField(null=True)
state = CharField(null=True)
created_at = DateTimeField(default=datetime.now())
modified_at = DateTimeField(default=datetime.now())
def has_address(self):
return (self.address_1 is not None and self.city is not None
and self.province is not None and self.zipcode is not None
and self.country is not None)
def is_active(self):
return self.state == 'ACTIVE'
class Group(BaseModel):
name = CharField(unique=True)
class UserModel(BaseModel):
name = CharField()
password = CharField(null=False)
email = CharField(unique=True, null=False)
class CfgNotify(BaseModel):
check_order = IntegerField() # 排序
notify_type = CharField() # 通知类型：MAIL/SMS
notify_name = CharField() # 通知人姓名
notify_number = CharField() # 通知号码
status = BooleanField(default=True) # 生效失效标识
class Donor(Model):
name = CharField(max_length=255, unique=True)
class Meta:
database = db
class PublishedVideos(BaseModel):
vid = CharField(unique=True)
class DbBarData(Model):
"""
Candlestick bar data for database storage.
Index is defined unique with vt_symbol, interval and datetime.
"""
symbol = CharField()
exchange = CharField()
datetime = DateTimeField()
interval = CharField()
volume = FloatField()
open_price = FloatField()
high_price = FloatField()
low_price = FloatField()
close_price = FloatField()
vt_symbol = CharField()
gateway_name = CharField()
class Meta:
database = DB
indexes = ((("vt_symbol", "interval", "datetime"), True),)
@staticmethod
def from_bar(bar: BarData):
"""
Generate DbBarData object from BarData.
"""
db_bar = DbBarData()
db_bar.symbol = bar.symbol
db_bar.exchange = bar.exchange.value
db_bar.datetime = bar.datetime
db_bar.interval = bar.interval.value
db_bar.volume = bar.volume
db_bar.open_price = bar.open_price
db_bar.high_price = bar.high_price
db_bar.low_price = bar.low_price
db_bar.close_price = bar.close_price
db_bar.vt_symbol = bar.vt_symbol
db_bar.gateway_name = "DB"
return db_bar
def to_bar(self):
"""
Generate BarData object from DbBarData.
"""
bar = BarData(
symbol=self.symbol,
exchange=Exchange(self.exchange),
datetime=self.datetime,
interval=Interval(self.interval),
volume=self.volume,
open_price=self.open_price,
high_price=self.high_price,
low_price=self.low_price,
close_price=self.close_price,
gateway_name=self.gateway_name,
)
return bar
class User(BaseModel):
username = CharField()
password = CharField()
score = ForeignKeyField(Score, related_name='score')
class User(BaseModel):
id = PrimaryKeyField(null=True)
username = CharField(max_length=32)
first_name = CharField(max_length=64, null=True)
last_name = CharField(max_length=64, null=True)
password = CharField(max_length=64, null=True)
image_url = CharField(max_length=200, null=True)
country = CharField(max_length=128, null=True)
province = CharField(max_length=128, null=True)
city = CharField(max_length=128, null=True)
region = CharField(max_length=128, null=True)
postal_code = CharField(max_length=32, null=True)
addr = CharField(max_length=200, null=True)
city_code = IntegerField(null=True)
tel = IntegerField(null=True)
is_active = BooleanField(default=False)
def __unicode__(self):
return self.username
class Proposal(GovernanceClass, BaseModel):
governance_object = ForeignKeyField(GovernanceObject, related_name='proposals', on_delete='CASCADE', on_update='CASCADE')
name = CharField(default='', max_length=40)
url = CharField(default='')
start_epoch = IntegerField()
end_epoch = IntegerField()
payment_address = CharField(max_length=36)
payment_amount = DecimalField(max_digits=16, decimal_places=8)
object_hash = CharField(max_length=64)
# src/governance-validators.cpp
MAX_DATA_SIZE = 512
govobj_type = AZARTD_GOVOBJ_TYPES['proposal']
class Meta:
db_table = 'proposals'
def is_valid(self):
import azartlib
printdbg("In Proposal#is_valid, for Proposal: %s" % self.__dict__)
try:
# proposal name exists and is not null/whitespace
if (len(self.name.strip()) == 0):
printdbg("\tInvalid Proposal name [%s], returning False" % self.name)
return False
# proposal name is normalized (something like "[a-zA-Z0-9-_]+")
if not re.match(r'^[-_a-zA-Z0-9]+$', self.name):
printdbg("\tInvalid Proposal name [%s] (does not match regex), returning False" % self.name)
return False
# end date < start date
if (self.end_epoch <= self.start_epoch):
printdbg("\tProposal end_epoch [%s] <= start_epoch [%s] , returning False" % (self.end_epoch, self.start_epoch))
return False
# amount must be numeric
if misc.is_numeric(self.payment_amount) is False:
printdbg("\tProposal amount [%s] is not valid, returning False" % self.payment_amount)
return False
# amount can't be negative or 0
if (float(self.payment_amount) <= 0):
printdbg("\tProposal amount [%s] is negative or zero, returning False" % self.payment_amount)
return False
# payment address is valid base58 azart addr, non-multisig
if not azartlib.is_valid_azart_address(self.payment_address, config.network):
printdbg("\tPayment address [%s] not a valid Azart address for network [%s], returning False" % (self.payment_address, config.network))
return False
# URL
if (len(self.url.strip()) < 4):
printdbg("\tProposal URL [%s] too short, returning False" % self.url)
return False
# proposal URL has any whitespace
if (re.search(r'\s', self.url)):
printdbg("\tProposal URL [%s] has whitespace, returning False" % self.name)
return False
# Azart Core restricts proposals to 512 bytes max
if len(self.serialise()) > (self.MAX_DATA_SIZE * 2):
printdbg("\tProposal [%s] is too big, returning False" % self.name)
return False
try:
parsed = urlparse.urlparse(self.url)
except Exception as e:
printdbg("\tUnable to parse Proposal URL, marking invalid: %s" % e)
return False
except Exception as e:
printdbg("Unable to validate in Proposal#is_valid, marking invalid: %s" % e.message)
return False
printdbg("Leaving Proposal#is_valid, Valid = True")
return True
def is_expired(self, superblockcycle=None):
from constants import SUPERBLOCK_FUDGE_WINDOW
import azartlib
if not superblockcycle:
raise Exception("Required field superblockcycle missing.")
printdbg("In Proposal#is_expired, for Proposal: %s" % self.__dict__)
now = misc.now()
printdbg("\tnow = %s" % now)
# half the SB cycle, converted to seconds
# add the fudge_window in seconds, defined elsewhere in Sentinel
expiration_window_seconds = int(
(azartlib.blocks_to_seconds(superblockcycle) / 2) +
SUPERBLOCK_FUDGE_WINDOW
)
printdbg("\texpiration_window_seconds = %s" % expiration_window_seconds)
# "fully expires" adds the expiration window to end time to ensure a
# valid proposal isn't excluded from SB by cutting it too close
fully_expires_at = self.end_epoch + expiration_window_seconds
printdbg("\tfully_expires_at = %s" % fully_expires_at)
if (fully_expires_at < now):
printdbg("\tProposal end_epoch [%s] < now [%s] , returning True" % (self.end_epoch, now))
return True
printdbg("Leaving Proposal#is_expired, Expired = False")
return False
@classmethod
def approved_and_ranked(self, proposal_quorum, next_superblock_max_budget):
# return all approved proposals, in order of descending vote count
#
# we need a secondary 'order by' in case of a tie on vote count, since
# superblocks must be deterministic
query = (self
.select(self, GovernanceObject) # Note that we are selecting both models.
.join(GovernanceObject)
.where(GovernanceObject.absolute_yes_count > proposal_quorum)
.order_by(GovernanceObject.absolute_yes_count.desc(), GovernanceObject.object_hash.desc())
)
ranked = []
for proposal in query:
proposal.max_budget = next_superblock_max_budget
if proposal.is_valid():
ranked.append(proposal)
return ranked
@classmethod
def expired(self, superblockcycle=None):
if not superblockcycle:
raise Exception("Required field superblockcycle missing.")
expired = []
for proposal in self.select():
if proposal.is_expired(superblockcycle):
expired.append(proposal)
return expired
@property
def rank(self):
rank = 0
if self.governance_object:
rank = self.governance_object.absolute_yes_count
return rank
class schema_version(BaseModel):
key = CharField()
val = SmallIntegerField()
class User(BaseModel):
id = BigIntegerField(primary_key=True)
steam_id = BigIntegerField()
authorization_key = CharField()
class TaskGoal(BaseModel):
id = PrimaryKeyField()
name = CharField()
description = CharField()
class Proposal(GovernanceClass, BaseModel):
governance_object = ForeignKeyField(GovernanceObject, related_name='proposals', on_delete='CASCADE', on_update='CASCADE')
name = CharField(default='', max_length=40)
url = CharField(default='')
start_epoch = IntegerField()
end_epoch = IntegerField()
payment_address = CharField(max_length=36)
payment_amount = DecimalField(max_digits=16, decimal_places=8)
object_hash = CharField(max_length=64)
govobj_type = SPRINTD_GOVOBJ_TYPES['proposal']
class Meta:
db_table = 'proposals'
def is_valid(self):
import sprintlib
printdbg("In Proposal#is_valid, for Proposal: %s" % self.__dict__)
try:
# proposal name exists and is not null/whitespace
if (len(self.name.strip()) == 0):
printdbg("\tInvalid Proposal name [%s], returning False" % self.name)
return False
# proposal name is normalized (something like "[a-zA-Z0-9-_]+")
if not re.match(r'^[-_a-zA-Z0-9]+$', self.name):
printdbg("\tInvalid Proposal name [%s] (does not match regex), returning False" % self.name)
return False
# end date < start date
if (self.end_epoch <= self.start_epoch):
printdbg("\tProposal end_epoch [%s] <= start_epoch [%s] , returning False" % (self.end_epoch, self.start_epoch))
return False
# amount must be numeric
if misc.is_numeric(self.payment_amount) is False:
printdbg("\tProposal amount [%s] is not valid, returning False" % self.payment_amount)
return False
# amount can't be negative or 0
if (float(self.payment_amount) <= 0):
printdbg("\tProposal amount [%s] is negative or zero, returning False" % self.payment_amount)
return False
# payment address is valid base58 sprint addr, non-multisig
if not sprintlib.is_valid_sprint_address(self.payment_address, config.network):
printdbg("\tPayment address [%s] not a valid Sprint address for network [%s], returning False" % (self.payment_address, config.network))
return False
# URL
if (len(self.url.strip()) < 4):
printdbg("\tProposal URL [%s] too short, returning False" % self.url)
return False
try:
parsed = urlparse.urlparse(self.url)
except Exception as e:
printdbg("\tUnable to parse Proposal URL, marking invalid: %s" % e)
return False
except Exception as e:
printdbg("Unable to validate in Proposal#is_valid, marking invalid: %s" % e.message)
return False
printdbg("Leaving Proposal#is_valid, Valid = True")
return True
def is_expired(self, superblockcycle=None):
from constants import SUPERBLOCK_FUDGE_WINDOW
import sprintlib
if not superblockcycle:
raise Exception("Required field superblockcycle missing.")
printdbg("In Proposal#is_expired, for Proposal: %s" % self.__dict__)
now = misc.now()
printdbg("\tnow = %s" % now)
# half the SB cycle, converted to seconds
# add the fudge_window in seconds, defined elsewhere in Sentinel
expiration_window_seconds = int(
(sprintlib.blocks_to_seconds(superblockcycle) / 2) +
SUPERBLOCK_FUDGE_WINDOW
)
printdbg("\texpiration_window_seconds = %s" % expiration_window_seconds)
# "fully expires" adds the expiration window to end time to ensure a
# valid proposal isn't excluded from SB by cutting it too close
fully_expires_at = self.end_epoch + expiration_window_seconds
printdbg("\tfully_expires_at = %s" % fully_expires_at)
if (fully_expires_at < now):
printdbg("\tProposal end_epoch [%s] < now [%s] , returning True" % (self.end_epoch, now))
return True
printdbg("Leaving Proposal#is_expired, Expired = False")
return False
def is_deletable(self):
# end_date < (current_date - 30 days)
thirty_days = (86400 * 30)
if (self.end_epoch < (misc.now() - thirty_days)):
return True
# TBD (item moved to external storage/SprintDrive, etc.)
return False
@classmethod
def approved_and_ranked(self, proposal_quorum, next_superblock_max_budget):
# return all approved proposals, in order of descending vote count
#
# we need a secondary 'order by' in case of a tie on vote count, since
# superblocks must be deterministic
query = (self
.select(self, GovernanceObject) # Note that we are selecting both models.
.join(GovernanceObject)
.where(GovernanceObject.absolute_yes_count > proposal_quorum)
.order_by(GovernanceObject.absolute_yes_count.desc(), GovernanceObject.object_hash.desc())
)
ranked = []
for proposal in query:
proposal.max_budget = next_superblock_max_budget
if proposal.is_valid():
ranked.append(proposal)
return ranked
@classmethod
def expired(self, superblockcycle=None):
if not superblockcycle:
raise Exception("Required field superblockcycle missing.")
expired = []
for proposal in self.select():
if proposal.is_expired(superblockcycle):
expired.append(proposal)
return expired
@property
def rank(self):
rank = 0
if self.governance_object:
rank = self.governance_object.absolute_yes_count
return rank
def get_prepare_command(self):
import sprintlib
obj_data = sprintlib.SHIM_serialise_for_sprintd(self.serialise())
# new superblocks won't have parent_hash, revision, etc...
cmd = ['gobject', 'prepare', '0', '1', str(int(time.time())), obj_data]
return cmd
def prepare(self, sprintd):
try:
object_hash = sprintd.rpc_command(*self.get_prepare_command())
printdbg("Submitted: [%s]" % object_hash)
self.go.object_fee_tx = object_hash
self.go.save()
manual_submit = ' '.join(self.get_submit_command())
print(manual_submit)
except JSONRPCException as e:
print("Unable to prepare: %s" % e.message)
class CharFreqTable(BaseModel):
id = IntegerField(primary_key=True)
char = CharField()
freq = IntegerField()
updatedt = DateTimeField("%Y-%m-%d %H:%M:%S")
class Superblock(BaseModel, GovernanceClass):
governance_object = ForeignKeyField(GovernanceObject, related_name='superblocks', on_delete='CASCADE', on_update='CASCADE')
event_block_height = IntegerField()
payment_addresses = TextField()
payment_amounts = TextField()
proposal_hashes = TextField(default='')
sb_hash = CharField()
object_hash = CharField(max_length=64)
govobj_type = SPRINTD_GOVOBJ_TYPES['superblock']
only_masternode_can_submit = True
class Meta:
db_table = 'superblocks'
def is_valid(self):
import sprintlib
import decimal
printdbg("In Superblock#is_valid, for SB: %s" % self.__dict__)
# it's a string from the DB...
addresses = self.payment_addresses.split('|')
for addr in addresses:
if not sprintlib.is_valid_sprint_address(addr, config.network):
printdbg("\tInvalid address [%s], returning False" % addr)
return False
amounts = self.payment_amounts.split('|')
for amt in amounts:
if not misc.is_numeric(amt):
printdbg("\tAmount [%s] is not numeric, returning False" % amt)
return False
# no negative or zero amounts allowed
damt = decimal.Decimal(amt)
if not damt > 0:
printdbg("\tAmount [%s] is zero or negative, returning False" % damt)
return False
# verify proposal hashes correctly formatted...
if len(self.proposal_hashes) > 0:
hashes = self.proposal_hashes.split('|')
for object_hash in hashes:
if not misc.is_hash(object_hash):
printdbg("\tInvalid proposal hash [%s], returning False" % object_hash)
return False
# ensure number of payment addresses matches number of payments
if len(addresses) != len(amounts):
printdbg("\tNumber of payment addresses [%s] != number of payment amounts [%s], returning False" % (len(addresses), len(amounts)))
return False
printdbg("Leaving Superblock#is_valid, Valid = True")
return True
def is_deletable(self):
# end_date < (current_date - 30 days)
# TBD (item moved to external storage/SprintDrive, etc.)
pass
def hash(self):
import sprintlib
return sprintlib.hashit(self.serialise())
def hex_hash(self):
return "%x" % self.hash()
# workaround for now, b/c we must uniquely ID a superblock with the hash,
# in case of differing superblocks
#
# this prevents sb_hash from being added to the serialised fields
@classmethod
def serialisable_fields(self):
return [
'event_block_height',
'payment_addresses',
'payment_amounts',
'proposal_hashes'
]
# has this masternode voted to fund *any* superblocks at the given
# event_block_height?
@classmethod
def is_voted_funding(self, ebh):
count = (self.select()
.where(self.event_block_height == ebh)
.join(GovernanceObject)
.join(Vote)
.join(Signal)
.switch(Vote) # switch join query context back to Vote
.join(Outcome)
.where(Vote.signal == VoteSignals.funding)
.where(Vote.outcome == VoteOutcomes.yes)
.count())
return count
@classmethod
def latest(self):
try:
obj = self.select().order_by(self.event_block_height).desc().limit(1)[0]
except IndexError as e:
obj = None
return obj
@classmethod
def at_height(self, ebh):
query = (self.select().where(self.event_block_height == ebh))
return query
@classmethod
def find_highest_deterministic(self, sb_hash):
# highest block hash wins
query = (self.select()
.where(self.sb_hash == sb_hash)
.order_by(self.object_hash.desc()))
try:
obj = query.limit(1)[0]
except IndexError as e:
obj = None
return obj
import yaml
import os
from peewee import (
PostgresqlDatabase,
Model,
CharField,
)
from playhouse.migrate import (
PostgresqlMigrator,
migrate,
)
with open("config.yaml") as cfg_file:
cfg = yaml.load(cfg_file, Loader=yaml.Loader).get(os.environ['ENV'])
db_cfg = cfg.get('postgres')
pg_db = PostgresqlDatabase(db_cfg.get('name'),
user=db_cfg.get('user'),
password=db_cfg.get('password'),
host=db_cfg.get('host'),
port=db_cfg.get('port'))
migrator = PostgresqlMigrator(pg_db)
migrate(migrator.add_column('students', 'state', CharField(default='TEST')))
class GovernanceObject(BaseModel):
parent_id = IntegerField(default=0)
object_creation_time = IntegerField(default=int(time.time()))
object_hash = CharField(max_length=64)
object_parent_hash = CharField(default='0')
object_type = IntegerField(default=0)
object_revision = IntegerField(default=1)
object_fee_tx = CharField(default='')
yes_count = IntegerField(default=0)
no_count = IntegerField(default=0)
abstain_count = IntegerField(default=0)
absolute_yes_count = IntegerField(default=0)
class Meta:
db_table = 'governance_objects'
# sync sprintd gobject list with our local relational DB backend
@classmethod
def sync(self, sprintd):
golist = sprintd.rpc_command('gobject', 'list')
# objects which are removed from the network should be removed from the DB
try:
for purged in self.purged_network_objects(list(golist.keys())):
# SOMEDAY: possible archive step here
purged.delete_instance(recursive=True, delete_nullable=True)
for item in golist.values():
(go, subobj) = self.import_gobject_from_sprintd(sprintd, item)
except Exception as e:
printdbg("Got an error upon import: %s" % e)
@classmethod
def purged_network_objects(self, network_object_hashes):
query = self.select()
if network_object_hashes:
query = query.where(~(self.object_hash << network_object_hashes))
return query
@classmethod
def import_gobject_from_sprintd(self, sprintd, rec):
import decimal
import sprintlib
import inflection
object_hex = rec['DataHex']
object_hash = rec['Hash']
gobj_dict = {
'object_hash': object_hash,
'object_fee_tx': rec['CollateralHash'],
'absolute_yes_count': rec['AbsoluteYesCount'],
'abstain_count': rec['AbstainCount'],
'yes_count': rec['YesCount'],
'no_count': rec['NoCount'],
}
# shim/sprintd conversion
object_hex = sprintlib.SHIM_deserialise_from_sprintd(object_hex)
objects = sprintlib.deserialise(object_hex)
subobj = None
obj_type, dikt = objects[0:2:1]
obj_type = inflection.pluralize(obj_type)
subclass = self._meta.reverse_rel[obj_type].model_class
# set object_type in govobj table
gobj_dict['object_type'] = subclass.govobj_type
# exclude any invalid model data from sprintd...
valid_keys = subclass.serialisable_fields()
subdikt = {k: dikt[k] for k in valid_keys if k in dikt}
# get/create, then sync vote counts from sprintd, with every run
govobj, created = self.get_or_create(object_hash=object_hash, defaults=gobj_dict)
if created:
printdbg("govobj created = %s" % created)
count = govobj.update(**gobj_dict).where(self.id == govobj.id).execute()
if count:
printdbg("govobj updated = %d" % count)
subdikt['governance_object'] = govobj
# get/create, then sync payment amounts, etc. from sprintd - Sprintd is the master
try:
newdikt = subdikt.copy()
newdikt['object_hash'] = object_hash
if subclass(**newdikt).is_valid() is False:
govobj.vote_delete(sprintd)
return (govobj, None)
subobj, created = subclass.get_or_create(object_hash=object_hash, defaults=subdikt)
except Exception as e:
# in this case, vote as delete, and log the vote in the DB
printdbg("Got invalid object from sprintd! %s" % e)
govobj.vote_delete(sprintd)
return (govobj, None)
if created:
printdbg("subobj created = %s" % created)
count = subobj.update(**subdikt).where(subclass.id == subobj.id).execute()
if count:
printdbg("subobj updated = %d" % count)
# ATM, returns a tuple w/gov attributes and the govobj
return (govobj, subobj)
def vote_delete(self, sprintd):
if not self.voted_on(signal=VoteSignals.delete, outcome=VoteOutcomes.yes):
self.vote(sprintd, VoteSignals.delete, VoteOutcomes.yes)
return
def get_vote_command(self, signal, outcome):
cmd = ['gobject', 'vote-conf', self.object_hash,
signal.name, outcome.name]
return cmd
def vote(self, sprintd, signal, outcome):
import sprintlib
# At this point, will probably never reach here. But doesn't hurt to
# have an extra check just in case objects get out of sync (people will
# muck with the DB).
if (self.object_hash == '0' or not misc.is_hash(self.object_hash)):
printdbg("No governance object hash, nothing to vote on.")
return
# have I already voted on this gobject with this particular signal and outcome?
if self.voted_on(signal=signal):
printdbg("Found a vote for this gobject/signal...")
vote = self.votes.where(Vote.signal == signal)[0]
# if the outcome is the same, move on, nothing more to do
if vote.outcome == outcome:
# move on.
printdbg("Already voted for this same gobject/signal/outcome, no need to re-vote.")
return
else:
printdbg("Found a STALE vote for this gobject/signal, deleting so that we can re-vote.")
vote.delete_instance()
else:
printdbg("Haven't voted on this gobject/signal yet...")
# now ... vote!
vote_command = self.get_vote_command(signal, outcome)
printdbg(' '.join(vote_command))
output = sprintd.rpc_command(*vote_command)
# extract vote output parsing to external lib
voted = sprintlib.did_we_vote(output)
if voted:
printdbg('VOTE success, saving Vote object to database')
Vote(governance_object=self, signal=signal, outcome=outcome,
object_hash=self.object_hash).save()
else:
printdbg('VOTE failed, trying to sync with network vote')
self.sync_network_vote(sprintd, signal)
def sync_network_vote(self, sprintd, signal):
printdbg('\tsyncing network vote for object %s with signal %s' % (self.object_hash, signal.name))
vote_info = sprintd.get_my_gobject_votes(self.object_hash)
for vdikt in vote_info:
if vdikt['signal'] != signal.name:
continue
# ensure valid outcome
outcome = VoteOutcomes.get(vdikt['outcome'])
if not outcome:
continue
printdbg('\tFound a matching valid vote on the network, outcome = %s' % vdikt['outcome'])
Vote(governance_object=self, signal=signal, outcome=outcome,
object_hash=self.object_hash).save()
def voted_on(self, **kwargs):
signal = kwargs.get('signal', None)
outcome = kwargs.get('outcome', None)
query = self.votes
if signal:
query = query.where(Vote.signal == signal)
if outcome:
query = query.where(Vote.outcome == outcome)
count = query.count()
return count
class Plugin(BaseModel):
id = AutoField()
name = CharField(unique=True)
version = CharField()
class DbTickData(ModelBase):
"""
Tick data for database storage.
Index is defined unique with (datetime, symbol)
"""
id = AutoField()
symbol: str = CharField()
exchange: str = CharField()
datetime: datetime = DateTimeField()
name: str = CharField()
volume: float = FloatField()
open_interest: float = FloatField()
last_price: float = FloatField()
last_volume: float = FloatField()
limit_up: float = FloatField()
limit_down: float = FloatField()
open_price: float = FloatField()
high_price: float = FloatField()
low_price: float = FloatField()
pre_close: float = FloatField()
bid_price_1: float = FloatField()
bid_price_2: float = FloatField(null=True)
bid_price_3: float = FloatField(null=True)
bid_price_4: float = FloatField(null=True)
bid_price_5: float = FloatField(null=True)
ask_price_1: float = FloatField()
ask_price_2: float = FloatField(null=True)
ask_price_3: float = FloatField(null=True)
ask_price_4: float = FloatField(null=True)
ask_price_5: float = FloatField(null=True)
bid_volume_1: float = FloatField()
bid_volume_2: float = FloatField(null=True)
bid_volume_3: float = FloatField(null=True)
bid_volume_4: float = FloatField(null=True)
bid_volume_5: float = FloatField(null=True)
ask_volume_1: float = FloatField()
ask_volume_2: float = FloatField(null=True)
ask_volume_3: float = FloatField(null=True)
ask_volume_4: float = FloatField(null=True)
ask_volume_5: float = FloatField(null=True)
class Meta:
database = db
indexes = ((("symbol", "exchange", "datetime"), True), )
@staticmethod
def from_tick(tick: TickData):
"""
Generate DbTickData object from TickData.
"""
db_tick = DbTickData()
db_tick.symbol = tick.symbol
db_tick.exchange = tick.exchange.value
db_tick.datetime = tick.datetime
db_tick.name = tick.name
db_tick.volume = tick.volume
db_tick.open_interest = tick.open_interest
db_tick.last_price = tick.last_price
db_tick.last_volume = tick.last_volume
db_tick.limit_up = tick.limit_up
db_tick.limit_down = tick.limit_down
db_tick.open_price = tick.open_price
db_tick.high_price = tick.high_price
db_tick.low_price = tick.low_price
db_tick.pre_close = tick.pre_close
db_tick.bid_price_1 = tick.bid_price_1
db_tick.ask_price_1 = tick.ask_price_1
db_tick.bid_volume_1 = tick.bid_volume_1
db_tick.ask_volume_1 = tick.ask_volume_1
if tick.bid_price_2:
db_tick.bid_price_2 = tick.bid_price_2
db_tick.bid_price_3 = tick.bid_price_3
db_tick.bid_price_4 = tick.bid_price_4
db_tick.bid_price_5 = tick.bid_price_5
db_tick.ask_price_2 = tick.ask_price_2
db_tick.ask_price_3 = tick.ask_price_3
db_tick.ask_price_4 = tick.ask_price_4
db_tick.ask_price_5 = tick.ask_price_5
db_tick.bid_volume_2 = tick.bid_volume_2
db_tick.bid_volume_3 = tick.bid_volume_3
db_tick.bid_volume_4 = tick.bid_volume_4
db_tick.bid_volume_5 = tick.bid_volume_5
db_tick.ask_volume_2 = tick.ask_volume_2
db_tick.ask_volume_3 = tick.ask_volume_3
db_tick.ask_volume_4 = tick.ask_volume_4
db_tick.ask_volume_5 = tick.ask_volume_5
return db_tick
def to_tick(self):
"""
Generate TickData object from DbTickData.
"""
tick = TickData(
symbol=self.symbol,
exchange=Exchange(self.exchange),
datetime=self.datetime,
name=self.name,
volume=self.volume,
open_interest=self.open_interest,
last_price=self.last_price,
last_volume=self.last_volume,
limit_up=self.limit_up,
limit_down=self.limit_down,
open_price=self.open_price,
high_price=self.high_price,
low_price=self.low_price,
pre_close=self.pre_close,
bid_price_1=self.bid_price_1,
ask_price_1=self.ask_price_1,
bid_volume_1=self.bid_volume_1,
ask_volume_1=self.ask_volume_1,
gateway_name="DB",
)
if self.bid_price_2:
tick.bid_price_2 = self.bid_price_2
tick.bid_price_3 = self.bid_price_3
tick.bid_price_4 = self.bid_price_4
tick.bid_price_5 = self.bid_price_5
tick.ask_price_2 = self.ask_price_2
tick.ask_price_3 = self.ask_price_3
tick.ask_price_4 = self.ask_price_4
tick.ask_price_5 = self.ask_price_5
tick.bid_volume_2 = self.bid_volume_2
tick.bid_volume_3 = self.bid_volume_3
tick.bid_volume_4 = self.bid_volume_4
tick.bid_volume_5 = self.bid_volume_5
tick.ask_volume_2 = self.ask_volume_2
tick.ask_volume_3 = self.ask_volume_3
tick.ask_volume_4 = self.ask_volume_4
tick.ask_volume_5 = self.ask_volume_5
return tick
@staticmethod
def save_all(objs: List["DbTickData"]):
dicts = [i.to_dict() for i in objs]
with db.atomic():
if driver is Driver.POSTGRESQL:
for tick in dicts:
DbTickData.insert(tick).on_conflict(
update=tick,
conflict_target=(
DbTickData.symbol,
DbTickData.exchange,
DbTickData.datetime,
),
).execute()
else:
for c in chunked(dicts, 50):
DbTickData.insert_many(
c).on_conflict_replace().execute()