def get_previous_states_metadata(self, limit: int = 20) -> List[Dict]: # dash threading means that connection needs to be established in same thread as use db_conn = sqlite3.connect(self._db_filepath) states_dict_list = [] try: result = db_conn.execute( f"SELECT * FROM {CrawlerNodeStorage.STATE_DB_NAME} " f"ORDER BY datetime(updated) DESC LIMIT {limit}") # TODO use `pandas` package instead to automatically get dict? column_names = [ description[0] for description in result.description ] for row in result: state_info = dict() for idx, value in enumerate(row): column_name = column_names[idx] if column_name == 'updated': # convert column from rfc3339 (for sorting) back to rfc2822 # TODO does this matter for displaying? - it doesn't, but rfc2822 is easier on the eyes state_info[column_name] = MayaDT.from_rfc3339( row[idx]).rfc2822() else: state_info[column_name] = row[idx] states_dict_list.append(state_info) return states_dict_list finally: db_conn.close()
def get_historical_locked_tokens_over_range(self, days: int): range_begin, range_end = self._get_range_bookends(days) results = list( self._client.query( f"SELECT SUM(locked_stake) " f"FROM (" f"SELECT staker_address, current_period, " f"LAST(locked_stake) " f"AS locked_stake " f"FROM {Crawler.NODE_MEASUREMENT} " f"WHERE time >= '{MayaDT.from_datetime(range_begin).rfc3339()}' " f"AND " f"time < '{MayaDT.from_datetime(range_end).rfc3339()}' " f"GROUP BY staker_address, time(1d)" f") " f"GROUP BY time(1d)").get_points()) # Note: all days may not have values eg. days before DB started getting populated # As time progresses this should be less of an issue locked_tokens_dict = OrderedDict() for r in results: locked_stake = r['sum'] if locked_stake: # Dash accepts datetime objects for graphs locked_tokens_dict[MayaDT.from_rfc3339( r['time']).datetime()] = locked_stake return locked_tokens_dict
def get_historical_num_stakers_over_range(self, days: int): range_begin, range_end = self._get_range_bookends(days) results = list( self._client.query( f"SELECT COUNT(staker_address) FROM " f"(" f"SELECT staker_address, LAST(locked_stake)" f"FROM {Crawler.NODE_MEASUREMENT} WHERE " f"time >= '{MayaDT.from_datetime(range_begin).rfc3339()}' AND " f"time < '{MayaDT.from_datetime(range_end).rfc3339()}' " f"GROUP BY staker_address, time(1d)" f") " "GROUP BY time(1d)").get_points()) # 1 day measurements # Note: all days may not have values eg. days before DB started getting populated # As time progresses this should be less of an issue num_stakers_dict = OrderedDict() for r in results: locked_stake = r['count'] if locked_stake: # Dash accepts datetime objects for graphs num_stakers_dict[MayaDT.from_rfc3339( r['time']).datetime()] = locked_stake return num_stakers_dict
def get_historical_locked_tokens_over_range(self, days: int): today = datetime.utcnow() range_end = datetime(year=today.year, month=today.month, day=today.day, hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1) # include today range_begin = range_end - timedelta(days=days) results = list(self._client.query(f"SELECT SUM(locked_stake) " f"FROM (" f"SELECT staker_address, current_period, " f"LAST(locked_stake) " f"AS locked_stake " f"FROM moe_network_info " f"WHERE time >= '{MayaDT.from_datetime(range_begin).rfc3339()}' " f"AND " f"time < '{MayaDT.from_datetime(range_end).rfc3339()}' " f"GROUP BY staker_address, time(1d)" f") " f"GROUP BY time(1d)").get_points()) # Note: all days may not have values eg. days before DB started getting populated # As time progresses this should be less of an issue locked_tokens_dict = OrderedDict() for r in results: locked_stake = r['sum'] if locked_stake: # Dash accepts datetime objects for graphs locked_tokens_dict[MayaDT.from_rfc3339(r['time']).datetime()] = locked_stake return locked_tokens_dict
def test_blockchain_client_get_historical_locked_tokens(new_influx_db): mock_influxdb_client = new_influx_db.return_value mock_query_object = MagicMock(spec=ResultSet, autospec=True) mock_influxdb_client.query.return_value = mock_query_object # fake results for 5 days days = 5 start_date = maya.now().subtract(days=days) base_amount = 45000 amount_increment = 10000 results = [] for day in range(0, days): results.append(dict(time=start_date.add(days=day).rfc3339(), sum=base_amount + (day * amount_increment))) mock_query_object.get_points.return_value = results blockchain_db_client = CrawlerInfluxClient(None, None, None) locked_tokens_dict = blockchain_db_client.get_historical_locked_tokens_over_range(days) # check query today = datetime.utcnow() range_end = datetime(year=today.year, month=today.month, day=today.day, hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1) # include today in range range_begin = range_end - timedelta(days=days) expected_in_query = [ "SELECT SUM(locked_stake)", "AS locked_stake", f"FROM {Crawler.NODE_MEASUREMENT} WHERE time >= '{MayaDT.from_datetime(range_begin).rfc3339()}' AND " f"time < '{MayaDT.from_datetime(range_end).rfc3339()}'", "GROUP BY staker_address, time(1d)) GROUP BY time(1d)", ] mock_influxdb_client.query.assert_called_once() mock_query_object.get_points.assert_called_once() call_args_list = mock_influxdb_client.query.call_args_list assert len(call_args_list) == 1 for idx, execute_call in enumerate(call_args_list): query = execute_call[0][0] for statement in expected_in_query: assert statement in query # check results assert len(locked_tokens_dict) == days for idx, key in enumerate(locked_tokens_dict): # use of rfc3339 loses milliseconds precision date = MayaDT.from_rfc3339(start_date.add(days=idx).rfc3339()).datetime() assert key == date locked_tokens = locked_tokens_dict[key] assert locked_tokens == base_amount + (idx * amount_increment) # close must be explicitly called on CrawlerInfluxClient mock_influxdb_client.close.assert_not_called()
def get_last_seen(node_info): try: slang_last_seen = MayaDT.from_rfc3339(node_info['last_seen']).slang_time() except ParserError: # Show whatever we have anyways slang_last_seen = str(node_info['last_seen']) if slang_last_seen == NO_CONNECTION_TO_NODE: slang_last_seen = NOT_YET_CONNECTED_TO_NODE return slang_last_seen
def generate_node_table_components(node_info: dict, registry) -> dict: identity = html.Td(children=html.Div([ html.A(node_info['nickname'], href=f'https://{node_info["rest_url"]}/status', target='_blank') ])) # Fleet State fleet_state_div = [] fleet_state_icon = node_info['fleet_state_icon'] if fleet_state_icon is not UNKNOWN_FLEET_STATE: icon_list = node_info['fleet_state_icon'] fleet_state_div = icon_list fleet_state = html.Td(children=html.Div(fleet_state_div)) staker_address = node_info['staker_address'] # Blockchainy (TODO) staking_agent = ContractAgency.get_agent(StakingEscrowAgent, registry=registry) current_period = staking_agent.get_current_period() last_confirmed_period = staking_agent.get_last_active_period( staker_address) status = get_node_status(staking_agent, staker_address, current_period, last_confirmed_period) etherscan_url = f'https://goerli.etherscan.io/address/{node_info["staker_address"]}' try: slang_last_seen = MayaDT.from_rfc3339( node_info['last_seen']).slang_time() except ParserError: slang_last_seen = node_info['last_seen'] components = { 'Status': status, 'Checksum': html.Td( html.A(f'{node_info["staker_address"][:10]}...', href=etherscan_url, target='_blank')), 'Nickname': identity, 'Launched': html.Td(node_info['timestamp']), 'Last Seen': html.Td([slang_last_seen, f" | Period {last_confirmed_period}"]), 'Fleet State': fleet_state } return components
def _get_rows_from_xml(filepath: str, creation_date_start: MayaDT): """Parse the comments xml file and yield all row elements after the given creation date.""" parser = iter(ElementTree.iterparse(filepath, events=['start', 'end'])) _, root = next(parser) month = 0 for event, elem in parser: if event == 'end' and elem.tag == 'row': cd = MayaDT.from_rfc3339(elem.attrib['CreationDate']) if cd.month != month: month = cd.month if creation_date_start is None or creation_date_start <= cd: yield elem root.clear()
def _post_xml_row_to_model(elem, question_ids: Set[int] = None, target_post_type: PostType = PostType.QUESTION): """Convert an xml row from the Posts.xml file to a model. Text is sanitized before conversion. question_ids is only applicable if the target post type is PostType.ANSWER. An answer is only added if its parent_id is contained in question_ids. """ try: post_type = PostType(int(elem.attrib['PostTypeId'])) except ValueError: # was not a question or answer return None # early returns if target_post_type != post_type: return None if target_post_type == PostType.ANSWER and int( elem.attrib['ParentId']) not in question_ids: return None try: sanitized = sanitize_post(elem.attrib['Body']) except ValueError: LOGGER.error( f"Sanitization failed for Post with Id={elem.attrib['Id']}") return None date = MayaDT.from_rfc3339(elem.attrib['CreationDate']).date if post_type == PostType.ANSWER: title = None tags = None parent_id = elem.attrib['ParentId'] else: # is question title = elem.attrib['Title'] tags = elem.attrib['Tags'] parent_id = None post = Post(id=elem.attrib['Id'], creation_date=date, post_type_id=post_type.value, title=title, text=sanitized, tags=tags, parent_id=parent_id) return post
def get_historical_work_orders_over_range(self, days: int): range_begin, range_end = self._get_range_bookends(days) results = list( self._client.query( f"SELECT SUM(work_orders) FROM " f"(" f"SELECT staker_address, LAST(work_orders)" f"FROM {Crawler.NODE_MEASUREMENT} WHERE " f"time >= '{MayaDT.from_datetime(range_begin).rfc3339()}' AND " f"time < '{MayaDT.from_datetime(range_end).rfc3339()}' " f"GROUP BY staker_address, time(1d)" f") " "GROUP BY time(1d)").get_points()) # 1 day measurements work_orders_dict = OrderedDict() for r in results: num_work_orders = r['sum'] work_orders_dict[MayaDT.from_rfc3339(r['time']).datetime( )] = num_work_orders if num_work_orders else 0 return work_orders_dict
def _comment_xml_row_to_model(elem, post_ids: Set[int]): """Convert an xml row from the Comments.xml file to a model. Text is sanitized before conversion. Return None if the post_id is not contained in post_ids. """ post_id = int(elem.attrib['PostId']) if post_id not in post_ids: return None try: sanitized = sanitize_comment(elem.attrib['Text']) except Exception as e: LOGGER.error( f"Sanitization failed for Comment with Id={elem.attrib['Id']}\n" f"{type(e).__name__}\n{str(e)}") return None date = MayaDT.from_rfc3339(elem.attrib['CreationDate']).date comment = Comment(id=elem.attrib['Id'], creation_date=date, text=sanitized, post_id=post_id) return comment
def to_maya(commit): return MayaDT.from_rfc3339(commit['timestamp'])
INITIAL_SUPPLY = NU(1_000_000_000, 'NU') UNIVERSITY_INITIAL_SUPPLY = NU(19_500_000, 'NU') SAFT2_INITIAL_SUPPLY = NU(value=(SAFT2_ALLOCATION_PERCENTAGE * INITIAL_SUPPLY.to_nunits()), denomination='NuNit') TEAM_INITIAL_SUPPLY = NU(value=(TEAM_ALLOCATION_PERCENTAGE * INITIAL_SUPPLY.to_nunits()), denomination='NuNit') NUCO_INITIAL_SUPPLY = NU(value=(NUCO_ALLOCATION_PERCENTAGE * INITIAL_SUPPLY.to_nunits()), denomination='NuNit') SAFT1_SUPPLY = NU(value=(SAFT1_ALLOCATION_PERCENTAGE * INITIAL_SUPPLY.to_nunits()), denomination='NuNit') CASI_SUPPLY = NU(9_000_000, 'NU') NUCO_VESTING_MONTHS = 5 * 12 WORKLOCK_VESTING_MONTHS = 6 UNIVERSITY_VESTING_MONTHS = 3 * 12 SAFT2_TEAM_VESTING_MONTHS = 24 LAUNCH_DATE = MayaDT.from_rfc3339('2020-10-15T00:00:00.0Z') DAYS_PER_MONTH = 30.416 # value used in csv allocations def months_transpired_since_launch(now: MayaDT) -> int: """ Determines the number of months transpired since the launch date, Oct 15, 2020 00:00:00 UTC, based on how monthly durations were calculated when allocations were distributed. """ days_transpired = (now - LAUNCH_DATE).days months_transpired = days_transpired / DAYS_PER_MONTH months_transpired_ceil = math.ceil(months_transpired) # calculation of vesting days (based on months) done during allocation rounded_up_months_min_duration_days = round(months_transpired_ceil * DAYS_PER_MONTH)