MappingNotAllowed,
ValidatingNotAllowed,
MappingPermission,
ValidationPermission,
TeamRoles,
)
from backend.models.postgis.task import Task, TaskHistory
from backend.models.postgis.utils import NotFound
from backend.services.users.user_service import UserService
from backend.services.project_search_service import ProjectSearchService
from backend.services.project_admin_service import ProjectAdminService
from backend.services.team_service import TeamService
from sqlalchemy import func, or_
from sqlalchemy.sql.expression import true
summary_cache = TTLCache(maxsize=1024, ttl=600)
class ProjectServiceError(Exception):
""" Custom Exception to notify callers an error occurred when handling projects """
def __init__(self, message):
if current_app:
current_app.logger.error(message)
class ProjectService:
@staticmethod
def get_project_by_id(project_id: int) -> Project:
project = Project.get(project_id)
if project is None:
class StoryscriptHub:
update_thread = None
retry_lock = Lock()
update_lock = Lock()
ttl_cache_for_services = TTLCache(maxsize=128, ttl=1 * 60)
ttl_cache_for_service_names = TTLCache(maxsize=1, ttl=1 * 60)
@staticmethod
def get_config_dir(app):
if sys.platform == 'win32':
p = os.getenv('APPDATA')
else:
p = os.getenv('XDG_DATA_HOME', os.path.expanduser('~/'))
return os.path.join(p, app)
def __init__(self, db_path: str = None, auto_update: bool = True,
service_wrapper=False):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
:param service_wrapper: Allows you to utilize safe ServiceData objects
"""
if db_path is None:
db_path = StoryscriptHub.get_config_dir('.storyscript')
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
self._service_wrapper = None
if service_wrapper:
self._service_wrapper = ServiceWrapper()
# we need to update the cache immediately for the
# service wrapper to initialize data.
self.update_cache()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
@cached(cache=ttl_cache_for_service_names)
def get_all_service_names(self) -> [str]:
"""
Get all service names and aliases from the database.
:return: An array of strings, which might look like:
["hello", "universe/hello"]
"""
services = []
with Database(self.db_path):
for s in Service.select(Service.name, Service.alias,
Service.username):
if s.alias:
services.append(s.alias)
services.append(f'{s.username}/{s.name}')
return services
@cached(cache=ttl_cache_for_services)
def get(self, alias=None, owner=None, name=None,
wrap_service=False) -> Union[Service, ServiceData]:
"""
Get a service from the database.
:param alias: Takes precedence when specified over owner/name
:param owner: The owner of the service
:param name: The name of the service
:param wrap_service: When set to true, it will return a
@ServiceData object
:return: Returns a Service instance, with all fields populated
"""
service = None
# check if the service_wrapper was initialized for automatic
# wrapping
if self._service_wrapper is not None:
service = self._service_wrapper.get(alias=alias, owner=owner,
name=name)
if service is None:
service = self._get(alias, owner, name)
if service is None:
# Maybe it's new in the Hub?
with self.retry_lock:
service = self._get(alias, owner, name)
if service is None:
self.update_cache()
service = self._get(alias, owner, name)
if service is not None:
# ensures test don't break
if isinstance(service, MagicMock):
return service
assert isinstance(service, Service) or \
isinstance(service, ServiceData)
# if the service wrapper is set, and the service doesn't exist
# we can safely convert this object since it was probably loaded
# from the cache
if wrap_service or self._service_wrapper is not None:
return ServiceData.from_dict(data={
"service_data": json.loads(service.raw_data)
})
if service.topics is not None:
service.topics = json.loads(service.topics)
if service.configuration is not None:
service.configuration = json.loads(service.configuration)
return service
def _get(self, alias: str = None, owner: str = None, name: str = None):
try:
if alias is not None and alias.count("/") == 1:
owner, name = alias.split("/")
alias = None
with Database(self.db_path):
if alias:
service = Service.select().where(Service.alias == alias)
else:
service = Service.select().where(
(Service.username == owner) & (Service.name == name))
return service.get()
except DoesNotExist:
return None
def update_cache(self):
services = GraphQL.get_all()
# tell the service wrapper to reload any services from the cache.
if self._service_wrapper is not None:
self._service_wrapper.reload_services(services)
with Database(self.db_path) as db:
with db.atomic(lock_type='IMMEDIATE'):
Service.delete().execute()
for service in services:
Service.create(
service_uuid=service['serviceUuid'],
name=service['service']['name'],
alias=service['service']['alias'],
username=service['service']['owner']['username'],
description=service['service']['description'],
certified=service['service']['isCertified'],
public=service['service']['public'],
topics=json.dumps(service['service']['topics']),
state=service['state'],
configuration=json.dumps(service['configuration']),
readme=service['readme'],
raw_data=json.dumps(service))
with self.update_lock:
self.ttl_cache_for_service_names.clear()
self.ttl_cache_for_services.clear()
return True
class GsDataApi(DataApi):
__definitions = {}
DEFAULT_SCROLL = '30s'
# DataApi interface
@classmethod
def query_data(cls, query: DataQuery, dataset_id: str = None, asset_id_type: Union[GsIdType, str] = None) \
-> Union[MDAPIDataBatchResponse, DataQueryResponse, tuple]:
if query.marketDataCoordinates:
# Don't use MDAPIDataBatchResponse for now - it doesn't handle quoting style correctly
results: Union[MDAPIDataBatchResponse,
dict] = GsSession.current._post(
'/data/coordinates/query', payload=query)
if isinstance(results, dict):
return results.get('responses', ())
else:
return results.responses if results.responses is not None else (
)
if query.where:
where = query.where.as_dict()
xref_keys = set(where.keys()).intersection(XRef.properties())
if xref_keys:
# Check that assetId is a symbol dimension of this data set. If not, we need to do a separate query
# to resolve xref --> assetId
if len(xref_keys) > 1:
raise MqValueError(
'Cannot not specify more than one type of asset identifier'
)
definition = cls.get_definition(dataset_id)
sd = definition.dimensions.symbolDimensions
if definition.parameters.symbolStrategy == 'MDAPI' or (
'assetId' not in sd and 'gsid' not in sd):
xref_type = min(xref_keys)
if asset_id_type is None:
asset_id_type = xref_type
xref_values = where[asset_id_type]
xref_values = (xref_values, ) if isinstance(
xref_values, str) else xref_values
asset_id_map = GsAssetApi.map_identifiers(
xref_type, GsIdType.id, xref_values)
if len(asset_id_map) != len(xref_values):
raise MqValueError(
'Not all {} were resolved to asset Ids'.format(
asset_id_type))
setattr(query.where, xref_type, None)
query.where.assetId = [
asset_id_map[x] for x in xref_values
]
results: Union[DataQueryResponse, dict] = GsSession.current._post(
'/data/{}/query'.format(dataset_id), payload=query)
if isinstance(results, dict):
results = results.get('data', ())
else:
results = results.data if results.data is not None else ()
if asset_id_type not in {GsIdType.id, None}:
asset_ids = tuple(
set(filter(None, (r.get('assetId') for r in results))))
if asset_ids:
xref_map = GsAssetApi.map_identifiers(GsIdType.id,
asset_id_type, asset_ids)
if len(xref_map) != len(asset_ids):
raise MqValueError(
'Not all asset Ids were resolved to {}'.format(
asset_id_type))
for result in results:
result[asset_id_type] = xref_map[result['assetId']]
return results
@classmethod
def last_data(cls,
query: DataQuery,
dataset_id: str = None) -> Union[list, tuple]:
if query.marketDataCoordinates:
result = GsSession.current._post('/data/coordinates/query/last',
payload=query)
return result.get('responses', ())
else:
result = GsSession.current._post(
'/data/{}/last/query'.format(dataset_id), payload=query)
return result.get('data', ())
@classmethod
def symbol_dimensions(cls, dataset_id: str) -> tuple:
definition = cls.get_definition(dataset_id)
return definition.dimensions.symbolDimensions
@classmethod
def time_field(cls, dataset_id: str) -> str:
definition = cls.get_definition(dataset_id)
return definition.dimensions.timeField
# GS-specific functionality
@classmethod
def get_coverage(cls,
dataset_id: str,
scroll: str = DEFAULT_SCROLL,
scroll_id: Optional[str] = None,
limit: int = None,
offset: int = None,
fields: List[str] = None) -> List[dict]:
params = {}
if scroll:
params['scroll'] = scroll
if scroll_id:
params['scrollId'] = scroll_id
if not limit:
limit = 4000
params['limit'] = limit
if offset:
params['offset'] = offset
if fields:
params['fields'] = fields
body = GsSession.current._get('/data/{}/coverage'.format(dataset_id),
payload=params)
results = body['results']
if len(results) > 0 and 'scrollId' in body:
return results + cls.get_coverage(dataset_id,
scroll_id=body['scrollId'],
scroll=GsDataApi.DEFAULT_SCROLL,
limit=limit)
else:
return results
@classmethod
def create(cls, definition: Union[DataSetEntity, dict]) -> DataSetEntity:
result = GsSession.current._post('/data/datasets', payload=definition)
return result
@classmethod
def update_definition(
cls, dataset_id: str, definition: Union[DataSetEntity,
dict]) -> DataSetEntity:
result = GsSession.current._put('/data/datasets/{}'.format(dataset_id),
payload=definition,
cls=DataSetEntity)
return result
@classmethod
def upload_data(cls, dataset_id: str, data: Union[pd.DataFrame, list,
tuple]) -> dict:
result = GsSession.current._post('/data/{}'.format(dataset_id),
payload=data)
return result
@classmethod
def get_definition(cls, dataset_id: str) -> DataSetEntity:
definition = cls.__definitions.get(dataset_id)
if not definition:
definition = GsSession.current._get(
'/data/datasets/{}'.format(dataset_id), cls=DataSetEntity)
if not definition:
raise MqValueError('Unknown dataset {}'.format(dataset_id))
cls.__definitions[dataset_id] = definition
return definition
@staticmethod
def build_market_data_query(asset_ids: List[str],
query_type: QueryType,
where: Union[FieldFilterMap] = None,
source: Union[str] = None,
real_time: bool = False):
inner = {
'assetIds': asset_ids,
'queryType': query_type.value,
'where': where or {},
'source': source or 'any',
'frequency': 'Real Time' if real_time else 'End Of Day',
'measures': ['Curve']
}
if real_time:
inner['startTime'] = DataContext.current.start_time
inner['endTime'] = DataContext.current.end_time
else:
inner['startDate'] = DataContext.current.start_date
inner['endDate'] = DataContext.current.end_date
return {'queries': [inner]}
@classmethod
def get_market_data(cls, query) -> pd.DataFrame:
GsSession.current: GsSession
body = GsSession.current._post('/data/markets', payload=query)
container = body['responses'][0]['queryResponse'][0]
if 'errorMessages' in container:
raise MqValueError(container['errorMessages'])
if 'response' not in container:
return pd.DataFrame()
df = pd.DataFrame(container['response']['data'])
df.set_index('date' if 'date' in df.columns else 'time', inplace=True)
df.index = pd.to_datetime(df.index)
return df
@classmethod
def __normalise_coordinate_data(
cls, data: Iterable[Union[MDAPIDataQueryResponse, dict]]
) -> Iterable[Iterable[dict]]:
ret = []
for response in data:
coord_data = []
rows = (r.as_dict() for r in response.data) if isinstance(
response, MDAPIDataQueryResponse) else response.get(
'data', ())
for pt in rows:
if not pt:
continue
if 'value' not in pt:
value_field = pt['mktQuotingStyle']
pt['value'] = pt.pop(value_field)
coord_data.append(pt)
ret.append(coord_data)
return ret
@classmethod
def __df_from_coordinate_data(cls, data: Iterable[dict]) -> pd.DataFrame:
df = cls._sort_coordinate_data(pd.DataFrame.from_records(data))
index_field = next((f for f in ('time', 'date') if f in df.columns),
None)
if index_field:
df = df.set_index(pd.DatetimeIndex(df.loc[:, index_field].values))
return df
@classmethod
def _sort_coordinate_data(
cls,
df: pd.DataFrame,
by: Tuple[str] = ('date', 'time', 'mktType', 'mktAsset', 'mktClass',
'mktPoint', 'mktQuotingStyle', 'value')
) -> pd.DataFrame:
columns = df.columns
field_order = [f for f in by if f in columns]
field_order.extend(f for f in columns if f not in field_order)
return df[field_order]
@classmethod
def _coordinate_from_str(cls, coordinate_str: str) -> MarketDataCoordinate:
tmp = coordinate_str.rsplit(".", 1)
dimensions = tmp[0].split("_")
if len(dimensions) < 2:
raise MqValueError('invalid coordinate ' + coordinate_str)
kwargs = {
'mkt_type': dimensions[0],
'mkt_asset': dimensions[1] or None,
'mkt_quoting_style': tmp[-1] if len(tmp) > 1 else None
}
if len(dimensions) > 2:
kwargs['mkt_class'] = dimensions[2] or None
if len(dimensions) > 3:
kwargs['mkt_point'] = tuple(dimensions[3:]) or None
return MarketDataCoordinate(**kwargs)
@classmethod
def coordinates_last(
cls,
coordinates: Union[Iterable[str], Iterable[MarketDataCoordinate]],
as_of: Optional[dt.datetime] = None,
vendor: str = 'Goldman Sachs',
as_dataframe: bool = False,
) -> Union[dict, pd.DataFrame]:
market_data_coordinates = tuple(
cls._coordinate_from_str(coord) if isinstance(coord, str
) else coord
for coord in coordinates)
ret = {coordinate: None for coordinate in market_data_coordinates}
query = cls.build_query(
end=as_of,
market_data_coordinates=market_data_coordinates,
vendor=vendor)
data = cls.last_data(query)
for idx, row in enumerate(cls.__normalise_coordinate_data(data)):
try:
ret[market_data_coordinates[idx]] = row[0]['value']
except IndexError:
ret[market_data_coordinates[idx]] = None
if as_dataframe:
data = [
dict(
chain(
c.as_dict(as_camel_case=True).items(),
(('value', v), ))) for c, v in ret.items()
]
return cls.__df_from_coordinate_data(data)
return ret
@classmethod
def coordinates_data(
cls,
coordinates: Union[str, MarketDataCoordinate, Iterable[str],
Iterable[MarketDataCoordinate]],
start: Optional[dt.datetime] = None,
end: Optional[dt.datetime] = None,
vendor: str = 'Goldman Sachs',
as_multiple_dataframes: bool = False
) -> Union[pd.DataFrame, Tuple[pd.DataFrame]]:
coordinates_iterable = (coordinates, ) if isinstance(
coordinates, (MarketDataCoordinate, str)) else coordinates
query = cls.build_query(market_data_coordinates=tuple(
cls._coordinate_from_str(coord) if isinstance(coord, str
) else coord
for coord in coordinates_iterable),
vendor=vendor,
start=start,
end=end)
results = cls.__normalise_coordinate_data(cls.query_data(query))
if as_multiple_dataframes:
return tuple(
GsDataApi.__df_from_coordinate_data(r) for r in results)
else:
return cls.__df_from_coordinate_data(chain.from_iterable(results))
@classmethod
def coordinates_data_series(
cls,
coordinates: Union[str, MarketDataCoordinate, Iterable[str],
Iterable[MarketDataCoordinate]],
start: Optional[dt.datetime] = None,
end: Optional[dt.datetime] = None,
vendor: str = 'Goldman Sachs',
) -> Union[pd.Series, Tuple[pd.Series]]:
dfs = cls.coordinates_data(coordinates,
start=start,
end=end,
vendor=vendor,
as_multiple_dataframes=True)
ret = tuple(pd.Series() if df.empty else pd.
Series(index=df.index, data=df.value.values) for df in dfs)
if isinstance(coordinates, (MarketDataCoordinate, str)):
return ret[0]
else:
return ret
@staticmethod
@cachetools.cached(TTLCache(ttl=3600, maxsize=128))
def get_types(dataset_id: str):
results = GsSession.current._get(f'/data/catalog/{dataset_id}')
fields = results.get("fields")
if fields:
field_types = {}
for key, value in fields.items():
field_type = value.get('type')
field_format = value.get('format')
field_types[key] = field_format or field_type
return field_types
raise RuntimeError(f"Unable to get Dataset schema for {dataset_id}")
@classmethod
def construct_dataframe_with_types(
cls, dataset_id: str, data: Union[Base, list,
tuple]) -> pd.DataFrame:
"""
Constructs a dataframe with correct date types.
:param data: data to convert with correct types
:return: dataframe with correct types
"""
if len(data):
dataset_types = cls.get_types(dataset_id)
df = pd.DataFrame(data)
for field_name, type_name in dataset_types.items():
if df.get(type_name) is not None and type_name in (
'date', 'date-time'):
df = df.astype({field_name: numpy.datetime64})
field_names = dataset_types.keys()
if 'date' in field_names:
df = df.set_index('date')
elif 'time' in field_names:
df = df.set_index('time')
return df
else:
return pd.DataFrame({})
"""Util functions to help run generic searches for craigslist toronto"""
from bs4 import BeautifulSoup as bs4
from cachetools import cached, TTLCache
from collections import OrderedDict
import json
import requests
import sys
URL_BASE = 'https://toronto.craigslist.org'
cache = TTLCache(maxsize=120, ttl=300)
AREAS = {
'toronto': 'tor',
'durham_region': 'drh',
'york_region': 'yrk',
'brampton': 'bra',
'mississuaga': 'mss',
'oakville': 'oak'
}
CATEGORY_TOPICS = {
'community': 'ccc',
'events': 'eee',
'for_sale': 'sss',
'gigs': 'ggg',
'housing': 'hhh',
'jobs': 'jjj',
'resume': 'rrr',
'services': 'bbb'
}
def __init__(self, check, config, shutdown_callback):
collection_interval = float(
config.statement_samples_config.get('collection_interval',
DEFAULT_COLLECTION_INTERVAL))
if collection_interval <= 0:
collection_interval = DEFAULT_COLLECTION_INTERVAL
super(PostgresStatementSamples, self).__init__(
check,
rate_limit=1 / collection_interval,
run_sync=is_affirmative(
config.statement_samples_config.get('run_sync', False)),
enabled=is_affirmative(
config.statement_samples_config.get('enabled', True)),
dbms="postgres",
min_collection_interval=config.min_collection_interval,
expected_db_exceptions=(psycopg2.errors.DatabaseError, ),
job_name="query-samples",
shutdown_callback=shutdown_callback,
)
self._check = check
self._config = config
self._tags_no_db = None
self._activity_last_query_start = None
# The value is loaded when connecting to the main database
self._explain_function = config.statement_samples_config.get(
'explain_function', 'datadog.explain_statement')
self._obfuscate_options = to_native_string(
json.dumps(self._config.obfuscator_options))
self._collection_strategy_cache = TTLCache(
maxsize=config.statement_samples_config.get(
'collection_strategy_cache_maxsize', 1000),
ttl=config.statement_samples_config.get(
'collection_strategy_cache_ttl', 300),
)
self._explain_errors_cache = TTLCache(
maxsize=config.statement_samples_config.get(
'explain_errors_cache_maxsize', 5000),
# only try to re-explain invalid statements once per day
ttl=config.statement_samples_config.get('explain_errors_cache_ttl',
24 * 60 * 60),
)
# explained_statements_ratelimiter: limit how often we try to re-explain the same query
self._explained_statements_ratelimiter = RateLimitingTTLCache(
maxsize=int(
config.statement_samples_config.get(
'explained_queries_cache_maxsize', 5000)),
ttl=60 * 60 / int(
config.statement_samples_config.get(
'explained_queries_per_hour_per_query', 60)),
)
# seen_samples_ratelimiter: limit the ingestion rate per (query_signature, plan_signature)
self._seen_samples_ratelimiter = RateLimitingTTLCache(
# assuming ~100 bytes per entry (query & plan signature, key hash, 4 pointers (ordered dict), expiry time)
# total size: 10k * 100 = 1 Mb
maxsize=int(
config.statement_samples_config.get(
'seen_samples_cache_maxsize', 10000)),
ttl=60 * 60 / int(
config.statement_samples_config.get(
'samples_per_hour_per_query', 15)),
)
self._activity_coll_enabled = is_affirmative(
self._config.statement_activity_config.get('enabled', True))
# activity events cannot be reported more often than regular samples
self._activity_coll_interval = max(
self._config.statement_activity_config.get(
'collection_interval', DEFAULT_ACTIVITY_COLLECTION_INTERVAL),
collection_interval,
)
self._activity_max_rows = self._config.statement_activity_config.get(
'payload_row_limit', 3500)
# Keep track of last time we sent an activity event
self._time_since_last_activity_event = 0
self._pg_stat_activity_cols = None
from flask import render_template, url_for, flash, redirect
from . import app, db
from .models import Post
from cachetools import cached, LRUCache, TTLCache
import re
import json
from . import _config
# from .post_orm import Post
@app.route("/")
@app.route("/home")
@cached(cache=TTLCache(maxsize=_config.CACHE_MAXSIZE, ttl=_config.CACHE_TTL))
def home():
posts = Post.query.filter(
Post.similar_post_info != json.dumps([])).order_by(
Post.id.desc()).all()
non_duplicate_post = []
for post in posts:
if post.id in non_duplicate_post:
continue
post.get_similar_post_info()
similar_post_json = post.similar_post_to_json()
non_duplicate_post.append(post.id)
for similar_post in similar_post_json:
if similar_post['score'] > _config.DD_THRESHOLD:
non_duplicate_post.append(similar_post['id'])
all_posts = Post.query.order_by(Post.id.desc()).all()
for p in all_posts:
if p.url:
p.domain = re.search('https?:\/\/([\w.]+)\/', p.url).group(1)
logger.info(f"running hook {url}")
data = {
"id": self.user_id,
"date": str(datetime.now()),
"type": "CutFileCompletedEvent",
"data": {
"room_id": int(self.room_id),
"path": str(self.out_path)
},
}
httpx.post(url, json=data)
logger.info(f"run hook {url} successfully")
if TYPE_CHECKING:
cut_tasks: TTLCache[str, CutTask] = TTLCache(maxsize=100, ttl=60 * 60 * 1)
else:
cut_tasks = TTLCache(maxsize=100, ttl=60 * 60 * 1)
def check_sub(user_id: str, uid: str) -> Optional[str]:
sub_list = get_sub_list(user_id)
if uid not in sub_list:
return "尚未订阅该主播"
if not sub_list[uid].get("record", False):
return "未开启该主播的自动录播"
async def check_task(room_id: str) -> Union[str, Path]:
task_info = await get_task(room_id)
if not task_info:
import requests
from cachetools import cached, TTLCache
api_url = "https://freegeoip.app/json/"
@cached(cache=TTLCache(maxsize=1000, ttl=345600)
) # removes a given entry from cache every 4 days
def get_ip_data(ip):
response = requests.get(api_url + ip)
return response.json()
from backend.models.postgis.project import Project
from backend.models.postgis.user import User, UserRole, MappingLevel, UserEmail
from backend.models.postgis.task import TaskHistory, TaskAction, Task
from backend.models.dtos.user_dto import UserTaskDTOs
from backend.models.dtos.stats_dto import Pagination
from backend.models.postgis.statuses import TaskStatus, ProjectStatus
from backend.models.postgis.utils import NotFound
from backend.services.users.osm_service import OSMService, OSMServiceError
from backend.services.messaging.smtp_service import SMTPService
from backend.services.messaging.template_service import (
get_template,
template_var_replacing,
)
user_filter_cache = TTLCache(maxsize=1024, ttl=600)
class UserServiceError(Exception):
""" Custom Exception to notify callers an error occurred when in the User Service """
def __init__(self, message):
if current_app:
current_app.logger.error(message)
class UserService:
@staticmethod
def get_user_by_id(user_id: int) -> User:
user = User.get_by_id(user_id)
import uuid
from cachetools import TTLCache
from datetime import datetime
from http import HTTPStatus
from sanic import (exceptions, response)
from random import randint
from application import APIApplication
from decorators import require_authorization
from schemas import (Command, Task, TaskView)
# Set up the application.
app = APIApplication('todos-api')
# Set up memory-based storage backend for the TODO items.
collection = TTLCache(32, 1800)
def _update_data(pk, data):
"""Updates data in collection.
Used via event loop's `call_later` to simulate eventually consistent architecture.
Arguments:
pk: Primary key for the item to update.
data: Data to set. Note: to delete items, set it to `None`.
"""
if data is None:
del collection[pk]
else:
collection[pk] = data
import os
import re
from collections import namedtuple
import pandas as pd
import requests
from bs4 import BeautifulSoup
from cachetools import cached, TTLCache
from helpers import config
from models.attendance import Attendance
from models.raid import Raid
from models.raidevent import RaidEvent
# CACHE CONFIGURATION
_cache = TTLCache(maxsize=100, ttl=60)
# PANDAS DISPLAY CONFIGURATION
pd.set_option('display.max_rows', 25)
pd.set_option('display.max_columns', 10)
pd.set_option('display.width', 1000)
pd.set_option('display.column_space', 25)
# EQDKP API CONFIGURATION
_URL = os.getenv('EQDKP_URL')
_API_URL = _URL + 'api.php'
_API_TOKEN = os.getenv('EQDKP_API_TOKEN')
_API_HEADERS = {'X-Custom-Authorization': f'token={_API_TOKEN}&type=api'}
# GET STANDINGS CONFIGURATION
EQDKP_COLUMNS = ['CHARACTER', 'CLASS', 'DKP', '30DAY', '60DAY', '90DAY']
def __init__(self, capacity, *indices):
self._cache = TTLCache(capacity, TTL, timer=time)
self._indices = indices
from flask import current_app
from sqlalchemy import text, func
from backend import create_app, db
from backend.models.dtos.message_dto import MessageDTO, MessagesDTO
from backend.models.dtos.stats_dto import Pagination
from backend.models.postgis.message import Message, MessageType, NotFound
from backend.models.postgis.notification import Notification
from backend.models.postgis.project import Project
from backend.models.postgis.task import TaskStatus, TaskAction, TaskHistory
from backend.services.messaging.smtp_service import SMTPService
from backend.services.messaging.template_service import get_template, get_profile_url
from backend.services.users.user_service import UserService, User
message_cache = TTLCache(maxsize=512, ttl=30)
class MessageServiceError(Exception):
""" Custom Exception to notify callers an error occurred when handling mapping """
def __init__(self, message):
if current_app:
current_app.logger.error(message)
class MessageService:
@staticmethod
def send_welcome_message(user: User):
""" Sends welcome message to all new users at Sign up"""
text_template = get_template("welcome_message_en.txt")
class ProjectService:
@staticmethod
def get_project_by_id(project_id: int) -> Project:
project = Project.get(project_id)
if project is None:
raise NotFound()
return project
@staticmethod
def exists(project_id: int) -> bool:
project = Project.exists(project_id)
if project is None:
raise NotFound()
return True
@staticmethod
def get_project_by_name(project_id: int) -> Project:
project = Project.get(project_id)
if project is None:
raise NotFound()
return project
@staticmethod
def auto_unlock_tasks(project_id: int):
Task.auto_unlock_tasks(project_id)
@staticmethod
def delete_tasks(project_id: int, tasks_ids):
# Validate project exists.
project = Project.get(project_id)
if project is None:
raise NotFound({"project": project_id})
tasks = [{"id": i, "obj": Task.get(i, project_id)} for i in tasks_ids]
# In case a task is not found.
not_found = [t["id"] for t in tasks if t["obj"] is None]
if len(not_found) > 0:
raise NotFound({"tasks": not_found})
# Delete task one by one.
[t["obj"].delete() for t in tasks]
@staticmethod
def get_contribs_by_day(project_id: int) -> ProjectContribsDTO:
# Validate that project exists
project = ProjectService.get_project_by_id(project_id)
# Fetch all state change with date and task ID
stats = (
TaskHistory.query.with_entities(
TaskHistory.action_text.label("action_text"),
func.DATE(TaskHistory.action_date).label("day"),
TaskHistory.task_id.label("task_id"),
)
.filter(TaskHistory.project_id == project_id)
.filter(
TaskHistory.action == "STATE_CHANGE",
or_(
TaskHistory.action_text == "MAPPED",
TaskHistory.action_text == "VALIDATED",
TaskHistory.action_text == "INVALIDATED",
),
)
.group_by("action_text", "day", "task_id")
.order_by("day")
).all()
contribs_dto = ProjectContribsDTO()
# Filter and store unique dates
dates = list(set(r[1] for r in stats))
dates.sort(
reverse=False
) # Why was this reversed? To have the dates in ascending order
dates_list = []
cumulative_mapped = 0
cumulative_validated = 0
# A hashmap to track task state change updates
tasks = {
"MAPPED": {"total": 0},
"VALIDATED": {"total": 0},
"INVALIDATED": {"total": 0},
}
for date in dates:
dto = ProjectContribDTO(
{
"date": date,
"mapped": 0,
"validated": 0,
"total_tasks": project.total_tasks,
}
)
# s -> ('LOCKED_FOR_MAPPING', datetime.date(2019, 4, 23), 1)
# s[0] -> action, s[1] -> date, s[2] -> task_id
values = [(s[0], s[2]) for s in stats if date == s[1]]
values.sort(reverse=True) # Most recent action comes first
for val in values:
task_id = val[1]
task_status = val[0]
if task_status == "MAPPED":
if task_id not in tasks["MAPPED"]:
tasks["MAPPED"][task_id] = 1
tasks["MAPPED"]["total"] += 1
dto.mapped += 1
elif task_status == "VALIDATED":
if task_id not in tasks["VALIDATED"]:
tasks["VALIDATED"][task_id] = 1
tasks["VALIDATED"]["total"] += 1
dto.validated += 1
if task_id in tasks["INVALIDATED"]:
del tasks["INVALIDATED"][task_id]
tasks["INVALIDATED"]["total"] -= 1
if task_id not in tasks["MAPPED"]:
tasks["MAPPED"][task_id] = 1
tasks["MAPPED"]["total"] += 1
dto.mapped += 1
else:
if task_id not in tasks["INVALIDATED"]:
tasks["INVALIDATED"][task_id] = 1
tasks["INVALIDATED"]["total"] += 1
if task_id in tasks["MAPPED"]:
del tasks["MAPPED"][task_id]
tasks["MAPPED"]["total"] -= 1
if dto.mapped > 0:
dto.mapped -= 1
if task_id in tasks["VALIDATED"]:
del tasks["VALIDATED"][task_id]
tasks["VALIDATED"]["total"] -= 1
if dto.validated > 0:
dto.validated -= 1
cumulative_mapped = tasks["MAPPED"]["total"]
cumulative_validated = tasks["VALIDATED"]["total"]
dto.cumulative_mapped = cumulative_mapped
dto.cumulative_validated = cumulative_validated
dates_list.append(dto)
contribs_dto.stats = dates_list
return contribs_dto
@staticmethod
def get_project_dto_for_mapper(
project_id, current_user_id, locale="en", abbrev=False
) -> ProjectDTO:
"""
Get the project DTO for mappers
:param project_id: ID of the Project mapper has requested
:param locale: Locale the mapper has requested
:raises ProjectServiceError, NotFound
"""
project = ProjectService.get_project_by_id(project_id)
# if project is public and is not draft, we don't need to check permissions
if not project.private and not project.status == ProjectStatus.DRAFT.value:
return project.as_dto_for_mapping(current_user_id, locale, abbrev)
is_allowed_user = True
is_team_member = None
is_manager_permission = False
if current_user_id:
is_manager_permission = ProjectAdminService.is_user_action_permitted_on_project(
current_user_id, project_id
)
# Draft Projects - admins, authors, org admins & team managers permitted
if project.status == ProjectStatus.DRAFT.value:
if not is_manager_permission:
is_allowed_user = False
raise ProjectServiceError("Unable to fetch project")
# Private Projects - allowed_users, admins, org admins &
# assigned teams (mappers, validators, project managers), authors permitted
if project.private and not is_manager_permission:
is_allowed_user = False
if current_user_id:
is_allowed_user = (
len(
[
user
for user in project.allowed_users
if user.id == current_user_id
]
)
> 0
)
if not (is_allowed_user or is_manager_permission):
if current_user_id:
allowed_roles = [
TeamRoles.MAPPER.value,
TeamRoles.VALIDATOR.value,
TeamRoles.PROJECT_MANAGER.value,
]
is_team_member = TeamService.check_team_membership(
project_id, allowed_roles, current_user_id
)
if is_allowed_user or is_manager_permission or is_team_member:
return project.as_dto_for_mapping(current_user_id, locale, abbrev)
else:
raise ProjectServiceError("Unable to fetch project")
@staticmethod
def get_project_tasks(
project_id,
task_ids_str: str,
order_by: str = None,
order_by_type: str = "ASC",
status: int = None,
):
project = ProjectService.get_project_by_id(project_id)
return project.tasks_as_geojson(task_ids_str, order_by, order_by_type, status)
@staticmethod
def get_project_aoi(project_id):
project = ProjectService.get_project_by_id(project_id)
return project.get_aoi_geometry_as_geojson()
@staticmethod
def get_project_priority_areas(project_id):
project = ProjectService.get_project_by_id(project_id)
geojson_areas = []
for priority_area in project.priority_areas:
geojson_areas.append(priority_area.get_as_geojson())
return geojson_areas
@staticmethod
def get_task_for_logged_in_user(user_id: int):
""" if the user is working on a task in the project return it """
tasks = Task.get_locked_tasks_for_user(user_id)
tasks_dto = tasks
return tasks_dto
@staticmethod
def get_task_details_for_logged_in_user(user_id: int, preferred_locale: str):
""" if the user is working on a task in the project return it """
tasks = Task.get_locked_tasks_details_for_user(user_id)
if len(tasks) == 0:
raise NotFound()
# TODO put the task details in to a DTO
dtos = []
for task in tasks:
dtos.append(task.as_dto_with_instructions(preferred_locale))
task_dtos = TaskDTOs()
task_dtos.tasks = dtos
return task_dtos
@staticmethod
def is_user_in_the_allowed_list(allowed_users: list, current_user_id: int):
"""For private projects, check if user is present in the allowed list"""
return (
len([user.id for user in allowed_users if user.id == current_user_id]) > 0
)
@staticmethod
def evaluate_mapping_permission(
project_id: int, user_id: int, mapping_permission: int
):
allowed_roles = [
TeamRoles.MAPPER.value,
TeamRoles.VALIDATOR.value,
TeamRoles.PROJECT_MANAGER.value,
]
is_team_member = TeamService.check_team_membership(
project_id, allowed_roles, user_id
)
# mapping_permission = 1(level),2(teams),3(teamsAndLevel)
if mapping_permission == MappingPermission.TEAMS.value:
if not is_team_member:
return False, MappingNotAllowed.USER_NOT_TEAM_MEMBER
elif mapping_permission == MappingPermission.LEVEL.value:
if not ProjectService._is_user_intermediate_or_advanced(user_id):
return False, MappingNotAllowed.USER_NOT_CORRECT_MAPPING_LEVEL
elif mapping_permission == MappingPermission.TEAMS_LEVEL.value:
if not ProjectService._is_user_intermediate_or_advanced(user_id):
return False, MappingNotAllowed.USER_NOT_CORRECT_MAPPING_LEVEL
if not is_team_member:
return False, MappingNotAllowed.USER_NOT_TEAM_MEMBER
@staticmethod
def is_user_permitted_to_map(project_id: int, user_id: int):
""" Check if the user is allowed to map the on the project in scope """
if UserService.is_user_blocked(user_id):
return False, MappingNotAllowed.USER_NOT_ON_ALLOWED_LIST
project = ProjectService.get_project_by_id(project_id)
if project.license_id:
if not UserService.has_user_accepted_license(user_id, project.license_id):
return False, MappingNotAllowed.USER_NOT_ACCEPTED_LICENSE
mapping_permission = project.mapping_permission
is_manager_permission = (
False # is_admin or is_author or is_org_manager or is_manager_team
)
if ProjectAdminService.is_user_action_permitted_on_project(user_id, project_id):
is_manager_permission = True
# Draft (public/private) accessible only for is_manager_permission
if (
ProjectStatus(project.status) == ProjectStatus.DRAFT
and not is_manager_permission
):
return False, MappingNotAllowed.PROJECT_NOT_PUBLISHED
is_restriction = None
if not is_manager_permission and mapping_permission:
is_restriction = ProjectService.evaluate_mapping_permission(
project_id, user_id, mapping_permission
)
tasks = Task.get_locked_tasks_for_user(user_id)
if len(tasks.locked_tasks) > 0:
return False, MappingNotAllowed.USER_ALREADY_HAS_TASK_LOCKED
is_allowed_user = None
if project.private and not is_manager_permission:
# Check if user is in allowed user list
is_allowed_user = ProjectService.is_user_in_the_allowed_list(
project.allowed_users, user_id
)
if is_allowed_user:
return True, "User allowed to map"
if not is_manager_permission and is_restriction:
return is_restriction
elif project.private and not (
is_manager_permission or is_allowed_user or not is_restriction
):
return False, MappingNotAllowed.USER_NOT_ON_ALLOWED_LIST
return True, "User allowed to map"
@staticmethod
def _is_user_intermediate_or_advanced(user_id):
""" Helper method to determine if user level is not beginner """
user_mapping_level = UserService.get_mapping_level(user_id)
if user_mapping_level not in [MappingLevel.INTERMEDIATE, MappingLevel.ADVANCED]:
return False
return True
@staticmethod
def evaluate_validation_permission(
project_id: int, user_id: int, validation_permission: int
):
allowed_roles = [TeamRoles.VALIDATOR.value, TeamRoles.PROJECT_MANAGER.value]
is_team_member = TeamService.check_team_membership(
project_id, allowed_roles, user_id
)
# validation_permission = 1(level),2(teams),3(teamsAndLevel)
if validation_permission == ValidationPermission.TEAMS.value:
if not is_team_member:
return False, ValidatingNotAllowed.USER_NOT_TEAM_MEMBER
elif validation_permission == ValidationPermission.LEVEL.value:
if not ProjectService._is_user_intermediate_or_advanced(user_id):
return False, ValidatingNotAllowed.USER_IS_BEGINNER
elif validation_permission == ValidationPermission.TEAMS_LEVEL.value:
if not ProjectService._is_user_intermediate_or_advanced(user_id):
return False, ValidatingNotAllowed.USER_IS_BEGINNER
if not is_team_member:
return False, ValidatingNotAllowed.USER_NOT_TEAM_MEMBER
@staticmethod
def is_user_permitted_to_validate(project_id, user_id):
""" Check if the user is allowed to validate on the project in scope """
if UserService.is_user_blocked(user_id):
return False, ValidatingNotAllowed.USER_NOT_ON_ALLOWED_LIST
project = ProjectService.get_project_by_id(project_id)
if project.license_id:
if not UserService.has_user_accepted_license(user_id, project.license_id):
return False, ValidatingNotAllowed.USER_NOT_ACCEPTED_LICENSE
validation_permission = project.validation_permission
# is_admin or is_author or is_org_manager or is_manager_team
is_manager_permission = False
if ProjectAdminService.is_user_action_permitted_on_project(user_id, project_id):
is_manager_permission = True
# Draft (public/private) accessible only for is_manager_permission
if (
ProjectStatus(project.status) == ProjectStatus.DRAFT
and not is_manager_permission
):
return False, ValidatingNotAllowed.PROJECT_NOT_PUBLISHED
is_restriction = None
if not is_manager_permission and validation_permission:
is_restriction = ProjectService.evaluate_validation_permission(
project_id, user_id, validation_permission
)
tasks = Task.get_locked_tasks_for_user(user_id)
if len(tasks.locked_tasks) > 0:
return False, ValidatingNotAllowed.USER_ALREADY_HAS_TASK_LOCKED
is_allowed_user = None
if project.private and not is_manager_permission:
# Check if user is in allowed user list
is_allowed_user = ProjectService.is_user_in_the_allowed_list(
project.allowed_users, user_id
)
if is_allowed_user:
return True, "User allowed to validate"
if not is_manager_permission and is_restriction:
return is_restriction
elif project.private and not (
is_manager_permission or is_allowed_user or not is_restriction
):
return False, ValidatingNotAllowed.USER_NOT_ON_ALLOWED_LIST
return True, "User allowed to validate"
@staticmethod
@cached(summary_cache)
def get_project_summary(
project_id: int, preferred_locale: str = "en"
) -> ProjectSummary:
""" Gets the project summary DTO """
project = ProjectService.get_project_by_id(project_id)
return project.get_project_summary(preferred_locale)
@staticmethod
def set_project_as_featured(project_id: int):
""" Sets project as featured """
project = ProjectService.get_project_by_id(project_id)
project.set_as_featured()
@staticmethod
def unset_project_as_featured(project_id: int):
""" Sets project as featured """
project = ProjectService.get_project_by_id(project_id)
project.unset_as_featured()
@staticmethod
def get_featured_projects(preferred_locale):
""" Sets project as featured """
query = ProjectSearchService.create_search_query()
projects = query.filter(Project.featured == true()).group_by(Project.id).all()
# Get total contributors.
contrib_counts = ProjectSearchService.get_total_contributions(projects)
zip_items = zip(projects, contrib_counts)
dto = ProjectSearchResultsDTO()
dto.results = [
ProjectSearchService.create_result_dto(p, preferred_locale, t)
for p, t in zip_items
]
return dto
@staticmethod
def is_favorited(project_id: int, user_id: int) -> bool:
project = ProjectService.get_project_by_id(project_id)
return project.is_favorited(user_id)
@staticmethod
def favorite(project_id: int, user_id: int):
project = ProjectService.get_project_by_id(project_id)
project.favorite(user_id)
@staticmethod
def unfavorite(project_id: int, user_id: int):
project = ProjectService.get_project_by_id(project_id)
project.unfavorite(user_id)
@staticmethod
def get_project_title(project_id: int, preferred_locale: str = "en") -> str:
""" Gets the project title DTO """
project = ProjectService.get_project_by_id(project_id)
return project.get_project_title(preferred_locale)
@staticmethod
@cached(TTLCache(maxsize=1024, ttl=600))
def get_project_stats(project_id: int) -> ProjectStatsDTO:
""" Gets the project stats DTO """
project = ProjectService.get_project_by_id(project_id)
return project.get_project_stats()
@staticmethod
def get_project_user_stats(project_id: int, username: str) -> ProjectUserStatsDTO:
""" Gets the user stats for a specific project """
project = ProjectService.get_project_by_id(project_id)
user = UserService.get_user_by_username(username)
return project.get_project_user_stats(user.id)
def get_project_teams(project_id: int):
project = ProjectService.get_project_by_id(project_id)
if project is None:
raise NotFound()
return project.teams
@staticmethod
def get_project_organisation(project_id: int) -> Organisation:
project = ProjectService.get_project_by_id(project_id)
if project is None:
raise NotFound()
return project.organisation
from datetime import date, timedelta
from typing import List, Union
from cachetools import TTLCache
from cachetools.keys import hashkey
from dateutil.relativedelta import relativedelta, FR, SA, SU, TH, TU, WE, MO
from numpy import busday_offset
from pandas import Series, to_datetime, DataFrame
from gs_quant.api.gs.data import GsDataApi
from gs_quant.markets.securities import ExchangeCode
from gs_quant.target.common import Currency
DATE_LOW_LIMIT = date(1952, 1, 1)
DATE_HIGH_LIMIT = date(2052, 12, 31)
_cache = TTLCache(maxsize=128, ttl=600)
_logger = logging.getLogger(__name__)
class RDateRule(ABC):
result: date
number: int
week_mask: str
currencies: List[Union[Currency, str]] = None
exchanges: List[Union[ExchangeCode, str]] = None
holiday_calendar: List[date] = None
def __init__(self, result: date, **params):
self.result = result
self.number = params.get('number')
self.week_mask = params.get('week_mask')
from telegram import Chat, ChatMember, ParseMode, Update
from telegram.ext import CallbackContext
from LEGEND import (
DEL_CMDS,
DEMONS,
DEV_USERS,
DRAGONS,
SUPPORT_CHAT,
TIGERS,
WOLVES,
dispatcher,
)
# stores admemes in memory for 10 min.
ADMIN_CACHE = TTLCache(maxsize=512, ttl=60 * 10)
THREAD_LOCK = RLock()
def is_whitelist_plus(chat: Chat,
user_id: int,
member: ChatMember = None) -> bool:
return any(user_id in user
for user in [WOLVES, TIGERS, DEMONS, DRAGONS, DEV_USERS])
def is_support_plus(chat: Chat,
user_id: int,
member: ChatMember = None) -> bool:
return user_id in DEMONS or user_id in DRAGONS or user_id in DEV_USERS
class PairListManager():
def __init__(self, exchange, config: dict) -> None:
self._exchange = exchange
self._config = config
self._whitelist = self._config['exchange'].get('pair_whitelist')
self._blacklist = self._config['exchange'].get('pair_blacklist', [])
self._pairlist_handlers: List[IPairList] = []
self._tickers_needed = False
for pairlist_handler_config in self._config.get('pairlists', None):
if 'method' not in pairlist_handler_config:
logger.warning(f"No method found in {pairlist_handler_config}, ignoring.")
continue
pairlist_handler = PairListResolver.load_pairlist(
pairlist_handler_config['method'],
exchange=exchange,
pairlistmanager=self,
config=config,
pairlistconfig=pairlist_handler_config,
pairlist_pos=len(self._pairlist_handlers)
)
self._tickers_needed |= pairlist_handler.needstickers
self._pairlist_handlers.append(pairlist_handler)
if not self._pairlist_handlers:
raise OperationalException("No Pairlist Handlers defined")
@property
def whitelist(self) -> List[str]:
"""
Has the current whitelist
"""
return self._whitelist
@property
def blacklist(self) -> List[str]:
"""
Has the current blacklist
-> no need to overwrite in subclasses
"""
return self._blacklist
@property
def name_list(self) -> List[str]:
"""
Get list of loaded Pairlist Handler names
"""
return [p.name for p in self._pairlist_handlers]
def short_desc(self) -> List[Dict]:
"""
List of short_desc for each Pairlist Handler
"""
return [{p.name: p.short_desc()} for p in self._pairlist_handlers]
@cached(TTLCache(maxsize=1, ttl=1800))
def _get_cached_tickers(self):
return self._exchange.get_tickers()
def refresh_pairlist(self) -> None:
"""
Run pairlist through all configured Pairlist Handlers.
"""
# Tickers should be cached to avoid calling the exchange on each call.
tickers: Dict = {}
if self._tickers_needed:
tickers = self._get_cached_tickers()
# Adjust whitelist if filters are using tickers
pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
# Generate the pairlist with first Pairlist Handler in the chain
pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
# Process all Pairlist Handlers in the chain
for pairlist_handler in self._pairlist_handlers:
pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)
# Validation against blacklist happens after the chain of Pairlist Handlers
# to ensure blacklist is respected.
pairlist = self.verify_blacklist(pairlist, logger.warning)
self._whitelist = pairlist
def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]:
"""
Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
pairs that do not have ticker available
"""
if self._tickers_needed:
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
if p not in tickers:
pairlist.remove(p)
return pairlist
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
"""
Verify and remove items from pairlist - returning a filtered pairlist.
Logs a warning or info depending on `aswarning`.
Pairlist Handlers explicitly using this method shall use
`logmethod=logger.info` to avoid spamming with warning messages
:param pairlist: Pairlist to validate
:param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
:return: pairlist - blacklisted pairs
"""
for pair in deepcopy(pairlist):
if pair in self._blacklist:
logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...")
pairlist.remove(pair)
return pairlist
def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes:
"""
Create list of pair tuples with (pair, timeframe)
"""
return [(pair, timeframe or self._config['timeframe']) for pair in pairs]
return self.get(item)
def return_attr(func):
def wrapper(*args, **kwargs):
res = func(*args, **kwargs)
if isinstance(res, dict):
return AttributeDict(res)
return res
return wrapper
@return_attr
@check_chat
@cached(TTLCache(maxsize=1024, ttl=30))
def get_group_setting(group_id):
conn, cur = get_cur()
try:
query = f"""
select jointime_pin as jtp,
game_started_pin as stp,
is_confirm_tsww_enable as cts,
is_confirm_score_enable as cs,
is_startnewgame_enable as stn,
start_mode as stm,
disabled_by
from v1.all_group_helper
where group_id = %s
"""
cur.execute(query, (group_id, ))
from chatbot.utils import dbg, jsonr
from toolbox import warn, error, json, st
db.init_app(app)
load_bot_configs(app.config)
setup_caching(app.config)
slack_events_adapter = SlackEventAdapter(app.config['SLACK_SIGNING_SECRET'],
endpoint="/slack/event",
server=app)
slack_client = SlackClient(app.config['SLACK_BOT_TOKEN'])
# TODO: better home and configurable for production
SLACK_CONVO_CACHE_SIZE = 1000
SLACK_CONVO_CACHE_TTL = 60 * 60
slack_convo_cache = TTLCache(SLACK_CONVO_CACHE_SIZE, SLACK_CONVO_CACHE_TTL)
@slack_events_adapter.on("app_mention")
def slack_app_mention(event_data):
dbg('App mention!')
dbg(event_data)
@slack_events_adapter.on("message")
def slack_message(event_data):
dbg('Message!')
dbg(event_data)
event = event_data["event"]
slack_app_id = event_data["api_app_id"]
class Schism(IStrategy):
"""
Strategy Configuration Items
"""
timeframe = '5m'
inf_timeframe = '1h'
buy_params = {
'inf-pct-adr': 0.83534,
'inf-rsi': 57,
'mp': 64,
'rmi-fast': 49,
'rmi-slow': 24,
'xinf-stake-rmi': 45,
'xtf-fiat-rsi': 28,
'xtf-stake-rsi': 90
}
sell_params = {}
minimal_roi = {
"0": 0.05,
"10": 0.025,
"20": 0.015,
"30": 0.01,
"720": 0.005,
"1440": 0
}
stoploss = -0.30
# Recommended
use_sell_signal = False
sell_profit_only = False
ignore_roi_if_buy_signal = True
startup_candle_count: int = 72
# Startegy Specific Variable Storage
custom_trade_info = {}
custom_fiat = "USD"
custom_current_price_cache: TTLCache = TTLCache(maxsize=100,
ttl=300) # 5 minutes
"""
Informative Pair Definitions
"""
def informative_pairs(self):
# add existing pairs from whitelist on the inf_timeframe
pairs = self.dp.current_whitelist()
informative_pairs = [(pair, self.inf_timeframe) for pair in pairs]
# add additional informative pairs based on certain stakes
if self.config['stake_currency'] in ('BTC', 'ETH'):
for pair in pairs:
# add in the COIN/FIAT pairs (e.g. XLM/USD) on base timeframe
coin, stake = pair.split('/')
coin_fiat = f"{coin}/{self.custom_fiat}"
informative_pairs += [(coin_fiat, self.timeframe)]
# add in the STAKE/FIAT pair (e.g. BTC/USD) on base and inf timeframes
stake_fiat = f"{self.config['stake_currency']}/{self.custom_fiat}"
informative_pairs += [(stake_fiat, self.timeframe)]
informative_pairs += [(stake_fiat, self.inf_timeframe)]
return informative_pairs
"""
Indicator Definitions
"""
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
# Populate/update the trade data if there is any, set trades to false if not live/dry
self.custom_trade_info[metadata['pair']] = self.populate_trades(
metadata['pair'])
# Set up primary indicators
dataframe['rmi-slow'] = RMI(dataframe, length=21, mom=5)
dataframe['rmi-fast'] = RMI(dataframe, length=8, mom=4)
# Momentum Pinball
dataframe['roc'] = ta.ROC(dataframe, timeperiod=6)
dataframe['mp'] = ta.RSI(dataframe['roc'], timeperiod=6)
# Trend Calculations
dataframe['rmi-up'] = np.where(
dataframe['rmi-slow'] >= dataframe['rmi-slow'].shift(), 1, 0)
dataframe['rmi-dn'] = np.where(
dataframe['rmi-slow'] <= dataframe['rmi-slow'].shift(), 1, 0)
dataframe['rmi-up-trend'] = np.where(
dataframe['rmi-up'].rolling(3, min_periods=1).sum() >= 2, 1, 0)
dataframe['rmi-dn-trend'] = np.where(
dataframe['rmi-dn'].rolling(3, min_periods=1).sum() >= 2, 1, 0)
# Informative for STAKE/FIAT and COIN/FIAT on default timeframe, only relevant if stake currency is BTC or ETH
if self.config['stake_currency'] in ('BTC', 'ETH'):
coin, stake = metadata['pair'].split('/')
fiat = self.custom_fiat
coin_fiat = f"{coin}/{fiat}"
stake_fiat = f"{stake}/{fiat}"
# COIN/FIAT (e.g. XLM/USD) - timeframe
coin_fiat_tf = self.dp.get_pair_dataframe(pair=coin_fiat,
timeframe=self.timeframe)
dataframe[f"{fiat}_rsi"] = ta.RSI(coin_fiat_tf, timeperiod=14)
# STAKE/FIAT (e.g. BTC/USD) - inf_timeframe
stake_fiat_tf = self.dp.get_pair_dataframe(
pair=stake_fiat, timeframe=self.timeframe)
stake_fiat_inf_tf = self.dp.get_pair_dataframe(
pair=stake_fiat, timeframe=self.inf_timeframe)
dataframe[f"{stake}_rsi"] = ta.RSI(stake_fiat_tf, timeperiod=14)
dataframe[f"{stake}_rmi_{self.inf_timeframe}"] = RMI(
stake_fiat_inf_tf, length=21, mom=5)
# Informative indicators for current pair on inf_timeframe
informative = self.dp.get_pair_dataframe(pair=metadata['pair'],
timeframe=self.inf_timeframe)
informative['rsi'] = ta.RSI(informative, timeperiod=14)
informative['1d_high'] = informative['close'].rolling(24).max()
informative['3d_low'] = informative['close'].rolling(72).min()
informative['adr'] = informative['1d_high'] - informative['3d_low']
dataframe = merge_informative_pair(dataframe,
informative,
self.timeframe,
self.inf_timeframe,
ffill=True)
return dataframe
"""
Buy Trigger Signals
"""
def populate_buy_trend(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
params = self.get_pair_params(metadata['pair'], 'buy')
trade_data = self.custom_trade_info[metadata['pair']]
conditions = []
# Persist a buy signal for existing trades to make use of ignore_roi_if_buy_signal = True
# when this buy signal is not present a sell can happen according to the defined ROI table
if trade_data['active_trade']:
# peak_profit factor f(x)=1-x/400, rmi 30 -> 0.925, rmi 80 -> 0.80
profit_factor = (1 - (dataframe['rmi-slow'].iloc[-1] / 400))
# grow from 30 -> 70 after 720 minutes starting after 180 minutes
rmi_grow = self.linear_growth(30, 70, 180, 720,
trade_data['open_minutes'])
conditions.append(dataframe['rmi-up-trend'] == 1)
conditions.append(
trade_data['current_profit'] > (trade_data['peak_profit'] *
profit_factor))
conditions.append(dataframe['rmi-slow'] >= rmi_grow)
# Normal buy triggers that apply to new trades we want to enter
else:
# Primary buy triggers
conditions.append(
# "buy the dip" based buy signal using momentum pinball and downward RMI
(dataframe['close'] <=
dataframe[f"3d_low_{self.inf_timeframe}"] +
(params['inf-pct-adr'] *
dataframe[f"adr_{self.inf_timeframe}"]))
& (dataframe[f"rsi_{self.inf_timeframe}"] >= params['inf-rsi'])
& (dataframe['rmi-dn-trend'] == 1)
& (dataframe['rmi-slow'] >= params['rmi-slow'])
& (dataframe['rmi-fast'] <= params['rmi-fast'])
& (dataframe['mp'] <= params['mp']))
# If the stake is BTC or ETH apply additional conditions
if self.config['stake_currency'] in ('BTC', 'ETH'):
# default timeframe conditions
conditions.append((
dataframe[f"{self.config['stake_currency']}_rsi"] <
params['xtf-stake-rsi'])
| (dataframe[f"{self.custom_fiat}_rsi"] >
params['xtf-fiat-rsi']))
# informative timeframe conditions
conditions.append(dataframe[
f"{self.config['stake_currency']}_rmi_{self.inf_timeframe}"]
< params['xinf-stake-rmi'])
# Anything below here applies to persisting and new buy signal
conditions.append(dataframe['volume'].gt(0))
if conditions:
dataframe.loc[reduce(lambda x, y: x & y, conditions), 'buy'] = 1
return dataframe
"""
Sell Trigger Signals
"""
def populate_sell_trend(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
params = self.get_pair_params(metadata['pair'], 'sell')
trade_data = self.custom_trade_info[metadata['pair']]
conditions = []
# In this strategy all sells for profit happen according to ROI
# This sell signal is designed only as a "dynamic stoploss"
# if we are in an active trade for this pair
if trade_data['active_trade']:
# grow from -0.03 -> 0 after 300 minutes starting immediately
loss_cutoff = self.linear_growth(-0.03, 0, 0, 300,
trade_data['open_minutes'])
# if we are at a loss, consider what the trend looks and preempt the stoploss
conditions.append((trade_data['current_profit'] < loss_cutoff)
& (trade_data['current_profit'] > self.stoploss)
& (dataframe['rmi-dn-trend'] == 1)
& (dataframe['volume'].gt(0)))
# if the peak profit was positive at some point but never reached ROI, set a higher cross point for exit
if trade_data['peak_profit'] > 0:
conditions.append(
qtpylib.crossed_below(dataframe['rmi-slow'], 50))
# if the trade was always negative, the bounce we expected didn't happen
else:
conditions.append(
qtpylib.crossed_below(dataframe['rmi-slow'], 10))
# if there are other open trades in addition to this one, consider the average profit
# across them all and how many free slots we have in our sell decision
if trade_data['other_trades']:
if trade_data['free_slots'] > 0:
"""
Less free slots, more willing to sell
1 / free_slots * x =
1 slot = 1/1 * -0.04 = -0.04 -> only allow sells if avg_other_proift above -0.04
4 slot = 1/4 * -0.04 = -0.01 -> only allow sells is avg_other_profit above -0.01
"""
max_market_down = -0.04
hold_pct = (1 / trade_data['free_slots']) * max_market_down
conditions.append(
trade_data['avg_other_profit'] >= hold_pct)
else:
# if were out of slots, allow the biggest losing trade to sell regardless of avg profit
conditions.append(trade_data['biggest_loser'] == True)
# Impossible condition to satisfy the bot when it looks here and theres no active trade
else:
conditions.append(dataframe['volume'].lt(0))
if conditions:
dataframe.loc[reduce(lambda x, y: x & y, conditions), 'sell'] = 1
return dataframe
"""
Super Legit Custom Methods
"""
# Populate trades_data from the database
def populate_trades(self, pair: str) -> dict:
# Initialize the trades dict if it doesn't exist, persist it otherwise
if not pair in self.custom_trade_info:
self.custom_trade_info[pair] = {}
# init the temp dicts and set the trade stuff to false
trade_data = {}
trade_data['active_trade'] = trade_data['other_trades'] = trade_data[
'biggest_loser'] = False
self.custom_trade_info['meta'] = {}
# active trade stuff only works in live and dry, not backtest
if self.config['runmode'].value in ('live', 'dry_run'):
# find out if we have an open trade for this pair
active_trade = Trade.get_trades([
Trade.pair == pair,
Trade.is_open.is_(True),
]).all()
# if so, get some information
if active_trade:
# get current price and update the min/max rate
current_rate = self.get_current_price(pair, True)
active_trade[0].adjust_min_max_rates(current_rate)
# get how long the trade has been open in minutes and candles
present = arrow.utcnow()
trade_start = arrow.get(active_trade[0].open_date)
open_minutes = (present -
trade_start).total_seconds() // 60 # floor
# set up the things we use in the strategy
trade_data['active_trade'] = True
trade_data['current_profit'] = active_trade[
0].calc_profit_ratio(current_rate)
trade_data['peak_profit'] = max(
0, active_trade[0].calc_profit_ratio(
active_trade[0].max_rate))
trade_data['open_minutes']: int = open_minutes
trade_data['open_candles']: int = (
open_minutes // active_trade[0].timeframe) # floor
else:
trade_data['current_profit'] = trade_data['peak_profit'] = 0.0
trade_data['open_minutes'] = trade_data['open_candles'] = 0
# if there are open trades not including the current pair, get some information
# future reference, for *all* open trades: open_trades = Trade.get_open_trades()
other_trades = Trade.get_trades([
Trade.pair != pair,
Trade.is_open.is_(True),
]).all()
if other_trades:
trade_data['other_trades'] = True
other_profit = tuple(
trade.calc_profit_ratio(
self.get_current_price(trade.pair, False))
for trade in other_trades)
trade_data['avg_other_profit'] = mean(other_profit)
# find which of our trades is the biggest loser
if trade_data['current_profit'] < min(other_profit):
trade_data['biggest_loser'] = True
else:
trade_data['avg_other_profit'] = 0
# get the number of free trade slots, storing in every pairs dict due to laziness
open_trades = len(Trade.get_open_trades())
trade_data['free_slots'] = max(
0, self.config['max_open_trades'] - open_trades)
return trade_data
# Get the current price from the exchange (or cache)
def get_current_price(self, pair: str, refresh: bool) -> float:
if not refresh:
rate = self.custom_current_price_cache.get(pair)
# Check if cache has been invalidated
if rate:
return rate
ask_strategy = self.config.get('ask_strategy', {})
if ask_strategy.get('use_order_book', False):
ob = self.dp.orderbook(pair, 1)
rate = ob[f"{ask_strategy['price_side']}s"][0][0]
else:
ticker = self.dp.ticker(pair)
rate = ticker['last']
self.custom_current_price_cache[pair] = rate
return rate
# linear growth, starts at X and grows to Y after A minutes (starting after B miniutes)
# f(t) = X + (rate * t), where rate = (Y - X) / (A - B)
def linear_growth(self, start: float, end: float, start_time: int,
end_time: int, trade_time: int) -> float:
time = max(0, trade_time - start_time)
rate = (end - start) / (end_time - start_time)
return min(end, start + (rate * time))
"""
Allow for buy/sell override parameters per pair. Testing, might remove.
TODO:
If good: make this more robust so you never have to edit this method.
Consider: per-pair ROI if it seems worthwhile?
"""
def get_pair_params(self, pair: str, side: str) -> Dict:
buy_params = self.buy_params
sell_params = self.sell_params
### Stake: USD
if pair in ('ABC/XYZ', 'DEF/XYZ'):
buy_params = self.buy_params_GROUP1
sell_params = self.sell_params_GROUP1
elif pair in ('QRD/WTF'):
buy_params = self.buy_params_QRD
sell_params = self.sell_params_QRD
if side == 'sell':
return sell_params
return buy_params
"""
Price protection on trade entry and timeouts, built-in Freqtrade functionality
https://www.freqtrade.io/en/latest/strategy-advanced/
"""
def check_buy_timeout(self, pair: str, trade: Trade, order: dict,
**kwargs) -> bool:
bid_strategy = self.config.get('bid_strategy', {})
ob = self.dp.orderbook(pair, 1)
current_price = ob[f"{bid_strategy['price_side']}s"][0][0]
# Cancel buy order if price is more than 1% above the order.
if current_price > order['price'] * 1.01:
return True
return False
def check_sell_timeout(self, pair: str, trade: Trade, order: dict,
**kwargs) -> bool:
ask_strategy = self.config.get('ask_strategy', {})
ob = self.dp.orderbook(pair, 1)
current_price = ob[f"{ask_strategy['price_side']}s"][0][0]
# Cancel sell order if price is more than 1% below the order.
if current_price < order['price'] * 0.99:
return True
return False
def confirm_trade_entry(self, pair: str, order_type: str, amount: float,
rate: float, time_in_force: str, **kwargs) -> bool:
bid_strategy = self.config.get('bid_strategy', {})
ob = self.dp.orderbook(pair, 1)
current_price = ob[f"{bid_strategy['price_side']}s"][0][0]
# Cancel buy order if price is more than 1% above the order.
if current_price > rate * 1.01:
return False
return True
from SaitamaRobot import (
DEL_CMDS,
DEV_USERS,
DRAGONS,
SUPPORT_CHAT,
DEMONS,
TIGERS,
WOLVES,
dispatcher,
)
from telegram import Chat, ChatMember, ParseMode, Update
from telegram.ext import CallbackContext
# stores admemes in memory for 10 min.
ADMIN_CACHE = TTLCache(maxsize=512, ttl=60 * 10, timer=perf_counter)
THREAD_LOCK = RLock()
def is_whitelist_plus(chat: Chat, user_id: int, member: ChatMember = None) -> bool:
return any(user_id in user for user in [WOLVES, TIGERS, DEMONS, DRAGONS, DEV_USERS])
def is_support_plus(chat: Chat, user_id: int, member: ChatMember = None) -> bool:
return user_id in DEMONS or user_id in DRAGONS or user_id in DEV_USERS
def is_sudo_plus(chat: Chat, user_id: int, member: ChatMember = None) -> bool:
return user_id in DRAGONS or user_id in DEV_USERS
class GsDataApi(DataApi):
__definitions = {}
__asset_coordinates_cache = TTLCache(10000, 86400)
DEFAULT_SCROLL = '30s'
# DataApi interface
@classmethod
def query_data(cls, query: Union[DataQuery, MDAPIDataQuery], dataset_id: str = None,
asset_id_type: Union[GsIdType, str] = None) \
-> Union[MDAPIDataBatchResponse, DataQueryResponse, tuple]:
if isinstance(query, MDAPIDataQuery) and query.market_data_coordinates:
# Don't use MDAPIDataBatchResponse for now - it doesn't handle quoting style correctly
results: Union[MDAPIDataBatchResponse,
dict] = cls.execute_query('coordinates', query)
if isinstance(results, dict):
return results.get('responses', ())
else:
return results.responses if results.responses is not None else (
)
elif isinstance(query, DataQuery) and query.where:
where = query.where.as_dict() if isinstance(
query.where, FieldFilterMap) else query.where
xref_keys = set(where.keys()).intersection(XRef.properties())
if xref_keys:
# Check that assetId is a symbol dimension of this data set. If not, we need to do a separate query
# to resolve xref pip install dtaidistance--> assetId
if len(xref_keys) > 1:
raise MqValueError(
'Cannot not specify more than one type of asset identifier'
)
definition = cls.get_definition(dataset_id)
sd = definition.dimensions.symbolDimensions
if definition.parameters.symbolStrategy == 'MDAPI' or (
'assetId' not in sd and 'gsid' not in sd):
xref_type = min(xref_keys)
if asset_id_type is None:
asset_id_type = xref_type
xref_values = where[asset_id_type]
xref_values = (xref_values, ) if isinstance(
xref_values, str) else xref_values
asset_id_map = GsAssetApi.map_identifiers(
xref_type, GsIdType.id, xref_values)
if len(asset_id_map) != len(xref_values):
raise MqValueError(
'Not all {} were resolved to asset Ids'.format(
asset_id_type))
setattr(query.where, xref_type, None)
query.where.assetId = [
asset_id_map[x] for x in xref_values
]
response: Union[DataQueryResponse,
dict] = cls.execute_query(dataset_id, query)
results = cls.get_results(dataset_id, response, query)
if asset_id_type not in {GsIdType.id, None}:
asset_ids = tuple(
set(filter(None, (r.get('assetId') for r in results))))
if asset_ids:
xref_map = GsAssetApi.map_identifiers(GsIdType.id,
asset_id_type, asset_ids)
if len(xref_map) != len(asset_ids):
raise MqValueError(
'Not all asset Ids were resolved to {}'.format(
asset_id_type))
for result in results:
result[asset_id_type] = xref_map[result['assetId']]
return results
@staticmethod
def execute_query(dataset_id: str, query: Union[DataQuery,
MDAPIDataQuery]):
return GsSession.current._post('/data/{}/query'.format(dataset_id),
payload=query)
@staticmethod
def get_results(dataset_id: str, response: Union[DataQueryResponse, dict],
query: DataQuery) -> list:
if isinstance(response, dict):
total_pages = response.get('totalPages')
results = response.get('data', ())
else:
total_pages = response.total_pages if response.total_pages is not None else 0
results = response.data if response.data is not None else ()
if total_pages:
if query.page is None:
query.page = total_pages - 1
results = results + GsDataApi.get_results(
dataset_id, GsDataApi.execute_query(dataset_id, query),
query)
elif query.page - 1 > 0:
query.page -= 1
results = results + GsDataApi.get_results(
dataset_id, GsDataApi.execute_query(dataset_id, query),
query)
else:
return results
return results
@classmethod
def last_data(cls,
query: Union[DataQuery, MDAPIDataQuery],
dataset_id: str = None) -> Union[list, tuple]:
if getattr(query, 'marketDataCoordinates', None):
result = GsSession.current._post('/data/coordinates/query/last',
payload=query)
return result.get('responses', ())
else:
result = GsSession.current._post(
'/data/{}/last/query'.format(dataset_id), payload=query)
return result.get('data', ())
@classmethod
def symbol_dimensions(cls, dataset_id: str) -> tuple:
definition = cls.get_definition(dataset_id)
return definition.dimensions.symbolDimensions
@classmethod
def time_field(cls, dataset_id: str) -> str:
definition = cls.get_definition(dataset_id)
return definition.dimensions.timeField
# GS-specific functionality
@classmethod
def get_coverage(cls,
dataset_id: str,
scroll: str = DEFAULT_SCROLL,
scroll_id: Optional[str] = None,
limit: int = None,
offset: int = None,
fields: List[str] = None,
include_history: bool = False) -> List[dict]:
params = {}
if scroll:
params['scroll'] = scroll
if scroll_id:
params['scrollId'] = scroll_id
if not limit:
limit = 4000
params['limit'] = limit
if offset:
params['offset'] = offset
if fields:
params['fields'] = fields
if include_history:
params['includeHistory'] = 'true'
body = GsSession.current._get('/data/{}/coverage'.format(dataset_id),
payload=params)
results = body['results']
if len(results) > 0 and 'scrollId' in body:
return results + cls.get_coverage(dataset_id,
scroll_id=body['scrollId'],
scroll=GsDataApi.DEFAULT_SCROLL,
limit=limit)
else:
return results
@classmethod
def create(cls, definition: Union[DataSetEntity, dict]) -> DataSetEntity:
result = GsSession.current._post('/data/datasets', payload=definition)
return result
@classmethod
def update_definition(
cls, dataset_id: str, definition: Union[DataSetEntity,
dict]) -> DataSetEntity:
result = GsSession.current._put('/data/datasets/{}'.format(dataset_id),
payload=definition,
cls=DataSetEntity)
return result
@classmethod
def upload_data(cls, dataset_id: str, data: Union[pd.DataFrame, list,
tuple]) -> dict:
result = GsSession.current._post('/data/{}'.format(dataset_id),
payload=data)
return result
@classmethod
def get_definition(cls, dataset_id: str) -> DataSetEntity:
definition = cls.__definitions.get(dataset_id)
if not definition:
definition = GsSession.current._get(
'/data/datasets/{}'.format(dataset_id), cls=DataSetEntity)
if not definition:
raise MqValueError('Unknown dataset {}'.format(dataset_id))
cls.__definitions[dataset_id] = definition
return definition
@classmethod
def get_many_definitions(
cls,
limit: int = 100,
dataset_id: str = None,
owner_id: str = None,
name: str = None,
mq_symbol: str = None) -> Tuple[DataSetEntity, ...]:
query_string = urlencode(
dict(
filter(
lambda item: item[1] is not None,
dict(id=dataset_id,
ownerId=owner_id,
name=name,
mqSymbol=mq_symbol,
limit=limit).items())))
res = GsSession.current._get(
'/data/datasets?{query}'.format(query=query_string),
cls=DataSetEntity)['results']
return res
@classmethod
@cachetools.cached(__asset_coordinates_cache)
def get_many_coordinates(
cls,
mkt_type: str = None,
mkt_asset: str = None,
mkt_class: str = None,
mkt_point: Tuple[str, ...] = (),
*,
limit: int = 100,
return_type: type = str,
) -> Union[Tuple[str, ...], Tuple[MarketDataCoordinate, ...]]:
where = FieldFilterMap(
mkt_type=mkt_type.upper() if mkt_type is not None else None,
mkt_asset=mkt_asset.upper() if mkt_asset is not None else None,
mkt_class=mkt_class.upper() if mkt_class is not None else None,
)
for index, point in enumerate(mkt_point):
setattr(where, 'mkt_point' + str(index + 1), point.upper())
query = EntityQuery(where=where, limit=limit)
results = GsSession.current._post('/data/mdapi/query',
query)['results']
if return_type is str:
return tuple(coordinate['name'] for coordinate in results)
elif return_type is MarketDataCoordinate:
return tuple(
MarketDataCoordinate(
mkt_type=coordinate['dimensions']['mktType'],
mkt_asset=coordinate['dimensions']['mktAsset'],
mkt_class=coordinate['dimensions']['mktClass'],
mkt_point=tuple(coordinate['dimensions']
['mktPoint'].values()),
mkt_quoting_style=coordinate['dimensions']
['mktQuotingStyle']) for coordinate in results)
else:
raise NotImplementedError('Unsupported return type')
@staticmethod
def build_market_data_query(asset_ids: List[str],
query_type: QueryType,
where: Union[FieldFilterMap, Dict] = None,
source: Union[str] = None,
real_time: bool = False):
inner = {
'assetIds': asset_ids,
'queryType': query_type.value,
'where': where or {},
'source': source or 'any',
'frequency': 'Real Time' if real_time else 'End Of Day',
'measures': ['Curve']
}
if real_time:
inner['startTime'] = DataContext.current.start_time
inner['endTime'] = DataContext.current.end_time
else:
inner['startDate'] = DataContext.current.start_date
inner['endDate'] = DataContext.current.end_date
return {'queries': [inner]}
@classmethod
def get_data_providers(cls, entity_id: str) -> Dict:
"""Return daily and real-time data providers
:param entity_id: identifier of entity i.e. asset, country, subdivision
:return: dictionary of available data providers
** Usage **
Return a dictionary containing a set of dataset providers for each available data field.
For each field will return a dict of daily and real-time dataset providers where available.
"""
GsSession.current: GsSession
body = GsSession.current._get(
f'/data/measures/{entity_id}/availability')
if 'errorMessages' in body:
raise MqValueError(
f"data availablity request {body['requestId']} failed: {body.get('errorMessages', '')}"
)
if 'data' not in body:
providers = {}
else:
providers = {}
all_data_mappings = sorted(body['data'],
key=lambda x: x['rank'],
reverse=True)
for source in all_data_mappings:
freq = source.get('frequency', 'End Of Day')
dataset_field = source.get('datasetField', '')
rank = source.get('rank')
providers.setdefault(dataset_field, {})
if rank:
if freq == 'End Of Day':
providers[dataset_field][
DataFrequency.DAILY] = source['datasetId']
elif freq == 'Real Time':
providers[dataset_field][
DataFrequency.REAL_TIME] = source['datasetId']
return providers
@classmethod
def get_market_data(cls, query) -> pd.DataFrame:
GsSession.current: GsSession
body = GsSession.current._post('/data/markets', payload=query)
container = body['responses'][0]['queryResponse'][0]
if 'errorMessages' in container:
raise MqValueError(
f"market data request {body['requestId']} failed: {container['errorMessages']}"
)
if 'response' not in container:
df = MarketDataResponseFrame()
else:
df = MarketDataResponseFrame(container['response']['data'])
df.set_index('date' if 'date' in df.columns else 'time',
inplace=True)
df.index = pd.to_datetime(df.index)
df.dataset_ids = tuple(container.get('dataSetIds', ()))
return df
@classmethod
def __normalise_coordinate_data(
cls,
data: Iterable[Union[MDAPIDataQueryResponse, Dict]],
fields: Optional[Tuple[MDAPIQueryField, ...]] = None
) -> Iterable[Iterable[Dict]]:
ret = []
for response in data:
coord_data = []
rows = (r.as_dict() for r in response.data) if isinstance(
response, MDAPIDataQueryResponse) else response.get(
'data', ())
for pt in rows:
if not pt:
continue
if not fields and 'value' not in pt:
value_field = pt['mktQuotingStyle']
pt['value'] = pt.pop(value_field)
coord_data.append(pt)
ret.append(coord_data)
return ret
@classmethod
def __df_from_coordinate_data(
cls,
data: Iterable[Dict],
*,
use_datetime_index: Optional[bool] = True) -> pd.DataFrame:
df = cls._sort_coordinate_data(pd.DataFrame.from_records(data))
index_field = next((f for f in ('time', 'date') if f in df.columns),
None)
if index_field and use_datetime_index:
df = df.set_index(pd.DatetimeIndex(df.loc[:, index_field].values))
return df
@classmethod
def _sort_coordinate_data(
cls,
df: pd.DataFrame,
by: Tuple[str] = ('date', 'time', 'mktType', 'mktAsset', 'mktClass',
'mktPoint', 'mktQuotingStyle', 'value')
) -> pd.DataFrame:
columns = df.columns
field_order = [f for f in by if f in columns]
field_order.extend(f for f in columns if f not in field_order)
return df[field_order]
@classmethod
def _coordinate_from_str(cls, coordinate_str: str) -> MarketDataCoordinate:
tmp = coordinate_str.rsplit(".", 1)
dimensions = tmp[0].split("_")
if len(dimensions) < 2:
raise MqValueError('invalid coordinate ' + coordinate_str)
kwargs = {
'mkt_type': dimensions[0],
'mkt_asset': dimensions[1] or None,
'mkt_quoting_style': tmp[-1] if len(tmp) > 1 else None
}
if len(dimensions) > 2:
kwargs['mkt_class'] = dimensions[2] or None
if len(dimensions) > 3:
kwargs['mkt_point'] = tuple(dimensions[3:]) or None
return MarketDataCoordinate(**kwargs)
@classmethod
def coordinates_last(
cls,
coordinates: Union[Iterable[str], Iterable[MarketDataCoordinate]],
as_of: Union[dt.datetime, dt.date] = None,
vendor: MarketDataVendor = MarketDataVendor.Goldman_Sachs,
as_dataframe: bool = False,
pricing_location: Optional[PricingLocation] = None
) -> Union[Dict, pd.DataFrame]:
"""
Get last value of coordinates data
:param coordinates: market data coordinate(s)
:param as_of: snapshot date or time
:param vendor: data vendor
:param as_dataframe: whether to return the result as Dataframe
:param pricing_location: the location where close data has been recorded (not used for real-time query)
:return: Dataframe or dictionary of the returned data
**Examples**
>>> coordinate = ("FX Fwd_USD/EUR_Fwd Pt_2y",)
>>> data = GsDataApi.coordinates_last(coordinate, dt.datetime(2019, 11, 19))
"""
market_data_coordinates = tuple(
cls._coordinate_from_str(coord) if isinstance(coord, str
) else coord
for coord in coordinates)
query = cls.build_query(
end=as_of,
market_data_coordinates=market_data_coordinates,
vendor=vendor,
pricing_location=pricing_location)
data = cls.last_data(query)
if not as_dataframe:
ret = {coordinate: None for coordinate in market_data_coordinates}
for idx, row in enumerate(cls.__normalise_coordinate_data(data)):
try:
ret[market_data_coordinates[idx]] = row[0]['value']
except IndexError:
ret[market_data_coordinates[idx]] = None
return ret
ret = []
for idx, row in enumerate(cls.__normalise_coordinate_data(data)):
coordinate_as_dict = market_data_coordinates[idx].as_dict(
as_camel_case=True)
try:
ret.append(
dict(
chain(coordinate_as_dict.items(),
(('value', row[0]['value']),
('time', row[0]['time'])))))
except IndexError:
ret.append(
dict(
chain(coordinate_as_dict.items(),
(('value', None), ('time', None)))))
return cls.__df_from_coordinate_data(ret, use_datetime_index=False)
@classmethod
def coordinates_data(
cls,
coordinates: Union[str, MarketDataCoordinate, Iterable[str],
Iterable[MarketDataCoordinate]],
start: Union[dt.datetime, dt.date] = None,
end: Union[dt.datetime, dt.date] = None,
vendor: MarketDataVendor = MarketDataVendor.Goldman_Sachs,
as_multiple_dataframes: bool = False,
pricing_location: Optional[PricingLocation] = None,
fields: Optional[Tuple[MDAPIQueryField, ...]] = None,
**kwargs) -> Union[pd.DataFrame, Tuple[pd.DataFrame]]:
"""
Get coordinates data
:param coordinates: market data coordinate(s)
:param start: start date or time
:param end: end date or time
:param vendor: data vendor
:param as_multiple_dataframes: whether to return the result as one or multiple Dataframe(s)
:param pricing_location: the location where close data has been recorded (not used for real-time query)
:param fields: value fields to return
:param kwargs: Extra query arguments
:return: Dataframe(s) of the returned data
**Examples**
>>> coordinate = ("FX Fwd_USD/EUR_Fwd Pt_2y",)
>>> data = GsDataApi.coordinates_data(coordinate, dt.datetime(2019, 11, 18), dt.datetime(2019, 11, 19))
"""
coordinates_iterable = (coordinates, ) if isinstance(
coordinates, (MarketDataCoordinate, str)) else coordinates
query = cls.build_query(market_data_coordinates=tuple(
cls._coordinate_from_str(coord) if isinstance(coord, str
) else coord
for coord in coordinates_iterable),
vendor=vendor,
start=start,
end=end,
pricing_location=pricing_location,
fields=fields,
**kwargs)
results = cls.__normalise_coordinate_data(cls.query_data(query),
fields=fields)
if as_multiple_dataframes:
return tuple(
GsDataApi.__df_from_coordinate_data(r) for r in results)
else:
return cls.__df_from_coordinate_data(chain.from_iterable(results))
@classmethod
def coordinates_data_series(
cls,
coordinates: Union[str, MarketDataCoordinate, Iterable[str],
Iterable[MarketDataCoordinate]],
start: Union[dt.datetime, dt.date] = None,
end: Union[dt.datetime, dt.date] = None,
vendor: MarketDataVendor = MarketDataVendor.Goldman_Sachs,
pricing_location: Optional[PricingLocation] = None,
**kwargs) -> Union[pd.Series, Tuple[pd.Series]]:
"""
Get coordinates data series
:param coordinates: market data coordinate(s)
:param start: start date or time
:param end: end date or time
:param vendor: data vendor
:param pricing_location: the location where close data has been recorded (not used for real-time query)
:param kwargs: Extra query arguments
:return: Series of the returned data
**Examples**
>>> coordinate = ("FX Fwd_USD/EUR_Fwd Pt_2y",)
>>> data = GsDataApi.coordinates_data_series(coordinate, dt.datetime(2019, 11, 18), dt.datetime(2019, 11, 19))
"""
dfs = cls.coordinates_data(coordinates,
start=start,
end=end,
pricing_location=pricing_location,
vendor=vendor,
as_multiple_dataframes=True,
**kwargs)
ret = tuple(pd.Series() if df.empty else pd.
Series(index=df.index, data=df.value.values) for df in dfs)
if isinstance(coordinates, (MarketDataCoordinate, str)):
return ret[0]
else:
return ret
@staticmethod
@cachetools.cached(TTLCache(ttl=3600, maxsize=128))
def get_types(dataset_id: str):
results = GsSession.current._get(f'/data/catalog/{dataset_id}')
fields = results.get("fields")
if fields:
field_types = {}
for key, value in fields.items():
field_type = value.get('type')
field_format = value.get('format')
field_types[key] = field_format or field_type
return field_types
raise RuntimeError(f"Unable to get Dataset schema for {dataset_id}")
@classmethod
def construct_dataframe_with_types(
cls, dataset_id: str, data: Union[Base, List,
Tuple]) -> pd.DataFrame:
"""
Constructs a dataframe with correct date types.
:param dataset_id: id of the dataset
:param data: data to convert with correct types
:return: dataframe with correct types
"""
if len(data):
dataset_types = cls.get_types(dataset_id)
df = pd.DataFrame(data)
for field_name, type_name in dataset_types.items():
if df.get(field_name) is not None and type_name in (
'date', 'date-time'):
df = df.astype({field_name: numpy.datetime64})
field_names = dataset_types.keys()
if 'date' in field_names:
df = df.set_index('date')
elif 'time' in field_names:
df = df.set_index('time')
return df
else:
return pd.DataFrame({})
import requests
import csv
from cachetools import cached, TTLCache
from app.utils import date as date_util
from . import countrycodes as cc
"""
Base URL for fetching data.
"""
base_url = 'https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-%s.csv'
@cached(cache=TTLCache(maxsize=1024, ttl=3600))
def get_data(category):
"""
Retrieves the data for the provided type. The data is cached for 1 hour.
"""
# Adhere to category naming standard.
category = category.lower().capitalize()
# Request the data
request = requests.get(base_url % category)
text = request.text
# Parse the CSV.
data = list(csv.DictReader(text.splitlines()))
# The normalized locations.
locations = []
specific language governing permissions and limitations under the License.
"""
from cachetools import cached, TTLCache
from cachetools.keys import hashkey
from django.utils.translation import ugettext_lazy as _
from pipeline.component_framework.models import ComponentModel
from plugin_service.plugin_client import PluginServiceApiClient
from plugin_service import env
from gcloud.analysis_statistics.models import TemplateNodeStatistics
@cached(cache=TTLCache(maxsize=1024, ttl=60), key=hashkey)
def get_remote_plugin_name(limit=100, offset=0):
"""
@summary: 拉取第三方插件名
@param: limit: 拉取第三方插件数据默认limit，默认为100
@param: offset: 拉取第三方插件分页起点，默认为0
return plugin_info:dict() 第三方插件信息
"""
TOTAL = limit
CUR_TOTAL = 0
plugin_info = {}
if not env.USE_PLUGIN_SERVICE == "1":
return plugin_info
while CUR_TOTAL < TOTAL:
result = PluginServiceApiClient.get_paas_plugin_info(
import pandas as pd
import streamlit as st
import urllib.request, json
import area as ar
from cachetools import cached, TTLCache
plz_name = None
cases = None
# Streamlit cache doesn't work here
@cached(cache=TTLCache(maxsize=1, ttl=60 * 60 * 1))
def load_data():
#PLZ to Name Mapping
plz_name = dict({})
plz_area = dict({})
districts = pd.read_csv('Districts.tsv', header=None, sep='\t')
plz_district = dict([(str(i), str(v))
for i, v in zip(districts[0], districts[2])])
with urllib.request.urlopen(
"https://raw.githubusercontent.com/openZH/covid_19/master/fallzahlen_plz/PLZ_gen_epsg4326_F_KTZH_2020.json"
) as url:
data = json.loads(url.read().decode())
for feature in data['features']:
plz = (feature['properties']['PLZ'])
name = feature['properties']['Ortschaftsname']
area = ar.area(feature['geometry']) / 1e6
if plz is not None and name is not None:
plz_str = str(int(plz))
plz_name[plz_str] = str(name)
def __init__(self, ttl):
self.ttl = ttl
self.values = TTLCache(maxsize=1024, ttl=ttl / 1000)
class BinanceAPIManager:
def __init__(self, config: Config, db: Database, logger: Logger):
# initializing the client class calls `ping` API endpoint, verifying the connection
self.binance_client = Client(
config.BINANCE_API_KEY,
config.BINANCE_API_SECRET_KEY,
tld=config.BINANCE_TLD,
)
self.db = db
self.logger = logger
self.config = config
self.cache = BinanceCache()
self.stream_manager: Optional[BinanceStreamManager] = None
self.setup_websockets()
def setup_websockets(self):
self.stream_manager = BinanceStreamManager(
self.cache,
self.config,
self.binance_client,
self.logger,
)
@cached(cache=TTLCache(maxsize=1, ttl=43200))
def get_trade_fees(self) -> Dict[str, float]:
return {
ticker["symbol"]: ticker["taker"]
for ticker in self.binance_client.get_trade_fee()["tradeFee"]
}
@cached(cache=TTLCache(maxsize=1, ttl=60))
def get_using_bnb_for_fees(self):
return self.binance_client.get_bnb_burn_spot_margin()["spotBNBBurn"]
def get_fee(self, origin_coin: Coin, target_coin: Coin, selling: bool):
fees = self.get_trade_fees()
if not fees:
base_fee = 0.001
else:
base_fee = fees[origin_coin + target_coin]
if not self.get_using_bnb_for_fees():
return base_fee
# The discount is only applied if we have enough BNB to cover the fee
amount_trading = (self._sell_quantity(origin_coin.symbol,
target_coin.symbol)
if selling else self._buy_quantity(
origin_coin.symbol, target_coin.symbol))
fee_amount = amount_trading * base_fee * 0.75
if origin_coin.symbol == "BNB":
fee_amount_bnb = fee_amount
else:
origin_price = self.get_ticker_price(origin_coin + Coin("BNB"))
if origin_price is None:
return base_fee
fee_amount_bnb = fee_amount * origin_price
bnb_balance = self.get_currency_balance("BNB")
if bnb_balance >= fee_amount_bnb:
return base_fee * 0.75
return base_fee
def get_account(self):
"""
Get account information
"""
return self.binance_client.get_account()
def get_ticker_price(self, ticker_symbol: str):
"""
Get ticker price of a specific coin
"""
price = self.cache.ticker_values.get(ticker_symbol, None)
if price is None and ticker_symbol not in self.cache.non_existent_tickers:
self.cache.ticker_values = {
ticker["symbol"]: float(ticker["price"])
for ticker in self.binance_client.get_symbol_ticker()
}
self.logger.debug(
f"Fetched all ticker prices: {self.cache.ticker_values}")
price = self.cache.ticker_values.get(ticker_symbol, None)
if price is None:
self.logger.debug(
f"Ticker does not exist: {ticker_symbol} - will not be fetched from now on"
)
self.cache.non_existent_tickers.add(ticker_symbol)
return price
def get_currency_balance(self, currency_symbol: str, force=False) -> float:
"""
Get balance of a specific coin
"""
with self.cache.open_balances() as cache_balances:
balance = cache_balances.get(currency_symbol, None)
if force or balance is None:
cache_balances.clear()
cache_balances.update({
currency_balance["asset"]: float(currency_balance["free"])
for currency_balance in self.binance_client.get_account()
["balances"]
})
self.logger.debug(f"Fetched all balances: {cache_balances}")
if currency_symbol not in cache_balances:
cache_balances[currency_symbol] = 0.0
return 0.0
return cache_balances.get(currency_symbol, 0.0)
return balance
def retry(self, func, *args, **kwargs):
time.sleep(1)
attempts = 0
while attempts < 20:
try:
return func(*args, **kwargs)
except Exception: # pylint: disable=broad-except
self.logger.warning(
f"Failed to Buy/Sell. Trying Again (attempt {attempts}/20)"
)
if attempts == 0:
self.logger.warning(traceback.format_exc())
attempts += 1
return None
def get_symbol_filter(self, origin_symbol: str, target_symbol: str,
filter_type: str):
return next(_filter for _filter in self.binance_client.get_symbol_info(
origin_symbol + target_symbol)["filters"]
if _filter["filterType"] == filter_type)
@cached(cache=TTLCache(maxsize=2000, ttl=43200))
def get_alt_tick(self, origin_symbol: str, target_symbol: str):
step_size = self.get_symbol_filter(origin_symbol, target_symbol,
"LOT_SIZE")["stepSize"]
if step_size.find("1") == 0:
return 1 - step_size.find(".")
return step_size.find("1") - 1
@cached(cache=TTLCache(maxsize=2000, ttl=43200))
def get_min_notional(self, origin_symbol: str, target_symbol: str):
return float(
self.get_symbol_filter(origin_symbol, target_symbol,
"MIN_NOTIONAL")["minNotional"])
def _wait_for_order(
self, order_id, origin_symbol: str, target_symbol: str
) -> Optional[BinanceOrder]: # pylint: disable=unsubscriptable-object
while True:
order_status: BinanceOrder = self.cache.orders.get(order_id, None)
if order_status is not None:
break
self.logger.debug(f"Waiting for order {order_id} to be created")
time.sleep(1)
self.logger.debug(f"Order created: {order_status}")
while order_status.status != "FILLED":
try:
order_status = self.cache.orders.get(order_id, order_status)
self.logger.debug(f"Waiting for order {order_id} to be filled")
if self._should_cancel_order(order_status):
cancel_order = None
while cancel_order is None:
cancel_order = self.binance_client.cancel_order(
symbol=origin_symbol + target_symbol,
orderId=order_id)
self.logger.info("Order timeout, canceled...")
# sell partially
if order_status.status == "PARTIALLY_FILLED" and order_status.side == "BUY":
self.logger.info("Sell partially filled amount")
order_quantity = self._sell_quantity(
origin_symbol, target_symbol)
partially_order = None
while partially_order is None:
partially_order = self.binance_client.order_market_sell(
symbol=origin_symbol + target_symbol,
quantity=order_quantity)
self.logger.info("Going back to scouting mode...")
return None
if order_status.status == "CANCELED":
self.logger.info(
"Order is canceled, going back to scouting mode...")
return None
time.sleep(1)
except BinanceAPIException as e:
self.logger.info(e)
time.sleep(1)
except Exception as e: # pylint: disable=broad-except
self.logger.info(f"Unexpected order error: {e}")
time.sleep(1)
self.logger.debug(f"Order filled: {order_status}")
return order_status
def wait_for_order(
self, order_id, origin_symbol: str, target_symbol: str,
order_guard: OrderGuard
) -> Optional[BinanceOrder]: # pylint: disable=unsubscriptable-object
with order_guard:
return self._wait_for_order(order_id, origin_symbol, target_symbol)
def _should_cancel_order(self, order_status):
minutes = (time.time() - order_status.time / 1000) / 60
timeout = 0
if order_status.side == "SELL":
timeout = float(self.config.SELL_TIMEOUT)
else:
timeout = float(self.config.BUY_TIMEOUT)
if timeout and minutes > timeout and order_status.status == "NEW":
return True
if timeout and minutes > timeout and order_status.status == "PARTIALLY_FILLED":
if order_status.side == "SELL":
return True
if order_status.side == "BUY":
current_price = self.get_ticker_price(order_status.symbol)
if float(current_price) * (1 - 0.001) > float(
order_status.price):
return True
return False
def buy_alt(self, origin_coin: Coin, target_coin: Coin) -> BinanceOrder:
return self.retry(self._buy_alt, origin_coin, target_coin)
def _buy_quantity(self,
origin_symbol: str,
target_symbol: str,
target_balance: float = None,
from_coin_price: float = None):
target_balance = target_balance or self.get_currency_balance(
target_symbol)
from_coin_price = from_coin_price or self.get_ticker_price(
origin_symbol + target_symbol)
origin_tick = self.get_alt_tick(origin_symbol, target_symbol)
return math.floor(target_balance * 10**origin_tick /
from_coin_price) / float(10**origin_tick)
def _buy_alt(self, origin_coin: Coin, target_coin: Coin):
"""
Buy altcoin
"""
trade_log = self.db.start_trade_log(origin_coin, target_coin, False)
origin_symbol = origin_coin.symbol
target_symbol = target_coin.symbol
with self.cache.open_balances() as balances:
balances.clear()
origin_balance = self.get_currency_balance(origin_symbol)
target_balance = self.get_currency_balance(target_symbol)
from_coin_price = self.get_ticker_price(origin_symbol + target_symbol)
order_quantity = self._buy_quantity(origin_symbol, target_symbol,
target_balance, from_coin_price)
self.logger.info(f"Buying roughly {order_quantity} {origin_symbol}")
# Try to buy until successful
order = None
order_guard = self.stream_manager.acquire_order_guard()
while order is None:
try:
order = self.binance_client.order_limit_buy(
symbol=origin_symbol + target_symbol,
quantity=order_quantity,
price=from_coin_price,
)
self.logger.debug(order)
except BinanceAPIException as e:
self.logger.info(e)
time.sleep(1)
except Exception as e: # pylint: disable=broad-except
self.logger.warning(f"Unexpected Error: {e}")
orderId = order["orderId"]
self.logger.info(
f"Placed buy order {orderId}, waiting for it to complete")
trade_log.set_ordered(origin_balance, target_balance, order_quantity)
order_guard.set_order(origin_symbol, target_symbol, int(orderId))
order = self.wait_for_order(orderId, origin_symbol, target_symbol,
order_guard)
if order is None:
return None
newBalance = self.get_currency_balance(origin_symbol)
self.logger.info(f"Bought {newBalance} {origin_symbol}")
trade_log.set_complete(order.cumulative_quote_qty)
return order
def sell_alt(self, origin_coin: Coin, target_coin: Coin) -> BinanceOrder:
return self.retry(self._sell_alt, origin_coin, target_coin)
def _sell_quantity(self,
origin_symbol: str,
target_symbol: str,
origin_balance: float = None):
origin_balance = origin_balance or self.get_currency_balance(
origin_symbol)
origin_tick = self.get_alt_tick(origin_symbol, target_symbol)
return math.floor(origin_balance * 10**origin_tick) / float(
10**origin_tick)
def _sell_alt(self, origin_coin: Coin, target_coin: Coin):
"""
Sell altcoin
"""
trade_log = self.db.start_trade_log(origin_coin, target_coin, True)
origin_symbol = origin_coin.symbol
target_symbol = target_coin.symbol
with self.cache.open_balances() as balances:
balances.clear()
origin_balance = self.get_currency_balance(origin_symbol)
target_balance = self.get_currency_balance(target_symbol)
from_coin_price = self.get_ticker_price(origin_symbol + target_symbol)
order_quantity = self._sell_quantity(origin_symbol, target_symbol,
origin_balance)
self.logger.info(f"Selling {order_quantity} {origin_symbol}")
self.logger.debug(f"Balance is {origin_balance}")
order = None
order_guard = self.stream_manager.acquire_order_guard()
while order is None:
# Should sell at calculated price to avoid lost coin
self.logger.debug("Attempting to place order")
order = self.binance_client.order_limit_sell(
symbol=origin_symbol + target_symbol,
quantity=order_quantity,
price=from_coin_price)
self.logger.debug(f"order: {order}")
orderId = order["orderId"]
self.logger.info(
f"Placed sell order {orderId}, waiting for it to complete")
trade_log.set_ordered(origin_balance, target_balance, order_quantity)
order_guard.set_order(origin_symbol, target_symbol,
int(order["orderId"]))
order = self.wait_for_order(order["orderId"], origin_symbol,
target_symbol, order_guard)
if order is None:
return None
new_balance = self.get_currency_balance(origin_symbol)
while new_balance >= origin_balance:
new_balance = self.get_currency_balance(origin_symbol, True)
new_target_balance = self.get_currency_balance(target_symbol)
self.logger.info(
f"Sold {origin_symbol} for {new_target_balance} {target_symbol}")
trade_log.set_complete(order.cumulative_quote_qty)
return order
load_dotenv()
TOKEN = os.getenv('TOKEN', None)
if TOKEN is None:
print('Set TOKEN="YOUR_TOKEN" in the .env file')
sys.exit(1)
game = discord.Game("BROWSING NANO DOCS")
description = '''NanoDocs BOT - Will send info upon request'''
bot = commands.Bot(command_prefix='#', description=description)
bot.remove_command("help")
datenow = datetime.datetime.now()
# 1 hour cache for RPC
cache = TTLCache(maxsize=10, ttl=3600)
@bot.event
async def on_ready():
print('Logged in as')
print(bot.user.name)
print(bot.user.id)
print('------')
print('NanoDocs Online')
print(datenow)
await bot.change_presence(status=discord.Status.online, activity=game)
def loadRPCdescr():
try:
async def get(self, loc_id): # pylint: disable=arguments-differ
# Get location at the index equal to provided id.
locations = await self.get_all()
return locations[loc_id]
# ---------------------------------------------------------------
# Base URL for fetching category.
BASE_URL = (
"https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/"
)
@cached(cache=TTLCache(maxsize=128, ttl=1800))
async def get_category(category):
"""
Retrieves the data for the provided category. The data is cached for 30 minutes locally, 1 hour via shared Redis.
:returns: The data for category.
:rtype: dict
"""
# Adhere to category naming standard.
category = category.lower()
data_id = f"jhu.{category}"
# check shared cache
cache_results = await check_cache(data_id)
if cache_results:
LOGGER.info(f"{data_id} using shared cache results")
roman += letters[i]
number -= numbers[i]
return roman
def settings_text(key):
def get(name):
if hasattr(settings, name):
return getattr(settings, name)
return None
return get('{}_{}'.format(
key, (get_language() or settings.LANGUAGE_CODE).upper())) or get(key)
@cached(TTLCache(100, ttl=30))
def get_url_ip_address_list(url):
"""
This function takes a full URL as a parameter and returns the IP addresses
of the host as a string.
It will cache results for 30 seconds, so repeated calls return fast
"""
hostname = urlsplit(url).hostname
assert hostname, "Invalid url: no hostname found"
ips = (a[4][0] for a in socket.getaddrinfo(
hostname, None, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP))
return tuple(set(ips))
def get_remote_addr(request):
