async def on_startup():
global trader_database
trader_database = get_client(asyncio.get_running_loop()).trader
client = get_client(asyncio.get_running_loop())
app.__dict__['mongo'] = client
app.__dict__['aiohttp_session'] = aiohttp.ClientSession()
users = await client.trader.users.find({}, {
'_id': True
}).to_list(length=1000)
for user in users:
_id = ObjectId(user['_id'])
await client.trader.users.update_one(
{'_id': _id}, {'$set': {
'loop_state': 'stopped'
}}, upsert=True)
logging.info('Set all loops to stopped')
tasks['gecko'] = asyncio.ensure_future(background_process.coin_gecko())
tasks['coin_gecko_hourly'] = asyncio.ensure_future(
background_process.coin_gecko_hourly())
logging.info('Started tasks')
async def login(request):
body = await request.json()
client = get_client(asyncio.get_running_loop())
user = await client.trader.users.find_one({'username': body['username']})
exchanges = await client.trader.exchanges.find({}).to_list(length=100)
wallet_types = await client.trader.wallet_types.find({}).to_list(length=100)
if not user:
raise Exception('User not found')
if not bcrypt.checkpw(body['password'].encode(), user['password'].encode()):
raise Exception('Wrong Username/Password')
user['password_decrypted'] = body['password']
user['exchanges'] = make_exchanges(user, body['password'], exchanges)
user['wallets'] = make_wallets(user, wallet_types)
if user['loop_state'] != 'running':
try:
payload = {'_id': str(user['_id']), 'wallets': user['wallets'], 'exchanges': user['exchanges']}
await fetch(request.app.aiohttp_session, f'http://{os.environ["WORKER"]}:8002', 'post', body={"user": payload})
except Exception as e:
print(e)
token = jwt.encode({'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=1800), 'id': str(user['_id']), 'access': 'write'}, 'secret', algorithm='HS256')
response = {'token': token}
return JSONResponse(response)
async def on_shutdown():
global trader_database
trader_database = get_client(asyncio.get_running_loop()).trader
client = get_client(asyncio.get_running_loop())
logging.info('Set all loops to stopped')
gecko_collection = client.trader.coin_gecko.find_one({})
coin_gecko = await gecko_collection
gecko_collection.update_one(
{'_id': coin_gecko['_id']},
{'$set': {
'hourly': 'stopped',
'loop_state': 'stopped'
}},
upsert=False)
def __init__(self, config):
self.config = config
self.cluster = self.config.get('cluster', 'test')
self.frequency = self.config.get('frequency', 5)
database_config = self.config['database']
self.db_client = database.get_client(database_config['type'],
database_config)
manager_config = self.config['workload_manager']
self.metrics_aggregator = metric.aggregator.get_aggregator(
manager_config['type'], manager_config)
def batch_insert(file, archive_name):
print("Connecting to database...")
db = database.get_client()["twitter"]
print("Creating indexes...")
db.tweets.create_index([("$**", pymongo.TEXT)],
name="text_index",
background=True)
db.tweets.create_index([("tweet_language", pymongo.ASCENDING)],
name="language_index",
background=True)
db.tweets.create_index([("_archive", pymongo.ASCENDING)],
name="archive_index",
background=True)
db.tweets.create_index([("like_count", pymongo.DESCENDING)],
name="like_index",
background=True)
db.tweets.create_index([("retweet_count", pymongo.DESCENDING)],
name="retweet_index",
background=True)
db.tweets.create_index([("follower_count", pymongo.DESCENDING)],
name="follower_index",
background=True)
print("Loading & inserting tweets...")
def _insert(queue):
try:
db.tweets.insert_many(queue)
except Exception as e:
print(e)
# Try individually
for item in queue:
try:
db.tweets.insert_one(item)
except Exception as e2:
# print(e2)
pass
csv_reader.process_csv(file, archive_name, _insert, batch_size=20000)
print("Done!")
async def coin_gecko():
session = aiohttp.ClientSession()
client = get_client(asyncio.get_running_loop())
gecko_collection = client.trader.coin_gecko
coin_gecko = await gecko_collection.find_one({})
coins_collection = client.trader.coins
await update_gecko(gecko_collection, coin_gecko, {'loop_state':'running'})
coin_gecko = await gecko_collection.find_one({})
logging.info('starting minute gecko sync...')
while coin_gecko['loop_state'] == 'running':
start = datetime.now()
try:
coin_gecko = await gecko_collection.find_one({})
subscriptions = ",".join(coin_gecko['subscriptions'])
if subscriptions != '':
response = await fetch(session, "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=250&page=1&sparkline=false&price_change_percentage=1h,24h,7d,14d,30d")
await update_coins(coins_collection, response)
await update_gecko(gecko_collection, coin_gecko, {'last_price_update': int(datetime.now().timestamp())})
except Exception as e:
print(e)
logging.exception(e)
while (datetime.now() - start).seconds < 60:
coin_gecko = await gecko_collection.find_one({})
if coin_gecko['loop_state'] != 'running':
break
await asyncio.sleep(5)
await update_gecko(gecko_collection, coin_gecko, {'loop_state': 'stopped'})
def decorator(*args, **kwargs):
logging.error("\nNEW REQUEST:")
if not "Authorization" in request.headers:
logging.error("Authorization header missing")
return json("fail", {"authorization": "authorization header is required"})
# Poimitaan Authorization-headerin parametrit:
auth_header = request.headers["Authorization"]
auth_params = [param.split("=") for param in auth_header.split(",")]
auth_dict = {k: v[1:-1] for k, v in auth_params}
# Tarkastetaan timestamp:
if time.time() - float(auth_dict["timestamp"]) > TIMESTAMP_LIMIT:
logging.error("Old timestamp")
return json("fail", {"timestamp": "old timestamp"})
# Etsitään käyttäjä tietokannasta:
user = db.get_user(urllib.unquote(auth_dict["username"]))
if not user:
logging.error("User not found")
return json("fail", {"username": "******"})
# Etsitään asiakassovellus tietokannasta:
client = db.get_client(urllib.unquote(auth_dict["client"]))
if not client:
logging.error("Client not found")
return json("fail", {"client": "client not found"})
# Poimitaan pyynnön data:
method = request.method
if method in ["GET", "DELETE"]:
data_src = request.args
else:
data_src = request.form
data = {escape(k): escape(v) for k, v in data_src.iteritems()}
logging.error("DATA=" + str(data))
# Kerätään parametrit allekirjoitusta varten:
params = {
"username": auth_dict["username"],
"client": auth_dict["client"],
"timestamp": auth_dict["timestamp"],
}
signature_params = dict(params.items() + data.items())
# Kääritään parametrit yhteen merkkijonoon:
root_url = request.url.split("?")[0]
if not root_url.startswith("http://"):
root_url = "http://" + root_url
params_str = "&".join(["%s=%s" % (key, signature_params[key])
for key in sorted(signature_params)])
base_string = "&".join([method, escape(root_url), escape(params_str)])
logging.error("BASE_STR=" + base_string)
# Luodaan allekirjoitus:
signing_key = client["key"] + "&" + user["key"]
hashed = hmac.new(signing_key.encode("utf-8"), base_string.encode("utf-8"), sha1)
signature = escape(base64.b64encode(hashed.hexdigest()))
# Tarkastetaan vastaako luotu allekirjoitus pyynnön allekirjoitusta:
if signature != auth_dict["signature"]:
logging.error("Incorrect signature, base_string=" + base_string)
return json("fail", {"signature": "incorrect signature"})
# Allekirjoitus oikein -> autorisointi onnistui:
logging.error("Auth success")
g.user = user
return f(*args, **kwargs)
#!/usr/bin/env python3
import database
import pymongo
import os
import datetime
import json
import dateparser
from flask import Flask, make_response
from flask_cors import CORS
from flask_restful import Resource, Api, reqparse
app = Flask(__name__)
api = Api(app)
db = database.get_client()["twitter"]
CORS(app)
@api.representation("application/json")
def output_json(data, code, headers=None):
def _clean(o):
if isinstance(o, list):
return [_clean(item) for item in o]
if isinstance(o, dict):
return {k: _clean(v) for k, v in o.items()}
if isinstance(o, datetime.datetime):
return o.__str__()
return o
resp = make_response(json.dumps(_clean(data)), code)
resp.headers.extend(headers or {})
def decorator(*args, **kwargs):
logging.error("\nNEW REQUEST:")
if not "Authorization" in request.headers:
logging.error("Authorization header missing")
return json("fail",
{"authorization": "authorization header is required"})
# Poimitaan Authorization-headerin parametrit:
auth_header = request.headers["Authorization"]
auth_params = [param.split("=") for param in auth_header.split(",")]
auth_dict = {k: v[1:-1] for k, v in auth_params}
# Tarkastetaan timestamp:
if time.time() - float(auth_dict["timestamp"]) > TIMESTAMP_LIMIT:
logging.error("Old timestamp")
return json("fail", {"timestamp": "old timestamp"})
# Etsitään käyttäjä tietokannasta:
user = db.get_user(urllib.unquote(auth_dict["username"]))
if not user:
logging.error("User not found")
return json("fail", {"username": "******"})
# Etsitään asiakassovellus tietokannasta:
client = db.get_client(urllib.unquote(auth_dict["client"]))
if not client:
logging.error("Client not found")
return json("fail", {"client": "client not found"})
# Poimitaan pyynnön data:
method = request.method
if method in ["GET", "DELETE"]:
data_src = request.args
else:
data_src = request.form
data = {escape(k): escape(v) for k, v in data_src.iteritems()}
logging.error("DATA=" + str(data))
# Kerätään parametrit allekirjoitusta varten:
params = {
"username": auth_dict["username"],
"client": auth_dict["client"],
"timestamp": auth_dict["timestamp"],
}
signature_params = dict(params.items() + data.items())
# Kääritään parametrit yhteen merkkijonoon:
root_url = request.url.split("?")[0]
if not root_url.startswith("http://"):
root_url = "http://" + root_url
params_str = "&".join([
"%s=%s" % (key, signature_params[key])
for key in sorted(signature_params)
])
base_string = "&".join([method, escape(root_url), escape(params_str)])
logging.error("BASE_STR=" + base_string)
# Luodaan allekirjoitus:
signing_key = client["key"] + "&" + user["key"]
hashed = hmac.new(signing_key.encode("utf-8"),
base_string.encode("utf-8"), sha1)
signature = escape(base64.b64encode(hashed.hexdigest()))
# Tarkastetaan vastaako luotu allekirjoitus pyynnön allekirjoitusta:
if signature != auth_dict["signature"]:
logging.error("Incorrect signature, base_string=" + base_string)
return json("fail", {"signature": "incorrect signature"})
# Allekirjoitus oikein -> autorisointi onnistui:
logging.error("Auth success")
g.user = user
return f(*args, **kwargs)
from weather import download_weather_measures
from wind import download_wind_measures
from webcam import take_screenshot
from datetime import datetime, timedelta
from database import get_client
import json
with open("configuration.json", "r", encoding="utf-8") as handler:
configuration = json.loads(handler.read())
def get_date_yesterday():
yesterday = datetime.today().date() - timedelta(days=1)
return yesterday
yesterday = get_date_yesterday()
client = get_client()
download_wind_measures(start=yesterday, end=yesterday +
timedelta(days=1), client=client)
download_weather_measures(start=yesterday, end=yesterday+timedelta(days=1),
client=client, api_key=configuration["weather_api_key"])
take_screenshot(date=datetime.today())
client.close()
async def on_startup():
client = get_client(asyncio.get_running_loop())
app.__dict__['mongo'] = client
app.__dict__['aiohttp_session'] = aiohttp.ClientSession()
async def coin_gecko_hourly():
session = aiohttp.ClientSession()
client = get_client(asyncio.get_running_loop())
gecko_collection = client.trader.coin_gecko
coin_gecko = await gecko_collection.find_one({})
coins_collection = client.trader.coins
value_history_collection = client.trader.value_history
await update_gecko(gecko_collection, coin_gecko, {'hourly':'running'})
coin_gecko = await gecko_collection.find_one({})
logging.info('starting hourly gecko sync...')
while coin_gecko['hourly'] == 'running':
start = datetime.now()
try:
coin_gecko = await gecko_collection.find_one({})
subscriptions = ",".join(coin_gecko['subscriptions'])
if subscriptions != '':
# response = requests.get('https://api.coingecko.com/api/v3/coins/bitcoin/ohlc?vs_currency=usd&days=1')
try:
coin_id = 'bitcoin'
response = await fetch(session, f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days=1')
hourly_ohlc = response[::2]
result = await coins_collection.find_one({"id": coin_id}, {"hourly_ohlc": 1})
update = {"id": coin_id}
if 'hourly_ohlc' in result:
update["hourly_ohlc"] = result['hourly_ohlc'][-20:] + hourly_ohlc
try:
response = await fetch(session, f'http://{os.environ["AI"]}:8003/', 'post', body=update["hourly_ohlc"])
update['prediction_20h'] = response['prediction']
prediction_usd = update["hourly_ohlc"][-1][-1] * response['prediction']
await update_value_history(value_history_collection, coin_gecko, {f'{coin_id}_predictions': {'usd': prediction_usd, 'prediction': response['prediction'], 'timestamp': datetime.utcnow() + timedelta(hours=20)}})
except Exception as e:
logging.exception(e)
await update_coins(coins_collection, [update])
await update_gecko(gecko_collection, coin_gecko, {'last_ohlc_update': int(datetime.now().timestamp())})
except Exception as e:
logging.exception(e)
except Exception as e:
logging.exception(e)
while (datetime.now() - start).seconds < 60 * 60:
coin_gecko = await gecko_collection.find_one({})
if coin_gecko['hourly'] != 'running':
break
await asyncio.sleep(5)
await update_gecko(gecko_collection, coin_gecko, {'hourly': 'stopped'})
async def user_hourly(app, user):
user_with_decrypted_keys = user
loop = asyncio.get_running_loop()
client = get_client(loop)
user_collection = client.trader.users
value_history_collection = client.trader.value_history
user = await user_collection.find_one({'_id': ObjectId(user['_id'])})
wallet_types = await client.trader.wallet_types.find({}).to_list(length=100)
logging.info(f'starting background sync for {user["username"]}')
await update_user(user_collection, user, {'hourly': "running"}, upsert=True)
user = await user_collection.find_one({'_id': user['_id']})
while user['hourly'] == "running":
start = datetime.now()
try:
user = await user_collection.find_one({'_id': user['_id']})
user_with_decrypted_keys['wallets'] = make_wallets(user, wallet_types)
query = ''
if 'accounts' in user:
for account in user['accounts']:
exchange = await client.trader.exchanges.find_one({'_id': account['exchange']})
query = query + create_exchange_balance_query(exchange['name'])
for wallet in user['wallets']:
wallet_type = await client.trader.wallet_types.find_one({'_id': wallet['wallet_type']})
query = query + create_exchange_balance_query(wallet_type['name'])
query = 'query{\n' + query + '\n}'
result = await schema.execute(query, executor=AsyncioExecutor(loop=loop), return_promise=True, context={'app': app, 'user': user_with_decrypted_keys})
if result.data:
balance = []
updates = []
for key in result.data.keys():
if result.data[key]['balance']:
for entry in result.data[key]['balance']:
entry['exchange'] = key
balance.append(entry)
updates.append(process_to_lower_with_underscore(entry))
total_usd = sum([float(currency['usd']) for exchange in result.data.values() if exchange['balance'] for currency in exchange['balance']])
# updates = aggregate_balance(updates)
await update_value_history(value_history_collection, user, {'portfolio_value': {'total_usd': total_usd, 'timestamp': datetime.utcnow()}})
# try:
# await handle_notifications(user=user, balance=balance, client=client)
# except Exception as e:
# logging.exception(f'notification: {e.message}')
# # user.total_value.append(models.TotalValue(usd_value=total_usd))
await update_user(user_collection, user, {'last_hourly_update': int(datetime.now().timestamp())}, upsert=True)
if result.errors:
[logging.exception(error.message) for error in result.errors if hasattr(error, 'message')]
except Exception as e:
logging.exception(e)
while (datetime.now() - start).seconds < 60 * 60:
user = await user_collection.find_one({'_id': user['_id']})
if user['hourly'] != 'running':
break
await asyncio.sleep(5)
await update_user(user_collection, user, {'hourly': "stopped"})