async def on_startup(): global trader_database trader_database = get_client(asyncio.get_running_loop()).trader client = get_client(asyncio.get_running_loop()) app.__dict__['mongo'] = client app.__dict__['aiohttp_session'] = aiohttp.ClientSession() users = await client.trader.users.find({}, { '_id': True }).to_list(length=1000) for user in users: _id = ObjectId(user['_id']) await client.trader.users.update_one( {'_id': _id}, {'$set': { 'loop_state': 'stopped' }}, upsert=True) logging.info('Set all loops to stopped') tasks['gecko'] = asyncio.ensure_future(background_process.coin_gecko()) tasks['coin_gecko_hourly'] = asyncio.ensure_future( background_process.coin_gecko_hourly()) logging.info('Started tasks')
async def login(request): body = await request.json() client = get_client(asyncio.get_running_loop()) user = await client.trader.users.find_one({'username': body['username']}) exchanges = await client.trader.exchanges.find({}).to_list(length=100) wallet_types = await client.trader.wallet_types.find({}).to_list(length=100) if not user: raise Exception('User not found') if not bcrypt.checkpw(body['password'].encode(), user['password'].encode()): raise Exception('Wrong Username/Password') user['password_decrypted'] = body['password'] user['exchanges'] = make_exchanges(user, body['password'], exchanges) user['wallets'] = make_wallets(user, wallet_types) if user['loop_state'] != 'running': try: payload = {'_id': str(user['_id']), 'wallets': user['wallets'], 'exchanges': user['exchanges']} await fetch(request.app.aiohttp_session, f'http://{os.environ["WORKER"]}:8002', 'post', body={"user": payload}) except Exception as e: print(e) token = jwt.encode({'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=1800), 'id': str(user['_id']), 'access': 'write'}, 'secret', algorithm='HS256') response = {'token': token} return JSONResponse(response)
async def on_shutdown(): global trader_database trader_database = get_client(asyncio.get_running_loop()).trader client = get_client(asyncio.get_running_loop()) logging.info('Set all loops to stopped') gecko_collection = client.trader.coin_gecko.find_one({}) coin_gecko = await gecko_collection gecko_collection.update_one( {'_id': coin_gecko['_id']}, {'$set': { 'hourly': 'stopped', 'loop_state': 'stopped' }}, upsert=False)
def __init__(self, config): self.config = config self.cluster = self.config.get('cluster', 'test') self.frequency = self.config.get('frequency', 5) database_config = self.config['database'] self.db_client = database.get_client(database_config['type'], database_config) manager_config = self.config['workload_manager'] self.metrics_aggregator = metric.aggregator.get_aggregator( manager_config['type'], manager_config)
def batch_insert(file, archive_name): print("Connecting to database...") db = database.get_client()["twitter"] print("Creating indexes...") db.tweets.create_index([("$**", pymongo.TEXT)], name="text_index", background=True) db.tweets.create_index([("tweet_language", pymongo.ASCENDING)], name="language_index", background=True) db.tweets.create_index([("_archive", pymongo.ASCENDING)], name="archive_index", background=True) db.tweets.create_index([("like_count", pymongo.DESCENDING)], name="like_index", background=True) db.tweets.create_index([("retweet_count", pymongo.DESCENDING)], name="retweet_index", background=True) db.tweets.create_index([("follower_count", pymongo.DESCENDING)], name="follower_index", background=True) print("Loading & inserting tweets...") def _insert(queue): try: db.tweets.insert_many(queue) except Exception as e: print(e) # Try individually for item in queue: try: db.tweets.insert_one(item) except Exception as e2: # print(e2) pass csv_reader.process_csv(file, archive_name, _insert, batch_size=20000) print("Done!")
async def coin_gecko(): session = aiohttp.ClientSession() client = get_client(asyncio.get_running_loop()) gecko_collection = client.trader.coin_gecko coin_gecko = await gecko_collection.find_one({}) coins_collection = client.trader.coins await update_gecko(gecko_collection, coin_gecko, {'loop_state':'running'}) coin_gecko = await gecko_collection.find_one({}) logging.info('starting minute gecko sync...') while coin_gecko['loop_state'] == 'running': start = datetime.now() try: coin_gecko = await gecko_collection.find_one({}) subscriptions = ",".join(coin_gecko['subscriptions']) if subscriptions != '': response = await fetch(session, "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=250&page=1&sparkline=false&price_change_percentage=1h,24h,7d,14d,30d") await update_coins(coins_collection, response) await update_gecko(gecko_collection, coin_gecko, {'last_price_update': int(datetime.now().timestamp())}) except Exception as e: print(e) logging.exception(e) while (datetime.now() - start).seconds < 60: coin_gecko = await gecko_collection.find_one({}) if coin_gecko['loop_state'] != 'running': break await asyncio.sleep(5) await update_gecko(gecko_collection, coin_gecko, {'loop_state': 'stopped'})
def decorator(*args, **kwargs): logging.error("\nNEW REQUEST:") if not "Authorization" in request.headers: logging.error("Authorization header missing") return json("fail", {"authorization": "authorization header is required"}) # Poimitaan Authorization-headerin parametrit: auth_header = request.headers["Authorization"] auth_params = [param.split("=") for param in auth_header.split(",")] auth_dict = {k: v[1:-1] for k, v in auth_params} # Tarkastetaan timestamp: if time.time() - float(auth_dict["timestamp"]) > TIMESTAMP_LIMIT: logging.error("Old timestamp") return json("fail", {"timestamp": "old timestamp"}) # Etsitään käyttäjä tietokannasta: user = db.get_user(urllib.unquote(auth_dict["username"])) if not user: logging.error("User not found") return json("fail", {"username": "******"}) # Etsitään asiakassovellus tietokannasta: client = db.get_client(urllib.unquote(auth_dict["client"])) if not client: logging.error("Client not found") return json("fail", {"client": "client not found"}) # Poimitaan pyynnön data: method = request.method if method in ["GET", "DELETE"]: data_src = request.args else: data_src = request.form data = {escape(k): escape(v) for k, v in data_src.iteritems()} logging.error("DATA=" + str(data)) # Kerätään parametrit allekirjoitusta varten: params = { "username": auth_dict["username"], "client": auth_dict["client"], "timestamp": auth_dict["timestamp"], } signature_params = dict(params.items() + data.items()) # Kääritään parametrit yhteen merkkijonoon: root_url = request.url.split("?")[0] if not root_url.startswith("http://"): root_url = "http://" + root_url params_str = "&".join(["%s=%s" % (key, signature_params[key]) for key in sorted(signature_params)]) base_string = "&".join([method, escape(root_url), escape(params_str)]) logging.error("BASE_STR=" + base_string) # Luodaan allekirjoitus: signing_key = client["key"] + "&" + user["key"] hashed = hmac.new(signing_key.encode("utf-8"), base_string.encode("utf-8"), sha1) signature = escape(base64.b64encode(hashed.hexdigest())) # Tarkastetaan vastaako luotu allekirjoitus pyynnön allekirjoitusta: if signature != auth_dict["signature"]: logging.error("Incorrect signature, base_string=" + base_string) return json("fail", {"signature": "incorrect signature"}) # Allekirjoitus oikein -> autorisointi onnistui: logging.error("Auth success") g.user = user return f(*args, **kwargs)
#!/usr/bin/env python3 import database import pymongo import os import datetime import json import dateparser from flask import Flask, make_response from flask_cors import CORS from flask_restful import Resource, Api, reqparse app = Flask(__name__) api = Api(app) db = database.get_client()["twitter"] CORS(app) @api.representation("application/json") def output_json(data, code, headers=None): def _clean(o): if isinstance(o, list): return [_clean(item) for item in o] if isinstance(o, dict): return {k: _clean(v) for k, v in o.items()} if isinstance(o, datetime.datetime): return o.__str__() return o resp = make_response(json.dumps(_clean(data)), code) resp.headers.extend(headers or {})
def decorator(*args, **kwargs): logging.error("\nNEW REQUEST:") if not "Authorization" in request.headers: logging.error("Authorization header missing") return json("fail", {"authorization": "authorization header is required"}) # Poimitaan Authorization-headerin parametrit: auth_header = request.headers["Authorization"] auth_params = [param.split("=") for param in auth_header.split(",")] auth_dict = {k: v[1:-1] for k, v in auth_params} # Tarkastetaan timestamp: if time.time() - float(auth_dict["timestamp"]) > TIMESTAMP_LIMIT: logging.error("Old timestamp") return json("fail", {"timestamp": "old timestamp"}) # Etsitään käyttäjä tietokannasta: user = db.get_user(urllib.unquote(auth_dict["username"])) if not user: logging.error("User not found") return json("fail", {"username": "******"}) # Etsitään asiakassovellus tietokannasta: client = db.get_client(urllib.unquote(auth_dict["client"])) if not client: logging.error("Client not found") return json("fail", {"client": "client not found"}) # Poimitaan pyynnön data: method = request.method if method in ["GET", "DELETE"]: data_src = request.args else: data_src = request.form data = {escape(k): escape(v) for k, v in data_src.iteritems()} logging.error("DATA=" + str(data)) # Kerätään parametrit allekirjoitusta varten: params = { "username": auth_dict["username"], "client": auth_dict["client"], "timestamp": auth_dict["timestamp"], } signature_params = dict(params.items() + data.items()) # Kääritään parametrit yhteen merkkijonoon: root_url = request.url.split("?")[0] if not root_url.startswith("http://"): root_url = "http://" + root_url params_str = "&".join([ "%s=%s" % (key, signature_params[key]) for key in sorted(signature_params) ]) base_string = "&".join([method, escape(root_url), escape(params_str)]) logging.error("BASE_STR=" + base_string) # Luodaan allekirjoitus: signing_key = client["key"] + "&" + user["key"] hashed = hmac.new(signing_key.encode("utf-8"), base_string.encode("utf-8"), sha1) signature = escape(base64.b64encode(hashed.hexdigest())) # Tarkastetaan vastaako luotu allekirjoitus pyynnön allekirjoitusta: if signature != auth_dict["signature"]: logging.error("Incorrect signature, base_string=" + base_string) return json("fail", {"signature": "incorrect signature"}) # Allekirjoitus oikein -> autorisointi onnistui: logging.error("Auth success") g.user = user return f(*args, **kwargs)
from weather import download_weather_measures from wind import download_wind_measures from webcam import take_screenshot from datetime import datetime, timedelta from database import get_client import json with open("configuration.json", "r", encoding="utf-8") as handler: configuration = json.loads(handler.read()) def get_date_yesterday(): yesterday = datetime.today().date() - timedelta(days=1) return yesterday yesterday = get_date_yesterday() client = get_client() download_wind_measures(start=yesterday, end=yesterday + timedelta(days=1), client=client) download_weather_measures(start=yesterday, end=yesterday+timedelta(days=1), client=client, api_key=configuration["weather_api_key"]) take_screenshot(date=datetime.today()) client.close()
async def on_startup(): client = get_client(asyncio.get_running_loop()) app.__dict__['mongo'] = client app.__dict__['aiohttp_session'] = aiohttp.ClientSession()
async def coin_gecko_hourly(): session = aiohttp.ClientSession() client = get_client(asyncio.get_running_loop()) gecko_collection = client.trader.coin_gecko coin_gecko = await gecko_collection.find_one({}) coins_collection = client.trader.coins value_history_collection = client.trader.value_history await update_gecko(gecko_collection, coin_gecko, {'hourly':'running'}) coin_gecko = await gecko_collection.find_one({}) logging.info('starting hourly gecko sync...') while coin_gecko['hourly'] == 'running': start = datetime.now() try: coin_gecko = await gecko_collection.find_one({}) subscriptions = ",".join(coin_gecko['subscriptions']) if subscriptions != '': # response = requests.get('https://api.coingecko.com/api/v3/coins/bitcoin/ohlc?vs_currency=usd&days=1') try: coin_id = 'bitcoin' response = await fetch(session, f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days=1') hourly_ohlc = response[::2] result = await coins_collection.find_one({"id": coin_id}, {"hourly_ohlc": 1}) update = {"id": coin_id} if 'hourly_ohlc' in result: update["hourly_ohlc"] = result['hourly_ohlc'][-20:] + hourly_ohlc try: response = await fetch(session, f'http://{os.environ["AI"]}:8003/', 'post', body=update["hourly_ohlc"]) update['prediction_20h'] = response['prediction'] prediction_usd = update["hourly_ohlc"][-1][-1] * response['prediction'] await update_value_history(value_history_collection, coin_gecko, {f'{coin_id}_predictions': {'usd': prediction_usd, 'prediction': response['prediction'], 'timestamp': datetime.utcnow() + timedelta(hours=20)}}) except Exception as e: logging.exception(e) await update_coins(coins_collection, [update]) await update_gecko(gecko_collection, coin_gecko, {'last_ohlc_update': int(datetime.now().timestamp())}) except Exception as e: logging.exception(e) except Exception as e: logging.exception(e) while (datetime.now() - start).seconds < 60 * 60: coin_gecko = await gecko_collection.find_one({}) if coin_gecko['hourly'] != 'running': break await asyncio.sleep(5) await update_gecko(gecko_collection, coin_gecko, {'hourly': 'stopped'})
async def user_hourly(app, user): user_with_decrypted_keys = user loop = asyncio.get_running_loop() client = get_client(loop) user_collection = client.trader.users value_history_collection = client.trader.value_history user = await user_collection.find_one({'_id': ObjectId(user['_id'])}) wallet_types = await client.trader.wallet_types.find({}).to_list(length=100) logging.info(f'starting background sync for {user["username"]}') await update_user(user_collection, user, {'hourly': "running"}, upsert=True) user = await user_collection.find_one({'_id': user['_id']}) while user['hourly'] == "running": start = datetime.now() try: user = await user_collection.find_one({'_id': user['_id']}) user_with_decrypted_keys['wallets'] = make_wallets(user, wallet_types) query = '' if 'accounts' in user: for account in user['accounts']: exchange = await client.trader.exchanges.find_one({'_id': account['exchange']}) query = query + create_exchange_balance_query(exchange['name']) for wallet in user['wallets']: wallet_type = await client.trader.wallet_types.find_one({'_id': wallet['wallet_type']}) query = query + create_exchange_balance_query(wallet_type['name']) query = 'query{\n' + query + '\n}' result = await schema.execute(query, executor=AsyncioExecutor(loop=loop), return_promise=True, context={'app': app, 'user': user_with_decrypted_keys}) if result.data: balance = [] updates = [] for key in result.data.keys(): if result.data[key]['balance']: for entry in result.data[key]['balance']: entry['exchange'] = key balance.append(entry) updates.append(process_to_lower_with_underscore(entry)) total_usd = sum([float(currency['usd']) for exchange in result.data.values() if exchange['balance'] for currency in exchange['balance']]) # updates = aggregate_balance(updates) await update_value_history(value_history_collection, user, {'portfolio_value': {'total_usd': total_usd, 'timestamp': datetime.utcnow()}}) # try: # await handle_notifications(user=user, balance=balance, client=client) # except Exception as e: # logging.exception(f'notification: {e.message}') # # user.total_value.append(models.TotalValue(usd_value=total_usd)) await update_user(user_collection, user, {'last_hourly_update': int(datetime.now().timestamp())}, upsert=True) if result.errors: [logging.exception(error.message) for error in result.errors if hasattr(error, 'message')] except Exception as e: logging.exception(e) while (datetime.now() - start).seconds < 60 * 60: user = await user_collection.find_one({'_id': user['_id']}) if user['hourly'] != 'running': break await asyncio.sleep(5) await update_user(user_collection, user, {'hourly': "stopped"})