def taskDownloadLatestMarketData(domain) -> bool:
dlog.d("Downloading as cache is old")
result, data = download(doamin=domain, period=1)
if result is True:
dglobaldata.saveMarketDataFormDayDF(domain, data)
return True
return False
def configure_celery(app: Flask) -> Celery:
"""Configures celery instance from app, using it's config"""
TaskBase: CeleryTask = extensions.celery.Task
class ContextTask(TaskBase): # pylint: disable=too-few-public-methods
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
# https://docs.celeryproject.org/en/stable/userguide/configuration.html
extensions.celery.conf.update(
broker_url=app.config['CELERY_BROKER_URL'],
result_backend=app.config['CELERY_RESULT_BACKEND'],
accept_content=app.config['CELERY_ACCEPT_CONTENT'],
task_serializer=app.config['CELERY_TASK_SERIALIZER'],
result_serializer=app.config['CELERY_TASK_SERIALIZER'],
worker_hijack_root_logger=False,
beat_schedule=app.config.get('CELERYBEAT_SCHEDULE', {}),
worker_redirect_stdouts_level='ERROR',
task_always_eager=app.config.get('CELERY_ALWAYS_EAGER', False)
)
extensions.celery.Task = ContextTask
dlog.d("Celery setup properly:{}".format(
app.config.get('CELERYBEAT_SCHEDULE', {})))
return extensions.celery
def performScreen(domain: str, condition: str, columns=[], sort_by: str = None, limit: int = None):
dlog.d('[INFO] Running screen for condition: {}, columns: {}'.format(
condition, columns))
indicatorHistory = getIndicatorHistory(domain)
result = []
sl = 0
columns = [resolveIndicatorExpression(c) for c in columns]
condition = resolveCondition(condition)
last_error = ''
try:
for symbol in indicatorHistory.keys():
indicator_map = indicatorHistory[symbol]
try:
if(eval(condition)):
sl += 1
selected_one = {}
selected_one['symbol'] = symbol
selected_one['name'] = getSymbolList(domain)[
symbol]['name']
selected_one['sector'] = getSymbolList(domain)[
symbol]['sector']
selected_one['close'] = str(
np.round(indicator_map['1d'][0]['close'], 2))
selected_one['volume'] = str(
np.round(indicator_map['1d'][0]['volume'], 2))
selected_one['rsi_14'] = str(
np.round(indicator_map['1d'][0]['rsi_14'], 2))
selected_one['change'] = str(
np.round(indicator_map['1d'][0]['close_change_percentage'], 2))
# extra col
for col in columns:
selected_one[col[0]] = str(
np.round(eval(col[1]), 2))
# add used defined data
result.append(fixDict(selected_one))
except Exception as e:
last_error = 'Are you passing right Filter {}'.format(
condition)
dlog.ex(e, showStack=False)
dlog.e(
"We faced the issue when we are running filter for symbol:{}".format(symbol))
# We just ignore this
except Exception as e:
raise e
# Filter None
# Do sort
if sort_by:
sort_key = sort_by.replace("-", "")
reverse = sort_by[0] != "-"
result = [x for x in result if x.get(sort_key) is not None]
result.sort(key=lambda x: float(x.get(sort_key)), reverse=reverse)
# Do limit
if limit:
result = result[:limit]
return {'result': result, 'last_error': last_error}
def wrapper(*args, **kwargs):
t = time.time()
dlog.d("\n\n>>>>>>>>>>>>>>>> STARTING {} <<<<<<<<<<<<<".format(
func.__name__))
res = func(*args, **kwargs)
dlog.d("\n>>>>>>>>>>>>>>>> ENDING {}, Time taken: {} sec <<<<<<<<<<<<<\n\n".format(
func.__name__, time.time() - t))
return res
def loadDataForCandle(candle_type: TCandleType):
try:
_candleTypeToDataFrameMap[
candle_type.value] = dstorage.load_data_to_disk(
dstorage.get_default_path_for_candle((candle_type)))
timetracker.mark_last_data_update_ts(candle_type)
dlog.d("updating data...")
except Exception as e:
dlog.ex(e, "not able to load data from storage.")
def buildTechnicalIndicators(df: DataFrame, domain: str):
# If somevalue is nan and all calculation just dont work
df.fillna(method='ffill', inplace=True)
for ticker in getSymbolList(domain).keys():
try:
computeIndicator(df, ticker, domain)
except Exception as e:
dlog.ex(e, "Not able to process data frame for {}".format(ticker))
dlog.d("Indicator compution done proper way")
return df
def getLatestMarketData(domain: str, reload: str = "0", sync: str = "0"):
# Build indicator if not exist
mayGetLatestStockData(domain, reload, sync)
mayBuildStockIndicatorInBackground(domain, TCandleType.DAY_1, reload, sync)
dlog.d("getting data from cache")
return {
'latest': dredis.getPickle("market_data_{}".format(domain)),
'indicator':
dredis.getPickle("indicator_data_{}_{}".format(domain, '1d'))
}
def taskBuildIndicator(domain: str, candle_type: str):
try:
pingCelery()
_candle_type = TCandleType(candle_type)
dglobaldata.downloadAndBuildIndicator(domain, _candle_type)
# Compute Summary
if _candle_type == TCandleType.DAY_1:
pass
# dhighlights.taskComputeSummary()
buildTaskSuccess("taskBuildIndicator Done", None)
except Exception as e:
dlog.d("Got exception in taskBuildIndicator")
dlog.ex(e)
def wrapper(*args, **kwargs):
t = time.time()
func_name = func.__name__
if remote_logging:
danalytics.reportAction(func_name + "_started")
dlog.d("\n\n>>>>>>>>>>>>>>>> STARTING {} <<<<<<<<<<<<<".format(
func.__name__))
res = func(*args, **kwargs)
dlog.d("\n>>>>>>>>>>>>>>>> ENDING {}, Time taken: {} sec <<<<<<<<<<<<<\n\n".format(
func.__name__, time.time() - t))
if remote_logging:
danalytics.reportAction(func_name + "_ended")
return res
def reportAction(tag: str, extra: dict = {}):
dlog.d("logging remore action with tag:" + tag)
if isDebug():
dlog.d("ignore remote log in debug mode")
return
try:
simplestore_post(
url="{}/api/analytics/action".format(SIMPLESTORE_ENDPOINT),
data={
"app_id": APP_ID,
"session": session,
"tag": tag,
"extra": extra
})
except Exception as ex:
dlog.ex(ex)
def reportException(ex: Exception, location=""):
if isDebug():
dlog.d("ignore remote log in debug mode")
return
try:
res = simplestore_post(
url="{}/api/analytics/exception".format(SIMPLESTORE_ENDPOINT),
data={
"app_id": APP_ID,
"session": session,
"location": traceback.format_exc().split("\n")[-4].strip(),
"type": "exception",
"stack": traceback.format_exc(),
"args": str(ex.args)
})
dlog.d(str(res))
except Exception as ex:
dlog.ex(ex)
def mayGetLatestStockData(domain: str, reload, sync: str):
if (reload == "1"):
dlog.d("taskDownloadLatestMarketData: submitting task")
if sync == "1":
tasks.taskDownloadLatestMarketData(domain)
else:
tasks.taskDownloadLatestMarketData.delay(domain)
return
last_update = dredis.get("market_ts_{}".format(domain), None)
if last_update is None or last_update == 'None':
dlog.d("No last update - submitting task")
tasks.taskDownloadLatestMarketData.delay(domain)
elif IfTimeIs5MinOld(last_update):
dlog.d("data is 5 min old... submitting task")
# tasks.taskDownloadLatestMarketData.delay(domain)
else:
dlog.d("Data is already there")
def mayBuildStockIndicatorInBackground(domain: str, candle_type: TCandleType,
reload: str, sync: str):
# reload
if reload == "1":
if sync == "1":
tasks.taskBuildIndicator(domain, candle_type=candle_type.value)
else:
tasks.taskBuildIndicator.delay(domain, candle_type.value)
dlog.d("taskBuildIndicator: task submitted")
return
if dredis.getPickle("indicator_data_{}_{}".format(domain, '1d')) is None:
# task submitted
tasks.taskBuildIndicator.delay(domain, candle_type.value)
dlog.d("task submitted")
else:
dlog.d("Data is already there")
def init():
if isDebug():
dlog.d("ignore remote log in debug mode")
return
global session
if session != '':
dlog.d("Already session exist")
return
try:
data: dict = simplestore_post(
url="{}/api/analytics/launch".format(SIMPLESTORE_ENDPOINT),
data={
"app_id": APP_ID,
"app_version": "1.0",
"device_os": "web",
"device_id": "null",
"device_api": "null"
})
session = data.get('out')[0].get('session')
dlog.d("Remote log is inited with session:{}".format(session))
except Exception as ex:
dlog.ex(e=ex)
pass
except Exception as e:
dlog.ex(e)
danalytics.reportException(e)
# More some info.
# for x in final_result.keys():
# final_result[x]['sector'] = getSymbolList(domain=domain)[x]['sector']
return final_result
# Rest all locks here - This is needed for restart the server
for candle_type in SUPPORTED_CANDLE:
for domain in SUPPORTED_DOMAIN:
dredis.set(
"downloadAndBuildindicator_{}_{}".format(domain,
candle_type.value), "0")
dlog.d("Reset downloadAndBuildIndicator locks")
def getLastUpdatedTimeStamp(domain: str):
result = {}
for candle_type in SUPPORTED_CANDLE:
result['{}-{}'.format(domain, candle_type.value)] = dredis.get(
"indicator_timestamp_{}_{}".format(domain, candle_type.value),
"Data not found")
return result
# It will create a cache for laest data frame
def saveMarketDataFormDayDF(domain: str, df: DataFrame):
def downloadAndBuildIndicator(domain, candle_type: TCandleType):
# Optimization
if not shouldBuildIndicator(domain, candle_type):
dlog.d("Ignore rebuilding shouldBuildIndicator")
return
# Locking
lockkey = "downloadAndBuildindicator_{}_{}".format(domain,
candle_type.value)
if dredis.get(lockkey) == "1":
dlog.d("downloadAndBuildIndicator locked for key {}".format(lockkey))
raise Exception("downloadAndBuildIndicator is progress")
dredis.set(lockkey, "1")
try:
dlog.d("downloadAndBuildIndicator start")
dlog.d("downloadAndBuildIndicator download start")
ret_value, download_data = ddownload.download(domain,
interval=candle_type)
if ret_value is False:
dlog.d("Download fails")
return {
"status": "error",
"msg": "something goes wrong",
"out": None
}
dlog.d("downloadAndBuildIndicator building start")
processed_df = dindicator.buildTechnicalIndicators(
download_data, domain)
# DONOT STOARE AS FILEdlog.d("downloadAndBuildIndicator: saving to storage start")
#path_to_store = dstorage.get_default_path_for_candle(candle_type)
#dstorage.store_data_to_disk(processed_df, path_to_store)
dlog.d("downloadAndBuildIndicator: building indicator history map")
# Building Indicator map for O(1) looks up.
# This will be a 4d map
# map[REL][1d][-1][close]...
last15SlotIndicator = getLastNIndicatorInJson(domain, processed_df)
indicator_history_key = "indicator_history_{}".format(domain)
olddata = dredis.getPickle(indicator_history_key)
if not olddata:
olddata = {}
for key in last15SlotIndicator.keys():
if key not in olddata:
olddata[key] = {}
olddata[key][candle_type.value] = last15SlotIndicator.get(key)
dredis.setPickle(indicator_history_key, olddata)
dlog.d(
"downloadAndBuildIndicator: saved indicator history to {}".format(
indicator_history_key))
dlog.d("downloadAndBuildIndicator: saving to redis start")
dredis.setPickle(
"indicator_data_{}_{}".format(domain, candle_type.value), {
'data': getLatestDataInJson(domain, processed_df),
'timestamp': getCurTimeStr()
})
# update market data
if candle_type == TCandleType.DAY_1:
saveMarketDataFormDayDF(domain, download_data)
# Set TimeStamp key
dredis.set(
"indicator_timestamp_{}_{}".format(domain, candle_type.value),
getCurTimeStr())
# unlock
dredis.set(lockkey, "0")
dlog.d("downloadAndBuildIndicator ends")
return {
"status": "success",
"msg": "Completed snapshot pipeline",
"out": None
}
except Exception as e:
dredis.set(lockkey, "0")
dlog.d("downloadAndBuildIndicator Exception happened")
danalytics.reportException(e, "Exception in downloadAndBuildIndicator")
dlog.ex(e)
raise e
finally:
dredis.set(lockkey, "0")
pass
def set(key, value):
dlog.d("Setting Key is redis :{}".format(key))
_redis.set(key, value)
def saveMarketDataFormDayDF(domain: str, df: DataFrame):
dlog.d("Saving market data...")
resultJSON = getLatestDataInJson(domain, df)
# Save this data
dredis.setPickle("market_data_{}".format(domain), {"data": resultJSON})
dredis.set("market_ts_{}".format(domain), getCurTimeStr())
from myapp.core.dlog import stack
from myapp.core.dtypes import TCandleType
from myapp.core.rootConfig import SUPPORTED_CANDLE
from myapp.core.sync import getSymbolList
from pandas.core.frame import DataFrame
### Here is how to test down your own#
"""
import yfinance as yf
yf.download("TCS.NS")
"""
# Rest all locks here
for candle_type in SUPPORTED_CANDLE:
dredis.set("download_progress_{}".format(candle_type.value), "0")
dlog.d("Reset download locks")
@trace_perf
def download(doamin="IN",
interval: TCandleType = TCandleType.DAY_1,
period=50) -> typing.Tuple[bool, DataFrame]:
key = "download_progress_" + interval.value
if (dredis.get(key) == "1"):
danalytics.reportAction(
"ignore_duplicate_fetch_download_already_progress")
return (False, None)
data = None
dredis.set(key, "1")
try:
# stack()
def taskPrintHello():
pingCelery()
dlog.d('task run for print')