def wrapper(*args, **kwargs):
print(args)
print(kwargs)
func_name = func.__name__
cache_key_loading = "{}_loading".format(cache_key)
if dredis.get(cache_key_loading) == "1":
raise Exception(
"{} is already in progress".format(func_name))
# Lock
dredis.set(cache_key_loading, "1")
# We need to use try catch to avoid unlock
res = None
try:
# Execute
res = func(*args, **kwargs)
except Exception as e:
dlog.ex(e, "exception happened while executing:{}".format(func_name))
pass
# Unlock
dredis.set(cache_key_loading, "0")
return res
def wrapper(*args, **kwargs):
print(args)
print(kwargs)
func_name = func.__name__
cache_key_loading = "{}_loading".format(cache_key)
cache_key_ts = "{}_ts".format(cache_key)
# Check if cache exist.
if kwargs.get('ignore_cache') != True:
cache = dredis.getPickle(cache_key, None)
if cache:
return cache
# Check global lock
if dredis.get(cache_key_loading) == "1":
raise Exception(
"{} is locked by smart cache".format(func_name))
# Lock
dredis.set(cache_key_loading, "1")
# We need to use try catch to avoid unlock
res = None
try:
# Execute
res = func(*args, **kwargs)
dredis.setPickle(cache_key, res)
dredis.set(cache_key_ts, time.time())
except Exception as e:
dlog.ex(e, "exception happened while executing:{}".format(func_name))
pass
# Unlock
dredis.set(cache_key_loading, "0")
return res
def mark_dataload_end(candle_type: TCandleType):
dredis.set("download_start_{}".format(candle_type.value), "0")
dredis.set("download_start_{}_ts".format(candle_type.value),
int(time.time()))
def mark_dataload_start(candle_type: TCandleType):
dredis.set("download_start_{}".format(candle_type.value), "1")
from myapp.core.ddecorators import trace_perf
from myapp.core.dlog import stack
from myapp.core.dtypes import TCandleType
from myapp.core.rootConfig import SUPPORTED_CANDLE
from myapp.core.sync import getSymbolList
from pandas.core.frame import DataFrame
### Here is how to test down your own#
"""
import yfinance as yf
yf.download("TCS.NS")
"""
# Rest all locks here
for candle_type in SUPPORTED_CANDLE:
dredis.set("download_progress_{}".format(candle_type.value), "0")
dlog.d("Reset download locks")
@trace_perf
def download(doamin="IN",
interval: TCandleType = TCandleType.DAY_1,
period=50) -> typing.Tuple[bool, DataFrame]:
key = "download_progress_" + interval.value
if (dredis.get(key) == "1"):
danalytics.reportAction(
"ignore_duplicate_fetch_download_already_progress")
return (False, None)
data = None
dredis.set(key, "1")
try:
def downloadAndBuildIndicator(domain, candle_type: TCandleType):
# Optimization
if not shouldBuildIndicator(domain, candle_type):
dlog.d("Ignore rebuilding shouldBuildIndicator")
return
# Locking
lockkey = "downloadAndBuildindicator_{}_{}".format(domain,
candle_type.value)
if dredis.get(lockkey) == "1":
dlog.d("downloadAndBuildIndicator locked for key {}".format(lockkey))
raise Exception("downloadAndBuildIndicator is progress")
dredis.set(lockkey, "1")
try:
dlog.d("downloadAndBuildIndicator start")
dlog.d("downloadAndBuildIndicator download start")
ret_value, download_data = ddownload.download(domain,
interval=candle_type)
if ret_value is False:
dlog.d("Download fails")
return {
"status": "error",
"msg": "something goes wrong",
"out": None
}
dlog.d("downloadAndBuildIndicator building start")
processed_df = dindicator.buildTechnicalIndicators(
download_data, domain)
# DONOT STOARE AS FILEdlog.d("downloadAndBuildIndicator: saving to storage start")
#path_to_store = dstorage.get_default_path_for_candle(candle_type)
#dstorage.store_data_to_disk(processed_df, path_to_store)
dlog.d("downloadAndBuildIndicator: building indicator history map")
# Building Indicator map for O(1) looks up.
# This will be a 4d map
# map[REL][1d][-1][close]...
last15SlotIndicator = getLastNIndicatorInJson(domain, processed_df)
indicator_history_key = "indicator_history_{}".format(domain)
olddata = dredis.getPickle(indicator_history_key)
if not olddata:
olddata = {}
for key in last15SlotIndicator.keys():
if key not in olddata:
olddata[key] = {}
olddata[key][candle_type.value] = last15SlotIndicator.get(key)
dredis.setPickle(indicator_history_key, olddata)
dlog.d(
"downloadAndBuildIndicator: saved indicator history to {}".format(
indicator_history_key))
dlog.d("downloadAndBuildIndicator: saving to redis start")
dredis.setPickle(
"indicator_data_{}_{}".format(domain, candle_type.value), {
'data': getLatestDataInJson(domain, processed_df),
'timestamp': getCurTimeStr()
})
# update market data
if candle_type == TCandleType.DAY_1:
saveMarketDataFormDayDF(domain, download_data)
# Set TimeStamp key
dredis.set(
"indicator_timestamp_{}_{}".format(domain, candle_type.value),
getCurTimeStr())
# unlock
dredis.set(lockkey, "0")
dlog.d("downloadAndBuildIndicator ends")
return {
"status": "success",
"msg": "Completed snapshot pipeline",
"out": None
}
except Exception as e:
dredis.set(lockkey, "0")
dlog.d("downloadAndBuildIndicator Exception happened")
danalytics.reportException(e, "Exception in downloadAndBuildIndicator")
dlog.ex(e)
raise e
finally:
dredis.set(lockkey, "0")
pass
def saveMarketDataFormDayDF(domain: str, df: DataFrame):
dlog.d("Saving market data...")
resultJSON = getLatestDataInJson(domain, df)
# Save this data
dredis.setPickle("market_data_{}".format(domain), {"data": resultJSON})
dredis.set("market_ts_{}".format(domain), getCurTimeStr())
final_result[symbol][offset_key][indicator] = value
# print(json.dumps(final_result, indent=4))
except Exception as e:
dlog.ex(e)
danalytics.reportException(e)
# More some info.
# for x in final_result.keys():
# final_result[x]['sector'] = getSymbolList(domain=domain)[x]['sector']
return final_result
# Rest all locks here - This is needed for restart the server
for candle_type in SUPPORTED_CANDLE:
for domain in SUPPORTED_DOMAIN:
dredis.set(
"downloadAndBuildindicator_{}_{}".format(domain,
candle_type.value), "0")
dlog.d("Reset downloadAndBuildIndicator locks")
def getLastUpdatedTimeStamp(domain: str):
result = {}
for candle_type in SUPPORTED_CANDLE:
result['{}-{}'.format(domain, candle_type.value)] = dredis.get(
"indicator_timestamp_{}_{}".format(domain, candle_type.value),
"Data not found")
return result
# It will create a cache for laest data frame
