def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
dlog.ex(e, "Unknown error in the server")
danalytics.reportException(e)
return RetHelper.buildException(e)
def download(doamin="IN",
interval: TCandleType = TCandleType.DAY_1,
period=50) -> typing.Tuple[bool, DataFrame]:
key = "download_progress_" + interval.value
if (dredis.get(key) == "1"):
danalytics.reportAction(
"ignore_duplicate_fetch_download_already_progress")
return (False, None)
data = None
dredis.set(key, "1")
try:
# stack()
ticker = [x for x in getSymbolList(doamin).keys()]
data = yf.download(tickers=ticker,
period=covert_to_period_from_duration(
interval, period),
interval=interval.value,
group_by='ticker',
auto_adjust=False,
prepost=False,
threads=True,
proxy=None,
rounding=True)
except Exception as e:
dlog.ex(e)
return (False, None)
finally:
dredis.set(key, "0")
# Sometime it ret duplicate results for the last row so drop it,
data = data[~data.index.duplicated(keep='last')]
return (True, data)
def wrapper(*args, **kwargs):
print(args)
print(kwargs)
func_name = func.__name__
cache_key_loading = "{}_loading".format(cache_key)
cache_key_ts = "{}_ts".format(cache_key)
# Check if cache exist.
if kwargs.get('ignore_cache') != True:
cache = dredis.getPickle(cache_key, None)
if cache:
return cache
# Check global lock
if dredis.get(cache_key_loading) == "1":
raise Exception(
"{} is locked by smart cache".format(func_name))
# Lock
dredis.set(cache_key_loading, "1")
# We need to use try catch to avoid unlock
res = None
try:
# Execute
res = func(*args, **kwargs)
dredis.setPickle(cache_key, res)
dredis.set(cache_key_ts, time.time())
except Exception as e:
dlog.ex(e, "exception happened while executing:{}".format(func_name))
pass
# Unlock
dredis.set(cache_key_loading, "0")
return res
def getLastNIndicatorInJson(domain, df: DataFrame, limit=15):
final_result = {}
try:
df = df.tail(limit)
result = df.to_json(orient="records")
parsed = json.loads(result)
for offset in range(limit):
row = parsed[offset]
# Key => 0, -1,-2....
offset_key = (limit - offset - 1) * -1
for x in row.keys():
pair = literal_eval(x)
symbol = pair[0]
indicator = pair[1]
value = row[x]
if symbol not in final_result:
final_result[symbol] = {}
if offset_key not in final_result[symbol]:
final_result[symbol][offset_key] = {}
final_result[symbol][offset_key][indicator] = value
# print(json.dumps(final_result, indent=4))
except Exception as e:
dlog.ex(e)
danalytics.reportException(e)
# More some info.
# for x in final_result.keys():
# final_result[x]['sector'] = getSymbolList(domain=domain)[x]['sector']
return final_result
def performScreen(domain: str, condition: str, columns=[], sort_by: str = None, limit: int = None):
dlog.d('[INFO] Running screen for condition: {}, columns: {}'.format(
condition, columns))
indicatorHistory = getIndicatorHistory(domain)
result = []
sl = 0
columns = [resolveIndicatorExpression(c) for c in columns]
condition = resolveCondition(condition)
last_error = ''
try:
for symbol in indicatorHistory.keys():
indicator_map = indicatorHistory[symbol]
try:
if(eval(condition)):
sl += 1
selected_one = {}
selected_one['symbol'] = symbol
selected_one['name'] = getSymbolList(domain)[
symbol]['name']
selected_one['sector'] = getSymbolList(domain)[
symbol]['sector']
selected_one['close'] = str(
np.round(indicator_map['1d'][0]['close'], 2))
selected_one['volume'] = str(
np.round(indicator_map['1d'][0]['volume'], 2))
selected_one['rsi_14'] = str(
np.round(indicator_map['1d'][0]['rsi_14'], 2))
selected_one['change'] = str(
np.round(indicator_map['1d'][0]['close_change_percentage'], 2))
# extra col
for col in columns:
selected_one[col[0]] = str(
np.round(eval(col[1]), 2))
# add used defined data
result.append(fixDict(selected_one))
except Exception as e:
last_error = 'Are you passing right Filter {}'.format(
condition)
dlog.ex(e, showStack=False)
dlog.e(
"We faced the issue when we are running filter for symbol:{}".format(symbol))
# We just ignore this
except Exception as e:
raise e
# Filter None
# Do sort
if sort_by:
sort_key = sort_by.replace("-", "")
reverse = sort_by[0] != "-"
result = [x for x in result if x.get(sort_key) is not None]
result.sort(key=lambda x: float(x.get(sort_key)), reverse=reverse)
# Do limit
if limit:
result = result[:limit]
return {'result': result, 'last_error': last_error}
def loadDataForCandle(candle_type: TCandleType):
try:
_candleTypeToDataFrameMap[
candle_type.value] = dstorage.load_data_to_disk(
dstorage.get_default_path_for_candle((candle_type)))
timetracker.mark_last_data_update_ts(candle_type)
dlog.d("updating data...")
except Exception as e:
dlog.ex(e, "not able to load data from storage.")
def buildTechnicalIndicators(df: DataFrame, domain: str):
# If somevalue is nan and all calculation just dont work
df.fillna(method='ffill', inplace=True)
for ticker in getSymbolList(domain).keys():
try:
computeIndicator(df, ticker, domain)
except Exception as e:
dlog.ex(e, "Not able to process data frame for {}".format(ticker))
dlog.d("Indicator compution done proper way")
return df
def taskBuildIndicator(domain: str, candle_type: str):
try:
pingCelery()
_candle_type = TCandleType(candle_type)
dglobaldata.downloadAndBuildIndicator(domain, _candle_type)
# Compute Summary
if _candle_type == TCandleType.DAY_1:
pass
# dhighlights.taskComputeSummary()
buildTaskSuccess("taskBuildIndicator Done", None)
except Exception as e:
dlog.d("Got exception in taskBuildIndicator")
dlog.ex(e)
def wrapper(*args, **kwargs):
func_name = func.__name__
try:
danalytics.reportAction("worker_task_start_{}".format(func_name))
ret = func(*args, **kwargs)
danalytics.reportAction("worker_task_success_{}".format(func_name))
return ret
except Exception as e:
dlog.ex(e, "Unknown error in the server")
danalytics.reportAction(
"worker_task_exception_{}".format(func_name))
danalytics.reportException(e)
return RetHelper.buildException(e)
def reportAction(tag: str, extra: dict = {}):
dlog.d("logging remore action with tag:" + tag)
if isDebug():
dlog.d("ignore remote log in debug mode")
return
try:
simplestore_post(
url="{}/api/analytics/action".format(SIMPLESTORE_ENDPOINT),
data={
"app_id": APP_ID,
"session": session,
"tag": tag,
"extra": extra
})
except Exception as ex:
dlog.ex(ex)
def reportException(ex: Exception, location=""):
if isDebug():
dlog.d("ignore remote log in debug mode")
return
try:
res = simplestore_post(
url="{}/api/analytics/exception".format(SIMPLESTORE_ENDPOINT),
data={
"app_id": APP_ID,
"session": session,
"location": traceback.format_exc().split("\n")[-4].strip(),
"type": "exception",
"stack": traceback.format_exc(),
"args": str(ex.args)
})
dlog.d(str(res))
except Exception as ex:
dlog.ex(ex)
def wrapper(*args, **kwargs):
print(args)
print(kwargs)
func_name = func.__name__
cache_key_loading = "{}_loading".format(cache_key)
if dredis.get(cache_key_loading) == "1":
raise Exception(
"{} is already in progress".format(func_name))
# Lock
dredis.set(cache_key_loading, "1")
# We need to use try catch to avoid unlock
res = None
try:
# Execute
res = func(*args, **kwargs)
except Exception as e:
dlog.ex(e, "exception happened while executing:{}".format(func_name))
pass
# Unlock
dredis.set(cache_key_loading, "0")
return res
def getLatestDataInJson(domain, df: DataFrame):
final_result = {}
try:
df = df.tail(1)
result = df.to_json(orient="records")
parsed = json.loads(result)
for x in parsed[0].keys():
pair = literal_eval(x)
symbol = pair[0]
indicator = pair[1]
value = parsed[0][x]
if symbol not in final_result:
final_result[symbol] = {}
final_result[symbol][indicator] = value
# print(json.dumps(final_result, indent=4))
except Exception as e:
dlog.ex(e)
danalytics.reportException(e)
# More some info.
for x in final_result.keys():
final_result[x]['sector'] = getSymbolList(domain=domain)[x]['sector']
return final_result
def init():
if isDebug():
dlog.d("ignore remote log in debug mode")
return
global session
if session != '':
dlog.d("Already session exist")
return
try:
data: dict = simplestore_post(
url="{}/api/analytics/launch".format(SIMPLESTORE_ENDPOINT),
data={
"app_id": APP_ID,
"app_version": "1.0",
"device_os": "web",
"device_id": "null",
"device_api": "null"
})
session = data.get('out')[0].get('session')
dlog.d("Remote log is inited with session:{}".format(session))
except Exception as ex:
dlog.ex(e=ex)
pass
def computeIndicator(df: DataFrame, ticker, domain: str):
# Make lower case << Validated
df[ticker, 'name'] = getSymbolList(domain)[ticker]['name']
df[ticker, 'symbol'] = getSymbolList(domain)[ticker]['symbol']
# df[ticker, 'sector'] = getSymbolList()[ticker]['sector']
df[ticker, 'open'] = np.round(df[ticker, 'Open'], 2)
df[ticker, 'date'] = df[ticker].index
df[ticker, 'date_iso'] = df[ticker, "date"].apply(
lambda x: x.strftime('%Y-%m-%dT%H:%M:%SZ'))
df[ticker, 'close'] = np.round(df[ticker, 'Close'], 2)
df[ticker, 'high'] = np.round(df[ticker, 'High'], 2)
df[ticker, 'low'] = np.round(df[ticker, 'Low'], 2)
df[ticker, 'volume'] = np.round(df[ticker, 'Volume'], 2)
# define changes
df[ticker, 'close_change_percentage'] = fixRound(
(df[ticker, 'close'] - df[ticker, 'close'].shift(1)) /
df[ticker, 'close'].shift(1) * 100)
df[ticker, 'volume_change_percentage'] = fixRound(
(df[ticker, 'volume'] - df[ticker, 'volume'].shift(1)) /
df[ticker, 'volume'].shift(1) * 100)
# Volatility
df[ticker, 'high_low_gap'] = df[ticker, 'high'] - df[ticker, 'low']
df[ticker, 'high_low_gap_percentage'] = np.round(
(df[ticker, 'high'] - df[ticker, 'low']) / df[ticker, 'close'] * 100,
2)
# write your own info here << Validated
for range in all_range:
df[ticker, "ema_{}".format(range)] = np.round(
df[ticker, "close"].ewm(span=range, adjust=False).mean(), 2)
df[ticker, "sma_{}".format(range)] = np.round(
df[ticker, "close"].rolling(range).mean(), 2)
try:
df[ticker, "wma_{}".format(range)] = talib.WMA(df[ticker, "close"],
timeperiod=range)
except Exception as e:
dlog.ex(e, "not able compute wma", showStack=False)
df[ticker, "wma_{}".format(range)] = 0
# validated
# macd and RSI
# df[ticker, 'macd'] = talib.MACD(df[ticker, 'close'].as_matrix())
# df[ticker, "macd_macd"], df[ticker, "macd_macdsignal"], df[ticker, "macd_macdhist"] = talib.MACD(
# df.close, fastperiod=12, slowperiod=26, signalperiod=9)
# TAlib cause exception so just be in the safe
try:
df[ticker,
'rsi_14'] = np.round(talib.RSI(df[ticker, 'close'].values, 14), 2)
df[ticker, 'rsi_18'] = talib.RSI(df[ticker, 'close'].values, 18)
# band
df[ticker, 'bb_up_5'], df[ticker,
'bb_mid_5'], df[ticker,
'bb_down_5'] = talib.BBANDS(
df[ticker, 'close'],
timeperiod=5)
# df[ticker, 'bb_up_15'], df[ticker, 'bb_mid_15'], df[ticker, 'bb_down_15'] = talib.BBANDS(
# df[ticker, 'close'], timeperiod=15)
# df[ticker, 'bb_up_60'], df[ticker, 'bb_mid_60'], df[ticker, 'bb_down_60'] = talib.BBANDS(
# df[ticker, 'close'], timeperiod=60)
df[ticker, 'sar'] = talib.SAR(df[ticker, 'high'],
df[ticker, 'low'],
acceleration=0.02,
maximum=0.2)
df[ticker, 'atr_14'] = talib.ATR(df[ticker, 'high'],
df[ticker, 'low'],
df[ticker, 'close'],
timeperiod=14)
df[ticker, 'natr_14'] = talib.NATR(df[ticker, 'high'],
df[ticker, 'low'],
df[ticker, 'close'],
timeperiod=14)
df[ticker, 'tr_14'] = talib.TRANGE(df[ticker, 'high'],
df[ticker, 'low'], df[ticker,
'close'])
except Exception as e:
dlog.ex(e,
"Not able to calculate RSI for Ticker".format(ticker),
showStack=False)
danalytics.reportAction("talib_exception_for_{}".format(ticker))
df[ticker, 'rsi_14'] = -1
df[ticker, 'rsi_18'] = -1
df[ticker, 'bb_up_5'], df[ticker,
'bb_mid_5'], df[ticker,
'bb_down_5'] = (-1, -1, -1)
df[ticker, 'sar'] = -1
df[ticker, 'atr_14'] = -1
df[ticker, 'natr_14'] = -1
df[ticker, 'tr_14'] = -1
def downloadAndBuildIndicator(domain, candle_type: TCandleType):
# Optimization
if not shouldBuildIndicator(domain, candle_type):
dlog.d("Ignore rebuilding shouldBuildIndicator")
return
# Locking
lockkey = "downloadAndBuildindicator_{}_{}".format(domain,
candle_type.value)
if dredis.get(lockkey) == "1":
dlog.d("downloadAndBuildIndicator locked for key {}".format(lockkey))
raise Exception("downloadAndBuildIndicator is progress")
dredis.set(lockkey, "1")
try:
dlog.d("downloadAndBuildIndicator start")
dlog.d("downloadAndBuildIndicator download start")
ret_value, download_data = ddownload.download(domain,
interval=candle_type)
if ret_value is False:
dlog.d("Download fails")
return {
"status": "error",
"msg": "something goes wrong",
"out": None
}
dlog.d("downloadAndBuildIndicator building start")
processed_df = dindicator.buildTechnicalIndicators(
download_data, domain)
# DONOT STOARE AS FILEdlog.d("downloadAndBuildIndicator: saving to storage start")
#path_to_store = dstorage.get_default_path_for_candle(candle_type)
#dstorage.store_data_to_disk(processed_df, path_to_store)
dlog.d("downloadAndBuildIndicator: building indicator history map")
# Building Indicator map for O(1) looks up.
# This will be a 4d map
# map[REL][1d][-1][close]...
last15SlotIndicator = getLastNIndicatorInJson(domain, processed_df)
indicator_history_key = "indicator_history_{}".format(domain)
olddata = dredis.getPickle(indicator_history_key)
if not olddata:
olddata = {}
for key in last15SlotIndicator.keys():
if key not in olddata:
olddata[key] = {}
olddata[key][candle_type.value] = last15SlotIndicator.get(key)
dredis.setPickle(indicator_history_key, olddata)
dlog.d(
"downloadAndBuildIndicator: saved indicator history to {}".format(
indicator_history_key))
dlog.d("downloadAndBuildIndicator: saving to redis start")
dredis.setPickle(
"indicator_data_{}_{}".format(domain, candle_type.value), {
'data': getLatestDataInJson(domain, processed_df),
'timestamp': getCurTimeStr()
})
# update market data
if candle_type == TCandleType.DAY_1:
saveMarketDataFormDayDF(domain, download_data)
# Set TimeStamp key
dredis.set(
"indicator_timestamp_{}_{}".format(domain, candle_type.value),
getCurTimeStr())
# unlock
dredis.set(lockkey, "0")
dlog.d("downloadAndBuildIndicator ends")
return {
"status": "success",
"msg": "Completed snapshot pipeline",
"out": None
}
except Exception as e:
dredis.set(lockkey, "0")
dlog.d("downloadAndBuildIndicator Exception happened")
danalytics.reportException(e, "Exception in downloadAndBuildIndicator")
dlog.ex(e)
raise e
finally:
dredis.set(lockkey, "0")
pass
