def taskDownloadLatestMarketData(domain) -> bool: dlog.d("Downloading as cache is old") result, data = download(doamin=domain, period=1) if result is True: dglobaldata.saveMarketDataFormDayDF(domain, data) return True return False
def configure_celery(app: Flask) -> Celery: """Configures celery instance from app, using it's config""" TaskBase: CeleryTask = extensions.celery.Task class ContextTask(TaskBase): # pylint: disable=too-few-public-methods abstract = True def __call__(self, *args, **kwargs): with app.app_context(): return TaskBase.__call__(self, *args, **kwargs) # https://docs.celeryproject.org/en/stable/userguide/configuration.html extensions.celery.conf.update( broker_url=app.config['CELERY_BROKER_URL'], result_backend=app.config['CELERY_RESULT_BACKEND'], accept_content=app.config['CELERY_ACCEPT_CONTENT'], task_serializer=app.config['CELERY_TASK_SERIALIZER'], result_serializer=app.config['CELERY_TASK_SERIALIZER'], worker_hijack_root_logger=False, beat_schedule=app.config.get('CELERYBEAT_SCHEDULE', {}), worker_redirect_stdouts_level='ERROR', task_always_eager=app.config.get('CELERY_ALWAYS_EAGER', False) ) extensions.celery.Task = ContextTask dlog.d("Celery setup properly:{}".format( app.config.get('CELERYBEAT_SCHEDULE', {}))) return extensions.celery
def performScreen(domain: str, condition: str, columns=[], sort_by: str = None, limit: int = None): dlog.d('[INFO] Running screen for condition: {}, columns: {}'.format( condition, columns)) indicatorHistory = getIndicatorHistory(domain) result = [] sl = 0 columns = [resolveIndicatorExpression(c) for c in columns] condition = resolveCondition(condition) last_error = '' try: for symbol in indicatorHistory.keys(): indicator_map = indicatorHistory[symbol] try: if(eval(condition)): sl += 1 selected_one = {} selected_one['symbol'] = symbol selected_one['name'] = getSymbolList(domain)[ symbol]['name'] selected_one['sector'] = getSymbolList(domain)[ symbol]['sector'] selected_one['close'] = str( np.round(indicator_map['1d'][0]['close'], 2)) selected_one['volume'] = str( np.round(indicator_map['1d'][0]['volume'], 2)) selected_one['rsi_14'] = str( np.round(indicator_map['1d'][0]['rsi_14'], 2)) selected_one['change'] = str( np.round(indicator_map['1d'][0]['close_change_percentage'], 2)) # extra col for col in columns: selected_one[col[0]] = str( np.round(eval(col[1]), 2)) # add used defined data result.append(fixDict(selected_one)) except Exception as e: last_error = 'Are you passing right Filter {}'.format( condition) dlog.ex(e, showStack=False) dlog.e( "We faced the issue when we are running filter for symbol:{}".format(symbol)) # We just ignore this except Exception as e: raise e # Filter None # Do sort if sort_by: sort_key = sort_by.replace("-", "") reverse = sort_by[0] != "-" result = [x for x in result if x.get(sort_key) is not None] result.sort(key=lambda x: float(x.get(sort_key)), reverse=reverse) # Do limit if limit: result = result[:limit] return {'result': result, 'last_error': last_error}
def wrapper(*args, **kwargs): t = time.time() dlog.d("\n\n>>>>>>>>>>>>>>>> STARTING {} <<<<<<<<<<<<<".format( func.__name__)) res = func(*args, **kwargs) dlog.d("\n>>>>>>>>>>>>>>>> ENDING {}, Time taken: {} sec <<<<<<<<<<<<<\n\n".format( func.__name__, time.time() - t)) return res
def loadDataForCandle(candle_type: TCandleType): try: _candleTypeToDataFrameMap[ candle_type.value] = dstorage.load_data_to_disk( dstorage.get_default_path_for_candle((candle_type))) timetracker.mark_last_data_update_ts(candle_type) dlog.d("updating data...") except Exception as e: dlog.ex(e, "not able to load data from storage.")
def buildTechnicalIndicators(df: DataFrame, domain: str): # If somevalue is nan and all calculation just dont work df.fillna(method='ffill', inplace=True) for ticker in getSymbolList(domain).keys(): try: computeIndicator(df, ticker, domain) except Exception as e: dlog.ex(e, "Not able to process data frame for {}".format(ticker)) dlog.d("Indicator compution done proper way") return df
def getLatestMarketData(domain: str, reload: str = "0", sync: str = "0"): # Build indicator if not exist mayGetLatestStockData(domain, reload, sync) mayBuildStockIndicatorInBackground(domain, TCandleType.DAY_1, reload, sync) dlog.d("getting data from cache") return { 'latest': dredis.getPickle("market_data_{}".format(domain)), 'indicator': dredis.getPickle("indicator_data_{}_{}".format(domain, '1d')) }
def taskBuildIndicator(domain: str, candle_type: str): try: pingCelery() _candle_type = TCandleType(candle_type) dglobaldata.downloadAndBuildIndicator(domain, _candle_type) # Compute Summary if _candle_type == TCandleType.DAY_1: pass # dhighlights.taskComputeSummary() buildTaskSuccess("taskBuildIndicator Done", None) except Exception as e: dlog.d("Got exception in taskBuildIndicator") dlog.ex(e)
def wrapper(*args, **kwargs): t = time.time() func_name = func.__name__ if remote_logging: danalytics.reportAction(func_name + "_started") dlog.d("\n\n>>>>>>>>>>>>>>>> STARTING {} <<<<<<<<<<<<<".format( func.__name__)) res = func(*args, **kwargs) dlog.d("\n>>>>>>>>>>>>>>>> ENDING {}, Time taken: {} sec <<<<<<<<<<<<<\n\n".format( func.__name__, time.time() - t)) if remote_logging: danalytics.reportAction(func_name + "_ended") return res
def reportAction(tag: str, extra: dict = {}): dlog.d("logging remore action with tag:" + tag) if isDebug(): dlog.d("ignore remote log in debug mode") return try: simplestore_post( url="{}/api/analytics/action".format(SIMPLESTORE_ENDPOINT), data={ "app_id": APP_ID, "session": session, "tag": tag, "extra": extra }) except Exception as ex: dlog.ex(ex)
def reportException(ex: Exception, location=""): if isDebug(): dlog.d("ignore remote log in debug mode") return try: res = simplestore_post( url="{}/api/analytics/exception".format(SIMPLESTORE_ENDPOINT), data={ "app_id": APP_ID, "session": session, "location": traceback.format_exc().split("\n")[-4].strip(), "type": "exception", "stack": traceback.format_exc(), "args": str(ex.args) }) dlog.d(str(res)) except Exception as ex: dlog.ex(ex)
def mayGetLatestStockData(domain: str, reload, sync: str): if (reload == "1"): dlog.d("taskDownloadLatestMarketData: submitting task") if sync == "1": tasks.taskDownloadLatestMarketData(domain) else: tasks.taskDownloadLatestMarketData.delay(domain) return last_update = dredis.get("market_ts_{}".format(domain), None) if last_update is None or last_update == 'None': dlog.d("No last update - submitting task") tasks.taskDownloadLatestMarketData.delay(domain) elif IfTimeIs5MinOld(last_update): dlog.d("data is 5 min old... submitting task") # tasks.taskDownloadLatestMarketData.delay(domain) else: dlog.d("Data is already there")
def mayBuildStockIndicatorInBackground(domain: str, candle_type: TCandleType, reload: str, sync: str): # reload if reload == "1": if sync == "1": tasks.taskBuildIndicator(domain, candle_type=candle_type.value) else: tasks.taskBuildIndicator.delay(domain, candle_type.value) dlog.d("taskBuildIndicator: task submitted") return if dredis.getPickle("indicator_data_{}_{}".format(domain, '1d')) is None: # task submitted tasks.taskBuildIndicator.delay(domain, candle_type.value) dlog.d("task submitted") else: dlog.d("Data is already there")
def init(): if isDebug(): dlog.d("ignore remote log in debug mode") return global session if session != '': dlog.d("Already session exist") return try: data: dict = simplestore_post( url="{}/api/analytics/launch".format(SIMPLESTORE_ENDPOINT), data={ "app_id": APP_ID, "app_version": "1.0", "device_os": "web", "device_id": "null", "device_api": "null" }) session = data.get('out')[0].get('session') dlog.d("Remote log is inited with session:{}".format(session)) except Exception as ex: dlog.ex(e=ex) pass
except Exception as e: dlog.ex(e) danalytics.reportException(e) # More some info. # for x in final_result.keys(): # final_result[x]['sector'] = getSymbolList(domain=domain)[x]['sector'] return final_result # Rest all locks here - This is needed for restart the server for candle_type in SUPPORTED_CANDLE: for domain in SUPPORTED_DOMAIN: dredis.set( "downloadAndBuildindicator_{}_{}".format(domain, candle_type.value), "0") dlog.d("Reset downloadAndBuildIndicator locks") def getLastUpdatedTimeStamp(domain: str): result = {} for candle_type in SUPPORTED_CANDLE: result['{}-{}'.format(domain, candle_type.value)] = dredis.get( "indicator_timestamp_{}_{}".format(domain, candle_type.value), "Data not found") return result # It will create a cache for laest data frame def saveMarketDataFormDayDF(domain: str, df: DataFrame):
def downloadAndBuildIndicator(domain, candle_type: TCandleType): # Optimization if not shouldBuildIndicator(domain, candle_type): dlog.d("Ignore rebuilding shouldBuildIndicator") return # Locking lockkey = "downloadAndBuildindicator_{}_{}".format(domain, candle_type.value) if dredis.get(lockkey) == "1": dlog.d("downloadAndBuildIndicator locked for key {}".format(lockkey)) raise Exception("downloadAndBuildIndicator is progress") dredis.set(lockkey, "1") try: dlog.d("downloadAndBuildIndicator start") dlog.d("downloadAndBuildIndicator download start") ret_value, download_data = ddownload.download(domain, interval=candle_type) if ret_value is False: dlog.d("Download fails") return { "status": "error", "msg": "something goes wrong", "out": None } dlog.d("downloadAndBuildIndicator building start") processed_df = dindicator.buildTechnicalIndicators( download_data, domain) # DONOT STOARE AS FILEdlog.d("downloadAndBuildIndicator: saving to storage start") #path_to_store = dstorage.get_default_path_for_candle(candle_type) #dstorage.store_data_to_disk(processed_df, path_to_store) dlog.d("downloadAndBuildIndicator: building indicator history map") # Building Indicator map for O(1) looks up. # This will be a 4d map # map[REL][1d][-1][close]... last15SlotIndicator = getLastNIndicatorInJson(domain, processed_df) indicator_history_key = "indicator_history_{}".format(domain) olddata = dredis.getPickle(indicator_history_key) if not olddata: olddata = {} for key in last15SlotIndicator.keys(): if key not in olddata: olddata[key] = {} olddata[key][candle_type.value] = last15SlotIndicator.get(key) dredis.setPickle(indicator_history_key, olddata) dlog.d( "downloadAndBuildIndicator: saved indicator history to {}".format( indicator_history_key)) dlog.d("downloadAndBuildIndicator: saving to redis start") dredis.setPickle( "indicator_data_{}_{}".format(domain, candle_type.value), { 'data': getLatestDataInJson(domain, processed_df), 'timestamp': getCurTimeStr() }) # update market data if candle_type == TCandleType.DAY_1: saveMarketDataFormDayDF(domain, download_data) # Set TimeStamp key dredis.set( "indicator_timestamp_{}_{}".format(domain, candle_type.value), getCurTimeStr()) # unlock dredis.set(lockkey, "0") dlog.d("downloadAndBuildIndicator ends") return { "status": "success", "msg": "Completed snapshot pipeline", "out": None } except Exception as e: dredis.set(lockkey, "0") dlog.d("downloadAndBuildIndicator Exception happened") danalytics.reportException(e, "Exception in downloadAndBuildIndicator") dlog.ex(e) raise e finally: dredis.set(lockkey, "0") pass
def set(key, value): dlog.d("Setting Key is redis :{}".format(key)) _redis.set(key, value)
def saveMarketDataFormDayDF(domain: str, df: DataFrame): dlog.d("Saving market data...") resultJSON = getLatestDataInJson(domain, df) # Save this data dredis.setPickle("market_data_{}".format(domain), {"data": resultJSON}) dredis.set("market_ts_{}".format(domain), getCurTimeStr())
from myapp.core.dlog import stack from myapp.core.dtypes import TCandleType from myapp.core.rootConfig import SUPPORTED_CANDLE from myapp.core.sync import getSymbolList from pandas.core.frame import DataFrame ### Here is how to test down your own# """ import yfinance as yf yf.download("TCS.NS") """ # Rest all locks here for candle_type in SUPPORTED_CANDLE: dredis.set("download_progress_{}".format(candle_type.value), "0") dlog.d("Reset download locks") @trace_perf def download(doamin="IN", interval: TCandleType = TCandleType.DAY_1, period=50) -> typing.Tuple[bool, DataFrame]: key = "download_progress_" + interval.value if (dredis.get(key) == "1"): danalytics.reportAction( "ignore_duplicate_fetch_download_already_progress") return (False, None) data = None dredis.set(key, "1") try: # stack()
def taskPrintHello(): pingCelery() dlog.d('task run for print')