def get_stock_by_name(self, name):
"""
get stocks from the redis with pattern matching and return in dict format
:param
name: name to search
:return:
success response : {"status":1,"data":"--dict of stocks--"}
fail response : {"status":0,"data":"--error message--"}
"""
res = {"status": 0, "data": ""}
try:
redis_conn_res = get_redis_connection()
if not redis_conn_res["status"]:
return redis_conn_res
redis_conn = redis_conn_res["data"]
keys = redis_conn.scan_iter(match='STOCK:*' + str(name).upper() +
'*')
stacks = []
print(type(keys))
for key in keys:
reg = redis_conn.hgetall(key)
stacks.append(reg)
print(stacks)
res["status"], res["data"] = 1, stacks
except Exception as e:
traceback.print_exc()
res["data"] = "Something went wrong, Kindly try again."
return res
def delay(*args, **kwargs):
db = config.get_redis_connection()
qkey = settings.TASK_QUEUE_NAME
task_id = str(uuid.uuid4())
key = '{0}:{1}'.format(qkey, task_id)
s = pickle.dumps((f, task_id, args, kwargs))
db.rpush(settings.TASK_QUEUE_NAME, s)
return DelayedResult(task_id)
def get_top_stocks(self, page_no):
"""
get top stocks from the redis and return in dict format
:param
page_no: page no for pagination
:return:
success response : {"status":1,"data":"--dict of stocks--","count":"--total count of stocks--"}
fail response : {"status":0,"data":"--error message--"}
"""
res = {"status": 0, "data": ""}
try:
redis_conn_res = get_redis_connection()
if not redis_conn_res["status"]:
return redis_conn_res
redis_conn = redis_conn_res["data"]
total_count = 0
if page_no == 0:
total_count = redis_conn.zcount(
"search_sorted", "-inf", "+inf"
) #the number of elements in the specified score range.
if total_count == 0:
"""
If data is not loaded, call parser to load it.
"""
res = self.get_latest_stocks()
if res["status"]:
total_count = redis_conn.zcount(
"search_sorted", "-inf", "+inf")
else:
return res
start_index = page_no * self.pagination_size
end_index = start_index + self.pagination_size - 1
keys = redis_conn.zrevrange("search_sorted",
start_index,
end_index,
withscores=False)
top_stacks = []
for key in keys:
reg = redis_conn.hgetall(key)
top_stacks.append(reg)
print(top_stacks)
date = redis_conn.get("latest_date")
res["status"], res["data"], res["count"], res[
"date"] = 1, top_stacks, total_count, date
except Exception as e:
traceback.print_exc()
res["data"] = "Something went wrong, Kindly try again."
return res
def get_top_stocks(self, page_no):
res = {"status": 0, "data": ""}
try:
redis_conn_res = get_redis_connection()
if not redis_conn_res["status"]:
return redis_conn_res
redis_conn = redis_conn_res["data"]
total_count = 0
if page_no == 0:
total_count = redis_conn.zcount("search_sorted", "-inf",
"+inf")
if total_count == 0:
"""
If data is not loaded, call parser to load it.
"""
res = self.get_latest_stocks()
if res["status"]:
total_count = redis_conn.zcount(
"search_sorted", "-inf", "+inf")
else:
return res
start_index = page_no * self.pagination_size
end_index = start_index + self.pagination_size - 1
keys = redis_conn.zrevrange("search_sorted",
start_index,
end_index,
withscores=False)
top_stacks = []
for key in keys:
reg = redis_conn.hgetall(key)
top_stacks.append(reg)
print(top_stacks)
date = redis_conn.get("latest_date")
res["status"], res["data"], res["count"], res[
"date"] = 1, top_stacks, total_count, date
except Exception as e:
traceback.print_exc()
res["data"] = "Something went wrong, Kindly try again."
return res
def get_stock_by_name(self, name):
res = {"status": 0, "data": ""}
try:
redis_conn_res = get_redis_connection()
if not redis_conn_res["status"]:
return redis_conn_res
redis_conn = redis_conn_res["data"]
keys = redis_conn.scan_iter(match='STOCK:*' + str(name).upper() +
'*')
stacks = []
print(type(keys))
for key in keys:
reg = redis_conn.hgetall(key)
stacks.append(reg)
print(stacks)
res["status"], res["data"] = 1, stacks
except Exception as e:
traceback.print_exc()
res["data"] = "Something went wrong, Kindly try again."
return res
def queue_daemon(app, rv_ttl=settings.TASK_QUEUE_KEY_TTL):
db = config.get_redis_connection()
while True:
msg = db.blpop(settings.TASK_QUEUE_NAME)
print('Running task: {0}'.format(msg))
func, task_id, args, kwargs = pickle.loads(msg[1])
qkey = settings.TASK_QUEUE_NAME
key = '{0}:{1}'.format(qkey, task_id)
data = {'date': time.time(), 'task_id': task_id, 'status': 'running', 'result': None}
db.set(key, json.dumps(data))
try:
rv = func(*args, **kwargs)
data['status'] = 'complete'
except Exception, e:
import traceback
traceback.print_exc()
rv = e
data['status'] = 'error'
if not isinstance(rv, dict):
rv = str(rv)
data['result'] = rv
if rv is not None:
db.set(key, json.dumps(data))
db.expire(key, rv_ttl)
def __init__(self, key):
self.db = config.get_redis_connection()
self.key = key
self._rv = None
def task_status(task_id=None):
rds = get_redis_connection()
v = rds.get('{0}:{1}'.format(app.config.get('TASK_QUEUE_NAME'), task_id))
if not v:
v = jsonify({'status': 'fail', 'result': 'invalid task'})
return v
from flask import Blueprint
from flask import request, jsonify, session, Response, json
from config import create_app, get_redis_connection
from decorators import api_key_required
import tempfile
from utils.applications import deploy_application, delete_application
import time
app = create_app()
bp = api_blueprint = Blueprint('api', __name__)
rds = get_redis_connection()
def generate_task_response(task_id=None):
return jsonify({'status': 'ok', 'task_id': task_id})
def wait_for_task(task_id=None):
key = '{0}:{1}'.format(app.config.get('TASK_QUEUE_NAME'), task_id)
while True:
try:
if json.loads(rds.get(key))['status'] != 'running':
break
except:
# if task hasn't started, the json.loads will fail because the task
# data is still pickled from the queue
pass
time.sleep(1)
return rds.get(key)
@bp.route('')
def index():
data = {
import json
import logging
from config import get_multivac_db, get_redis_connection
from flask import Blueprint, request, Response
from multivac.process_data import process_data
from rq import Queue
db = get_multivac_db()
redis_connection = get_redis_connection()
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.debug(db)
multivac_bp = Blueprint('multivac', __name__, template_folder="templates")
@multivac_bp.route("/")
def homepage():
return "Welcome to application!"
@multivac_bp.route("/multivac", methods=['GET'])
def get_multivac():
n_answer = db.entropy.find().count()
if n_answer == 0:
return "INSUFFICIENT DATA FOR MEANINGFUL ANSWER."
else:
answer = db.entropy.find_one()['data']
return answer
def load_zip_to_redis(self):
res = {"status": 0, "data": ""}
try:
#Fetch Zip URL
zip_url_res = self.get_zip_url()
if not zip_url_res["status"]:
return zip_url_res
# fetch and extract zip file
csv_file_res = self.extract_csv_file(zip_url_res['data'])
if not csv_file_res["status"]:
return csv_file_res
csv_path = csv_file_res["data"]
csv_file_name = csv_file_res["name"]
csv_file_date = csv_file_name[2:4] + "-" + csv_file_name[
4:6] + "-20" + csv_file_name[6:8]
# Get redis connection
redis_conn_res = get_redis_connection()
if not redis_conn_res["status"]:
return redis_conn_res
redis_conn = redis_conn_res["data"]
loaded_data_date = redis_conn.get("latest_date")
if csv_file_date == loaded_data_date:
res["data"] = "Database already have latest data."
return res
# Open csv file and read it
csv_list = csv.DictReader(open(csv_path, 'r'))
# start redis pipeline, with transaction
redis_pipeline = redis_conn.pipeline(transaction=True)
redis_pipeline.flushdb()
for row in csv_list:
#
stripped_key = "STOCK:" + row['SC_CODE'].rstrip(
) + ":" + row['SC_NAME'].rstrip()
value = {
'name': row['SC_NAME'].rstrip(),
'code': row['SC_CODE'],
'open': float(row['OPEN']),
'close': float(row['CLOSE']),
'high': float(row['HIGH']),
'low': float(row['LOW'])
}
percentage = round(
((value['close'] - value['open']) / value['open']) * 100,
2)
value['percentage'] = round(percentage, 2)
redis_pipeline.hmset(stripped_key, dict(value))
#Assuming logic of top 10 stock entries must be "Highest positive percentage movement first"
#Using sorted set and putting percentage as score
redis_pipeline.zadd("search_sorted",
{stripped_key: percentage})
#Storing date for withc bhavcopy has been loaded to redis
print(
"latest_date", csv_file_name[2:4] + "-" + csv_file_name[4:6] +
"-20" + csv_file_name[6:8])
redis_pipeline.set("latest_date", csv_file_date)
#execute the pipeline
redis_pipeline.execute()
res["status"] = 1
res["data"] = "Done."
except Exception as e:
traceback.print_exc()
res["data"] = "Something went wrong, Kindly try again."
return res
