async def get_receipts(username: str):
receipts = await retrieve_user_receipts(username)
receipts.sort(
key=lambda x: datetime.strptime(x["date"], '%d/%m/%Y %H:%M:%S'),
reverse=True)
receipts_without_time = []
for receipt in receipts:
receipt["date"] = receipt["date"].split(" ")[0]
receipts_without_time.append(receipt)
if receipts:
return ResponseModel(receipts_without_time, 200,
"Receipts data retrieved successfully")
return ResponseModel(receipts, 200, "Empty list returned")
async def get_category_expense_analytics(username: str):
result = defaultdict(dict)
# Everything is monthly for now
receipts = await retrieve_user_receipts(username)
if receipts:
for receipt in receipts:
receipt_date = receipt["date"]
category = receipt["category"]
amount = receipt["amount"]
date_time_obj = datetime.strptime(receipt_date,
'%d/%m/%Y %H:%M:%S')
month_and_year = date_time_obj.strftime('%b %Y')
if category not in result[month_and_year]:
result[month_and_year][category] = 0.0
try:
result[month_and_year][category] += float(amount)
except ValueError:
result[month_and_year][category] += 0
result = OrderedDict(
sorted(result.items(),
key=lambda t: datetime.strptime(t[0], '%b %Y'),
reverse=True))
return ResponseModel(result, 200,
"Receipts data retrieved successfully")
return ErrorResponseModel([], 400, "No Receipts")
# NOTE: For sample response
# receipt = Receipt("Pasta Express\t\r\nSMU, 40 Stamford\t23 February 2021\t\r\nRoad S(178908)\t1:42 pm\t\r\nSMUCONNEXION\t\r\n+65 9711 9906\t\r\nReceipt: ZThw\t\r\nTicket: 85\t\r\n2 Vegetable + 1 Meat+\tS$5.80\t\r\nSpaghetti\t\r\nSMU Staff\tSS0.58\t\r\n(10% off)\t\r\nSubtotal\tS$5.22\t\r\nRounding\t-S$0.02\t\r\nTotal\tS$5.20\t\r\nCash\tS$5.20\t\r\nChange\tSSO.00\t\r\nThank you!\t\r\n")
# print(receipt)
async def get_expense_analytics(username: str):
result = defaultdict(float)
# Initialise past 6 months
this_month = date.today().replace(day=1)
for i in range(6):
if i == 0:
result[this_month.strftime('%b %Y')] = 0.0
else:
this_month = (this_month - timedelta(days=1)).replace(day=1)
result[this_month.strftime('%b %Y')] = 0.0
# Everything is monthly for now
receipts = await retrieve_user_receipts(username)
if receipts:
for receipt in receipts:
receipt_date = receipt["date"]
date_time_obj = datetime.strptime(receipt_date,
'%d/%m/%Y %H:%M:%S')
month_and_year = date_time_obj.strftime('%b %Y')
if month_and_year in result:
result[month_and_year] += float(receipt["amount"])
else:
continue
result = OrderedDict(
sorted(result.items(),
key=lambda t: datetime.strptime(t[0], '%b %Y')))
return ResponseModel(result, 200,
"Receipts data retrieved successfully")
return ErrorResponseModel([], 400, "No Receipts")
async def add_image_info(file: UploadFile = File(...)):
img = Image.open(file.file)
exif = {
ExifTags.TAGS[k]: v
for k, v in img._getexif().items() if k in ExifTags.TAGS
}
return ResponseModel(exif, "image added successfully.")
async def delete_receipt_data(username: str, receipt_id: str):
deleted_receipt = await delete_receipt(username, receipt_id)
if deleted_receipt:
return ResponseModel("Receipt with ID: {} removed".format(receipt_id),
200, "Receipt deleted successfully")
return ErrorResponseModel(
"An error occurred", 400,
"Receipt with id {} doesn't exist".format(receipt_id))
def work(self):
try:
url = "http://search.sunbiz.org/Inquiry/CorporationSearch/SearchResults"
payload = "inquiryType=EntityName&searchTerm=Shayne"
headers = {'Content-Type': "application/x-www-form-urlencoded"}
response = requests.request("GET",
url,
data=payload,
headers=headers)
soup = bs(response.text, 'html.parser')
elems = soup.select('tbody tr')
result = ResponseModel()
result.Return_Count = len(elems)
url = "http://search.sunbiz.org/Inquiry/CorporationSearch/SearchResults"
querystring = {
'inquiryType': 'EntityName',
'searchTerm': 'SMITH COMPANY, INC.'
}
headers = {'Content-Type': "application/x-www-form-urlencoded"}
response = requests.request("GET",
url,
params=querystring,
headers=headers)
soup = bs(response.text, 'html.parser')
elems = soup.select('tbody tr')
for row in elems:
elem = row.select_one('td a')
if elem == None:
continue
link = elem['href']
entity_name = elem.text.strip()
print(entity_name)
corp = self.get_corp_FL('http://search.sunbiz.org/' + link)
if corp is not None:
result.Return_Result.append(corp)
result.Return_Count = len(result.Return_Result)
except Exception as e:
print(str(e))
return False
finally:
self.quit_browser()
def mutate(self, info, **kwargs):
req_kw = kwargs['request']
request_instance = RequestModel.objects.get(**req_kw)
kwargs['request'] = request_instance
del kwargs['request']
new_response_instance = ResponseModel(**kwargs)
print 'new response instance: '
print new_response_instance
new_response_instance.save()
return CreateResponse(response=new_response_instance)
async def upload_ocrreceipt(
username: str, request: dict
): # if never specify, its gonna be request body from call
# Get json name from request
image = request["image"]
filepath = request["filepath"]
try:
name = request["name"]
except KeyError:
name = ""
try:
category = request["category"]
except KeyError:
category = ""
# Call OCRSpace
response = call_ocrspace(image)
response_json = response.json()
try:
raw_response = response_json["ParsedResults"][0]["ParsedText"]
print(raw_response)
receipt_dict = Receipt(raw_response, filepath, name,
category).to_dict()
# success, receipt = await add_receipt(username, receipt_dict)
return ResponseModel(receipt_dict, 200,
"Receipt Successfully Uploaded")
except:
empty_error_dict = {
"name": "",
"amount": "",
"date": "",
"items": {},
"image": filepath,
"category": ""
}
return ResponseModel(empty_error_dict, 200,
"Receipt Successfully Uploaded")
def post(self):
json_data = request.get_json(force=True)
project_id = json_data['project_id']
domain_id = json_data['domain_id']
result = ResponseModel.create_response(json_data)
# Clear redis cache
r.delete("responses_" + str(project_id) + "_" + str(domain_id))
return result
async def create_user(username: str, request: dict):
# initialise empty receipt ids
request["receipt_ids"] = []
request["username"] = username
created, user = await add_user(request)
if not (created):
return ErrorResponseModel('Username already exists', 400,
'Please choose another username')
return ResponseModel(user, 201, "User successfully created.")
def response_model_from_yaml(yml):
resp = ResponseModel()
for obj in yml:
if 'send' in obj:
resp.send(obj['send']['message'])
if 'receive' in obj:
resp.receive(
obj['receive']['key'],
getattr(sys.modules[__name__], obj['receive']['expect_type']),
getattr(Actions, obj['receive']['on_failure']))
if 'conditional' in obj:
resp.conditional(
lambda message: eval(obj['conditional']['condition']),
response_model_from_yaml(
obj['conditional']['response_if_true']),
response_model_from_yaml(
obj['conditional']['response_if_false']))
if 'stop' in obj:
resp.build(None)
return resp
async def get_receipt(username: str, receipt_id: str):
receipt = await retrieve_user_receipt(receipt_id)
if receipt:
return ResponseModel(
receipt,
200,
"Success",
)
return ErrorResponseModel(None, 400,
"There was an error updating the receipt data")
def __init__(self):
self.ProjectsModel = ProjectsModel()
self.DomainModel = DomainsModel()
self.IntentsModel = IntentsModel()
self.IntentDetailModel = IntentDetailModel()
self.StoryModel = StoryModel()
self.StoryDetailModel = StoryDetailModel()
self.ResponseModel = ResponseModel()
self.ResponseDetailModel = ResponseDetailModel()
self.EntityModel = EntityModel()
self.CustomActionsModel = CustomActionsModel()
self.ValidateData = ValidateData()
self.project_home = ''
#self.project_base_path = '../vol_chatbot_data/temp/trainer-sessions/'
self.project_base_path = ''
self.session_id = ''
self.master_nlu = {"rasa_nlu_data": {"common_examples": []}}
self.master_stories = ""
self.master_domain_intents = ""
self.master_domain_actions = ""
self.master_domain_templates = ""
self.master_domain_entities = ""
async def upload_qrreceipt(
username: str, request: dict
): # if never specify, its gonna be request body from call
# add Hours minutes and seconds to date
request["date"] += " 00:00:00"
success, receipt = await add_receipt(username, request)
if success:
return ResponseModel(receipt, 201, "Receipt Successfully Uploaded")
return ErrorResponseModel("An error occurred.", 400,
"There was an error uploading the receipt data")
async def post_dialogflow(username: str, request: dict):
text = request["text"]
name = request["name"]
session_client = dialogflow.SessionsClient()
session = session_client.session_path('gitrich-9txq', str(uuid.uuid4()))
text_input = dialogflow.TextInput(text=text, language_code='en')
query_input = dialogflow.QueryInput(text=text_input)
response = session_client.detect_intent(request={
"session": session,
"query_input": query_input
})
params = response.query_result.parameters
try:
price = params["unit-currency"]["amount"]
price = f"{price:.2f}"
except TypeError:
return ErrorResponseModel("Price not found within text", 400,
"Please provide the price of the item")
if params["foods"] != "":
category = "Food & Drinks"
else:
category = "Entertainment"
receipt = {
"name":
name,
"amount":
price,
"items": {},
"category":
category,
"image":
"",
"date":
datetime.now(
pytz.timezone('Asia/Singapore')).strftime('%d/%m/%Y %H:%M:%S')
}
success, receipt = await add_receipt(username, receipt)
if not (success):
return ErrorResponseModel("Unexpected Error Occured", 400,
"Please check request body again")
return ResponseModel(receipt, 201, "Successfully uploaded adhoc receipt")
def mutate(self, info, **kwargs):
print "getting request info: "
req_kw = kwargs['request']
print req_kw
request_instance = RequestModel.objects.get(**req_kw)
print "request instance: "
print request_instance.id
#kwargs['request'] = request_instance
del kwargs['request']
new_response_instance = ResponseModel(**kwargs)
new_response_instance.request = request_instance
print 'new response instance: '
print new_response_instance
new_response_instance.save()
return CreateResponse(response=new_response_instance)
async def login_user(username: str, request: dict):
password = request.get("password", False)
# never send password
if not (password):
return ErrorResponseModel('Request body incomplete', 400,
'Please send the correct request')
exist, user = await check_user_existence(username, password)
if not (exist):
return ErrorResponseModel('User not found', 400,
'Please enter the correct username')
return ResponseModel(user, 200, "User logged in.")
async def update_receipt_data(username: str,
receipt_id: str,
req: UpdateReceiptModel = Body(...)):
req = {k: v for k, v in req.dict().items() if v is not None}
if " " not in req["date"]:
req["date"] += " 00:00:00"
updated_receipts = await update_receipt(receipt_id, req)
if updated_receipts:
return ResponseModel(
"Receipt with ID: {} update is successful".format(receipt_id),
200,
"Success",
)
return ErrorResponseModel("An error occurred.", 400,
"There was an error updating the receipt data")
def get(self):
project_id = request.args.getlist('project_id')[0]
domain_id = request.args.getlist('domain_id')[0]
# check if result can be served from cache
if r.exists("responses_"+str(project_id)+"_"+str(domain_id)):
return json.loads(r.get("responses_"+str(project_id)+"_"+str(domain_id)))
else:
# Get results and update the cache with new values
logging.debug('getting Data from DB')
result = ResponseModel.get_responses(project_id, domain_id)
r.set("responses_"+str(project_id)+"_"+str(domain_id), json.dumps(result), ex=GLOBAL_EXPIRY)
return result
class Export:
def __init__(self):
self.ProjectsModel = ProjectsModel()
self.DomainModel = DomainsModel()
self.IntentsModel = IntentsModel()
self.IntentDetailModel = IntentDetailModel()
self.StoryModel = StoryModel()
self.StoryDetailModel = StoryDetailModel()
self.ResponseModel = ResponseModel()
self.ResponseDetailModel = ResponseDetailModel()
self.EntityModel = EntityModel()
self.CustomActionsModel = CustomActionsModel()
self.ValidateData = ValidateData()
self.project_home = ''
#self.project_base_path = '../vol_chatbot_data/temp/trainer-sessions/'
self.project_base_path = ''
self.session_id = ''
self.master_nlu = {"rasa_nlu_data": {"common_examples": []}}
self.master_stories = ""
self.master_domain_intents = ""
self.master_domain_actions = ""
self.master_domain_templates = ""
self.master_domain_entities = ""
async def reset_globals(self):
self.master_nlu = {"rasa_nlu_data": {"common_examples": []}}
self.master_stories = ""
self.master_domain_intents = ""
self.master_domain_actions = ""
self.master_domain_templates = ""
self.master_domain_entities = ""
self.session_id = ""
async def clean_up(self, sid):
print(
"Cleaning up on session disconnect for session ID {}".format(sid))
if os.path.isdir(self.project_base_path + sid):
shutil.rmtree(self.project_base_path + sid)
return 1
def call_main(self, project_id):
return asyncio.run(self.main(project_id))
async def main(self, project_id):
# Invoke Method to start parallel activity
print("Starting export for project ID {}".format(project_id))
if not os.path.exists('/rasa_projects/'):
os.makedirs('/rasa_projects/')
self.project_base_path = '/rasa_projects/'
self.project_home = self.project_base_path + project_id
await self.reset_globals()
if os.path.isdir(self.project_home):
print("Directory already exists , Empty the directory")
print("Working on project home {}".format(self.project_home))
try:
shutil.rmtree(self.project_home + '/data')
shutil.rmtree(self.project_home + '/skills')
os.remove(self.project_home + '/config.yml')
os.remove(self.project_home + '/domain.yml')
except FileNotFoundError:
print("Folder already clear")
else:
print("Working on project home {}".format(self.project_home))
if not os.path.exists(self.project_home):
os.mkdir(self.project_home)
os.mkdir(self.project_home + '/data')
result = await self.start_export(project_id)
print(result)
async def start_export(self, project_id):
project_details = self.ProjectsModel.get_project_details(project_id)
print("Project Details Sent*********************", project_details)
domain_details = self.DomainModel.get_all_domains(project_id)
domains_list = []
# Makefile file and folder structures
for domain in domain_details:
domain_id = domain['_id']['$oid']
domains_list.append([domain_id, domain['domain_name']])
# check for path for skills folder
if not os.path.exists(self.project_home + '/skills/' +
domain['domain_name'] + '/data'):
os.makedirs(self.project_home + '/skills/' +
domain['domain_name'] + '/data')
# check for path for Data folder
if not os.path.exists(self.project_home + '/data'):
os.makedirs(self.project_home + '/data')
result = await self.write_domain_file(project_id, domain_id,
domain['domain_name'])
# Starting export of each Domain in Parallel - This is disabled till multi skills is proven
#result = await asyncio.gather(*(self.write_domain_file(project_id, domain[0], domain[1]) for domain in domains_list))
# Write master consolidated NLU file to disk
print("Writing Consolidated files to Disk")
async with aiofiles.open(self.project_home + '/data/nlu.json',
"w") as out:
await out.write(
json.dumps(self.master_nlu, indent=4, sort_keys=False))
await out.flush()
async with aiofiles.open(self.project_home + '/data/stories.md',
"w") as out:
await out.write(self.master_stories)
await out.flush()
async with aiofiles.open(self.project_home + '/domain.yml',
"w") as out:
await out.write("intents:" + "\n")
await out.write(self.master_domain_intents + "\n" + "\n")
await out.write("slots:" + "\n")
await out.write(self.master_domain_entities + "\n" + "\n")
await out.write("actions:" + "\n")
await out.write(self.master_domain_actions + "\n" + "\n")
await out.write("templates:" + "\n")
await out.write(self.master_domain_templates + "\n" + "\n")
await out.flush()
print("Writing Default config file to Disk")
async with aiofiles.open(self.project_home + '/config.yml',
"w") as out:
await out.write("# Configuration for Rasa NLU & Core." + "\n\n")
print(
yaml.dump(yaml.load(
json.dumps(project_details['configuration'])),
default_flow_style=False) + "\n")
await out.write(
yaml.dump(yaml.load(
json.dumps(project_details['configuration'])),
default_flow_style=False) + "\n")
await out.flush()
return result
async def write_domain_file(self, project_id, domain_id, domain_name):
print("writing domain File for id {} {}".format(
domain_id, domain_name))
# Starting with export for NLU , Stories and Domains files
result = await asyncio.gather(
self.export_nlu_data(project_id, domain_id, domain_name),
self.export_domain_yml_data(project_id, domain_id, domain_name),
self.export_stories(project_id, domain_id, domain_name))
return result
async def export_nlu_data(self, project_id, domain_id, domain_name):
print("Exporting NLU Data {} {}".format(project_id, domain_id))
intents_list = self.IntentsModel.get_intents(project_id, domain_id)
base_nlu_stub = {"rasa_nlu_data": {"common_examples": []}}
for intents in intents_list:
intent_details = self.IntentDetailModel.get_intent_details(
intents['_id']['$oid'])
for intent in intent_details['text_entities']:
json_record = {
"text": intent['text'],
"intent": intents['intent_name'],
"entities": intent['entities']
}
base_nlu_stub['rasa_nlu_data']['common_examples'].append(
json_record)
self.master_nlu['rasa_nlu_data']['common_examples'].append(
json_record)
async with aiofiles.open(
self.project_home + '/skills/' + domain_name +
'/data/nlu.json', "w") as out:
await out.write(
json.dumps(base_nlu_stub, indent=4, sort_keys=False))
await out.flush()
async def export_stories(self, project_id, domain_id, domain_name):
stories_list = self.StoryModel.get_stories(project_id, domain_id)
async with aiofiles.open(
self.project_home + '/skills/' + domain_name +
'/data/stories.md', "w") as out:
print("Writing files ")
entity_list = None
for stories in stories_list:
story_detail = self.StoryDetailModel.get_story_details(
stories['_id']['$oid'])
await out.write("##" + stories['story_name'] + "\n")
self.master_stories = self.master_stories + "##" + stories[
'story_name'] + "\n"
for story_rec in story_detail['story']:
if story_rec['type'] == 'intent':
await out.write("* " + story_rec['key'] + "\n")
for entities in story_rec['entities']:
entity_list = f'"' + entities[
'entity_name'] + f'"' + ":" + f'"' + entities[
'entity_value'] + f'"' + ","
print("Entity value ".format(
entities['entity_name']))
if entity_list is None:
self.master_stories = self.master_stories + "* " + story_rec[
'key'] + "\n"
else:
self.master_stories = self.master_stories + "* " + story_rec[
'key'] + "{" + entity_list[:-1] + "}" + "\n"
else:
await out.write(" - " + story_rec['key'] + "\n")
self.master_stories = self.master_stories + " - " + story_rec[
'key'] + "\n"
await out.write("\n")
await out.write("\n")
self.master_stories = self.master_stories + "\n" + "\n"
await out.flush()
async def export_domain_yml_data(self, project_id, domain_id, domain_name):
print("Creating Domain.yml files for domain {} {}".format(
project_id, domain_id))
# Writing intents List, responses list and responses details in Domains.yml
intents_list = self.IntentsModel.get_intents(project_id, domain_id)
async with aiofiles.open(
self.project_home + '/skills/' + domain_name + '/domain.yml',
"w") as out:
await out.write("intents:" + "\n")
for intents in intents_list:
await out.write("- " + intents['intent_name'] + "\n")
self.master_domain_intents = self.master_domain_intents + "- " + intents[
'intent_name'] + "\n"
await out.write("\n")
await out.write("\n")
slots_list = self.EntityModel.get_entities(project_id)
for slots in slots_list:
if slots['entity_name'] not in self.master_domain_entities:
self.master_domain_entities = self.master_domain_entities + " " + slots[
'entity_name'] + ":" + "\n"
self.master_domain_entities = self.master_domain_entities + " " + "type: " + slots[
'entity_slot']['type'] + "\n"
else:
print("Entity Already exists ")
response_list = self.ResponseModel.get_responses(
project_id, domain_id)
await out.write("actions:" + "\n")
for resp in response_list:
await out.write("- " + resp['response_name'] + "\n")
self.master_domain_actions = self.master_domain_actions + "- " + resp[
'response_name'] + "\n"
# Import custom actions from Actions collection
default_actions = [
'action_listen', 'action_restart', 'action_default_fallback',
'action_deactivate_form', 'action_revert_fallback_events',
'action_default_ask_rephrase', 'action_back'
]
custom_actions = self.CustomActionsModel.get_all_custom_actions()
for record in custom_actions:
if record['action_name'] not in default_actions:
await out.write("- " + record['action_name'] + "\n")
if record['action_name'] not in self.master_domain_actions:
self.master_domain_actions = self.master_domain_actions + "- " + record[
'action_name'] + "\n"
await out.write("\n")
await out.write("\n")
await out.write("templates:" + "\n")
for response in response_list:
await out.write(" " + response['response_name'] + ":\n")
self.master_domain_templates = self.master_domain_templates + " " + response[
'response_name'] + ":\n"
response_detail = self.ResponseDetailModel.get_response_details(
response['_id']['$oid'])
for resp_value in response_detail['text_entities']:
await out.write(" - " + "text: " + f'"' + resp_value +
f'"')
await out.write("\n")
self.master_domain_templates = self.master_domain_templates + " - " + "text: " + f'"' + resp_value + f'"' + "\n"
await out.write("\n")
await out.write("\n")
await out.flush()
from __main__ import sio
from models import ProjectsModel, DomainsModel, IntentsModel, ResponseModel, StoryModel, EntityModel, RefreshDb, RasaConversations, CustomActionsModel
from export_project import ExportProject
from config import CONFIG
import os
import json
import sys
import time
from aiohttp import web
from bson.json_util import dumps
EntityModel = EntityModel()
ProjectsModel = ProjectsModel()
DomainsModel = DomainsModel()
IntentsModel = IntentsModel()
ResponseModel = ResponseModel()
StoryModel = StoryModel()
ExportProject = ExportProject()
RefreshDb = RefreshDb()
RasaConversations = RasaConversations()
CustomActionsModel = CustomActionsModel()
@sio.on('connect')
async def connect(sid, environ):
print('connect ', sid)
print('Environment ', environ)
@sio.on('disconnect')
async def disconnect(sid):
async def bootstrap_receipts_endpoint(username: str):
status = await bootstrap_receipts(username)
if status:
return ResponseModel(None, 200, "Receipts data bootstraped")
return ErrorResponseModel(None, 400, "Error")