def get(self, project_id): logger.debug("Training model " + str(project_id)) logger.debug("Validating project ID " + str(project_id)) result = self.ValidateData.validate_data(project_id) if result != '': logger.debug("Validation failed for the project ") return {"status": "Error", "message": result} # Set Project Status as "Training" so that all users can see model is under training # Clear redis cache r.delete("all_projects") ProjectsModel.set_project_mode(mode="Training", project_id=project_id) result = Export.call_main(project_id) logger.debug(result) # Start Training for the model task_obj = trainer_app.send_task('tasks.train_model', kwargs={'project_id': project_id}) logger.debug("Task ID " + str(task_obj.id)) # get status status = trainer_app.AsyncResult(task_obj.id, app=trainer_app) logger.debug("Status of the task " + str(status.state)) return { "status": "Success", "message": str(status.state), "task_id": str(task_obj.id) }
def put(self,uid,pid): user = UsersModel.fetch_by_id(uid) record = user.projects for each in record: if each.id == pid: body = projectUpdateParser.parse_args() title = body['title'] if body['title'] else None description = body['description'] if body['description'] else None status = int(body['status']) if body['status'] else None startDate = None endDate = None if status == 1: startDate = datetime.now() elif status == 2: endDate = datetime.now() updated = ProjectsModel.update_by_id(id=pid, newTitle=title, newDescription=description, newStartDate=startDate, newEndDate=endDate, newStatus=status) if updated: record = ProjectsModel.fetch_by_id(pid) return record,200 else: return {},400 return {},404
def get(self, task_id): result = trainer_app.AsyncResult(task_id).result if result['Status'] == "Success": # Update the model path to Projects collection ProjectsModel.update_trained_model(result['Message']) ProjectsModel.set_project_mode(mode="Done", project_id=result['project_id']) # Clear redis cache r.delete("all_projects") return {"Status": result['Status'], "Message": result['Message']}
def post(self,uid): args = projectParser.parse_args() dateformat = '%Y-%m-%d' title = args['title'] description = args['description'] status = 0 userId = uid if UsersModel.fetch_by_id(uid) == None: return {"message":"User not found"},404 if ProjectsModel.fetch_by_title(title): return 'project "{}" already exist'.format(title),409 project = ProjectsModel(title=title, description=description,status=status,userId=userId) record = project.create_record() print(record) return record, 201
def post(self): json_data = request.get_json(force=True) result = ProjectsModel.create_projects(json_data) # Clear redis cache r.delete("all_projects") return result
def delete(self): # Deleting record object_id = request.get_json() result = ProjectsModel.delete_project(object_id) # Clear redis cache r.delete("all_projects") return result
def put(self): # Updating record json_data = request.get_json(force=True) result = ProjectsModel.update_project(json_data) # Clear redis cache r.delete("all_projects") return result
def delete(self,uid,pid): user = UsersModel.fetch_by_id(uid) record = user.projects for each in record: if each.id == pid: deleted = ProjectsModel.delete_by_id(pid) if deleted: return {"Message": "Task deleted Successfully"}, 204 return {"Message":"something went wrong"},500 return {"Message":"Record not found"},404
def __init__(self): self.ProjectsModel = ProjectsModel() self.DomainModel = DomainsModel() self.IntentsModel = IntentsModel() self.IntentDetailModel = IntentDetailModel() self.StoryModel = StoryModel() self.StoryDetailModel = StoryDetailModel() self.ResponseModel = ResponseModel() self.ResponseDetailModel = ResponseDetailModel() self.EntityModel = EntityModel() self.CustomActionsModel = CustomActionsModel() self.ValidateData = ValidateData() self.project_home = '' #self.project_base_path = '../vol_chatbot_data/temp/trainer-sessions/' self.project_base_path = '' self.session_id = '' self.master_nlu = {"rasa_nlu_data": {"common_examples": []}} self.master_stories = "" self.master_domain_intents = "" self.master_domain_actions = "" self.master_domain_templates = "" self.master_domain_entities = ""
def get(self): # check if result can be served from cache if r.exists("all_projects"): return json.loads(r.get("all_projects")) else: # Get results and update the cache with new values logging.debug('getting Data from DB') result = ProjectsModel.get_all_projects() r.set("all_projects", json.dumps(result), ex=GLOBAL_EXPIRY) return result
from __main__ import sio from models import ProjectsModel, DomainsModel, IntentsModel, ResponseModel, StoryModel, EntityModel, RefreshDb, RasaConversations, CustomActionsModel from export_project import ExportProject from config import CONFIG import os import json import sys import time from aiohttp import web from bson.json_util import dumps EntityModel = EntityModel() ProjectsModel = ProjectsModel() DomainsModel = DomainsModel() IntentsModel = IntentsModel() ResponseModel = ResponseModel() StoryModel = StoryModel() ExportProject = ExportProject() RefreshDb = RefreshDb() RasaConversations = RasaConversations() CustomActionsModel = CustomActionsModel() @sio.on('connect') async def connect(sid, environ): print('connect ', sid) print('Environment ', environ) @sio.on('disconnect') async def disconnect(sid):
def delete(self,uid): deleted = ProjectsModel.delete_all(uid) if deleted: return {'Message':'Records Deleted Successfully'},204 else: return {'Message':'Something is wrong with the server'},500
class Export: def __init__(self): self.ProjectsModel = ProjectsModel() self.DomainModel = DomainsModel() self.IntentsModel = IntentsModel() self.IntentDetailModel = IntentDetailModel() self.StoryModel = StoryModel() self.StoryDetailModel = StoryDetailModel() self.ResponseModel = ResponseModel() self.ResponseDetailModel = ResponseDetailModel() self.EntityModel = EntityModel() self.CustomActionsModel = CustomActionsModel() self.ValidateData = ValidateData() self.project_home = '' #self.project_base_path = '../vol_chatbot_data/temp/trainer-sessions/' self.project_base_path = '' self.session_id = '' self.master_nlu = {"rasa_nlu_data": {"common_examples": []}} self.master_stories = "" self.master_domain_intents = "" self.master_domain_actions = "" self.master_domain_templates = "" self.master_domain_entities = "" async def reset_globals(self): self.master_nlu = {"rasa_nlu_data": {"common_examples": []}} self.master_stories = "" self.master_domain_intents = "" self.master_domain_actions = "" self.master_domain_templates = "" self.master_domain_entities = "" self.session_id = "" async def clean_up(self, sid): print( "Cleaning up on session disconnect for session ID {}".format(sid)) if os.path.isdir(self.project_base_path + sid): shutil.rmtree(self.project_base_path + sid) return 1 def call_main(self, project_id): return asyncio.run(self.main(project_id)) async def main(self, project_id): # Invoke Method to start parallel activity print("Starting export for project ID {}".format(project_id)) if not os.path.exists('/rasa_projects/'): os.makedirs('/rasa_projects/') self.project_base_path = '/rasa_projects/' self.project_home = self.project_base_path + project_id await self.reset_globals() if os.path.isdir(self.project_home): print("Directory already exists , Empty the directory") print("Working on project home {}".format(self.project_home)) try: shutil.rmtree(self.project_home + '/data') shutil.rmtree(self.project_home + '/skills') os.remove(self.project_home + '/config.yml') os.remove(self.project_home + '/domain.yml') except FileNotFoundError: print("Folder already clear") else: print("Working on project home {}".format(self.project_home)) if not os.path.exists(self.project_home): os.mkdir(self.project_home) os.mkdir(self.project_home + '/data') result = await self.start_export(project_id) print(result) async def start_export(self, project_id): project_details = self.ProjectsModel.get_project_details(project_id) print("Project Details Sent*********************", project_details) domain_details = self.DomainModel.get_all_domains(project_id) domains_list = [] # Makefile file and folder structures for domain in domain_details: domain_id = domain['_id']['$oid'] domains_list.append([domain_id, domain['domain_name']]) # check for path for skills folder if not os.path.exists(self.project_home + '/skills/' + domain['domain_name'] + '/data'): os.makedirs(self.project_home + '/skills/' + domain['domain_name'] + '/data') # check for path for Data folder if not os.path.exists(self.project_home + '/data'): os.makedirs(self.project_home + '/data') result = await self.write_domain_file(project_id, domain_id, domain['domain_name']) # Starting export of each Domain in Parallel - This is disabled till multi skills is proven #result = await asyncio.gather(*(self.write_domain_file(project_id, domain[0], domain[1]) for domain in domains_list)) # Write master consolidated NLU file to disk print("Writing Consolidated files to Disk") async with aiofiles.open(self.project_home + '/data/nlu.json', "w") as out: await out.write( json.dumps(self.master_nlu, indent=4, sort_keys=False)) await out.flush() async with aiofiles.open(self.project_home + '/data/stories.md', "w") as out: await out.write(self.master_stories) await out.flush() async with aiofiles.open(self.project_home + '/domain.yml', "w") as out: await out.write("intents:" + "\n") await out.write(self.master_domain_intents + "\n" + "\n") await out.write("slots:" + "\n") await out.write(self.master_domain_entities + "\n" + "\n") await out.write("actions:" + "\n") await out.write(self.master_domain_actions + "\n" + "\n") await out.write("templates:" + "\n") await out.write(self.master_domain_templates + "\n" + "\n") await out.flush() print("Writing Default config file to Disk") async with aiofiles.open(self.project_home + '/config.yml', "w") as out: await out.write("# Configuration for Rasa NLU & Core." + "\n\n") print( yaml.dump(yaml.load( json.dumps(project_details['configuration'])), default_flow_style=False) + "\n") await out.write( yaml.dump(yaml.load( json.dumps(project_details['configuration'])), default_flow_style=False) + "\n") await out.flush() return result async def write_domain_file(self, project_id, domain_id, domain_name): print("writing domain File for id {} {}".format( domain_id, domain_name)) # Starting with export for NLU , Stories and Domains files result = await asyncio.gather( self.export_nlu_data(project_id, domain_id, domain_name), self.export_domain_yml_data(project_id, domain_id, domain_name), self.export_stories(project_id, domain_id, domain_name)) return result async def export_nlu_data(self, project_id, domain_id, domain_name): print("Exporting NLU Data {} {}".format(project_id, domain_id)) intents_list = self.IntentsModel.get_intents(project_id, domain_id) base_nlu_stub = {"rasa_nlu_data": {"common_examples": []}} for intents in intents_list: intent_details = self.IntentDetailModel.get_intent_details( intents['_id']['$oid']) for intent in intent_details['text_entities']: json_record = { "text": intent['text'], "intent": intents['intent_name'], "entities": intent['entities'] } base_nlu_stub['rasa_nlu_data']['common_examples'].append( json_record) self.master_nlu['rasa_nlu_data']['common_examples'].append( json_record) async with aiofiles.open( self.project_home + '/skills/' + domain_name + '/data/nlu.json', "w") as out: await out.write( json.dumps(base_nlu_stub, indent=4, sort_keys=False)) await out.flush() async def export_stories(self, project_id, domain_id, domain_name): stories_list = self.StoryModel.get_stories(project_id, domain_id) async with aiofiles.open( self.project_home + '/skills/' + domain_name + '/data/stories.md', "w") as out: print("Writing files ") entity_list = None for stories in stories_list: story_detail = self.StoryDetailModel.get_story_details( stories['_id']['$oid']) await out.write("##" + stories['story_name'] + "\n") self.master_stories = self.master_stories + "##" + stories[ 'story_name'] + "\n" for story_rec in story_detail['story']: if story_rec['type'] == 'intent': await out.write("* " + story_rec['key'] + "\n") for entities in story_rec['entities']: entity_list = f'"' + entities[ 'entity_name'] + f'"' + ":" + f'"' + entities[ 'entity_value'] + f'"' + "," print("Entity value ".format( entities['entity_name'])) if entity_list is None: self.master_stories = self.master_stories + "* " + story_rec[ 'key'] + "\n" else: self.master_stories = self.master_stories + "* " + story_rec[ 'key'] + "{" + entity_list[:-1] + "}" + "\n" else: await out.write(" - " + story_rec['key'] + "\n") self.master_stories = self.master_stories + " - " + story_rec[ 'key'] + "\n" await out.write("\n") await out.write("\n") self.master_stories = self.master_stories + "\n" + "\n" await out.flush() async def export_domain_yml_data(self, project_id, domain_id, domain_name): print("Creating Domain.yml files for domain {} {}".format( project_id, domain_id)) # Writing intents List, responses list and responses details in Domains.yml intents_list = self.IntentsModel.get_intents(project_id, domain_id) async with aiofiles.open( self.project_home + '/skills/' + domain_name + '/domain.yml', "w") as out: await out.write("intents:" + "\n") for intents in intents_list: await out.write("- " + intents['intent_name'] + "\n") self.master_domain_intents = self.master_domain_intents + "- " + intents[ 'intent_name'] + "\n" await out.write("\n") await out.write("\n") slots_list = self.EntityModel.get_entities(project_id) for slots in slots_list: if slots['entity_name'] not in self.master_domain_entities: self.master_domain_entities = self.master_domain_entities + " " + slots[ 'entity_name'] + ":" + "\n" self.master_domain_entities = self.master_domain_entities + " " + "type: " + slots[ 'entity_slot']['type'] + "\n" else: print("Entity Already exists ") response_list = self.ResponseModel.get_responses( project_id, domain_id) await out.write("actions:" + "\n") for resp in response_list: await out.write("- " + resp['response_name'] + "\n") self.master_domain_actions = self.master_domain_actions + "- " + resp[ 'response_name'] + "\n" # Import custom actions from Actions collection default_actions = [ 'action_listen', 'action_restart', 'action_default_fallback', 'action_deactivate_form', 'action_revert_fallback_events', 'action_default_ask_rephrase', 'action_back' ] custom_actions = self.CustomActionsModel.get_all_custom_actions() for record in custom_actions: if record['action_name'] not in default_actions: await out.write("- " + record['action_name'] + "\n") if record['action_name'] not in self.master_domain_actions: self.master_domain_actions = self.master_domain_actions + "- " + record[ 'action_name'] + "\n" await out.write("\n") await out.write("\n") await out.write("templates:" + "\n") for response in response_list: await out.write(" " + response['response_name'] + ":\n") self.master_domain_templates = self.master_domain_templates + " " + response[ 'response_name'] + ":\n" response_detail = self.ResponseDetailModel.get_response_details( response['_id']['$oid']) for resp_value in response_detail['text_entities']: await out.write(" - " + "text: " + f'"' + resp_value + f'"') await out.write("\n") self.master_domain_templates = self.master_domain_templates + " - " + "text: " + f'"' + resp_value + f'"' + "\n" await out.write("\n") await out.write("\n") await out.flush()