def content(): msg = '' err = False try: # channelid refers to the channel. email = 1, cisco.com = 2 if 'channelid' in request.args: chid = int(request.args['channelid']) else: msg = msg + 'Channel ID not found in request' err = True # offerid refers to the campaignid for emails and an equivalent for cisco.com if 'offerid' in request.args: ofid = int(request.args['offerid']) else: msg = msg + 'Offer ID not found in request' err = True d = request.get_json (force=True) data = d['data'] data = {k.lower (): v for k, v in data.items ()} except Exception as e: err = True msg = msg + e.__str__ () return jsonify ({'error': msg}) # Input data check and output format check required before responding. # Input data check data_sourcing_needed = InputDataCheck(data) if data_sourcing_needed != 0: # sourceData(data) return jsonify ({'error': 'Discrepancies found in input data'}) s = GenerateNarrative (None, None, varList = list(data.keys())) res = s.GenNarrTag (CampaignID=int (ofid), data=data) # Output format check # As of now, check if the channel is different from email. If so, format the output. if chid == 2: res = format_output(chid, ofid, res) return json.dumps(res) return jsonify (res)
def post(self): try: pool = mp.Pool(processes=mp.cpu_count() * 5) logger.info("Export Narrative Process started") content = request.get_json() username = content['user'] campaignname = content['campaign'][0] personas = content['exportPersonas'] language = content['language'] filename = content['filename'] if personas == []: logger.info("No Profile is Selected for Export Narrative") final = { "message": "Please Select a Profile to Export Narratives" } elif filename == "": logger.info("Table Name is not provided") final = { "message": "Please Provide a Table Name to Export Narratives" } else: logger.info( "Generating narrative for Persona %s and Language '%s'", personas, language) conn = mysql.connect() cur = conn.cursor() # Checking if given filename is present in table `exportinfo` # And Inserting Expoting table info into `exportinfo` table cur.execute("SELECT table_name from exportinfo") tables = cur.fetchall() if filename in [x[0] for x in tables]: cur.execute( "INSERT INTO exportinfo(campaign,table_name,status,create_date,user) VALUES (%s,%s,'Failed(Table already exists)',NOW(),%s)", (campaignname, filename, username)) conn.commit() logger.info("Table %s already exists", filename) final = {"message": "Table already exists."} else: cur.execute( "INSERT INTO exportinfo(campaign,table_name,status,create_date,user) VALUES (%s,%s,'In Process',NOW(),%s)", (campaignname, filename, username)) conn.commit() cur.execute( "SELECT table_id FROM exportinfo ORDER BY table_id DESC LIMIT 1" ) tableid = cur.fetchall() try: t0 = time.time() logger.info("Generating narrative to export") df_old = getData(campaignname) NarrGen = GenerateNarrative(None, df_old) # Filters dataframe to include only the data that falls into the personas df = NarrGen.getUpdatedDF(username, campaignname, personas) NarrGen = GenerateNarrative(None, df) # Selecting any one language from database for selected campaign and creating table for export cur.execute( "SELECT l.language FROM user u JOIN user_campaign_xref us ON u.user_id=us.user_user_id JOIN campaign s ON us.campaign_campaign_id=s.campaign_id JOIN campaign_feature_xref cf ON s.campaign_id=cf.campaign_campaign_id JOIN feature f ON cf.feature_feature_id=f.feature_id JOIN feature_language_xref fl ON f.feature_id=fl.feature_feature_id JOIN language l ON fl.language_language_id=l.language_id WHERE u.email='" + username + "' AND s.campaign_name='" + campaignname + "'") language = cur.fetchall() # if no language is present in database for this campaign then ending the export process if language == (): cur.execute( "UPDATE exportinfo SET status = 'Failed(No rule to export narrtive)' WHERE table_id = %s", tableid) conn.commit() else: # Getting column names for table from narrative tags sdict = NarrGen.GenNarrTag( data=df.iloc[1].to_dict(), CampaignName=campaignname, Language=language[0][0]) createSQLconstruct(dict=sdict, table=filename, conn=conn) # for row in range (len (df)): # self.process(df = df, row = row, NarrGen = NarrGen, filename = filename, conn = conn, campaignname = campaignname ) # iterate row by row and export. *Performance optimization to be explored.* results = [ pool.apply_async(self.process, args=(df, row, NarrGen, filename, campaignname)) for row in range(len(df)) ] results = [p.get() for p in results] tf = time.time() - t0 s = "%8.2f" % tf msg = str("Exported Successfully " + str(len(df)) + " records in " + str(s) + " seconds") final = {"message": msg} logger.info("Narrative %s", msg) # Updating status of completion of export narrative cur.execute( "UPDATE exportinfo SET status = 'Successful' WHERE table_id = %s", tableid) conn.commit() except Exception as e: # Updating status as Unsuccessfull for export narrative cur.execute( "UPDATE exportinfo SET status = 'Unsuccessful' WHERE table_id = %s", tableid) conn.commit() logger.error(e) final = {"message": e.__str__()} logger.info("Export narrative process is completed") return final except Exception as e: logger.error(e) return (e.__str__())
def narrative_generation(username, campaignname, personas, language, template=None): conn = mysql.connect() cur = conn.cursor() df = getData(campaignname, preview=True) #print("user for get narrative",username) # Get the list of persona rules for all personas rv_pr_all = query_persona_rule_all(username, campaignname) print("all rule", rv_pr_all) pr_list = [] for item in rv_pr_all: for ite in item: pr_list.append(ite) # narrative is combined narrative for all features # header has column names to show in data viewer # finalval stores column values for data viewer narrative = "" header = [] finalval = [] # Generating Narrative for each persona one by one for personaname in personas: print("persona", personaname) if (template == None): # Get all features mapped with selected Persona cur.execute( "SELECT f.feature_name FROM user u JOIN user_campaign_xref uc ON u.user_id=uc.user_user_id JOIN campaign c ON uc.campaign_campaign_id=c.campaign_id JOIN campaign_persona_xref cp ON c.campaign_id=cp.campaign_campaign_id JOIN persona p ON cp.persona_persona_id=p.persona_id JOIN persona_feature_xref pf ON p.persona_id=pf.persona_persona_id JOIN feature f ON pf.feature_feature_id=f.feature_id WHERE u.email='" + username + "' AND c.campaign_name='" + campaignname + "' AND p.persona_name='" + personaname + "'") features = cur.fetchall() # Get the rule for all features associated with a persona rv = query_rule(username, campaignname, features, language) #print("featurerule",rv) # joining all feature rule with new line parameter template = "\n".join(ite for item in rv for ite in item) print("here is rule", template) values = [] # Get the persona rule for the selected persona rv_pr = query_persona_rule(username, campaignname, personaname) #print(rv_pr) pr = " ".join(ite for item in rv_pr for ite in item) #print("onep",pr) # passing combined feature rule, dataframe, personarule and all persona rule for narrative generation # return_str = validate_GenNarr(template, df, pr, pr_list) NarrGen = GenerateNarrative(template, df) return_str = NarrGen.validate_GenNarr(pr, pr_list) # return_str have three parameter narrative, column names and column values narrative = "\n".join((narrative, return_str[0], "\n")) data = return_str[1] # print("here is the data",data) header = data[0] tempvalues = data[1] # For now Generating Html Preview for email security campaign only html_content = '' unused_tags_in_data = [] unknown_tags_in_html = [] html_data = dict(zip(header, tempvalues)) cur.execute( "SELECT template FROM htmltemplate WHERE campaign_name = '" + campaignname + "'") rv = cur.fetchall() # print("rv",rv[0][0]) if (rv != ()): sdict = NarrGen.GenNarrTag(html_data, CampaignName=campaignname, Language=language) final = html_preview(sdict, rv[0][0]) unknown_tags_in_html = final['unknown_tags_in_html'] unused_tags_in_data = final['unused_tags_in_data'] # print("data b",unused_tags_in_data) # print("html b 2",unknown_tags_in_html) html_content = final['html_preview'] print("fail") # else : # print ("successfully") # if (campaignname == 'Email Security') : # sdict = NarrGen.GenNarrTag (html_data, CampaignName=campaignname,Language = language) # # print("sdict",sdict) # sdict['first_name'] = html_data ['first_name'] # # print(sdict) # # print("button",sdict['clicktoactivatebutton']) # # print("link",sdict['clicktoactivatelink']) # # print("sicoon",sdict['statusicon']) # html_content = htmlPreview(sdict) # elif (campaignname == 'Digital Revolution - Ransomware') : # sdict = NarrGen.GenNarrTag (html_data, CampaignName=campaignname,Language = language) # # print("sdict",sdict) # final = html_preview(sdict, campaignname) # unknown_tags_in_html = final['unknown_tags_in_html'] # unused_tags_in_data = final['unused_tags_in_data'] # # print("data b",unused_tags_in_data) # # print("html b 2",unknown_tags_in_html) # html_content = final['html_preview'] # elif (campaignname == 'Digital Revolution - Ransomware') : # print("came here") # final = html_preview(sdict) # data_tags_missing = final['data_tag_missing'] # html_tags_missing = final['html_tag_missing'] # print("data b",data_tags_missing) # print("html b 2",html_tags_missing) # html_content = final['html_preview'] # print("s",html_content) # Relpacing Null values with None and removing Brackets present in data row selected for generating narrative for i in tempvalues: if pd.isnull(i): values.append("None") else: clean_val = str(i).replace("(", "") clean_val = str(clean_val).replace(")", "") values.append(clean_val) finalval.append(values) # Replacing tab and new line character as per html encoding character narrative = sub("\\\\t", "\\t", sub("\\\\n", "\\n", narrative)) data = { "narrative": [narrative], "header": header, "values": finalval, "rule": [template], "html": html_content, "unused_tags_in_data": unused_tags_in_data, "unknown_tags_in_html": unknown_tags_in_html } # print("data",data['html']) return data