def content():
msg = ''
err = False
try:
# channelid refers to the channel. email = 1, cisco.com = 2
if 'channelid' in request.args:
chid = int(request.args['channelid'])
else:
msg = msg + 'Channel ID not found in request'
err = True
# offerid refers to the campaignid for emails and an equivalent for cisco.com
if 'offerid' in request.args:
ofid = int(request.args['offerid'])
else:
msg = msg + 'Offer ID not found in request'
err = True
d = request.get_json (force=True)
data = d['data']
data = {k.lower (): v for k, v in data.items ()}
except Exception as e:
err = True
msg = msg + e.__str__ ()
return jsonify ({'error': msg})
# Input data check and output format check required before responding.
# Input data check
data_sourcing_needed = InputDataCheck(data)
if data_sourcing_needed != 0:
# sourceData(data)
return jsonify ({'error': 'Discrepancies found in input data'})
s = GenerateNarrative (None, None, varList = list(data.keys()))
res = s.GenNarrTag (CampaignID=int (ofid), data=data)
# Output format check
# As of now, check if the channel is different from email. If so, format the output.
if chid == 2:
res = format_output(chid, ofid, res)
return json.dumps(res)
return jsonify (res)
def post(self):
try:
pool = mp.Pool(processes=mp.cpu_count() * 5)
logger.info("Export Narrative Process started")
content = request.get_json()
username = content['user']
campaignname = content['campaign'][0]
personas = content['exportPersonas']
language = content['language']
filename = content['filename']
if personas == []:
logger.info("No Profile is Selected for Export Narrative")
final = {
"message": "Please Select a Profile to Export Narratives"
}
elif filename == "":
logger.info("Table Name is not provided")
final = {
"message":
"Please Provide a Table Name to Export Narratives"
}
else:
logger.info(
"Generating narrative for Persona %s and Language '%s'",
personas, language)
conn = mysql.connect()
cur = conn.cursor()
# Checking if given filename is present in table `exportinfo`
# And Inserting Expoting table info into `exportinfo` table
cur.execute("SELECT table_name from exportinfo")
tables = cur.fetchall()
if filename in [x[0] for x in tables]:
cur.execute(
"INSERT INTO exportinfo(campaign,table_name,status,create_date,user) VALUES (%s,%s,'Failed(Table already exists)',NOW(),%s)",
(campaignname, filename, username))
conn.commit()
logger.info("Table %s already exists", filename)
final = {"message": "Table already exists."}
else:
cur.execute(
"INSERT INTO exportinfo(campaign,table_name,status,create_date,user) VALUES (%s,%s,'In Process',NOW(),%s)",
(campaignname, filename, username))
conn.commit()
cur.execute(
"SELECT table_id FROM exportinfo ORDER BY table_id DESC LIMIT 1"
)
tableid = cur.fetchall()
try:
t0 = time.time()
logger.info("Generating narrative to export")
df_old = getData(campaignname)
NarrGen = GenerateNarrative(None, df_old)
# Filters dataframe to include only the data that falls into the personas
df = NarrGen.getUpdatedDF(username, campaignname,
personas)
NarrGen = GenerateNarrative(None, df)
# Selecting any one language from database for selected campaign and creating table for export
cur.execute(
"SELECT l.language FROM user u JOIN user_campaign_xref us ON u.user_id=us.user_user_id JOIN campaign s ON us.campaign_campaign_id=s.campaign_id JOIN campaign_feature_xref cf ON s.campaign_id=cf.campaign_campaign_id JOIN feature f ON cf.feature_feature_id=f.feature_id JOIN feature_language_xref fl ON f.feature_id=fl.feature_feature_id JOIN language l ON fl.language_language_id=l.language_id WHERE u.email='"
+ username + "' AND s.campaign_name='" +
campaignname + "'")
language = cur.fetchall()
# if no language is present in database for this campaign then ending the export process
if language == ():
cur.execute(
"UPDATE exportinfo SET status = 'Failed(No rule to export narrtive)' WHERE table_id = %s",
tableid)
conn.commit()
else:
# Getting column names for table from narrative tags
sdict = NarrGen.GenNarrTag(
data=df.iloc[1].to_dict(),
CampaignName=campaignname,
Language=language[0][0])
createSQLconstruct(dict=sdict,
table=filename,
conn=conn)
# for row in range (len (df)):
# self.process(df = df, row = row, NarrGen = NarrGen, filename = filename, conn = conn, campaignname = campaignname )
# iterate row by row and export. *Performance optimization to be explored.*
results = [
pool.apply_async(self.process,
args=(df, row, NarrGen,
filename, campaignname))
for row in range(len(df))
]
results = [p.get() for p in results]
tf = time.time() - t0
s = "%8.2f" % tf
msg = str("Exported Successfully " + str(len(df)) +
" records in " + str(s) + " seconds")
final = {"message": msg}
logger.info("Narrative %s", msg)
# Updating status of completion of export narrative
cur.execute(
"UPDATE exportinfo SET status = 'Successful' WHERE table_id = %s",
tableid)
conn.commit()
except Exception as e:
# Updating status as Unsuccessfull for export narrative
cur.execute(
"UPDATE exportinfo SET status = 'Unsuccessful' WHERE table_id = %s",
tableid)
conn.commit()
logger.error(e)
final = {"message": e.__str__()}
logger.info("Export narrative process is completed")
return final
except Exception as e:
logger.error(e)
return (e.__str__())
def narrative_generation(username,
campaignname,
personas,
language,
template=None):
conn = mysql.connect()
cur = conn.cursor()
df = getData(campaignname, preview=True)
#print("user for get narrative",username)
# Get the list of persona rules for all personas
rv_pr_all = query_persona_rule_all(username, campaignname)
print("all rule", rv_pr_all)
pr_list = []
for item in rv_pr_all:
for ite in item:
pr_list.append(ite)
# narrative is combined narrative for all features
# header has column names to show in data viewer
# finalval stores column values for data viewer
narrative = ""
header = []
finalval = []
# Generating Narrative for each persona one by one
for personaname in personas:
print("persona", personaname)
if (template == None):
# Get all features mapped with selected Persona
cur.execute(
"SELECT f.feature_name FROM user u JOIN user_campaign_xref uc ON u.user_id=uc.user_user_id JOIN campaign c ON uc.campaign_campaign_id=c.campaign_id JOIN campaign_persona_xref cp ON c.campaign_id=cp.campaign_campaign_id JOIN persona p ON cp.persona_persona_id=p.persona_id JOIN persona_feature_xref pf ON p.persona_id=pf.persona_persona_id JOIN feature f ON pf.feature_feature_id=f.feature_id WHERE u.email='"
+ username + "' AND c.campaign_name='" + campaignname +
"' AND p.persona_name='" + personaname + "'")
features = cur.fetchall()
# Get the rule for all features associated with a persona
rv = query_rule(username, campaignname, features, language)
#print("featurerule",rv)
# joining all feature rule with new line parameter
template = "\n".join(ite for item in rv for ite in item)
print("here is rule", template)
values = []
# Get the persona rule for the selected persona
rv_pr = query_persona_rule(username, campaignname, personaname)
#print(rv_pr)
pr = " ".join(ite for item in rv_pr for ite in item)
#print("onep",pr)
# passing combined feature rule, dataframe, personarule and all persona rule for narrative generation
# return_str = validate_GenNarr(template, df, pr, pr_list)
NarrGen = GenerateNarrative(template, df)
return_str = NarrGen.validate_GenNarr(pr, pr_list)
# return_str have three parameter narrative, column names and column values
narrative = "\n".join((narrative, return_str[0], "\n"))
data = return_str[1]
# print("here is the data",data)
header = data[0]
tempvalues = data[1]
# For now Generating Html Preview for email security campaign only
html_content = ''
unused_tags_in_data = []
unknown_tags_in_html = []
html_data = dict(zip(header, tempvalues))
cur.execute(
"SELECT template FROM htmltemplate WHERE campaign_name = '" +
campaignname + "'")
rv = cur.fetchall()
# print("rv",rv[0][0])
if (rv != ()):
sdict = NarrGen.GenNarrTag(html_data,
CampaignName=campaignname,
Language=language)
final = html_preview(sdict, rv[0][0])
unknown_tags_in_html = final['unknown_tags_in_html']
unused_tags_in_data = final['unused_tags_in_data']
# print("data b",unused_tags_in_data)
# print("html b 2",unknown_tags_in_html)
html_content = final['html_preview']
print("fail")
# else :
# print ("successfully")
# if (campaignname == 'Email Security') :
# sdict = NarrGen.GenNarrTag (html_data, CampaignName=campaignname,Language = language)
# # print("sdict",sdict)
# sdict['first_name'] = html_data ['first_name']
# # print(sdict)
# # print("button",sdict['clicktoactivatebutton'])
# # print("link",sdict['clicktoactivatelink'])
# # print("sicoon",sdict['statusicon'])
# html_content = htmlPreview(sdict)
# elif (campaignname == 'Digital Revolution - Ransomware') :
# sdict = NarrGen.GenNarrTag (html_data, CampaignName=campaignname,Language = language)
# # print("sdict",sdict)
# final = html_preview(sdict, campaignname)
# unknown_tags_in_html = final['unknown_tags_in_html']
# unused_tags_in_data = final['unused_tags_in_data']
# # print("data b",unused_tags_in_data)
# # print("html b 2",unknown_tags_in_html)
# html_content = final['html_preview']
# elif (campaignname == 'Digital Revolution - Ransomware') :
# print("came here")
# final = html_preview(sdict)
# data_tags_missing = final['data_tag_missing']
# html_tags_missing = final['html_tag_missing']
# print("data b",data_tags_missing)
# print("html b 2",html_tags_missing)
# html_content = final['html_preview']
# print("s",html_content)
# Relpacing Null values with None and removing Brackets present in data row selected for generating narrative
for i in tempvalues:
if pd.isnull(i):
values.append("None")
else:
clean_val = str(i).replace("(", "")
clean_val = str(clean_val).replace(")", "")
values.append(clean_val)
finalval.append(values)
# Replacing tab and new line character as per html encoding character
narrative = sub("\\\\t", "\\t", sub("\\\\n", "\\n", narrative))
data = {
"narrative": [narrative],
"header": header,
"values": finalval,
"rule": [template],
"html": html_content,
"unused_tags_in_data": unused_tags_in_data,
"unknown_tags_in_html": unknown_tags_in_html
}
# print("data",data['html'])
return data