def createVariable(parent, options, identity):
print ">", parent, options, identity
try:
entity = Item.get("entity", parent)
except Exception:
raise errors.ObjectNotFound(parent)
if entity.doc.status != ENTITYSTATES.READY:
raise errors.InvalidParameter(
"Could not add variable to unparsed entity")
bot = Item.get("bot", entity.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
name = options["name"]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
print uri
data = {}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "variable"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get(
"tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {
"field": options["field"],
"type": options.get("type", "dimension"),
"aliases": options["aliases"]
}
item = Item(**data)
item.save()
syncVariableValues(variableid=uri)
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
'''
def listIdentities(uri, identity, options):
bot = Item.get("bot", uri)
if options is None:
iterator = Item.parent_index.query("entity", Item.parent == uri)
else:
limit = options.get("limit", 5)
if "search" in options.keys():
term = options.get("search", )
iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term))
else:
iterator = Item.parent_index.query("entity", Item.parent == uri)
items=[i for i in apply_options(iterator, options)]
for entity in items :
print entity
if entity.doc.status==ENTITYSTATES.PARSING :
print " found an entity with status", ENTITYSTATES.PARSING
crawler_name = i.doc.crawler
crawler = Crawler.get(name=crawler_name)
if crawler["State"] in ["READY", "STOPPING"]:
doc = bot.doc
database=doc.database
tablename = entity.doc.tablename
columns = Crawler.get_table(database,tablename)
print "updating entity status and columns"
entity.update(actions=[Item.doc.columns.set(columns),Item.doc.status.set(ENTITYSTATES.READY)])
if options is None:
iterator = Item.parent_index.query("entity", Item.parent == uri)
else:
limit = options.get("limit", 5)
if "search" in options.keys():
term = options.get("search", )
iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term))
else:
iterator = Item.parent_index.query("entity", Item.parent == uri)
items = [merge_doc(i) for i in apply_options(iterator, options)]
return items
def test_aa_clean(self):
event = {
"identity": {
"claims": TestAccount.get_claim()
},
"field": "listMyAccounts",
"path": "Query/listMyAccounts",
"arguments": {}
}
accounts = App.handler(event, {})
if type(accounts) == type(["a", "list"]):
print "Found", len(accounts), "accounts"
print accounts
for accountdoc in accounts:
a = Item.get('account', accountdoc["ID"])
a.delete()
x = App.handler(event, {})
self.assertTrue(len(x) == 0)
else:
self.assertTrue(True, "no accounts")
def test_ec_get_variable(self):
entity = TestAccount.get_entity()
entityid = entity["ID"]
candidates = [
c.attribute_values
for c in Item.query("variable", Item.parent.__eq__(entityid))
]
variable = candidates[0]
variableid = variable["ID"]
print ">>", variableid
event = {
"identity": {
"claims": TestAccount.get_claim()
},
"field": "getVariable",
"path": "Query/getVariable",
"arguments": {
"variableid": variableid
}
}
r = App.handler(event, {})
print r
self.assertEqual(r["name"], variable["name"])
def test_fc_get_rule(self):
intent = TestAccount.get_intent()
intentid = intent["ID"]
candidates = [
c.attribute_values
for c in Item.query("rule", Item.parent.__eq__(intentid))
]
rule = candidates[0]
ruleid = rule["ID"]
print ">>", ruleid
event = {
"identity": {
"claims": TestAccount.get_claim()
},
"field": "getRule",
"path": "Query/getRule",
"arguments": {
"ruleid": ruleid
}
}
r = App.handler(event, {})
print r
self.assertEqual(r["name"], rule["name"])
def askBot(identity, botid, question) :
try :
bot = Item.get("bot", botid)
except Exception as e :
return "Seems i do not exists anymore :("
response= Bot.send(bot=bot.botname,input=question)
if response["dialogState"] :
parts = botid.split(":")
accountid=parts[2]
botname = parts[3]
cube = getBotCube(bot)
semantics= cube.resolve(response["slots"])
print pprint.pformat(semantics)
sql = semantics["sql"]
print semantics["sql"]
results = AthenaQuery.run(sql=sql)
print pprint.pformat(results)
return json.dumps({
"display" : semantics["display"],
"data" : results["data"]["records"]
})
else :
message = random.choice([
"Sorry, this is not clear for and old bot",
"Can you try to reformulate ?",
"Hmm, i'm just a bot",
"Well, this is clear, i'm not really awake right now"
])
return json.dumps({
"display" : "error",
"data" : message
})
def createLexBot(botid, identity) :
bot = Item.get("bot", botid)
c = getBotCube(bot)
c.deploy_bot()
Bot.put_alias(bot.botname)
return "Done"
def askBot(identity, botid, question) :
import random
try :
bot = Item.get("bot", botid)
except Exception as e :
return "Seems i do not exists anymore :("
response= Bot.send(bot=bot.name,input=question)
print response
if response["dialogState"] :
parts = botid.split(":")
accountid=parts[2]
botname = parts[3]
intentname = response["intentName"]
uri = "uri:intent:%(accountid)s:%(botname)s:%(intentname)s"%vars()
intent = Item.get("intent", uri)
database= bot.doc.database
SQL = intent.doc.sql
print "SQL )= ", SQL
graph = {
"bot" : {
"entities": {}
}
}
entities = list(Item.parent_index.query( "entity", Item.parent==intent.parent))
print "??",entities
variables = {}
for e in entities :
print ">>>>>>>>>>>>>>>>>>>",">"+e.name+"<"
SQL = SQL.replace(e.name, '"'+database+'".'+'"'+e.doc.tablename+'"')
graph["bot"]["entities"][e.name] ={
"name" : e.name,
"tablename" : '"'+database+'"' +'."'+ e.name+'"',
"database" : database,
"variables" : {
}
}
for v in Item.parent_index.query("variable", Item.parent == e.ID):
graph["bot"]["entities"][e.name]["variables"][v.name] = {
"name" : v.name,
"field" : v.doc.field
}
print pprint.pformat(graph)
filters = get_filters(graph=graph, slots=response["slots"])["filters"]
explains= get_filters(graph=graph, slots=response["slots"])["explained"]
print "filters = ",filters
print "explains = ",explains
sql = "select count(*) as NB from ("+SQL+" where "+filters+") as T"
results = AthenaQuery.run(sql=sql)
print "+-/"*30
print results["data"]["records"][0]
replytpl = random.choice(intent.doc.reply)
reply = replytpl.replace("{NB}", results["data"]["records"][0]["NB"])+" "+explains
return reply
else :
return random.choice([
"Sorry, this is not clear for and old bot",
"Can you try to reformulate ?",
"Hmm, i'm just a bot",
"Well, this is clear, i'm not really awake right now"
])
def get_item(identifier, identity):
try:
bot = Item.get(objecttype, identifier)
return bot.attribute_values
except Exception:
raise errors.ObjectNotFound(identifier)
def create_item(parent, input, identity):
print "create %(objectype)s received", parent, input, identity
try:
name = input["name"]
except Exception:
raise errors.InvalidParameter("Missing `name` field in input" %
vars())
if parent is None:
parent = "service"
else:
try:
parent_item = Item.get(parentobjecttype, parent)
except Exception:
raise errors.ObjectNotFound(
"No parent object with type `%(parentobjecttype)s` found with Id :`%(parent)s`"
% vars())
candidates = get_children_by_name(parent,
objecttype=objecttype,
name=input["name"])
if len(candidates):
if parentobjecttype is None:
parent = "accounts"
raise errors.DuplicateError(
"An object with name `%(name)s`and type `%(objecttype)s` already exists in `%(parent)s`"
% vars())
objectid = ID.get()
if objecttype == "account":
uri = "uri:account:%(objectid)s" % vars()
if objecttype == "bot":
accountid = parent.split(":")[2]
uri = "uri:bot:%(accountid)s:%(name)s" % vars()
if objecttype == "entity":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
uri = "uri:entity:%(accountid)s:%(botname)s:%(name)s" % vars()
if objecttype == "intent":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
uri = "uri:intent:%(accountid)s:%(botname)s:%(name)s" % vars()
if objecttype == "reply":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
intentname = parent.split(":")[4]
uri = "uri:reply:%(accountid)s:%(botname)s:%(intenname)s:(name)s" % vars(
)
if objecttype == "variable":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
if objecttype == "rule":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
intentname = parent.split(":")[4]
uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars(
)
input["ID"] = uri
input["objecttype"] = objecttype
input["parent"] = parent if parent is not None else "service"
input["search"] = input.get("name") + input.get(
"description", "-") + input.get("tags", "-")
input["createdat"] = ID.now()
input["creator"] = identity
input["doc"] = input.get("doc") or '{"":""}'
print pprint.pformat(input)
item = Item(**input)
item.save()
return item.attribute_values
def createOrUpdateLexBot(identity, botid):
bot = Item.get("bot", botid)
entities = Item.query("entity", Item.parent.__eq__(botid))
botgraph = {"slots": {}, "intents": {}}
for e in entities:
botgraph["slots"][e.name] = {
"name": e.name,
"ref": "{%(name)s}" % e.attribute_values,
"values": e.doc.aliases,
"type": "entity"
}
variables = Item.query("variable", Item.parent.__eq__(e.ID))
for v in variables:
nameslot = v.name
botgraph["slots"][nameslot] = {
"name": nameslot,
"ref": "{%(name)s}" % v.attribute_values,
"values": v.doc.aliases, #json.loads(v.doc)["aliases"]
"type": "variable"
}
opslot = v.name + "op"
botgraph["slots"][opslot] = {
"name":
opslot,
"ref":
"{%(name)sop}" % v.attribute_values,
"type":
"operator",
"values": [
"in", "outside", "greater", "equals", "different",
"smaller", "bigger", "taller"
]
}
valslot = v.name + "val"
botgraph["slots"][valslot] = {
"type": "value",
"name": valslot,
"ref": "{%(name)sval}" % v.attribute_values,
"values": list(get_sample_values(v.ID)) #sample_values(v.ID)
}
intents = Item.query("intent", Item.parent.__eq__(botid))
for i in intents:
botgraph["intents"][i.name] = {}
#print " ",i
rules = Item.query("rule", Item.parent.__eq__(i.ID))
for r in rules:
botgraph["intents"][i.name][
r.name] = r.doc.replacements #json.loads(r.doc)["expressions"]
#print "~BG~%~~"*20
#print pprint.pformat(botgraph)
logger().info("Creating bot %s", botid)
intents = []
for intent in botgraph["intents"].keys():
logger().critical("Adding Intent %s", intent)
intent_config = get_intent_config(intent, botgraph, "lex")
#print "** ** ** "*30
#print pprint.pformat(intent_config)
#print "-- -- -- "*30
intents.append(intent_config)
for slot in intent_config["slots"]:
logger().critical("Adding Slot Type `%s`", slot["name"])
#print pprint.pformat(slot["enumerationValues"])
Bot.add_slot_type(slot["name"], slot["enumerationValues"])
logger().critical("Saving Intent `%s` to Lex", intent)
Bot.add_intent(intent_config)
logger().info("Putting Bot %s", bot.name)
Bot.build(bot.name, intents=[i["name"] for i in intents])
logger().info("Putting Bot Alias %s %s", bot.name, "demo")
Bot.put_alias(bot.name)
Bot.put_bot_version(bot.name)
return botgraph
def get_intents(cls, botid):
intents = Item.query("intent",
filter_condition=Item.parent.__eq__(botid))
return [i.attribute_values for i in intents]
def get_entities(cls, botid):
entities = Item.query("entity",
filter_condition=Item.parent.__eq__(botid))
return [e.attribute_values for e in entities]
def get_bots(cls, user="******"):
accountid = cls.get_accountid(user)
bots = Item.query("bot",
filter_condition=Item.parent.__eq__(accountid))
return [b.attribute_values for b in bots]
def get_presigned_url(entityid, identity):
entity = Item.get("entity", entityid)
bot = Item.get("bot", entity.parent)
doc = bot.doc
return S3Helpers.get_presigned_url(
doc.bucket, doc.prefix + "/" + entity.name + "/INPUT.txt")
def createVariable(parent, options, identity):
print ">", parent, options, identity
try:
entity = Item.get("entity", parent)
except Exception:
raise errors.ObjectNotFound(parent)
if entity.doc.status != ENTITYSTATES.READY:
raise errors.InvalidParameter(
"Could not add variable to unparsed entity")
bot = Item.get("bot", entity.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
name = options["name"]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
print uri
data = {}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "variable"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get(
"tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {"aliases": options["aliases"], "field": options["field"]}
item = Item(**data)
item.save()
database = bot.doc.database
tablename = entity.doc.tablename
columnname = options["field"]
sql = 'select %(columnname)s from "%(database)s"."%(tablename)s" group by %(columnname)s limit 30' % vars(
)
response = AthenaQuery.run(**{"sql": sql})
values = [
r[columnname].replace('"', '') for r in response["data"]["records"]
]
for i, val in enumerate(values):
if len(val):
slug = unicodedata.normalize('NFKD', val).encode('ascii', 'ignore')
cache = Item(
**{
"name": slug,
"parent": uri,
"doc": {},
"createdat": ID.now(),
"objecttype": "value",
"ID": uri + ":" + str(i),
"search": slug
})
print cache
cache.save()
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
