def createBot(parent,identity,options) :
accountid = parent.split(":")[2]
name = options.get("name","unnamedbot"+datetime.now().strftime("%Y%m%D%H%M%S"))
uri = "uri:bot:%(accountid)s:%(name)s" % vars()
try :
existing = Item.get("bot", uri)
raise errors.DuplicateError(uri)
except Exception :
pass
input= {}
input["ID"] = uri
input["objecttype"] = "bot"
input["parent"] = parent
input["name"] = options.get("name","untitled")
input["description"] = options.get("description","no description available")
input["search"] = input["name"]+"-"+input["description"]
input["createdat"] = ID.now()
input["creator"] = identity
input["doc"] = {
"bucket" : "airbot2018",
"database" : "airbot"+input["name"],
"status" : "NOTBUILT",
"prefix" : input["name"]
}
item = Item(**input)
item.save()
bot = Item.get("bot", uri)
return bot.attribute_values
def getEntity(identity, entityid):
try:
entity = Item.get("entity", entityid)
bot = Item.get("bot", entity.parent)
except Exception:
raise errors.ObjectNotFound(entityid)
print ENTITYSTATES.PARSING, ">>", entity.doc.status, ENTITYSTATES.PARSING == entity.doc.status
if entity.doc.status == ENTITYSTATES.PARSING:
logger().info("entity is parsing")
if len(entity.doc.crawler):
crawler_name = entity.doc.crawler
crawler = Crawler.get(name=crawler_name)
if crawler["State"] in ["READY", "STOPPING"]:
doc = bot.doc
database = doc.database
tablename = entity.doc.tablename
columns = Crawler.get_table(database, tablename)
print "columns", columns
entity.update(actions=[
Item.doc.columns.set(columns),
Item.doc.status.set(ENTITYSTATES.READY)
])
entity_doc = entity.json()
print "entity_doc ", pprint.pformat(entity_doc)
entity_doc.update(entity_doc["doc"])
del entity_doc["doc"]
return entity_doc
def createRule(parent,options, identity):
print ">", parent, options, identity
try :
intent = Item.get("intent",parent)
except Exception:
raise errors.ObjectNotFound(parent)
bot = Item.get("bot",intent.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
intentname = parent.split(":")[4]
name = options["name"]
uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars()
print uri
data={}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "rule"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get("tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {
"replacements" : options["replacements"]
}
item = Item(**data)
item.save()
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
def getVariable(identity, variableid):
try:
variable = Item.get("variable", variableid)
entity = Item.get("entity", variable.parent)
except Exception:
raise errors.ObjectNotFound(variableid)
variable_doc = variable.json()
variable_doc.update(variable_doc["doc"])
del variable_doc["doc"]
return variable_doc
def sample_values(variableid):
variable = Item.get("variable", variableid)
columnname = json.loads(variable.doc)["field"]
entity = Item.get("entity", variable.parent)
tablename = entity.name
bot = Item.get("bot", entity.parent)
doc = json.loads(bot.doc)
database = doc["database"]
sql = 'select %(columnname)s from "%(database)s"."%(tablename)s" group by %(columnname)s limit 15'%vars()
response = AthenaQuery.run(**{"sql": sql})
return [r[columnname].replace('"','') for r in response["data"]["records"]]
def syncVariableValues(variableid):
variable = Item.get("variable", variableid)
valueiterator = Item.query("value", Item.parent == variableid)
with Item.batch_write() as batch:
for v in valueiterator:
batch.delete(v)
entity = Item.get("entity", variable.parent)
bot = Item.get("bot", entity.parent)
database = bot.doc.database
tablename = entity.doc.tablename
column = variable.doc.field
if variable.doc.type.lower() == "dimension":
sql = 'select distinct "%(column)s" from "%(database)s"."%(tablename)s" ' % vars(
)
response = AthenaQuery.run(**{"sql": sql})
#logger().info("Athena returned %s", pprint.pformat(response))
values = [
r[column].replace('"', '') for r in response["data"]["records"]
]
elif variable.doc.type.lower() == "metric":
sql = 'select max( "%(column)s") as mx, min("%(column)s") as mn from "%(database)s"."%(tablename)s" ' % vars(
)
response = AthenaQuery.run(**{"sql": sql})
r = response["data"]["records"][0]
M = r["mx"]
m = r["mn"]
print m, M
values = scale(m, M)
print ">>", values
with Item.batch_write() as batch:
for i, val in enumerate(values):
if type(val) == types.IntType:
val = unicode(val)
if len(val):
slug = unicodedata.normalize('NFKD',
val).encode('ascii', 'ignore')
cache = Item(
**{
"name": slug,
"parent": variableid,
"doc": {},
"createdat": ID.now(),
"objecttype": "value",
"ID": variableid + ":" + str(i),
"search": slug
})
batch.save(cache)
return
def preview(entityid, identity):
logger().info("Preview of %s", entityid)
entity = Item.get("entity", entityid)
bot = Item.get("bot", entity.parent)
doc = bot.doc
error = True
logger().info(" Querying ", )
database = doc.database
tablename = entity.doc.tablename
sql = 'select * from "%(database)s"."%(tablename)s" limit 100;' % vars()
logger().info("SQL = %s", sql)
response = AthenaQuery.run(**{"sql": sql})
#response["data"]["schema"] = response["data"]["records"][0].keys()
logger().info("Response = %s", pprint.pformat(response))
return json.dumps(response["data"]["records"])
def deleteVariable(identity, variableid):
try:
v = Item.get("variable", variableid)
v.delete()
return True
except Exception:
return False
def createEntity(identity, botid, data):
try:
bot = Item.get("bot", botid)
except Exception as e:
raise errors.ObjectNotFound(botid)
print ">>", data
parts = botid.split(":")
accountid = parts[2]
botname = parts[3]
entityname = data["name"]
uri = "uri:entity:%(accountid)s:%(botname)s:%(entityname)s" % vars()
entity = Item(parent=botid,
ID=uri,
createdat=ID.now(),
name=data["name"],
description=data.get("description", "-"),
search=data["name"] + data.get("description", "-"),
objecttype="entity",
doc={
"aliases": data["aliases"],
"status": ENTITYSTATES.NOT_READY,
"database": bot.doc.database,
"tablename": data["name"]
})
print entity
entity.save()
return entity.attribute_values
def preview(entityid, identity):
logger().info("Preview of %s", entityid)
entity = Item.get("entity", entityid)
if entity.doc.status != ENTITYSTATES.READY:
raise errors.FileStatusError("File has not been parsed")
bot = Item.get("bot", entity.parent)
doc = bot.doc
error = True
logger().info(" Querying ", )
database = doc.database
tablename = entity.doc.tablename
sql = 'select * from "%(database)s"."%(tablename)s" limit 100;' % vars()
logger().info("SQL = %s", sql)
response = AthenaQuery.run(**{"sql": sql})
logger().info("Response = %s,", pprint.pformat(response))
return response
def deleteRule(identity, ruleid):
try:
v= Item.get("rule", ruleid)
v.delete()
return True
except Exception:
return False
return False
def delete_item(identifier, identity):
try:
bot = Item.get(objecttype, identifier)
except Exception:
raise errors.ObjectNotFound(
"Could not find %(objecttype)s with ID `%(botid)s`" % vars())
bot.delete()
return True
def getRule(identity, ruleid) :
try:
rule = Item.get("rule", ruleid)
except Exception:
raise errors.ObjectNotFound(ruleid)
rule_doc = rule.json()
rule_doc.update(rule_doc["doc"])
del rule_doc["doc"]
return rule_doc
def getBotStatus(identity, botid) :
try :
bot = Item.get("bot", botid)
except Exception as e :
return "NOT READY"
try :
response = Bot.get_bot(bot.botname)
return response["status"]
except Exception as e :
return "NOT READY"
def createVariable(parent, options, identity):
print ">", parent, options, identity
try:
entity = Item.get("entity", parent)
except Exception:
raise errors.ObjectNotFound(parent)
if entity.doc.status != ENTITYSTATES.READY:
raise errors.InvalidParameter(
"Could not add variable to unparsed entity")
bot = Item.get("bot", entity.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
name = options["name"]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
print uri
data = {}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "variable"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get(
"tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {
"field": options["field"],
"type": options.get("type", "dimension"),
"aliases": options["aliases"]
}
item = Item(**data)
item.save()
syncVariableValues(variableid=uri)
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
'''
def list_accounts(identity, options):
accounts = []
iterator = apply_options(Item.query("user", Item.name.__eq__(identity)),
options)
for user in iterator:
try:
print user.attribute_values
account = Item.get("account", user.parent)
accounts.append(account.attribute_values)
except Exception, e:
pass
def deleteEntity(identity, entityid):
try:
e = Item.get("entity", entityid)
e.delete()
vars = Item.parent_index.query(entityid)
for r in vars:
r.delete()
return True
except Exception as e:
print e
return False
def createBot(parent,identity,options) :
accountid = parent.split(":")[2]
name = options.get("name","unnamedbot"+datetime.now().strftime("%Y%m%D%H%M%S"))
uri = "uri:bot:%(accountid)s:%(name)s" % vars()
try :
existing = Item.get("bot", uri)
raise errors.DuplicateError(uri)
except Exception :
pass
lexbotname=(random.choice([chr(i) for i in range(65, 65 + 26)]) + random.choice([chr(i) for i in range(65, 65 + 26)])).lower()
assigned = True
while assigned :
try :
Item.get("lex",lexbotname)
lexbotname = (random.choice([chr(i) for i in range(65, 65 + 26)]) + random.choice(
[chr(i) for i in range(65, 65 + 26)])).lower()
except Exception :
assigned=False
Item(objecttype="lex", ID=lexbotname,name=lexbotname,parent=uri, search=lexbotname,doc={},createdat=ID.now()).save()
input= {}
input["ID"] = uri
input["objecttype"] = "bot"
input["botname"] = lexbotname
input["parent"] = parent
input["name"] = options.get("name","untitled")
input["description"] = options.get("description","no description available")
input["search"] = input["name"]+"-"+input["description"]
input["createdat"] = ID.now()
input["creator"] = identity
input["doc"] = {
"bucket" : "airbot2018",
"database" : "airbot"+input["name"],
"status" : "NOTBUILT",
"prefix" : input["name"]
}
item = Item(**input)
item.save()
bot = Item.get("bot", uri)
return bot.attribute_values
def updateEntity(identity, entityid, data):
try:
entity = Item.get("entity", entityid)
except Exception as e:
raise errors.ObjectNotFound(entityid)
entity.update(actions=[
Item.doc.aliases.set(data.get("aliases", entity.doc.aliases)),
Item.description.set(data.get("description", entity.description))
])
return entity.attribute_values
def deleteIntent(identity, intentid):
try:
intent = Item.get("intent", intentid)
intent.delete()
rules = Item.parent_index.query(intentid)
for r in rules:
r.delete()
return True
except Exception:
return False
return False
def getIntent(identity, intentid):
print "%/" * 30
try:
intent = Item.get("intent", intentid)
except Exception:
raise errors.ObjectNotFound(intentid)
doc = intent.json()
doc.update(doc["doc"])
del doc["doc"]
print doc
return doc
def refreshSchema(entityid, identity):
entity = Item.get("entity", entityid)
print "??", entity.doc.attribute_values
print "==>", entity.doc.status
if entity.doc.status == ENTITYSTATES.PARSING:
return False
logger().info("Creating crawler ")
bot = Item.get("bot", entity.parent)
doc = bot.doc
database = doc["database"]
tablename = entity.name
name = Crawler.crawl_bucket(database=doc["database"],
bucket=doc["bucket"],
prefix=doc["prefix"] + "/" + entity.name)
done = False
nbtry = 10
entity.update(actions=[
Item.doc.status.set(ENTITYSTATES.PARSING),
Item.doc.crawler.set(name)
])
return True
def updateIntent(identity, intentid, data):
try:
v = Item.get("intent", intentid)
except Exception as e:
raise errors.ObjectNotFound(intentid)
v.update(actions=[
Item.doc.reply.set(data.get("reply", v.doc.reply)),
Item.doc.sql.set(data.get("sql", v.doc.sql)),
Item.description.set(data.get("description", v.description))
])
d = v.json()
d.update(d["doc"])
del d["doc"]
return d
def updateVariable(identity, variableid, data):
try:
v = Item.get("variable", variableid)
except Exception as e:
raise errors.ObjectNotFound(variableid)
v.update(actions=[
Item.doc.aliases.set(data.get("aliases", v.doc.aliases)),
Item.doc.field.set(data.get("field", v.doc.field)),
Item.description.set(data.get("description", v.description))
])
d = v.json()
d.update(d["doc"])
del d["doc"]
return d
def updateRule(identity, ruleid, data) :
try:
v = Item.get("rule",ruleid)
except Exception as e :
raise errors.ObjectNotFound(ruleid)
v.update(actions=[
Item.doc.replacements.set(data.get("replacements",v.doc.replacements)),
Item.description.set(data.get("description", v.description))
])
d = v.json()
d.update(d["doc"])
del d["doc"]
return d
def getVariableValues(variableid, identity):
try:
variable = Item.get("variable", variableid)
print variable
except Exception as e:
raise errors.ObjectNotFound(variableid)
if variable.doc.type.lower() == "dimension":
values = []
valueiterator = Item.query("value",
Item.parent == variableid,
limit=100)
for v in valueiterator:
if v.name not in values:
values.append(v.name)
return values
else:
return [10, 100, 1000, 5000, 10000]
def update_item(identifier, input, identity):
print "update_item <-", identifier, input, identity
try:
item = Item.get(objecttype, identifier)
doc = item.attribute_values
except Exception:
raise errors.ObjectNotFound(
"Could not find %(objecttype)s with ID `%(botid)s`" % vars())
actions = []
for k in input.keys():
attribute = getattr(Item, k)
print "attribute = ", attribute
actions.append(attribute.set(input[k]))
term = doc["name"] + input.get(
"description", doc.get("description", "-")) + input.get(
"tags", doc.get("tags", "-"))
actions.append(Item.search.set(term))
print actions
item.update(actions=actions)
return item.attribute_values
def listIdentities(uri, identity, options):
bot = Item.get("bot", uri)
if options is None:
iterator = Item.parent_index.query("entity", Item.parent == uri)
else:
limit = options.get("limit", 5)
if "search" in options.keys():
term = options.get("search", )
iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term))
else:
iterator = Item.parent_index.query("entity", Item.parent == uri)
items=[i for i in apply_options(iterator, options)]
for entity in items :
print entity
if entity.doc.status==ENTITYSTATES.PARSING :
print " found an entity with status", ENTITYSTATES.PARSING
crawler_name = i.doc.crawler
crawler = Crawler.get(name=crawler_name)
if crawler["State"] in ["READY", "STOPPING"]:
doc = bot.doc
database=doc.database
tablename = entity.doc.tablename
columns = Crawler.get_table(database,tablename)
print "updating entity status and columns"
entity.update(actions=[Item.doc.columns.set(columns),Item.doc.status.set(ENTITYSTATES.READY)])
if options is None:
iterator = Item.parent_index.query("entity", Item.parent == uri)
else:
limit = options.get("limit", 5)
if "search" in options.keys():
term = options.get("search", )
iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term))
else:
iterator = Item.parent_index.query("entity", Item.parent == uri)
items = [merge_doc(i) for i in apply_options(iterator, options)]
return items
def test_aa_clean(self):
event = {
"identity": {
"claims": TestAccount.get_claim()
},
"field": "listMyAccounts",
"path": "Query/listMyAccounts",
"arguments": {}
}
accounts = App.handler(event, {})
if type(accounts) == type(["a", "list"]):
print "Found", len(accounts), "accounts"
print accounts
for accountdoc in accounts:
a = Item.get('account', accountdoc["ID"])
a.delete()
x = App.handler(event, {})
self.assertTrue(len(x) == 0)
else:
self.assertTrue(True, "no accounts")
def askBot(identity, botid, question) :
try :
bot = Item.get("bot", botid)
except Exception as e :
return "Seems i do not exists anymore :("
response= Bot.send(bot=bot.botname,input=question)
if response["dialogState"] :
parts = botid.split(":")
accountid=parts[2]
botname = parts[3]
cube = getBotCube(bot)
semantics= cube.resolve(response["slots"])
print pprint.pformat(semantics)
sql = semantics["sql"]
print semantics["sql"]
results = AthenaQuery.run(sql=sql)
print pprint.pformat(results)
return json.dumps({
"display" : semantics["display"],
"data" : results["data"]["records"]
})
else :
message = random.choice([
"Sorry, this is not clear for and old bot",
"Can you try to reformulate ?",
"Hmm, i'm just a bot",
"Well, this is clear, i'm not really awake right now"
])
return json.dumps({
"display" : "error",
"data" : message
})
