def getEntity(identity, entityid):
try:
entity = Item.get("entity", entityid)
bot = Item.get("bot", entity.parent)
except Exception:
raise errors.ObjectNotFound(entityid)
print ENTITYSTATES.PARSING, ">>", entity.doc.status, ENTITYSTATES.PARSING == entity.doc.status
if entity.doc.status == ENTITYSTATES.PARSING:
logger().info("entity is parsing")
if len(entity.doc.crawler):
crawler_name = entity.doc.crawler
crawler = Crawler.get(name=crawler_name)
if crawler["State"] in ["READY", "STOPPING"]:
doc = bot.doc
database = doc.database
tablename = entity.doc.tablename
columns = Crawler.get_table(database, tablename)
print "columns", columns
entity.update(actions=[
Item.doc.columns.set(columns),
Item.doc.status.set(ENTITYSTATES.READY)
])
entity_doc = entity.json()
print "entity_doc ", pprint.pformat(entity_doc)
entity_doc.update(entity_doc["doc"])
del entity_doc["doc"]
return entity_doc
def createEntity(identity, botid, data):
try:
bot = Item.get("bot", botid)
except Exception as e:
raise errors.ObjectNotFound(botid)
print ">>", data
parts = botid.split(":")
accountid = parts[2]
botname = parts[3]
entityname = data["name"]
uri = "uri:entity:%(accountid)s:%(botname)s:%(entityname)s" % vars()
entity = Item(parent=botid,
ID=uri,
createdat=ID.now(),
name=data["name"],
description=data.get("description", "-"),
search=data["name"] + data.get("description", "-"),
objecttype="entity",
doc={
"aliases": data["aliases"],
"status": ENTITYSTATES.NOT_READY,
"database": bot.doc.database,
"tablename": data["name"]
})
print entity
entity.save()
return entity.attribute_values
def get_table(cls, database, tablename):
client = cls.glue()
try:
response = client.get_table(DatabaseName=database, Name=tablename)
return response["Table"]["StorageDescriptor"]["Columns"]
except Exception:
raise errors.ObjectNotFound(database + "." + tablename)
def createRule(parent,options, identity):
print ">", parent, options, identity
try :
intent = Item.get("intent",parent)
except Exception:
raise errors.ObjectNotFound(parent)
bot = Item.get("bot",intent.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
intentname = parent.split(":")[4]
name = options["name"]
uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars()
print uri
data={}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "rule"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get("tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {
"replacements" : options["replacements"]
}
item = Item(**data)
item.save()
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
def delete_item(identifier, identity):
try:
bot = Item.get(objecttype, identifier)
except Exception:
raise errors.ObjectNotFound(
"Could not find %(objecttype)s with ID `%(botid)s`" % vars())
bot.delete()
return True
def getRule(identity, ruleid) :
try:
rule = Item.get("rule", ruleid)
except Exception:
raise errors.ObjectNotFound(ruleid)
rule_doc = rule.json()
rule_doc.update(rule_doc["doc"])
del rule_doc["doc"]
return rule_doc
def getVariable(identity, variableid):
try:
variable = Item.get("variable", variableid)
entity = Item.get("entity", variable.parent)
except Exception:
raise errors.ObjectNotFound(variableid)
variable_doc = variable.json()
variable_doc.update(variable_doc["doc"])
del variable_doc["doc"]
return variable_doc
def getIntent(identity, intentid):
print "%/" * 30
try:
intent = Item.get("intent", intentid)
except Exception:
raise errors.ObjectNotFound(intentid)
doc = intent.json()
doc.update(doc["doc"])
del doc["doc"]
print doc
return doc
def updateEntity(identity, entityid, data):
try:
entity = Item.get("entity", entityid)
except Exception as e:
raise errors.ObjectNotFound(entityid)
entity.update(actions=[
Item.doc.aliases.set(data.get("aliases", entity.doc.aliases)),
Item.description.set(data.get("description", entity.description))
])
return entity.attribute_values
def run(cls, **kwargs):
client = boto3.client('glue', region_name="eu-west-1")
try:
response = client.get_table(DatabaseName=kwargs["database"],
Name=kwargs["tablename"])
schema = response["Table"]["StorageDescriptor"]["Columns"]
return {"success": True, "data": schema}
except Exception:
return {
"success": False,
"data": errors.ObjectNotFound(kwargs["entity"])
}
def updateIntent(identity, intentid, data):
try:
v = Item.get("intent", intentid)
except Exception as e:
raise errors.ObjectNotFound(intentid)
v.update(actions=[
Item.doc.reply.set(data.get("reply", v.doc.reply)),
Item.doc.sql.set(data.get("sql", v.doc.sql)),
Item.description.set(data.get("description", v.description))
])
d = v.json()
d.update(d["doc"])
del d["doc"]
return d
def updateRule(identity, ruleid, data) :
try:
v = Item.get("rule",ruleid)
except Exception as e :
raise errors.ObjectNotFound(ruleid)
v.update(actions=[
Item.doc.replacements.set(data.get("replacements",v.doc.replacements)),
Item.description.set(data.get("description", v.description))
])
d = v.json()
d.update(d["doc"])
del d["doc"]
return d
def updateVariable(identity, variableid, data):
try:
v = Item.get("variable", variableid)
except Exception as e:
raise errors.ObjectNotFound(variableid)
v.update(actions=[
Item.doc.aliases.set(data.get("aliases", v.doc.aliases)),
Item.doc.field.set(data.get("field", v.doc.field)),
Item.description.set(data.get("description", v.description))
])
d = v.json()
d.update(d["doc"])
del d["doc"]
return d
def getVariableValues(variableid, identity):
try:
variable = Item.get("variable", variableid)
print variable
except Exception as e:
raise errors.ObjectNotFound(variableid)
if variable.doc.type.lower() == "dimension":
values = []
valueiterator = Item.query("value",
Item.parent == variableid,
limit=100)
for v in valueiterator:
if v.name not in values:
values.append(v.name)
return values
else:
return [10, 100, 1000, 5000, 10000]
def createVariable(parent, options, identity):
print ">", parent, options, identity
try:
entity = Item.get("entity", parent)
except Exception:
raise errors.ObjectNotFound(parent)
if entity.doc.status != ENTITYSTATES.READY:
raise errors.InvalidParameter(
"Could not add variable to unparsed entity")
bot = Item.get("bot", entity.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
name = options["name"]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
print uri
data = {}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "variable"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get(
"tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {
"field": options["field"],
"type": options.get("type", "dimension"),
"aliases": options["aliases"]
}
item = Item(**data)
item.save()
syncVariableValues(variableid=uri)
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
'''
def update_item(identifier, input, identity):
print "update_item <-", identifier, input, identity
try:
item = Item.get(objecttype, identifier)
doc = item.attribute_values
except Exception:
raise errors.ObjectNotFound(
"Could not find %(objecttype)s with ID `%(botid)s`" % vars())
actions = []
for k in input.keys():
attribute = getattr(Item, k)
print "attribute = ", attribute
actions.append(attribute.set(input[k]))
term = doc["name"] + input.get(
"description", doc.get("description", "-")) + input.get(
"tags", doc.get("tags", "-"))
actions.append(Item.search.set(term))
print actions
item.update(actions=actions)
return item.attribute_values
def createVariable(parent, options, identity):
print ">", parent, options, identity
try:
entity = Item.get("entity", parent)
except Exception:
raise errors.ObjectNotFound(parent)
if entity.doc.status != ENTITYSTATES.READY:
raise errors.InvalidParameter(
"Could not add variable to unparsed entity")
bot = Item.get("bot", entity.parent)
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
name = options["name"]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
print uri
data = {}
data["ID"] = uri
data["name"] = options["name"]
data["objecttype"] = "variable"
data["parent"] = parent if parent is not None else "service"
data["search"] = name + options.get("description", "-") + options.get(
"tags", "-")
data["createdat"] = ID.now()
data["creator"] = identity
data["doc"] = {"aliases": options["aliases"], "field": options["field"]}
item = Item(**data)
item.save()
database = bot.doc.database
tablename = entity.doc.tablename
columnname = options["field"]
sql = 'select %(columnname)s from "%(database)s"."%(tablename)s" group by %(columnname)s limit 30' % vars(
)
response = AthenaQuery.run(**{"sql": sql})
values = [
r[columnname].replace('"', '') for r in response["data"]["records"]
]
for i, val in enumerate(values):
if len(val):
slug = unicodedata.normalize('NFKD', val).encode('ascii', 'ignore')
cache = Item(
**{
"name": slug,
"parent": uri,
"doc": {},
"createdat": ID.now(),
"objecttype": "value",
"ID": uri + ":" + str(i),
"search": slug
})
print cache
cache.save()
d = item.json()
d.update(d["doc"])
del d["doc"]
return d
def get_item(identifier, identity):
try:
bot = Item.get(objecttype, identifier)
return bot.attribute_values
except Exception:
raise errors.ObjectNotFound(identifier)
def create_item(parent, input, identity):
print "create %(objectype)s received", parent, input, identity
try:
name = input["name"]
except Exception:
raise errors.InvalidParameter("Missing `name` field in input" %
vars())
if parent is None:
parent = "service"
else:
try:
parent_item = Item.get(parentobjecttype, parent)
except Exception:
raise errors.ObjectNotFound(
"No parent object with type `%(parentobjecttype)s` found with Id :`%(parent)s`"
% vars())
candidates = get_children_by_name(parent,
objecttype=objecttype,
name=input["name"])
if len(candidates):
if parentobjecttype is None:
parent = "accounts"
raise errors.DuplicateError(
"An object with name `%(name)s`and type `%(objecttype)s` already exists in `%(parent)s`"
% vars())
objectid = ID.get()
if objecttype == "account":
uri = "uri:account:%(objectid)s" % vars()
if objecttype == "bot":
accountid = parent.split(":")[2]
uri = "uri:bot:%(accountid)s:%(name)s" % vars()
if objecttype == "entity":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
uri = "uri:entity:%(accountid)s:%(botname)s:%(name)s" % vars()
if objecttype == "intent":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
uri = "uri:intent:%(accountid)s:%(botname)s:%(name)s" % vars()
if objecttype == "reply":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
intentname = parent.split(":")[4]
uri = "uri:reply:%(accountid)s:%(botname)s:%(intenname)s:(name)s" % vars(
)
if objecttype == "variable":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
entityname = parent.split(":")[4]
uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars(
)
if objecttype == "rule":
accountid = parent.split(":")[2]
botname = parent.split(":")[3]
intentname = parent.split(":")[4]
uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars(
)
input["ID"] = uri
input["objecttype"] = objecttype
input["parent"] = parent if parent is not None else "service"
input["search"] = input.get("name") + input.get(
"description", "-") + input.get("tags", "-")
input["createdat"] = ID.now()
input["creator"] = identity
input["doc"] = input.get("doc") or '{"":""}'
print pprint.pformat(input)
item = Item(**input)
item.save()
return item.attribute_values
