def createBot(parent,identity,options) : accountid = parent.split(":")[2] name = options.get("name","unnamedbot"+datetime.now().strftime("%Y%m%D%H%M%S")) uri = "uri:bot:%(accountid)s:%(name)s" % vars() try : existing = Item.get("bot", uri) raise errors.DuplicateError(uri) except Exception : pass input= {} input["ID"] = uri input["objecttype"] = "bot" input["parent"] = parent input["name"] = options.get("name","untitled") input["description"] = options.get("description","no description available") input["search"] = input["name"]+"-"+input["description"] input["createdat"] = ID.now() input["creator"] = identity input["doc"] = { "bucket" : "airbot2018", "database" : "airbot"+input["name"], "status" : "NOTBUILT", "prefix" : input["name"] } item = Item(**input) item.save() bot = Item.get("bot", uri) return bot.attribute_values
def getEntity(identity, entityid): try: entity = Item.get("entity", entityid) bot = Item.get("bot", entity.parent) except Exception: raise errors.ObjectNotFound(entityid) print ENTITYSTATES.PARSING, ">>", entity.doc.status, ENTITYSTATES.PARSING == entity.doc.status if entity.doc.status == ENTITYSTATES.PARSING: logger().info("entity is parsing") if len(entity.doc.crawler): crawler_name = entity.doc.crawler crawler = Crawler.get(name=crawler_name) if crawler["State"] in ["READY", "STOPPING"]: doc = bot.doc database = doc.database tablename = entity.doc.tablename columns = Crawler.get_table(database, tablename) print "columns", columns entity.update(actions=[ Item.doc.columns.set(columns), Item.doc.status.set(ENTITYSTATES.READY) ]) entity_doc = entity.json() print "entity_doc ", pprint.pformat(entity_doc) entity_doc.update(entity_doc["doc"]) del entity_doc["doc"] return entity_doc
def createRule(parent,options, identity): print ">", parent, options, identity try : intent = Item.get("intent",parent) except Exception: raise errors.ObjectNotFound(parent) bot = Item.get("bot",intent.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] intentname = parent.split(":")[4] name = options["name"] uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars() print uri data={} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "rule" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get("tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = { "replacements" : options["replacements"] } item = Item(**data) item.save() d = item.json() d.update(d["doc"]) del d["doc"] return d
def getVariable(identity, variableid): try: variable = Item.get("variable", variableid) entity = Item.get("entity", variable.parent) except Exception: raise errors.ObjectNotFound(variableid) variable_doc = variable.json() variable_doc.update(variable_doc["doc"]) del variable_doc["doc"] return variable_doc
def sample_values(variableid): variable = Item.get("variable", variableid) columnname = json.loads(variable.doc)["field"] entity = Item.get("entity", variable.parent) tablename = entity.name bot = Item.get("bot", entity.parent) doc = json.loads(bot.doc) database = doc["database"] sql = 'select %(columnname)s from "%(database)s"."%(tablename)s" group by %(columnname)s limit 15'%vars() response = AthenaQuery.run(**{"sql": sql}) return [r[columnname].replace('"','') for r in response["data"]["records"]]
def syncVariableValues(variableid): variable = Item.get("variable", variableid) valueiterator = Item.query("value", Item.parent == variableid) with Item.batch_write() as batch: for v in valueiterator: batch.delete(v) entity = Item.get("entity", variable.parent) bot = Item.get("bot", entity.parent) database = bot.doc.database tablename = entity.doc.tablename column = variable.doc.field if variable.doc.type.lower() == "dimension": sql = 'select distinct "%(column)s" from "%(database)s"."%(tablename)s" ' % vars( ) response = AthenaQuery.run(**{"sql": sql}) #logger().info("Athena returned %s", pprint.pformat(response)) values = [ r[column].replace('"', '') for r in response["data"]["records"] ] elif variable.doc.type.lower() == "metric": sql = 'select max( "%(column)s") as mx, min("%(column)s") as mn from "%(database)s"."%(tablename)s" ' % vars( ) response = AthenaQuery.run(**{"sql": sql}) r = response["data"]["records"][0] M = r["mx"] m = r["mn"] print m, M values = scale(m, M) print ">>", values with Item.batch_write() as batch: for i, val in enumerate(values): if type(val) == types.IntType: val = unicode(val) if len(val): slug = unicodedata.normalize('NFKD', val).encode('ascii', 'ignore') cache = Item( **{ "name": slug, "parent": variableid, "doc": {}, "createdat": ID.now(), "objecttype": "value", "ID": variableid + ":" + str(i), "search": slug }) batch.save(cache) return
def preview(entityid, identity): logger().info("Preview of %s", entityid) entity = Item.get("entity", entityid) bot = Item.get("bot", entity.parent) doc = bot.doc error = True logger().info(" Querying ", ) database = doc.database tablename = entity.doc.tablename sql = 'select * from "%(database)s"."%(tablename)s" limit 100;' % vars() logger().info("SQL = %s", sql) response = AthenaQuery.run(**{"sql": sql}) #response["data"]["schema"] = response["data"]["records"][0].keys() logger().info("Response = %s", pprint.pformat(response)) return json.dumps(response["data"]["records"])
def deleteVariable(identity, variableid): try: v = Item.get("variable", variableid) v.delete() return True except Exception: return False
def createEntity(identity, botid, data): try: bot = Item.get("bot", botid) except Exception as e: raise errors.ObjectNotFound(botid) print ">>", data parts = botid.split(":") accountid = parts[2] botname = parts[3] entityname = data["name"] uri = "uri:entity:%(accountid)s:%(botname)s:%(entityname)s" % vars() entity = Item(parent=botid, ID=uri, createdat=ID.now(), name=data["name"], description=data.get("description", "-"), search=data["name"] + data.get("description", "-"), objecttype="entity", doc={ "aliases": data["aliases"], "status": ENTITYSTATES.NOT_READY, "database": bot.doc.database, "tablename": data["name"] }) print entity entity.save() return entity.attribute_values
def preview(entityid, identity): logger().info("Preview of %s", entityid) entity = Item.get("entity", entityid) if entity.doc.status != ENTITYSTATES.READY: raise errors.FileStatusError("File has not been parsed") bot = Item.get("bot", entity.parent) doc = bot.doc error = True logger().info(" Querying ", ) database = doc.database tablename = entity.doc.tablename sql = 'select * from "%(database)s"."%(tablename)s" limit 100;' % vars() logger().info("SQL = %s", sql) response = AthenaQuery.run(**{"sql": sql}) logger().info("Response = %s,", pprint.pformat(response)) return response
def deleteRule(identity, ruleid): try: v= Item.get("rule", ruleid) v.delete() return True except Exception: return False return False
def delete_item(identifier, identity): try: bot = Item.get(objecttype, identifier) except Exception: raise errors.ObjectNotFound( "Could not find %(objecttype)s with ID `%(botid)s`" % vars()) bot.delete() return True
def getRule(identity, ruleid) : try: rule = Item.get("rule", ruleid) except Exception: raise errors.ObjectNotFound(ruleid) rule_doc = rule.json() rule_doc.update(rule_doc["doc"]) del rule_doc["doc"] return rule_doc
def getBotStatus(identity, botid) : try : bot = Item.get("bot", botid) except Exception as e : return "NOT READY" try : response = Bot.get_bot(bot.botname) return response["status"] except Exception as e : return "NOT READY"
def createVariable(parent, options, identity): print ">", parent, options, identity try: entity = Item.get("entity", parent) except Exception: raise errors.ObjectNotFound(parent) if entity.doc.status != ENTITYSTATES.READY: raise errors.InvalidParameter( "Could not add variable to unparsed entity") bot = Item.get("bot", entity.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] name = options["name"] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) print uri data = {} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "variable" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get( "tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = { "field": options["field"], "type": options.get("type", "dimension"), "aliases": options["aliases"] } item = Item(**data) item.save() syncVariableValues(variableid=uri) d = item.json() d.update(d["doc"]) del d["doc"] return d '''
def list_accounts(identity, options): accounts = [] iterator = apply_options(Item.query("user", Item.name.__eq__(identity)), options) for user in iterator: try: print user.attribute_values account = Item.get("account", user.parent) accounts.append(account.attribute_values) except Exception, e: pass
def deleteEntity(identity, entityid): try: e = Item.get("entity", entityid) e.delete() vars = Item.parent_index.query(entityid) for r in vars: r.delete() return True except Exception as e: print e return False
def createBot(parent,identity,options) : accountid = parent.split(":")[2] name = options.get("name","unnamedbot"+datetime.now().strftime("%Y%m%D%H%M%S")) uri = "uri:bot:%(accountid)s:%(name)s" % vars() try : existing = Item.get("bot", uri) raise errors.DuplicateError(uri) except Exception : pass lexbotname=(random.choice([chr(i) for i in range(65, 65 + 26)]) + random.choice([chr(i) for i in range(65, 65 + 26)])).lower() assigned = True while assigned : try : Item.get("lex",lexbotname) lexbotname = (random.choice([chr(i) for i in range(65, 65 + 26)]) + random.choice( [chr(i) for i in range(65, 65 + 26)])).lower() except Exception : assigned=False Item(objecttype="lex", ID=lexbotname,name=lexbotname,parent=uri, search=lexbotname,doc={},createdat=ID.now()).save() input= {} input["ID"] = uri input["objecttype"] = "bot" input["botname"] = lexbotname input["parent"] = parent input["name"] = options.get("name","untitled") input["description"] = options.get("description","no description available") input["search"] = input["name"]+"-"+input["description"] input["createdat"] = ID.now() input["creator"] = identity input["doc"] = { "bucket" : "airbot2018", "database" : "airbot"+input["name"], "status" : "NOTBUILT", "prefix" : input["name"] } item = Item(**input) item.save() bot = Item.get("bot", uri) return bot.attribute_values
def updateEntity(identity, entityid, data): try: entity = Item.get("entity", entityid) except Exception as e: raise errors.ObjectNotFound(entityid) entity.update(actions=[ Item.doc.aliases.set(data.get("aliases", entity.doc.aliases)), Item.description.set(data.get("description", entity.description)) ]) return entity.attribute_values
def deleteIntent(identity, intentid): try: intent = Item.get("intent", intentid) intent.delete() rules = Item.parent_index.query(intentid) for r in rules: r.delete() return True except Exception: return False return False
def getIntent(identity, intentid): print "%/" * 30 try: intent = Item.get("intent", intentid) except Exception: raise errors.ObjectNotFound(intentid) doc = intent.json() doc.update(doc["doc"]) del doc["doc"] print doc return doc
def refreshSchema(entityid, identity): entity = Item.get("entity", entityid) print "??", entity.doc.attribute_values print "==>", entity.doc.status if entity.doc.status == ENTITYSTATES.PARSING: return False logger().info("Creating crawler ") bot = Item.get("bot", entity.parent) doc = bot.doc database = doc["database"] tablename = entity.name name = Crawler.crawl_bucket(database=doc["database"], bucket=doc["bucket"], prefix=doc["prefix"] + "/" + entity.name) done = False nbtry = 10 entity.update(actions=[ Item.doc.status.set(ENTITYSTATES.PARSING), Item.doc.crawler.set(name) ]) return True
def updateIntent(identity, intentid, data): try: v = Item.get("intent", intentid) except Exception as e: raise errors.ObjectNotFound(intentid) v.update(actions=[ Item.doc.reply.set(data.get("reply", v.doc.reply)), Item.doc.sql.set(data.get("sql", v.doc.sql)), Item.description.set(data.get("description", v.description)) ]) d = v.json() d.update(d["doc"]) del d["doc"] return d
def updateVariable(identity, variableid, data): try: v = Item.get("variable", variableid) except Exception as e: raise errors.ObjectNotFound(variableid) v.update(actions=[ Item.doc.aliases.set(data.get("aliases", v.doc.aliases)), Item.doc.field.set(data.get("field", v.doc.field)), Item.description.set(data.get("description", v.description)) ]) d = v.json() d.update(d["doc"]) del d["doc"] return d
def updateRule(identity, ruleid, data) : try: v = Item.get("rule",ruleid) except Exception as e : raise errors.ObjectNotFound(ruleid) v.update(actions=[ Item.doc.replacements.set(data.get("replacements",v.doc.replacements)), Item.description.set(data.get("description", v.description)) ]) d = v.json() d.update(d["doc"]) del d["doc"] return d
def getVariableValues(variableid, identity): try: variable = Item.get("variable", variableid) print variable except Exception as e: raise errors.ObjectNotFound(variableid) if variable.doc.type.lower() == "dimension": values = [] valueiterator = Item.query("value", Item.parent == variableid, limit=100) for v in valueiterator: if v.name not in values: values.append(v.name) return values else: return [10, 100, 1000, 5000, 10000]
def update_item(identifier, input, identity): print "update_item <-", identifier, input, identity try: item = Item.get(objecttype, identifier) doc = item.attribute_values except Exception: raise errors.ObjectNotFound( "Could not find %(objecttype)s with ID `%(botid)s`" % vars()) actions = [] for k in input.keys(): attribute = getattr(Item, k) print "attribute = ", attribute actions.append(attribute.set(input[k])) term = doc["name"] + input.get( "description", doc.get("description", "-")) + input.get( "tags", doc.get("tags", "-")) actions.append(Item.search.set(term)) print actions item.update(actions=actions) return item.attribute_values
def listIdentities(uri, identity, options): bot = Item.get("bot", uri) if options is None: iterator = Item.parent_index.query("entity", Item.parent == uri) else: limit = options.get("limit", 5) if "search" in options.keys(): term = options.get("search", ) iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term)) else: iterator = Item.parent_index.query("entity", Item.parent == uri) items=[i for i in apply_options(iterator, options)] for entity in items : print entity if entity.doc.status==ENTITYSTATES.PARSING : print " found an entity with status", ENTITYSTATES.PARSING crawler_name = i.doc.crawler crawler = Crawler.get(name=crawler_name) if crawler["State"] in ["READY", "STOPPING"]: doc = bot.doc database=doc.database tablename = entity.doc.tablename columns = Crawler.get_table(database,tablename) print "updating entity status and columns" entity.update(actions=[Item.doc.columns.set(columns),Item.doc.status.set(ENTITYSTATES.READY)]) if options is None: iterator = Item.parent_index.query("entity", Item.parent == uri) else: limit = options.get("limit", 5) if "search" in options.keys(): term = options.get("search", ) iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term)) else: iterator = Item.parent_index.query("entity", Item.parent == uri) items = [merge_doc(i) for i in apply_options(iterator, options)] return items
def test_aa_clean(self): event = { "identity": { "claims": TestAccount.get_claim() }, "field": "listMyAccounts", "path": "Query/listMyAccounts", "arguments": {} } accounts = App.handler(event, {}) if type(accounts) == type(["a", "list"]): print "Found", len(accounts), "accounts" print accounts for accountdoc in accounts: a = Item.get('account', accountdoc["ID"]) a.delete() x = App.handler(event, {}) self.assertTrue(len(x) == 0) else: self.assertTrue(True, "no accounts")
def askBot(identity, botid, question) : try : bot = Item.get("bot", botid) except Exception as e : return "Seems i do not exists anymore :(" response= Bot.send(bot=bot.botname,input=question) if response["dialogState"] : parts = botid.split(":") accountid=parts[2] botname = parts[3] cube = getBotCube(bot) semantics= cube.resolve(response["slots"]) print pprint.pformat(semantics) sql = semantics["sql"] print semantics["sql"] results = AthenaQuery.run(sql=sql) print pprint.pformat(results) return json.dumps({ "display" : semantics["display"], "data" : results["data"]["records"] }) else : message = random.choice([ "Sorry, this is not clear for and old bot", "Can you try to reformulate ?", "Hmm, i'm just a bot", "Well, this is clear, i'm not really awake right now" ]) return json.dumps({ "display" : "error", "data" : message })