def createVariable(parent, options, identity): print ">", parent, options, identity try: entity = Item.get("entity", parent) except Exception: raise errors.ObjectNotFound(parent) if entity.doc.status != ENTITYSTATES.READY: raise errors.InvalidParameter( "Could not add variable to unparsed entity") bot = Item.get("bot", entity.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] name = options["name"] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) print uri data = {} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "variable" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get( "tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = { "field": options["field"], "type": options.get("type", "dimension"), "aliases": options["aliases"] } item = Item(**data) item.save() syncVariableValues(variableid=uri) d = item.json() d.update(d["doc"]) del d["doc"] return d '''
def listIdentities(uri, identity, options): bot = Item.get("bot", uri) if options is None: iterator = Item.parent_index.query("entity", Item.parent == uri) else: limit = options.get("limit", 5) if "search" in options.keys(): term = options.get("search", ) iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term)) else: iterator = Item.parent_index.query("entity", Item.parent == uri) items=[i for i in apply_options(iterator, options)] for entity in items : print entity if entity.doc.status==ENTITYSTATES.PARSING : print " found an entity with status", ENTITYSTATES.PARSING crawler_name = i.doc.crawler crawler = Crawler.get(name=crawler_name) if crawler["State"] in ["READY", "STOPPING"]: doc = bot.doc database=doc.database tablename = entity.doc.tablename columns = Crawler.get_table(database,tablename) print "updating entity status and columns" entity.update(actions=[Item.doc.columns.set(columns),Item.doc.status.set(ENTITYSTATES.READY)]) if options is None: iterator = Item.parent_index.query("entity", Item.parent == uri) else: limit = options.get("limit", 5) if "search" in options.keys(): term = options.get("search", ) iterator = Item.parent_index.query("entity", Item.parent == uri& Item.search.contains(term)) else: iterator = Item.parent_index.query("entity", Item.parent == uri) items = [merge_doc(i) for i in apply_options(iterator, options)] return items
def test_aa_clean(self): event = { "identity": { "claims": TestAccount.get_claim() }, "field": "listMyAccounts", "path": "Query/listMyAccounts", "arguments": {} } accounts = App.handler(event, {}) if type(accounts) == type(["a", "list"]): print "Found", len(accounts), "accounts" print accounts for accountdoc in accounts: a = Item.get('account', accountdoc["ID"]) a.delete() x = App.handler(event, {}) self.assertTrue(len(x) == 0) else: self.assertTrue(True, "no accounts")
def test_ec_get_variable(self): entity = TestAccount.get_entity() entityid = entity["ID"] candidates = [ c.attribute_values for c in Item.query("variable", Item.parent.__eq__(entityid)) ] variable = candidates[0] variableid = variable["ID"] print ">>", variableid event = { "identity": { "claims": TestAccount.get_claim() }, "field": "getVariable", "path": "Query/getVariable", "arguments": { "variableid": variableid } } r = App.handler(event, {}) print r self.assertEqual(r["name"], variable["name"])
def test_fc_get_rule(self): intent = TestAccount.get_intent() intentid = intent["ID"] candidates = [ c.attribute_values for c in Item.query("rule", Item.parent.__eq__(intentid)) ] rule = candidates[0] ruleid = rule["ID"] print ">>", ruleid event = { "identity": { "claims": TestAccount.get_claim() }, "field": "getRule", "path": "Query/getRule", "arguments": { "ruleid": ruleid } } r = App.handler(event, {}) print r self.assertEqual(r["name"], rule["name"])
def askBot(identity, botid, question) : try : bot = Item.get("bot", botid) except Exception as e : return "Seems i do not exists anymore :(" response= Bot.send(bot=bot.botname,input=question) if response["dialogState"] : parts = botid.split(":") accountid=parts[2] botname = parts[3] cube = getBotCube(bot) semantics= cube.resolve(response["slots"]) print pprint.pformat(semantics) sql = semantics["sql"] print semantics["sql"] results = AthenaQuery.run(sql=sql) print pprint.pformat(results) return json.dumps({ "display" : semantics["display"], "data" : results["data"]["records"] }) else : message = random.choice([ "Sorry, this is not clear for and old bot", "Can you try to reformulate ?", "Hmm, i'm just a bot", "Well, this is clear, i'm not really awake right now" ]) return json.dumps({ "display" : "error", "data" : message })
def createLexBot(botid, identity) : bot = Item.get("bot", botid) c = getBotCube(bot) c.deploy_bot() Bot.put_alias(bot.botname) return "Done"
def askBot(identity, botid, question) : import random try : bot = Item.get("bot", botid) except Exception as e : return "Seems i do not exists anymore :(" response= Bot.send(bot=bot.name,input=question) print response if response["dialogState"] : parts = botid.split(":") accountid=parts[2] botname = parts[3] intentname = response["intentName"] uri = "uri:intent:%(accountid)s:%(botname)s:%(intentname)s"%vars() intent = Item.get("intent", uri) database= bot.doc.database SQL = intent.doc.sql print "SQL )= ", SQL graph = { "bot" : { "entities": {} } } entities = list(Item.parent_index.query( "entity", Item.parent==intent.parent)) print "??",entities variables = {} for e in entities : print ">>>>>>>>>>>>>>>>>>>",">"+e.name+"<" SQL = SQL.replace(e.name, '"'+database+'".'+'"'+e.doc.tablename+'"') graph["bot"]["entities"][e.name] ={ "name" : e.name, "tablename" : '"'+database+'"' +'."'+ e.name+'"', "database" : database, "variables" : { } } for v in Item.parent_index.query("variable", Item.parent == e.ID): graph["bot"]["entities"][e.name]["variables"][v.name] = { "name" : v.name, "field" : v.doc.field } print pprint.pformat(graph) filters = get_filters(graph=graph, slots=response["slots"])["filters"] explains= get_filters(graph=graph, slots=response["slots"])["explained"] print "filters = ",filters print "explains = ",explains sql = "select count(*) as NB from ("+SQL+" where "+filters+") as T" results = AthenaQuery.run(sql=sql) print "+-/"*30 print results["data"]["records"][0] replytpl = random.choice(intent.doc.reply) reply = replytpl.replace("{NB}", results["data"]["records"][0]["NB"])+" "+explains return reply else : return random.choice([ "Sorry, this is not clear for and old bot", "Can you try to reformulate ?", "Hmm, i'm just a bot", "Well, this is clear, i'm not really awake right now" ])
def get_item(identifier, identity): try: bot = Item.get(objecttype, identifier) return bot.attribute_values except Exception: raise errors.ObjectNotFound(identifier)
def create_item(parent, input, identity): print "create %(objectype)s received", parent, input, identity try: name = input["name"] except Exception: raise errors.InvalidParameter("Missing `name` field in input" % vars()) if parent is None: parent = "service" else: try: parent_item = Item.get(parentobjecttype, parent) except Exception: raise errors.ObjectNotFound( "No parent object with type `%(parentobjecttype)s` found with Id :`%(parent)s`" % vars()) candidates = get_children_by_name(parent, objecttype=objecttype, name=input["name"]) if len(candidates): if parentobjecttype is None: parent = "accounts" raise errors.DuplicateError( "An object with name `%(name)s`and type `%(objecttype)s` already exists in `%(parent)s`" % vars()) objectid = ID.get() if objecttype == "account": uri = "uri:account:%(objectid)s" % vars() if objecttype == "bot": accountid = parent.split(":")[2] uri = "uri:bot:%(accountid)s:%(name)s" % vars() if objecttype == "entity": accountid = parent.split(":")[2] botname = parent.split(":")[3] uri = "uri:entity:%(accountid)s:%(botname)s:%(name)s" % vars() if objecttype == "intent": accountid = parent.split(":")[2] botname = parent.split(":")[3] uri = "uri:intent:%(accountid)s:%(botname)s:%(name)s" % vars() if objecttype == "reply": accountid = parent.split(":")[2] botname = parent.split(":")[3] intentname = parent.split(":")[4] uri = "uri:reply:%(accountid)s:%(botname)s:%(intenname)s:(name)s" % vars( ) if objecttype == "variable": accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) if objecttype == "rule": accountid = parent.split(":")[2] botname = parent.split(":")[3] intentname = parent.split(":")[4] uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars( ) input["ID"] = uri input["objecttype"] = objecttype input["parent"] = parent if parent is not None else "service" input["search"] = input.get("name") + input.get( "description", "-") + input.get("tags", "-") input["createdat"] = ID.now() input["creator"] = identity input["doc"] = input.get("doc") or '{"":""}' print pprint.pformat(input) item = Item(**input) item.save() return item.attribute_values
def createOrUpdateLexBot(identity, botid): bot = Item.get("bot", botid) entities = Item.query("entity", Item.parent.__eq__(botid)) botgraph = {"slots": {}, "intents": {}} for e in entities: botgraph["slots"][e.name] = { "name": e.name, "ref": "{%(name)s}" % e.attribute_values, "values": e.doc.aliases, "type": "entity" } variables = Item.query("variable", Item.parent.__eq__(e.ID)) for v in variables: nameslot = v.name botgraph["slots"][nameslot] = { "name": nameslot, "ref": "{%(name)s}" % v.attribute_values, "values": v.doc.aliases, #json.loads(v.doc)["aliases"] "type": "variable" } opslot = v.name + "op" botgraph["slots"][opslot] = { "name": opslot, "ref": "{%(name)sop}" % v.attribute_values, "type": "operator", "values": [ "in", "outside", "greater", "equals", "different", "smaller", "bigger", "taller" ] } valslot = v.name + "val" botgraph["slots"][valslot] = { "type": "value", "name": valslot, "ref": "{%(name)sval}" % v.attribute_values, "values": list(get_sample_values(v.ID)) #sample_values(v.ID) } intents = Item.query("intent", Item.parent.__eq__(botid)) for i in intents: botgraph["intents"][i.name] = {} #print " ",i rules = Item.query("rule", Item.parent.__eq__(i.ID)) for r in rules: botgraph["intents"][i.name][ r.name] = r.doc.replacements #json.loads(r.doc)["expressions"] #print "~BG~%~~"*20 #print pprint.pformat(botgraph) logger().info("Creating bot %s", botid) intents = [] for intent in botgraph["intents"].keys(): logger().critical("Adding Intent %s", intent) intent_config = get_intent_config(intent, botgraph, "lex") #print "** ** ** "*30 #print pprint.pformat(intent_config) #print "-- -- -- "*30 intents.append(intent_config) for slot in intent_config["slots"]: logger().critical("Adding Slot Type `%s`", slot["name"]) #print pprint.pformat(slot["enumerationValues"]) Bot.add_slot_type(slot["name"], slot["enumerationValues"]) logger().critical("Saving Intent `%s` to Lex", intent) Bot.add_intent(intent_config) logger().info("Putting Bot %s", bot.name) Bot.build(bot.name, intents=[i["name"] for i in intents]) logger().info("Putting Bot Alias %s %s", bot.name, "demo") Bot.put_alias(bot.name) Bot.put_bot_version(bot.name) return botgraph
def get_intents(cls, botid): intents = Item.query("intent", filter_condition=Item.parent.__eq__(botid)) return [i.attribute_values for i in intents]
def get_entities(cls, botid): entities = Item.query("entity", filter_condition=Item.parent.__eq__(botid)) return [e.attribute_values for e in entities]
def get_bots(cls, user="******"): accountid = cls.get_accountid(user) bots = Item.query("bot", filter_condition=Item.parent.__eq__(accountid)) return [b.attribute_values for b in bots]
def get_presigned_url(entityid, identity): entity = Item.get("entity", entityid) bot = Item.get("bot", entity.parent) doc = bot.doc return S3Helpers.get_presigned_url( doc.bucket, doc.prefix + "/" + entity.name + "/INPUT.txt")
def createVariable(parent, options, identity): print ">", parent, options, identity try: entity = Item.get("entity", parent) except Exception: raise errors.ObjectNotFound(parent) if entity.doc.status != ENTITYSTATES.READY: raise errors.InvalidParameter( "Could not add variable to unparsed entity") bot = Item.get("bot", entity.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] name = options["name"] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) print uri data = {} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "variable" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get( "tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = {"aliases": options["aliases"], "field": options["field"]} item = Item(**data) item.save() database = bot.doc.database tablename = entity.doc.tablename columnname = options["field"] sql = 'select %(columnname)s from "%(database)s"."%(tablename)s" group by %(columnname)s limit 30' % vars( ) response = AthenaQuery.run(**{"sql": sql}) values = [ r[columnname].replace('"', '') for r in response["data"]["records"] ] for i, val in enumerate(values): if len(val): slug = unicodedata.normalize('NFKD', val).encode('ascii', 'ignore') cache = Item( **{ "name": slug, "parent": uri, "doc": {}, "createdat": ID.now(), "objecttype": "value", "ID": uri + ":" + str(i), "search": slug }) print cache cache.save() d = item.json() d.update(d["doc"]) del d["doc"] return d