def getEntity(identity, entityid): try: entity = Item.get("entity", entityid) bot = Item.get("bot", entity.parent) except Exception: raise errors.ObjectNotFound(entityid) print ENTITYSTATES.PARSING, ">>", entity.doc.status, ENTITYSTATES.PARSING == entity.doc.status if entity.doc.status == ENTITYSTATES.PARSING: logger().info("entity is parsing") if len(entity.doc.crawler): crawler_name = entity.doc.crawler crawler = Crawler.get(name=crawler_name) if crawler["State"] in ["READY", "STOPPING"]: doc = bot.doc database = doc.database tablename = entity.doc.tablename columns = Crawler.get_table(database, tablename) print "columns", columns entity.update(actions=[ Item.doc.columns.set(columns), Item.doc.status.set(ENTITYSTATES.READY) ]) entity_doc = entity.json() print "entity_doc ", pprint.pformat(entity_doc) entity_doc.update(entity_doc["doc"]) del entity_doc["doc"] return entity_doc
def createEntity(identity, botid, data): try: bot = Item.get("bot", botid) except Exception as e: raise errors.ObjectNotFound(botid) print ">>", data parts = botid.split(":") accountid = parts[2] botname = parts[3] entityname = data["name"] uri = "uri:entity:%(accountid)s:%(botname)s:%(entityname)s" % vars() entity = Item(parent=botid, ID=uri, createdat=ID.now(), name=data["name"], description=data.get("description", "-"), search=data["name"] + data.get("description", "-"), objecttype="entity", doc={ "aliases": data["aliases"], "status": ENTITYSTATES.NOT_READY, "database": bot.doc.database, "tablename": data["name"] }) print entity entity.save() return entity.attribute_values
def get_table(cls, database, tablename): client = cls.glue() try: response = client.get_table(DatabaseName=database, Name=tablename) return response["Table"]["StorageDescriptor"]["Columns"] except Exception: raise errors.ObjectNotFound(database + "." + tablename)
def createRule(parent,options, identity): print ">", parent, options, identity try : intent = Item.get("intent",parent) except Exception: raise errors.ObjectNotFound(parent) bot = Item.get("bot",intent.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] intentname = parent.split(":")[4] name = options["name"] uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars() print uri data={} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "rule" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get("tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = { "replacements" : options["replacements"] } item = Item(**data) item.save() d = item.json() d.update(d["doc"]) del d["doc"] return d
def delete_item(identifier, identity): try: bot = Item.get(objecttype, identifier) except Exception: raise errors.ObjectNotFound( "Could not find %(objecttype)s with ID `%(botid)s`" % vars()) bot.delete() return True
def getRule(identity, ruleid) : try: rule = Item.get("rule", ruleid) except Exception: raise errors.ObjectNotFound(ruleid) rule_doc = rule.json() rule_doc.update(rule_doc["doc"]) del rule_doc["doc"] return rule_doc
def getVariable(identity, variableid): try: variable = Item.get("variable", variableid) entity = Item.get("entity", variable.parent) except Exception: raise errors.ObjectNotFound(variableid) variable_doc = variable.json() variable_doc.update(variable_doc["doc"]) del variable_doc["doc"] return variable_doc
def getIntent(identity, intentid): print "%/" * 30 try: intent = Item.get("intent", intentid) except Exception: raise errors.ObjectNotFound(intentid) doc = intent.json() doc.update(doc["doc"]) del doc["doc"] print doc return doc
def updateEntity(identity, entityid, data): try: entity = Item.get("entity", entityid) except Exception as e: raise errors.ObjectNotFound(entityid) entity.update(actions=[ Item.doc.aliases.set(data.get("aliases", entity.doc.aliases)), Item.description.set(data.get("description", entity.description)) ]) return entity.attribute_values
def run(cls, **kwargs): client = boto3.client('glue', region_name="eu-west-1") try: response = client.get_table(DatabaseName=kwargs["database"], Name=kwargs["tablename"]) schema = response["Table"]["StorageDescriptor"]["Columns"] return {"success": True, "data": schema} except Exception: return { "success": False, "data": errors.ObjectNotFound(kwargs["entity"]) }
def updateIntent(identity, intentid, data): try: v = Item.get("intent", intentid) except Exception as e: raise errors.ObjectNotFound(intentid) v.update(actions=[ Item.doc.reply.set(data.get("reply", v.doc.reply)), Item.doc.sql.set(data.get("sql", v.doc.sql)), Item.description.set(data.get("description", v.description)) ]) d = v.json() d.update(d["doc"]) del d["doc"] return d
def updateRule(identity, ruleid, data) : try: v = Item.get("rule",ruleid) except Exception as e : raise errors.ObjectNotFound(ruleid) v.update(actions=[ Item.doc.replacements.set(data.get("replacements",v.doc.replacements)), Item.description.set(data.get("description", v.description)) ]) d = v.json() d.update(d["doc"]) del d["doc"] return d
def updateVariable(identity, variableid, data): try: v = Item.get("variable", variableid) except Exception as e: raise errors.ObjectNotFound(variableid) v.update(actions=[ Item.doc.aliases.set(data.get("aliases", v.doc.aliases)), Item.doc.field.set(data.get("field", v.doc.field)), Item.description.set(data.get("description", v.description)) ]) d = v.json() d.update(d["doc"]) del d["doc"] return d
def getVariableValues(variableid, identity): try: variable = Item.get("variable", variableid) print variable except Exception as e: raise errors.ObjectNotFound(variableid) if variable.doc.type.lower() == "dimension": values = [] valueiterator = Item.query("value", Item.parent == variableid, limit=100) for v in valueiterator: if v.name not in values: values.append(v.name) return values else: return [10, 100, 1000, 5000, 10000]
def createVariable(parent, options, identity): print ">", parent, options, identity try: entity = Item.get("entity", parent) except Exception: raise errors.ObjectNotFound(parent) if entity.doc.status != ENTITYSTATES.READY: raise errors.InvalidParameter( "Could not add variable to unparsed entity") bot = Item.get("bot", entity.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] name = options["name"] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) print uri data = {} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "variable" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get( "tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = { "field": options["field"], "type": options.get("type", "dimension"), "aliases": options["aliases"] } item = Item(**data) item.save() syncVariableValues(variableid=uri) d = item.json() d.update(d["doc"]) del d["doc"] return d '''
def update_item(identifier, input, identity): print "update_item <-", identifier, input, identity try: item = Item.get(objecttype, identifier) doc = item.attribute_values except Exception: raise errors.ObjectNotFound( "Could not find %(objecttype)s with ID `%(botid)s`" % vars()) actions = [] for k in input.keys(): attribute = getattr(Item, k) print "attribute = ", attribute actions.append(attribute.set(input[k])) term = doc["name"] + input.get( "description", doc.get("description", "-")) + input.get( "tags", doc.get("tags", "-")) actions.append(Item.search.set(term)) print actions item.update(actions=actions) return item.attribute_values
def createVariable(parent, options, identity): print ">", parent, options, identity try: entity = Item.get("entity", parent) except Exception: raise errors.ObjectNotFound(parent) if entity.doc.status != ENTITYSTATES.READY: raise errors.InvalidParameter( "Could not add variable to unparsed entity") bot = Item.get("bot", entity.parent) accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] name = options["name"] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) print uri data = {} data["ID"] = uri data["name"] = options["name"] data["objecttype"] = "variable" data["parent"] = parent if parent is not None else "service" data["search"] = name + options.get("description", "-") + options.get( "tags", "-") data["createdat"] = ID.now() data["creator"] = identity data["doc"] = {"aliases": options["aliases"], "field": options["field"]} item = Item(**data) item.save() database = bot.doc.database tablename = entity.doc.tablename columnname = options["field"] sql = 'select %(columnname)s from "%(database)s"."%(tablename)s" group by %(columnname)s limit 30' % vars( ) response = AthenaQuery.run(**{"sql": sql}) values = [ r[columnname].replace('"', '') for r in response["data"]["records"] ] for i, val in enumerate(values): if len(val): slug = unicodedata.normalize('NFKD', val).encode('ascii', 'ignore') cache = Item( **{ "name": slug, "parent": uri, "doc": {}, "createdat": ID.now(), "objecttype": "value", "ID": uri + ":" + str(i), "search": slug }) print cache cache.save() d = item.json() d.update(d["doc"]) del d["doc"] return d
def get_item(identifier, identity): try: bot = Item.get(objecttype, identifier) return bot.attribute_values except Exception: raise errors.ObjectNotFound(identifier)
def create_item(parent, input, identity): print "create %(objectype)s received", parent, input, identity try: name = input["name"] except Exception: raise errors.InvalidParameter("Missing `name` field in input" % vars()) if parent is None: parent = "service" else: try: parent_item = Item.get(parentobjecttype, parent) except Exception: raise errors.ObjectNotFound( "No parent object with type `%(parentobjecttype)s` found with Id :`%(parent)s`" % vars()) candidates = get_children_by_name(parent, objecttype=objecttype, name=input["name"]) if len(candidates): if parentobjecttype is None: parent = "accounts" raise errors.DuplicateError( "An object with name `%(name)s`and type `%(objecttype)s` already exists in `%(parent)s`" % vars()) objectid = ID.get() if objecttype == "account": uri = "uri:account:%(objectid)s" % vars() if objecttype == "bot": accountid = parent.split(":")[2] uri = "uri:bot:%(accountid)s:%(name)s" % vars() if objecttype == "entity": accountid = parent.split(":")[2] botname = parent.split(":")[3] uri = "uri:entity:%(accountid)s:%(botname)s:%(name)s" % vars() if objecttype == "intent": accountid = parent.split(":")[2] botname = parent.split(":")[3] uri = "uri:intent:%(accountid)s:%(botname)s:%(name)s" % vars() if objecttype == "reply": accountid = parent.split(":")[2] botname = parent.split(":")[3] intentname = parent.split(":")[4] uri = "uri:reply:%(accountid)s:%(botname)s:%(intenname)s:(name)s" % vars( ) if objecttype == "variable": accountid = parent.split(":")[2] botname = parent.split(":")[3] entityname = parent.split(":")[4] uri = "uri:variable:%(accountid)s:%(botname)s:%(entityname)s:%(name)s" % vars( ) if objecttype == "rule": accountid = parent.split(":")[2] botname = parent.split(":")[3] intentname = parent.split(":")[4] uri = "uri:rule:%(accountid)s:%(botname)s:%(intentname)s:%(name)s" % vars( ) input["ID"] = uri input["objecttype"] = objecttype input["parent"] = parent if parent is not None else "service" input["search"] = input.get("name") + input.get( "description", "-") + input.get("tags", "-") input["createdat"] = ID.now() input["creator"] = identity input["doc"] = input.get("doc") or '{"":""}' print pprint.pformat(input) item = Item(**input) item.save() return item.attribute_values