def new_application(app_name, fields, s3_bucket):
ok, message = fields_check(fields)
if not ok:
raise ArgsCheckError(message, "")
try:
# check application exist
if MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, app_name):
raise ExistError(f"application <{app_name}> had exist", "")
except ExistError:
raise
try:
for _, value in fields.items():
if value.get("type") == "pipeline":
pipe = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME,
value.get("value"))[0]
ei = identity(
pipe.get("encoder").get("instance").get("endpoint"))
name = f"{app_name}_{pipe.get('encoder').get('instance').get('name').replace('phantoscope_', '')}"
MilvusIns.new_milvus_collection(name, int(ei["dimension"]),
1024, "l2")
# create a application entity collection
MongoIns.new_mongo_collection(f"{app_name}_entity")
S3Ins.new_s3_buckets(s3_bucket)
# create milvus collections
app = Application(name=app_name, fields=fields, bucket=s3_bucket)
app.metadata = app._metadata()
MongoIns.insert_documents(APPLICATION_COLLECTION_NAME, app.to_dict())
return app
except Exception as e:
logger.error("error happen during create app: %s",
str(e),
exc_info=True)
raise e
def create_pipeline(name, processors=None, encoder=None, description=None):
try:
p = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME, name)
if p:
raise ExistError(f"pipeline <{name}> already exists", "")
pro = []
encoder_res = {}
processor_res = {}
for processor in processors:
pr = operator_detail(processor["name"])
processor_res["operator"] = pr.to_dict()
processor_res["instance"] = pr.inspect_instance(
processor["instance"])
pro.append(processor_res)
encoder_info = operator_detail(encoder["name"])
encoder_res["operator"] = encoder_info.to_dict()
encoder_res["instance"] = encoder_info.inspect_instance(
encoder["instance"])
pipe = Pipeline(name, description, pro, encoder_res)
pipe.metadata = pipe._metadata()
if pipeline_illegal(pipe):
raise PipelineIllegalError("Pipeline illegal check error", "")
MongoIns.insert_documents(PIPELINE_COLLECTION_NAME, pipe.to_dict())
return pipe
except Exception as e:
logger.error(e, exc_info=True)
raise e
def delete_milvus_collections_by_fields(app):
for _, field in app['fields'].items():
if field["type"] == "pipeline":
pipe = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME,
field.get("value"))[0]
name = f"{app.get('name')}_{pipe.get('encoder').get('instance').get('name').replace('phantoscope_', '')}"
MilvusIns.del_milvus_collection(name)
def register_operators(name, addr, author, version, type, description):
try:
op = Operator(name, addr, author, version, type, description)
op.metadata = op._metadata()
if MongoIns.search_by_name(OPERATOR_COLLECTION_NAME, name):
raise ExistError(f"operator {name} had exist", "")
MongoIns.insert_documents(OPERATOR_COLLECTION_NAME, op.to_dict())
return op.to_dict()
except Exception as e:
logger.error(f"Unexpected error happen during register operator, {str(e)}", exc_info=True)
raise e
def pipeline_detail(name):
try:
p = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME, name)
if not p:
raise NotExistError("pipeline %s is not exist" % name, "")
p = p[0]
pipe = Pipeline(p["name"], p["description"], p["processors"],
p["encoder"], p["input"], p["output"])
pipe.metadata = p["metadata"]
return pipe
except Exception as e:
raise e
def test_pipeline(name, data=None, url=None):
try:
pipe = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME, name)
if not pipe:
raise NotExistError("pipeline %s is not exist" % name, "")
pipe = pipe[0]
p = Pipeline(pipe["name"], pipe["description"], pipe["processors"],
pipe["encoder"], pipe["input"], pipe["output"])
p.metadata = pipe["metadata"]
return {"result": run_pipeline(p, data=data, url=url)}
except Exception as e:
raise e
def operator_detail(name):
try:
op = MongoIns.search_by_name(OPERATOR_COLLECTION_NAME, name)
if not op:
raise NotExistError(f"operator {name} not exist", "")
op = op[0]
operator = Operator(op["name"], op["addr"], op["author"], op["version"], op["type"], op["description"])
operator.metadata = op["metadata"]
return operator
except Exception as e:
logger.error(e)
raise e
def application_detail(name):
try:
app = MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, name)
if not app:
raise NotExistError(f"application {name} not exist", "")
app = app[0]
application = Application(app["name"], app["fields"], app["bucket"])
application.metadata = app["metadata"]
return application
except Exception as e:
logger.error(e)
raise e
def delete_application(name, force=False):
try:
if not force:
if not entities_list(name, 100, 0):
raise RequestError(
"Prevent to delete application with entity not deleted",
"")
app = MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, name)
if not app:
raise NotExistError(f"application {name} not exist", "")
app = app[0]
delete_milvus_collections_by_fields(app)
S3Ins.del_s3_buckets(app['bucket'])
MongoIns.delete_mongo_collection(f"{name}_entity")
MongoIns.delete_by_name(APPLICATION_COLLECTION_NAME, name)
logger.info("delete application %s", name)
application = Application(app["name"], app["fields"], app["bucket"])
application.metadata = app["metadata"]
return application
except Exception as e:
logger.error(e)
raise e