def new_pipeline(name,
input,
index_file_size,
processors,
encoder,
description=None):
try:
encoder = operator_detail(encoder)
pipe = Pipeline(name=name,
input=input,
output=encoder.output,
dimension=encoder.dimension,
index_file_size=index_file_size,
metric_type=encoder.metric_type,
description=description,
processors=processors.split(","),
encoder=encoder.name)
if pipeline_ilegal(pipe):
return PipelineIlegalError("Pipeline ilegal check error", "")
milvus_collection_name = f"{name}_{encoder.name}"
MilvusIns.new_milvus_collection(milvus_collection_name,
encoder.dimension, index_file_size,
encoder.metric_type)
return pipe.save()
except Exception as e:
print(e)
logger.error(e)
return e
def new_application(app_name, fields, s3_bucket):
ok, message = fields_check(fields)
if not ok:
raise ArgsCheckError(message, "")
try:
# check application exist
if MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, app_name):
raise ExistError(f"application <{app_name}> had exist", "")
except ExistError:
raise
try:
for _, value in fields.items():
if value.get("type") == "pipeline":
pipe = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME,
value.get("value"))[0]
ei = identity(
pipe.get("encoder").get("instance").get("endpoint"))
name = f"{app_name}_{pipe.get('encoder').get('instance').get('name').replace('phantoscope_', '')}"
MilvusIns.new_milvus_collection(name, int(ei["dimension"]),
1024, "l2")
# create a application entity collection
MongoIns.new_mongo_collection(f"{app_name}_entity")
S3Ins.new_s3_buckets(s3_bucket)
# create milvus collections
app = Application(name=app_name, fields=fields, bucket=s3_bucket)
app.metadata = app._metadata()
MongoIns.insert_documents(APPLICATION_COLLECTION_NAME, app.to_dict())
return app
except Exception as e:
logger.error("error happen during create app: %s",
str(e),
exc_info=True)
raise e
def create_milvus_collections_by_fields(app):
for field in search_fields(app.fields):
if field.type == "pipeline":
pipe = pipeline_detail(field.value)
name = pipe.encoder.get("name")
instance_name = pipe.encoder.get("instance")
encoder = operator_detail(name)
instance = encoder.inspect_instance(instance_name)
ei = identity(instance.endpoint)
MilvusIns.new_milvus_collection(
f"{app.name}_{name}_{instance_name}", int(ei["dimension"]),
1024, "l2")
def test_new_collection(self):
"""create new collection"""
rv = MilvusIns.new_milvus_collection(self.name, self.dimension,
self.index_file_size,
self.metric_type)
assert rv == None