def test_op_grpc_client(self):
error_endpoint = '127.0.0.1:80'
with pytest.raises(Exception):
identity(error_endpoint)
op = new_operator_instance(1, "name", "running", "127.0.0.1", "222")
with pytest.raises(Exception):
health(op)
with pytest.raises(Exception):
execute(op)
def pipeline_illegal(pipe):
input, output = None, None
try:
operators = pipe.processors.copy()
operators.append(pipe.encoder)
for num, operator in enumerate(operators):
# check operator and instance exist
# use identity check container health
logging.info(f"now identity {operator['instance'].endpoint}")
info = identity(operator['instance'].endpoint)
if num == len(operators) - 1:
if info.get("type") != OPERATOR_TYPE_ENCODER:
raise PipelineIllegalError("Pipeline illegal check error",
"")
else:
if info.get("type") != OPERATOR_TYPE_PROCESSOR:
raise PipelineIllegalError("Pipeline illegal check error",
"")
# check input and output
# if not input and not output:
# input, output = info.get("input"), info.get("output")
# else:
# if output != info.get("input"):
# raise PipelineIllegalError("Pipeline illegal check error", "")
# pipe.input, pipe.output = input, output
except Exception as e:
raise e
def new_application(app_name, fields, s3_bucket):
ok, message = fields_check(fields)
if not ok:
raise ArgsCheckError(message, "")
try:
# check application exist
if MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, app_name):
raise ExistError(f"application <{app_name}> had exist", "")
except ExistError:
raise
try:
for _, value in fields.items():
if value.get("type") == "pipeline":
pipe = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME,
value.get("value"))[0]
ei = identity(
pipe.get("encoder").get("instance").get("endpoint"))
name = f"{app_name}_{pipe.get('encoder').get('instance').get('name').replace('phantoscope_', '')}"
MilvusIns.new_milvus_collection(name, int(ei["dimension"]),
1024, "l2")
# create a application entity collection
MongoIns.new_mongo_collection(f"{app_name}_entity")
S3Ins.new_s3_buckets(s3_bucket)
# create milvus collections
app = Application(name=app_name, fields=fields, bucket=s3_bucket)
app.metadata = app._metadata()
MongoIns.insert_documents(APPLICATION_COLLECTION_NAME, app.to_dict())
return app
except Exception as e:
logger.error("error happen during create app: %s",
str(e),
exc_info=True)
raise e
def create_milvus_collections_by_fields(app):
for field in search_fields(app.fields):
if field.type == "pipeline":
pipe = pipeline_detail(field.value)
name = pipe.encoder.get("name")
instance_name = pipe.encoder.get("instance")
encoder = operator_detail(name)
instance = encoder.inspect_instance(instance_name)
ei = identity(instance.endpoint)
MilvusIns.new_milvus_collection(
f"{app.name}_{name}_{instance_name}", int(ei["dimension"]),
1024, "l2")
def regist_operators(endpoint, name):
try:
res = identity(endpoint)
op = DB(name=name, backend=res['name'], type=res['type'], input=res['input'],
output=res['output'], dimension=res['dimension'],
metric_type=res['metric_type'], endpoint=res['endpoint'])
insert_operator(op)
logger.info("regist operator %s" % res['name'])
return new_operator(name=op.name, backend=op.backend, type=op.type,
input=op.input,
output=op.output,
endpoint=op.endpoint,
dimension=op.dimension,
metric_type=op.metric_type)
except Exception as e:
logger.error(e)
return OperatorRegistError("opeartor %s regist error" % name, e)