def upload(name, **kwargs):
try:
app = application_detail(name)
if not app:
raise NotExistError("application not exist", "application %s not exist" % name)
bucket_name = app.buckets.split(",")[0]
accept_fields = [x for x, y in app.fields.items() if y.get('type') != "object"]
pipeline_fields = {x: y['pipeline'] for x, y in app.fields.items() if y.get('type') == "object"}
new_fields = app.fields.copy()
for k, v in kwargs.items():
if k in accept_fields:
new_fields[k]['value'] = v
res = []
for k, _ in kwargs.get('fields').items():
if k not in accept_fields and k not in pipeline_fields:
raise RequestError(f"fields {k} not in application", "")
for n, p in pipeline_fields.items():
pipe = pipeline_detail(p)
if not pipe:
raise NotExistError("pipeline not exist", "pipeline %s not exist" % p)
value = kwargs['fields'].get(n)
file_data = value.get('data')
url = value.get('url')
if not file_data and not url:
raise RequestError("can't find data or url from request", "")
file_name = "{}-{}".format(name, uuid.uuid4().hex)
file_path = save_tmp_file(file_name, file_data, url)
# begin to timing
start = time.time()
S3Ins.upload2bucket(bucket_name, file_path, file_name)
upload_time = time.time()
logger.debug("[timing] upload image to bucket costs: {:.3f}s".format(upload_time - start))
vectors = run_pipeline(pipe, data=file_data, url=url)
pipeline_time = time.time()
logger.debug("[timing] run pipeline costs: {:.3f}s".format(pipeline_time - upload_time))
milvus_collection_name = f"{pipe.name}_{pipe.encoder}"
vids = MilvusIns.insert_vectors(milvus_collection_name, vectors)
insert_time = time.time()
logger.debug("[timing] insert to milvus costs: {:.3f}s".format(insert_time - pipeline_time))
for vid in vids:
m = DB(id=vid, app_name=name,
image_url=gen_url(bucket_name, file_name),
fields=new_fields)
add_mapping_data(m)
res.append(new_mapping_ins(id=vid, app_name=name,
image_url=gen_url(bucket_name, file_name),
fields=new_fields))
final_time = time.time()
logger.debug("[timing] prepare result costs: {:.3f}s".format(final_time - insert_time))
return res
except Exception as e:
print(e)
return e
def upload(name, **kwargs):
try:
app = application_detail(name)
if not app:
raise NotExistError("application not exist",
"application %s not exist" % name)
bucket_name = app.buckets.split(",")[0]
accept_fields = [
x for x, y in app.fields.items() if y.get('type') != "object"
]
pipeline_fields = {
x: y['pipeline']
for x, y in app.fields.items() if y.get('type') == "object"
}
new_fields = app.fields.copy()
for k, v in kwargs.items():
if k in accept_fields:
new_fields[k]['value'] = v
res = []
for k, _ in kwargs.get('fields').items():
if k not in accept_fields and k not in pipeline_fields:
raise RequestError(f"fields {k} not in application", "")
for n, p in pipeline_fields.items():
pipe = pipeline_detail(p)
if not pipe:
raise NotExistError("pipeline not exist",
"pipeline %s not exist" % p)
value = kwargs['fields'].get(n)
file_data = value.get('data')
url = value.get('url')
if not file_data and not url:
raise RequestError("can't find data or url from request", "")
file_name = "{}-{}".format(name, uuid.uuid4().hex)
file_path = save_tmp_file(file_name, file_data, url)
S3Ins.upload2bucket(bucket_name, file_path, file_name)
vectors = run_pipeline(pipe, data=file_data, url=url)
if not vectors:
raise NoneVectorError(
"can't encode data by encoder, check input or encoder", "")
milvus_collection_name = f"{pipe.name}_{pipe.encoder}"
vids = MilvusIns.insert_vectors(milvus_collection_name, vectors)
for vid in vids:
m = DB(id=vid,
app_name=name,
image_url=gen_url(bucket_name, file_name),
fields=new_fields)
add_mapping_data(m)
res.append(
new_mapping_ins(id=vid,
app_name=name,
image_url=gen_url(bucket_name, file_name),
fields=new_fields))
return res
except Exception as e:
print(e)
return e
def upload(name, **kwargs):
try:
app = application_detail(name)
if not app:
raise NotExistError("application not exist", "application %s not exist" % name)
bucket_name = app.buckets.split(",")[0]
accept_fields = [x for x, y in app.fields.items() if y.get('type') != "pipeline"]
pipeline_fields = {x: y['value'] for x, y in app.fields.items() if y.get('type') == "pipeline"}
new_fields = app.fields.copy()
for k, v in kwargs.items():
if k in accept_fields:
new_fields[k]['value'] = v
res = []
for k, _ in kwargs.get('fields').items():
if k not in accept_fields and k not in pipeline_fields:
raise RequestError(f"fields {k} not in application", "")
docs = {}
valid_field_flag = False
for n, p in pipeline_fields.items():
pipe = pipeline_detail(p)
if not pipe:
raise NotExistError("pipeline not exist", "pipeline %s not exist" % p)
value = kwargs['fields'].get(n)
if not value:
continue
valid_field_flag = True
file_data = value.get('data')
url = value.get('url')
if not file_data and not url:
raise RequestError("can't find data or url from request", "")
file_name = "{}-{}".format(name, uuid.uuid4().hex)
file_path = save_tmp_file(file_name, file_data, url)
S3Ins.upload2bucket(bucket_name, file_path, file_name)
vectors = run_pipeline(pipe, data=file_data, url=url)
if not vectors:
raise NoneVectorError("can't encode data by encoder, check input or encoder", "")
milvus_collection_name = f"{app.name}_{pipe.encoder['name']}_{pipe.encoder['instance']}"
vids = MilvusIns.insert_vectors(milvus_collection_name, vectors)
docs[n] = {"ids": vids, "url": gen_url(bucket_name, file_name)}
doc_id = MongoIns.insert_documents(f"{app.name}_entity", docs)
res.append(new_mapping_ins(docs))
if not valid_field_flag:
raise RequestError("none valid field exist", "")
return res
except Exception as e:
err_msg = f"Unexpected error happen when upload: {str(e)}"
logger.error(err_msg, exc_info=True)
raise UnexpectedError(err_msg, e)
def delete_entity(app_name, entity_id):
try:
mongo_ins_name = f"{app_name}_entity"
entity = MongoIns.search_by_id(mongo_ins_name, entity_id)
if not entity.count():
raise NotExistError("Entity %s not exist" % entity_id,
"NotExistError")
for item in entity:
en = new_mapping_ins(item)
for name, fields in en._docs.items():
# delete s3 object
bucket_name = fields.get("url").split("/")[-2]
object_name = fields.get("url").split("/")[-1]
S3Ins.del_object(bucket_name, object_name)
# delete vector from milvus
vids = fields.get("ids")
app = application_detail(app_name)
pipe_name = app.fields[name]["value"]
pipe = pipeline_detail(pipe_name)
instance_name = pipe.encoder.get("instance")
MilvusIns.del_vectors(f"{app_name}_{name}_{instance_name}",
vids)
# delete from mongodb
MongoIns.delete_by_id(mongo_ins_name, entity_id)
logger.info("delete entity %s in application %s", entity_id,
app_name)
return en
except Exception as e:
logger.error(e)
raise e
def operator_health(name):
try:
op = search_operator(name)
if not op:
raise NotExistError("operator %s not exist" % name, "")
return health(op)
except Exception as e:
logger.error(e)
return e
def _shopValidCheck(self, id_shop=None, upc_shop=None):
if not id_shop and not upc_shop:
return
if id_shop and int(id_shop) == 0:
return
try:
assert CachedShop(id_shop, upc_shop), id_shop or upc_shop
except AssertionError, e:
raise NotExistError('ORDER_ERR_SHOP_%s_NOT_EXIST' % e)
def application_detail(name):
try:
x = search_application(name)
if not x:
raise NotExistError(f"application {name} not exist", "")
fields = json.loads(x.fields)
app = Application(name=x.name, fields=fields, buckets=x.s3_buckets)
return app
except Exception as e:
logger.error(e)
raise e
def test_pipeline(name, data=None, url=None):
try:
pipe = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME, name)
if not pipe:
raise NotExistError("pipeline %s is not exist" % name, "")
pipe = pipe[0]
p = Pipeline(pipe["name"], pipe["description"], pipe["processors"],
pipe["encoder"], pipe["input"], pipe["output"])
p.metadata = pipe["metadata"]
return {"result": run_pipeline(p, data=data, url=url)}
except Exception as e:
raise e
def operator_detail(name):
try:
op = MongoIns.search_by_name(OPERATOR_COLLECTION_NAME, name)
if not op:
raise NotExistError(f"operator {name} not exist", "")
op = op[0]
operator = Operator(op["name"], op["addr"], op["author"], op["version"], op["type"], op["description"])
operator.metadata = op["metadata"]
return operator
except Exception as e:
logger.error(e)
raise e
def application_detail(name):
try:
app = MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, name)
if not app:
raise NotExistError(f"application {name} not exist", "")
app = app[0]
application = Application(app["name"], app["fields"], app["bucket"])
application.metadata = app["metadata"]
return application
except Exception as e:
logger.error(e)
raise e
def pipeline_detail(name):
try:
p = MongoIns.search_by_name(PIPELINE_COLLECTION_NAME, name)
if not p:
raise NotExistError("pipeline %s is not exist" % name, "")
p = p[0]
pipe = Pipeline(p["name"], p["description"], p["processors"],
p["encoder"], p["input"], p["output"])
pipe.metadata = p["metadata"]
return pipe
except Exception as e:
raise e
def upload(name, **kwargs):
try:
app = application_detail(name)
if not app:
raise NotExistError("application not exist", "application %s not exist" % name)
bucket_name = app.buckets.split(",")[0]
accept_fields = [x for x, y in app.fields.items() if y.get('type') != "pipeline"]
pipeline_fields = {x: y['value'] for x, y in app.fields.items() if y.get('type') == "pipeline"}
new_fields = app.fields.copy()
for k, v in kwargs.items():
if k in accept_fields:
new_fields[k]['value'] = v
res = []
for k, _ in kwargs.get('fields').items():
if k not in accept_fields and k not in pipeline_fields:
raise RequestError(f"fields {k} not in application", "")
docs = {}
for n, p in pipeline_fields.items():
pipe = pipeline_detail(p)
if not pipe:
raise NotExistError("pipeline not exist", "pipeline %s not exist" % p)
value = kwargs['fields'].get(n)
file_data = value.get('data')
url = value.get('url')
if not file_data and not url:
raise RequestError("can't find data or url from request", "")
file_name = "{}-{}".format(name, uuid.uuid4().hex)
file_path = save_tmp_file(file_name, file_data, url)
S3Ins.upload2bucket(bucket_name, file_path, file_name)
vectors = run_pipeline(pipe, data=file_data, url=url)
milvus_collection_name = f"{app.name}_{pipe.encoder['name']}_{pipe.encoder['instance']}"
vids = MilvusIns.insert_vectors(milvus_collection_name, vectors)
docs[n] = {"ids": vids, "url": gen_url(bucket_name, file_name)}
doc_id = MongoIns.insert_documents(f"{app.name}_entity", docs)
res.append(new_mapping_ins(docs))
fields=new_fields))
return res
def update_pipeline(name, p):
try:
old = db.session.query(Pipeline).filter(Pipeline.name==name).first()
if not old:
raise NotExistError("update target not exist", "pipeline with %s not exist" % name)
for k, v in p.__dict__.items():
if v and k != "_sa_instance_state":
setattr(old, k, v)
db.session.commit()
return old
except Exception as e:
if isinstance(e, NotExistError):
raise e
raise UpdateFromSQLError("update from sql error", e.orig.args[-1])
def pipeline_detail(name):
try:
p = search_pipeline(name)
if not p:
raise NotExistError("pipeline %s is not exist" % name, "")
pipe = Pipeline(name=p.name,
input=p.input,
output=p.output,
description=p.description,
processors=json.loads(p.processors),
encoder=json.loads(p.encoder))
return pipe
except Exception as e:
raise e
def patch_application(name, fields, s3_buckets):
try:
app_model = DB(name=name,
fields=json.dumps(fields),
s3_buckets=s3_buckets)
x = update_application(name, app_model)
if not x:
raise NotExistError(f"application {name} not exist", "")
app = Application(name=x.name, fields=fields, buckets=s3_buckets)
logger.info("change appication %s config", name)
return app
except Exception as e:
logger.error(e)
return e
def delete_application(name):
try:
x = del_application(name)
if not x:
raise NotExistError("application %s not exist" % name, "")
x = x[0]
fields = json.loads(x.fields)
app = Application(name=x.name, fields=fields, buckets=x.s3_buckets)
S3Ins.del_s3_buckets(x.s3_buckets.split(","))
logger.info("delete application %s", name)
return app
except Exception as e:
logger.error(e)
return e
def delete_operators(name):
try:
op = del_operator(name)
if not op:
raise NotExistError("operator %s not exist" % name, "")
return new_operator(name=op.name,
type=op.type,
addr=op.addr,
author=op.author,
version=op.version,
description=op.description)
except Exception as e:
logger.error(e)
raise e
def operator_detail(name):
try:
op = search_operator(name)
if not op:
raise NotExistError("operator %s not exist" % name, "")
return new_operator(name=op.name, backend=op.backend,
type=op.type,
input=op.input,
output=op.output,
endpoint=op.endpoint,
dimension=op.dimension,
metric_type=op.metric_type)
except Exception as e:
logger.error(e)
raise e
def delete_pipeline(name):
try:
p = del_pipeline(name)
if not p:
raise NotExistError("pipeline %s is not exist" % name, "")
p = p[0]
pipe = Pipeline(name=p.name,
input=p.input,
output=p.output,
description=p.description,
processors=json.loads(p.processors),
encoder=json.loads(p.encoder))
return pipe
except Exception as e:
logger.error(e)
raise e
def new_application(name, fields, s3_buckets):
app = Application(name=name, fields=fields, buckets=s3_buckets)
try:
all_pipelines_names = [x.name for x in _all_pipelines()]
for _, v in fields.items():
if v.get("type") == "object":
if v.get("pipeline") not in all_pipelines_names:
return NotExistError(
"pipeline not exist",
"pipeline %s not exist" % v.get("value"))
if v.get("type") != "object":
if "value" not in v:
return RequestError("key 'value' not exist",
"request error")
return app.save()
except Exception as e:
return e
def delete_entity(app_name, entity_name):
try:
entity = search_from_mapping(entity_name)
if not entity:
raise NotExistError("Entity %s not exist" % entity_name,
"NotExistError")
MilvusIns.del_vectors(app_name, [int(entity_name)])
bucket_name = entity.image_url.split("/")[-2]
object_name = entity.image_url.split("/")[-1]
S3Ins.del_object(bucket_name, object_name)
del_mapping(entity_name)
logger.info("delete entity %s in application %s", entity_name,
app_name)
return new_mapping_ins(id=entity.id,
app_name=entity.app_name,
image_url=entity.image_url,
fields=entity.fields)
except Exception as e:
logger.error(e)
return e
def delete_pipeline(name):
try:
p = del_pipeline(name)
if not p:
raise NotExistError("pipeline %s is not exist" % name, "")
p = p[0]
milvus_collection_name = f"{name}_{p.encoder}"
MilvusIns.del_milvus_collection(milvus_collection_name)
pipe = Pipeline(name=p.name,
input=p.input,
output=p.output,
dimension=p.dimension,
index_file_size=p.index_file_size,
metric_type=p.metric_type,
description=p.description,
processors=p.processors.split(","),
encoder=p.encoder)
return pipe
except Exception as e:
logger.error(e)
return e
def pipeline_detail(name):
try:
p = search_pipeline(name)
if not p:
raise NotExistError("pipeline %s is not exist" % name, "")
if not p.processors:
pr = []
else:
pr = p.processors.split(",")
pipe = Pipeline(name=p.name,
input=p.input,
output=p.output,
dimension=p.dimension,
index_file_size=p.index_file_size,
metric_type=p.metric_type,
description=p.description,
processors=pr,
encoder=p.encoder)
return pipe
except Exception as e:
raise e
def delete_application(name, force=False):
try:
if not force:
if not entities_list(name, 100, 0):
raise RequestError(
"Prevent to delete application with entity not deleted",
"")
app = MongoIns.search_by_name(APPLICATION_COLLECTION_NAME, name)
if not app:
raise NotExistError(f"application {name} not exist", "")
app = app[0]
delete_milvus_collections_by_fields(app)
S3Ins.del_s3_buckets(app['bucket'])
MongoIns.delete_mongo_collection(f"{name}_entity")
MongoIns.delete_by_name(APPLICATION_COLLECTION_NAME, name)
logger.info("delete application %s", name)
application = Application(app["name"], app["fields"], app["bucket"])
application.metadata = app["metadata"]
return application
except Exception as e:
logger.error(e)
raise e
def delete_application(name):
try:
if len(entities_list(name, 100, 0)):
raise RequestError(
"Prevent to delete application with entity not deleted", "")
# TODO rewrite clean all resource before change metadata
x = del_application(name)
if not x:
raise NotExistError(f"application {name} not exist", "")
x = x[0]
fields = search_fields(json.loads(x.fields))
app = Application(name=x.name,
fields=fields2dict(fields),
buckets=x.s3_buckets)
delete_milvus_collections_by_fields(app)
delete_fields(json.loads(x.fields))
S3Ins.del_s3_buckets(x.s3_buckets.split(","))
MongoIns.delete_mongo_collection(f"{name}_entity")
logger.info("delete application %s", name)
return app
except Exception as e:
logger.error(e)
raise e
def _telephoneValidCheck(self, conn, id_phone):
where = {'id': id_phone, 'users_id': self.users_id}
telephone = select(conn, 'users_phone_num', where=where)
if not telephone:
raise NotExistError('ORDER_ERR_PHONE_%s_NOT_EXIST' % id_phone)
def _addressValidCheck(self, conn, addr_id):
where = {'id': addr_id, 'users_id': self.users_id}
addr = select(conn, 'users_address', where=where)
if not addr:
raise NotExistError('ORDER_ERR_ADDR_%s_NOT_EXIST' % addr_id)