def upload(): #获取用户ip地址 app.logger.info('start upload') start_time = time.time() # id = str(start_time) + '_' + request.remote_addr id = uuid.uuid4().hex es_model = EsModel(meta={'id': id}, ip=request.remote_addr) es_model.save() session['id'] = id if request.method == 'POST': file = request.files['file'] if file and check_file_extension(file.filename): app.logger.info('file type is allow') producer = Producer('Producer', uploading_queue, uploaded_queue, app) if producer.add_task(file): producer.start() producer.join() else: return jsonify(name=file.filename, status='waited') print(threading.enumerate()) updir = os.path.join(basedir, 'upload/') updir = os.path.join(updir, id) es_model = EsModel.get(id=session.get('id')) es_model.update(models_dir=os.path.join(updir, file.filename)) return jsonify(name=file.filename, status='success')
def x2paddle(cmd, model_name, save_base_dir): save_dir = os.path.join(save_base_dir, model_name) p = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True, universal_newlines=True) cmd_result = '' for line in p.stdout.readlines(): if 'Converting node' in line: continue cmd_result += str(line).rstrip() + '<br/>\n' sys.stdout.flush() # zip_dir = os.path.join(save_dir, model_name + '.tar.gz') zip_dir = save_dir + '.tar.gz' # zip_dir = model_name + '.tar.gz' es_model = EsModel.get(id=session.get('id')) es_model.update(log=cmd_result) if os.path.exists(os.path.join(save_dir, 'inference_model/__model__')): os.system('tar -C ' + save_base_dir + ' -zcvf ' + zip_dir + ' ' + model_name) app.logger.info(model_name + ' convert success') return jsonify(name=model_name + '.tar.gz', status='success', cmd_result=cmd_result) else: app.logger.info(model_name + ' convert failed') return jsonify(name='', status='failed', cmd_result=cmd_result)
def run_script(cmd, model_name, save_base_dir, id): p = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True, universal_newlines=True) cmd_result = '' for line in p.stdout.readlines(): if "Converting node" in line: continue cmd_result += str(line).rstrip() + '<br/>\n' sys.stdout.flush() zip_dir = os.path.join(save_base_dir, model_name + '.tar.gz') save_dir = os.path.join(save_base_dir, model_name) es_model = EsModel.get(id=id) es_model.update(log=cmd_result) es_model.update(save_dir=save_dir) if os.path.exists(os.path.join(save_dir, 'inference_model/__model__')): os.system('tar -C ' + save_base_dir + ' -cvzf ' + zip_dir + ' ' + model_name) res = { 'name': model_name + '.tar.gz', 'status': 'success', 'cmd_result': cmd_result } return res else: res = {'status': 'failed', 'cmd_result': cmd_result} return res
def save(self): updir = os.path.join(self.upload_base_dir, self.id) if not os.path.exists(updir): os.mkdir(updir) try: self.files['caffe_weight'].id = self.id caffe_weight = self.files['caffe_weight'] caffe_weight_filename = secure_filename(caffe_weight.filename) caffe_weight_dir = os.path.join(updir, caffe_weight_filename) caffe_weight.save(caffe_weight_dir) self.files['caffe_weight'].upload_dir = caffe_weight_dir save_base_dir = os.path.join(self.convert_base_dir, self.id) save_dir = os.path.join(save_base_dir + '/' + self.files['caffe_weight'].filename) file_size = os.path.getsize(caffe_weight_dir) es_model = EsModel.get(id=self.id) es_model.update(upload_dir=caffe_weight_dir) es_model.update(file_size=file_size) self.save_dir = save_dir except: self.files['caffe_model'].id = self.id caffe_model = self.files['caffe_model'] caffe_model_filename = secure_filename(caffe_model.filename) caffe_model_dir = os.path.join(updir, caffe_model_filename) caffe_model.save(caffe_model_dir) self.files['caffe_model'].upload_dir = caffe_model_dir save_base_dir = os.path.join(self.convert_base_dir, self.id) save_dir = os.path.join(save_base_dir + '/' + self.files['caffe_model'].filename) self.save_dir = save_dir
def convert(): ''' {0:'tensorflow',1:'onnx',2:'caffe'} :return: ''' print(request.get_data(), 123456) data = json.loads(request.get_data().decode('utf-8')) id = session.get('id') updir = os.path.join(basedir, 'upload/') updir = os.path.join(updir, id) es_model = EsModel.get(id=id) es_model.update(email=data['email']) es_model.update(framework=data['framework']) app.logger.info('start convert') if data['framework'] == '0': #tensorflow model_full_name = data['tf_name'] if model_full_name == '': return jsonify(status='failed') save_base_dir = os.path.join(convert_base_dir, id) model_name = model_full_name.split('.')[0] save_dir = os.path.join(save_base_dir, model_name) model_path = os.path.join(updir, model_full_name) cmd = 'x2paddle' + ' --framework=tensorflow' + ' --model=' + model_path + ' --save_dir=' + save_dir return x2paddle(cmd, model_name, save_base_dir) elif data['framework'] == '1': #onnx model_full_name = data['onnx_name'] if model_full_name == '': return jsonify(status='failed') save_base_dir = os.path.join(convert_base_dir, id) model_name = model_full_name.split('.')[0] save_dir = os.path.join(save_base_dir, model_name) model_path = os.path.join(updir, model_full_name) cmd = 'x2paddle' + ' --framework=onnx' + ' --model=' + model_path + ' --save_dir=' + save_dir return x2paddle(cmd, model_name, save_base_dir) elif data['framework'] == '1': # caffe caffe_weight_name = data['caffe_weight_name'] caffe_model_name = data['caffe_model_name'] if caffe_weight_name == '' or caffe_model_name == '': return jsonify(status='failed') save_base_dir = os.path.join(convert_base_dir, id) model_name = caffe_model_name.split('.')[0] save_dir = os.path.join(save_base_dir, model_name) weight_path = os.path.join(updir, caffe_weight_name) model_path = os.path.join(updir, caffe_model_name) cmd = 'x2paddle' + ' --framework=caffe' + ' --prototxt=' + model_path + ' --weight=' + weight_path + ' --save_dir=' + save_dir return x2paddle(cmd, model_name, save_base_dir)
def save(self): updir = os.path.join(self.upload_base_dir, self.id) if not os.path.exists(updir): os.mkdir(updir) file = self.file['object'] filename = secure_filename(file.filename) file_dir = os.path.join(updir, filename) file.save(file_dir) self.file['upload_dir'] = file_dir save_base_dir = os.path.join(self.convert_base_dir, self.id) save_dir = os.path.join(save_base_dir + '/' + self.file['filename']) file_size = os.path.getsize(file_dir) es_model = EsModel.get(id=self.id) es_model.update(upload_dir=file_dir) es_model.update(file_size=file_size) self.save_dir = save_dir
def convert(): # model = get_model(session) data = json.loads(request.get_data().decode('utf-8')) model = get_model(data['model_id']) global model_id model_id = data['model_id'] es_model = EsModel.get(id=model.id) es_model.update(email=data['email']) es_model.update(framework=data['framework']) es_model.update(model_class=data['model_class']) localtime = time.localtime(time.time()) es_model.update(time=localtime) producer = ConvertProducer('Producer', uploaded_queue, model_pool, app) if producer.add_task(model): producer.start() producer.join() return jsonify(model.result) else: return jsonify(name=model.id, status='failed', message='waiting')