def train(df): # https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html clf = LogisticRegression(C=50. / 5000, penalty='l1', solver='saga', tol=0.1) x_train, y_train = df[0], df[1] clf.fit(x_train, y_train) model_path = get_file_dir(__file__) + '/saved_model' if not os.path.exists(model_path): os.makedirs(model_path) model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime()) model_path = model_path + '/' + model_timestamp dump(clf, model_path) model = function_context.node_spec.output_model print(model.name) print(model_timestamp) # When registering a model, corresponding type of event will be sent to downstream job as well. af.register_model_version( model=model, model_path=model_path, current_stage=ModelVersionStage.GENERATED) print( af.get_latest_generated_model_version( model_name=model.name).model_path) return df
def execute(self, function_context: FunctionContext, input_list: List) -> List: x_train = input_list[0] input_dim = 512 encoding_dim = 128 x_test = np.random.rand(30, input_dim) model_input = Input(shape=(input_dim, )) encoder = Dense(encoding_dim)(model_input) decoder = Dense(input_dim)(encoder) model = Model(model_input, decoder) model.compile(loss='binary_crossentropy', optimizer=Adam()) model.fit(x_train, x_train, validation_data=(x_test, x_test), epochs=1) encoder = Model(model_input, encoder) model_path = os.path.dirname(os.path.abspath(__file__)) + '/model' print('Save trained model to {}'.format(model_path)) if os.path.exists(model_path): shutil.rmtree(model_path) tf.saved_model.simple_save(tf.keras.backend.get_session(), model_path, inputs={"aaa_input": encoder.input}, outputs={"bbb": encoder.output}) model_meta: ModelMeta = function_context.node_spec.output_model # Register model version to notify that cluster serving is ready to start loading the registered model version. register_model_version(model=model_meta, model_path=model_path) return []
def execute(self, function_context: FunctionContext, input_list: List) -> List: model = get_compiled_model() model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy', 'mse']) x_train, y_train = input_list[0][0] / 255.0, input_list[0][1] model.fit(x_train, y_train, epochs=1) model_meta: ModelMeta = function_context.node_spec.output_model save_path = 'saved_models/{}'.format(round(time.time() * 1000)) model.save(save_path, save_format='tf') af.register_model_version(model=model_meta, model_path=save_path)
def train(df, model, sess, graph): x_train, y_train = df[0] / 255.0, df[1] with graph.as_default(): tf.compat.v1.keras.backend.set_session(sess) model.fit(x_train, y_train, epochs=1) model_meta: ModelMeta = function_context.node_spec.output_model save_path = 'saved_models/{}'.format((round(time.time()))) model.save(save_path, save_format='tf') af.register_model_version(model=model_meta.uuid, model_path=save_path) return df
def execute(self, function_context: FunctionContext, input_list: List) -> List: # https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html clf = LogisticRegression(C=50. / 5000, penalty='l1', solver='saga', tol=0.1) x_train, y_train = input_list[0][0], input_list[0][1] clf.fit(x_train, y_train) model_path = get_file_dir(__file__) + '/saved_model' if not os.path.exists(model_path): os.makedirs(model_path) model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime()) model_path = model_path + '/' + model_timestamp dump(clf, model_path) af.register_model_version(model=function_context.node_spec.output_model, model_path=model_path) return []
def execute(self, function_context: FunctionContext, input_list: List) -> List: x_train = input_list[0] y_label = input_list[1] input_dim = 512 x_train_columns = list() x_train_columns.append('face_id') for i in range(1, input_dim + 1): x_train_columns.append('col' + str(i)) trainDf = pd.DataFrame(x_train, columns=x_train_columns) labelDf = pd.DataFrame(y_label, columns=('face_id', 'label')) trainDf = pd.merge(trainDf, labelDf, on=['face_id'], how='inner', suffixes=('_x', '_y')) y_label = trainDf['label'].values.astype(int) trainDf = trainDf.drop('face_id', 1) x_train = trainDf.drop('label', 1).values label_prop_model = None score = 0.0 while score < 0.95: print('before train ACC:', score) random_unlabeled_points = np.random.rand(len(y_label)) random_unlabeled_points = random_unlabeled_points < 0.3 # 0-1的随机数,小于0.7返回1,大于等于0.7返回0 Y = y_label[random_unlabeled_points] # label转换之前的 y_label[random_unlabeled_points] = -1 # 标签重置,将标签为1的变为-1 label_prop_model = LabelPropagation() label_prop_model.fit(x_train, y_label) Y_pred = label_prop_model.predict(x_train) Y_pred = Y_pred[random_unlabeled_points] score = accuracy_score(Y, Y_pred) y_label[random_unlabeled_points] = Y model_path = os.path.dirname(os.path.abspath(__file__)) + '/model' print('Save trained model to {}'.format(model_path)) if not os.path.exists(model_path): joblib.dump(label_prop_model, model_path) model_meta: ModelMeta = function_context.node_spec.output_model # Register model version to notify that cluster serving is ready to start loading the registered model version. register_model_version(model=model_meta, model_path=model_path) return []
def test_stream_with_external_trigger_with_model_control(self): print(sys._getframe().f_code.co_name) model_name = 'test_create_model_version' model_desc = 'test create model version' response = af.register_model(model_name=model_name, model_type=af.ModelType.CHECKPOINT, model_desc=model_desc) trigger = af.external_trigger(name='stream_trigger') job_config = af.BaseJobConfig('local', 'cmd_line') job_config.job_name = 'test_cmd' with af.config(job_config): cmd_executor = af.user_define_operation( output_num=0, executor=CmdExecutor( cmd_line="echo 'hello world' && sleep {}".format(1))) af.model_version_control_dependency( src=cmd_executor, dependency=trigger, model_name=model_name, model_version_event_type='MODEL_DEPLOYED') workflow_id = af.submit_ai_flow() model_path1 = 'fs://source1.pkl' model_metric1 = 'http://metric1' model_flavor1 = '{"flavor.version":1}' version_desc1 = 'test create model version1' time.sleep(1) response = af.register_model_version( model=model_name, model_path=model_path1, model_metric=model_metric1, model_flavor=model_flavor1, version_desc=version_desc1, current_stage=af.ModelVersionStage.DEPLOYED) time.sleep(5) response = af.register_model_version( model=model_name, model_path=model_path1, model_metric=model_metric1, model_flavor=model_flavor1, version_desc=version_desc1, current_stage=af.ModelVersionStage.DEPLOYED) time.sleep(10) af.stop_execution_by_id(workflow_id) res = af.get_ai_flow_client().list_job(5, 0) self.assertEqual(3, len(res))
def train(df): # https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html clf = LogisticRegression(C=50. / 5000, penalty='l1', solver='saga', tol=0.1) x_train, y_train = df[0], df[1] clf.fit(x_train, y_train) model_path = get_file_dir(__file__) + '/saved_model' if not os.path.exists(model_path): os.makedirs(model_path) model_version = time.strftime("%Y%m%d%H%M%S", time.localtime()) model_path = model_path + '/' + model_version dump(clf, model_path) model = function_context.node_spec.output_model print(model.name) print(model_version) af.register_model_version(model=model, model_path=model_path, current_stage=ModelVersionStage.GENERATED) return df
def process(self, execution_context: ExecutionContext, input_list: List) -> List: """ Train and save KNN model """ model_meta: af.ModelMeta = execution_context.config.get('model_info') clf = KNeighborsClassifier(n_neighbors=5) x_train, y_train = input_list[0][0], input_list[0][1] clf.fit(x_train, y_train) # Save model to local model_path = get_file_dir(__file__) + '/saved_model' if not os.path.exists(model_path): os.makedirs(model_path) model_timestamp = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) model_path = model_path + '/' + model_timestamp dump(clf, model_path) af.register_model_version(model=model_meta, model_path=model_path) return []
def process(self, execution_context: ExecutionContext, input_list: List) -> List: # https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html clf = LogisticRegression(C=50. / 5000, penalty='l1', solver='saga', tol=1) x_train, y_train = input_list[0][0], input_list[0][1] clf.fit(x_train, y_train) model_path = get_file_dir(__file__) + '/saved_model' if not os.path.exists(model_path): os.makedirs(model_path) model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime()) model_path = model_path + '/' + model_timestamp dump(clf, model_path) model_meta: af.ModelMeta = execution_context.config.get('model_info') af.register_model_version(model=model_meta, model_path=model_path) print("Done for {}".format(self.__class__.__name__)) return []
def run_task_function(client: NotificationClient): with af.global_config_file(workflow_config_file()): with af.config('task_2'): executor_1 = af.user_define_operation( af.PythonObjectExecutor(SimpleExecutor())) with af.config('task_3'): executor_2 = af.user_define_operation( af.PythonObjectExecutor(SimpleExecutor())) af.model_version_control_dependency( src=executor_2, dependency=executor_1, model_name='model_1', model_version_event_type=ModelVersionEventType. MODEL_GENERATED) workflow_info = af.workflow_operation.submit_workflow( workflow_name) af.workflow_operation.start_new_workflow_execution(workflow_name) r_flag = True while True: with create_session() as session: tes2 = session.query(TaskExecution).filter( TaskExecution.dag_id == 'test_project.test_workflow', TaskExecution.task_id == 'task_2').all() if len(tes2) == 1 and r_flag: af.register_model_version( model='model_1', model_path='/tmp/model/v1', current_stage=af.ModelVersionStage.GENERATED) r_flag = False dag_run = session.query(DagRun).filter( DagRun.dag_id == 'test_project.test_workflow').first() if dag_run is not None and dag_run.state in State.finished: break else: time.sleep(1)