def train(df):
# https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html
clf = LogisticRegression(C=50. / 5000,
penalty='l1',
solver='saga',
tol=0.1)
x_train, y_train = df[0], df[1]
clf.fit(x_train, y_train)
model_path = get_file_dir(__file__) + '/saved_model'
if not os.path.exists(model_path):
os.makedirs(model_path)
model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
model_path = model_path + '/' + model_timestamp
dump(clf, model_path)
model = function_context.node_spec.output_model
print(model.name)
print(model_timestamp)
# When registering a model, corresponding type of event will be sent to downstream job as well.
af.register_model_version(
model=model,
model_path=model_path,
current_stage=ModelVersionStage.GENERATED)
print(
af.get_latest_generated_model_version(
model_name=model.name).model_path)
return df
def execute(self, function_context: FunctionContext,
input_list: List) -> List:
x_train = input_list[0]
input_dim = 512
encoding_dim = 128
x_test = np.random.rand(30, input_dim)
model_input = Input(shape=(input_dim, ))
encoder = Dense(encoding_dim)(model_input)
decoder = Dense(input_dim)(encoder)
model = Model(model_input, decoder)
model.compile(loss='binary_crossentropy', optimizer=Adam())
model.fit(x_train, x_train, validation_data=(x_test, x_test), epochs=1)
encoder = Model(model_input, encoder)
model_path = os.path.dirname(os.path.abspath(__file__)) + '/model'
print('Save trained model to {}'.format(model_path))
if os.path.exists(model_path):
shutil.rmtree(model_path)
tf.saved_model.simple_save(tf.keras.backend.get_session(),
model_path,
inputs={"aaa_input": encoder.input},
outputs={"bbb": encoder.output})
model_meta: ModelMeta = function_context.node_spec.output_model
# Register model version to notify that cluster serving is ready to start loading the registered model version.
register_model_version(model=model_meta, model_path=model_path)
return []
def execute(self, function_context: FunctionContext,
input_list: List) -> List:
model = get_compiled_model()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'mse'])
x_train, y_train = input_list[0][0] / 255.0, input_list[0][1]
model.fit(x_train, y_train, epochs=1)
model_meta: ModelMeta = function_context.node_spec.output_model
save_path = 'saved_models/{}'.format(round(time.time() * 1000))
model.save(save_path, save_format='tf')
af.register_model_version(model=model_meta, model_path=save_path)
def train(df, model, sess, graph):
x_train, y_train = df[0] / 255.0, df[1]
with graph.as_default():
tf.compat.v1.keras.backend.set_session(sess)
model.fit(x_train, y_train, epochs=1)
model_meta: ModelMeta = function_context.node_spec.output_model
save_path = 'saved_models/{}'.format((round(time.time())))
model.save(save_path, save_format='tf')
af.register_model_version(model=model_meta.uuid,
model_path=save_path)
return df
def execute(self, function_context: FunctionContext, input_list: List) -> List:
# https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html
clf = LogisticRegression(C=50. / 5000, penalty='l1', solver='saga', tol=0.1)
x_train, y_train = input_list[0][0], input_list[0][1]
clf.fit(x_train, y_train)
model_path = get_file_dir(__file__) + '/saved_model'
if not os.path.exists(model_path):
os.makedirs(model_path)
model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
model_path = model_path + '/' + model_timestamp
dump(clf, model_path)
af.register_model_version(model=function_context.node_spec.output_model, model_path=model_path)
return []
def execute(self, function_context: FunctionContext,
input_list: List) -> List:
x_train = input_list[0]
y_label = input_list[1]
input_dim = 512
x_train_columns = list()
x_train_columns.append('face_id')
for i in range(1, input_dim + 1):
x_train_columns.append('col' + str(i))
trainDf = pd.DataFrame(x_train, columns=x_train_columns)
labelDf = pd.DataFrame(y_label, columns=('face_id', 'label'))
trainDf = pd.merge(trainDf,
labelDf,
on=['face_id'],
how='inner',
suffixes=('_x', '_y'))
y_label = trainDf['label'].values.astype(int)
trainDf = trainDf.drop('face_id', 1)
x_train = trainDf.drop('label', 1).values
label_prop_model = None
score = 0.0
while score < 0.95:
print('before train ACC:', score)
random_unlabeled_points = np.random.rand(len(y_label))
random_unlabeled_points = random_unlabeled_points < 0.3 # 0-1的随机数，小于0.7返回1，大于等于0.7返回0
Y = y_label[random_unlabeled_points] # label转换之前的
y_label[random_unlabeled_points] = -1 # 标签重置，将标签为1的变为-1
label_prop_model = LabelPropagation()
label_prop_model.fit(x_train, y_label)
Y_pred = label_prop_model.predict(x_train)
Y_pred = Y_pred[random_unlabeled_points]
score = accuracy_score(Y, Y_pred)
y_label[random_unlabeled_points] = Y
model_path = os.path.dirname(os.path.abspath(__file__)) + '/model'
print('Save trained model to {}'.format(model_path))
if not os.path.exists(model_path):
joblib.dump(label_prop_model, model_path)
model_meta: ModelMeta = function_context.node_spec.output_model
# Register model version to notify that cluster serving is ready to start loading the registered model version.
register_model_version(model=model_meta, model_path=model_path)
return []
def test_stream_with_external_trigger_with_model_control(self):
print(sys._getframe().f_code.co_name)
model_name = 'test_create_model_version'
model_desc = 'test create model version'
response = af.register_model(model_name=model_name,
model_type=af.ModelType.CHECKPOINT,
model_desc=model_desc)
trigger = af.external_trigger(name='stream_trigger')
job_config = af.BaseJobConfig('local', 'cmd_line')
job_config.job_name = 'test_cmd'
with af.config(job_config):
cmd_executor = af.user_define_operation(
output_num=0,
executor=CmdExecutor(
cmd_line="echo 'hello world' && sleep {}".format(1)))
af.model_version_control_dependency(
src=cmd_executor,
dependency=trigger,
model_name=model_name,
model_version_event_type='MODEL_DEPLOYED')
workflow_id = af.submit_ai_flow()
model_path1 = 'fs://source1.pkl'
model_metric1 = 'http://metric1'
model_flavor1 = '{"flavor.version":1}'
version_desc1 = 'test create model version1'
time.sleep(1)
response = af.register_model_version(
model=model_name,
model_path=model_path1,
model_metric=model_metric1,
model_flavor=model_flavor1,
version_desc=version_desc1,
current_stage=af.ModelVersionStage.DEPLOYED)
time.sleep(5)
response = af.register_model_version(
model=model_name,
model_path=model_path1,
model_metric=model_metric1,
model_flavor=model_flavor1,
version_desc=version_desc1,
current_stage=af.ModelVersionStage.DEPLOYED)
time.sleep(10)
af.stop_execution_by_id(workflow_id)
res = af.get_ai_flow_client().list_job(5, 0)
self.assertEqual(3, len(res))
def train(df):
# https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html
clf = LogisticRegression(C=50. / 5000, penalty='l1', solver='saga', tol=0.1)
x_train, y_train = df[0], df[1]
clf.fit(x_train, y_train)
model_path = get_file_dir(__file__) + '/saved_model'
if not os.path.exists(model_path):
os.makedirs(model_path)
model_version = time.strftime("%Y%m%d%H%M%S", time.localtime())
model_path = model_path + '/' + model_version
dump(clf, model_path)
model = function_context.node_spec.output_model
print(model.name)
print(model_version)
af.register_model_version(model=model, model_path=model_path, current_stage=ModelVersionStage.GENERATED)
return df
def process(self, execution_context: ExecutionContext,
input_list: List) -> List:
"""
Train and save KNN model
"""
model_meta: af.ModelMeta = execution_context.config.get('model_info')
clf = KNeighborsClassifier(n_neighbors=5)
x_train, y_train = input_list[0][0], input_list[0][1]
clf.fit(x_train, y_train)
# Save model to local
model_path = get_file_dir(__file__) + '/saved_model'
if not os.path.exists(model_path):
os.makedirs(model_path)
model_timestamp = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime())
model_path = model_path + '/' + model_timestamp
dump(clf, model_path)
af.register_model_version(model=model_meta, model_path=model_path)
return []
def process(self, execution_context: ExecutionContext,
input_list: List) -> List:
# https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html
clf = LogisticRegression(C=50. / 5000,
penalty='l1',
solver='saga',
tol=1)
x_train, y_train = input_list[0][0], input_list[0][1]
clf.fit(x_train, y_train)
model_path = get_file_dir(__file__) + '/saved_model'
if not os.path.exists(model_path):
os.makedirs(model_path)
model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
model_path = model_path + '/' + model_timestamp
dump(clf, model_path)
model_meta: af.ModelMeta = execution_context.config.get('model_info')
af.register_model_version(model=model_meta, model_path=model_path)
print("Done for {}".format(self.__class__.__name__))
return []
def run_task_function(client: NotificationClient):
with af.global_config_file(workflow_config_file()):
with af.config('task_2'):
executor_1 = af.user_define_operation(
af.PythonObjectExecutor(SimpleExecutor()))
with af.config('task_3'):
executor_2 = af.user_define_operation(
af.PythonObjectExecutor(SimpleExecutor()))
af.model_version_control_dependency(
src=executor_2,
dependency=executor_1,
model_name='model_1',
model_version_event_type=ModelVersionEventType.
MODEL_GENERATED)
workflow_info = af.workflow_operation.submit_workflow(
workflow_name)
af.workflow_operation.start_new_workflow_execution(workflow_name)
r_flag = True
while True:
with create_session() as session:
tes2 = session.query(TaskExecution).filter(
TaskExecution.dag_id == 'test_project.test_workflow',
TaskExecution.task_id == 'task_2').all()
if len(tes2) == 1 and r_flag:
af.register_model_version(
model='model_1',
model_path='/tmp/model/v1',
current_stage=af.ModelVersionStage.GENERATED)
r_flag = False
dag_run = session.query(DagRun).filter(
DagRun.dag_id == 'test_project.test_workflow').first()
if dag_run is not None and dag_run.state in State.finished:
break
else:
time.sleep(1)
