def train(df):
# https://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html
clf = LogisticRegression(C=50. / 5000,
penalty='l1',
solver='saga',
tol=0.1)
x_train, y_train = df[0], df[1]
clf.fit(x_train, y_train)
model_path = get_file_dir(__file__) + '/saved_model'
if not os.path.exists(model_path):
os.makedirs(model_path)
model_timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
model_path = model_path + '/' + model_timestamp
dump(clf, model_path)
model = function_context.node_spec.output_model
print(model.name)
print(model_timestamp)
# When registering a model, corresponding type of event will be sent to downstream job as well.
af.register_model_version(
model=model,
model_path=model_path,
current_stage=ModelVersionStage.GENERATED)
print(
af.get_latest_generated_model_version(
model_name=model.name).model_path)
return df
def setup(self, execution_context: ExecutionContext):
model_meta: af.ModelMeta = execution_context.config.get('model_info')
self.model_name = model_meta.name
self.model_meta = af.get_latest_generated_model_version(
self.model_name)
self.model_path = self.model_meta.model_path
self.model_version = self.model_meta.version
def process(self, execution_context: ExecutionContext,
input_list: List) -> List:
"""
Validate and deploy model if necessary
"""
current_model_meta: af.ModelMeta = execution_context.config.get(
'model_info')
deployed_model_version = af.get_deployed_model_version(
model_name=current_model_meta.name)
new_model_meta = af.get_latest_generated_model_version(
current_model_meta.name)
uri = af.get_artifact_by_name(self.artifact).uri
if deployed_model_version is None:
# If there is no deployed model for now, update the current generated model to be deployed.
af.update_model_version(model_name=current_model_meta.name,
model_version=new_model_meta.version,
current_stage=ModelVersionStage.VALIDATED)
af.update_model_version(model_name=current_model_meta.name,
model_version=new_model_meta.version,
current_stage=ModelVersionStage.DEPLOYED)
else:
x_validate = input_list[0][0]
y_validate = input_list[0][1]
knn = load(new_model_meta.model_path)
scores = knn.score(x_validate, y_validate)
deployed_knn = load(deployed_model_version.model_path)
deployed_scores = deployed_knn.score(x_validate, y_validate)
with open(uri, 'a') as f:
f.write('deployed model version: {} scores: {}\n'.format(
deployed_model_version.version, deployed_scores))
f.write('generated model version: {} scores: {}\n'.format(
new_model_meta.version, scores))
if scores >= deployed_scores:
# Deprecate current model and deploy better new model
af.update_model_version(
model_name=current_model_meta.name,
model_version=deployed_model_version.version,
current_stage=ModelVersionStage.DEPRECATED)
af.update_model_version(
model_name=current_model_meta.name,
model_version=new_model_meta.version,
current_stage=ModelVersionStage.VALIDATED)
af.update_model_version(
model_name=current_model_meta.name,
model_version=new_model_meta.version,
current_stage=ModelVersionStage.DEPLOYED)
return []
def setup(self, function_context: FunctionContext):
print(function_context.node_spec.model.name)
model = af.get_latest_generated_model_version(
function_context.node_spec.model.name)
self.model_path = model.model_path
self.model_version = model.version
def setup(self, function_context: FunctionContext):
self.model_name = function_context.node_spec.model.name
self.model_meta = af.get_latest_generated_model_version(
self.model_name)
self.model_path = self.model_meta.model_path
self.model_version = self.model_meta.version
