class ModelTraining:
AZURE_MODEL_CONTAINER = 'https://codeday.blob.core.windows.net/codeday-ml-ci-cd'
def __init__(self):
self.app_config = AppConfig()
self.db_config = DBConfig()
def train(self):
LOGGER.info('Started model training..')
connection = self.db_config.get_connection()
with connection.cursor() as _:
data = pd.read_sql('select * from grad_admission', con=connection)
labels = data['admit']
features = data.drop(columns='admit')
x_train, x_val, y_train, y_val = train_test_split(features,
labels,
test_size=0.25,
random_state=42)
max_iter = self.app_config.config['model']['train']['hyperparams'][
'max_iter']
model = LogisticRegression(max_iter=max_iter)
model = model.fit(x_train, y_train)
val_score = model.score(x_val, y_val)
logging.info('Model trained, val score: {}'.format(val_score))
return model, val_score
def evaluate_model(self, val_score):
eval_threshold = self.app_config.config['model']['train'][
'eval_threshold']
return val_score > eval_threshold
def save_model(self, model):
model_list_url = ModelTraining.AZURE_MODEL_CONTAINER + '?restype=container&comp=list&prefix=models'
model_list_res = requests.get(model_list_url)
root = ET.fromstring(model_list_res.text)
all_models = [
blob.find('Name').text
for blob in root.find('.').find('Blobs').findall('Blob')
]
all_versions = [int(x.split('.')[1][1:]) for x in all_models]
latest_version = max(all_versions)
new_version_num = latest_version + 1
saved_model_path = 'model.v' + str(new_version_num)
dump(model, saved_model_path)
return os.path.abspath(saved_model_path)
def ingest():
input_data = sys.stdin.readlines()
connection = DBConfig().get_connection()
for row in input_data:
row = row.rstrip()
try:
columns = _split_and_validate(row)
_ingest_row(connection, columns)
except Exception:
LOGGER.warning('Error in row: {}, skipping...'.format(row),
exc_info=sys.exc_info())
LOGGER.info('all standard input records consumed...')
sys.exit()
def __init__(self):
self.app_config = AppConfig()
self.db_config = DBConfig()
loss_layer = KL.Lambda(db_loss, name='db_loss')([
input_gt, input_mask, input_thresh, input_thresh_mask,
binarize_map, thresh_binary, threshold_map
])
db_model = K.Model(inputs=[
input_image, input_gt, input_mask, input_thresh, input_thresh_mask
],
outputs=[loss_layer])
loss_names = ["db_loss"]
for layer_name in loss_names:
layer = db_model.get_layer(layer_name)
db_model.add_loss(layer.output)
# db_model.add_metric(layer.output, name=layer_name, aggregation="mean")
else:
db_model = K.Model(inputs=input_image, outputs=binarize_map)
"""
db_model = K.Model(inputs=input_image,
outputs=thresh_binary)
"""
return db_model
if __name__ == '__main__':
from config import DBConfig
cfg = DBConfig()
model = DBNet(cfg, model='inference')
model.summary()
if not args or (not "-c" in opts and not "-d" in opts):
print (Usage)
sys.exit(2)
verbose = "-v" in opts
all_replicas = "-a" in opts
long_output = "-l" in opts or all_replicas
out_prefix = opts.get("-o")
zout = "-z" in opts
stats_file = opts.get("-s")
stats_key = opts.get("-S", "db_dump")
rse_name = args[0]
if "-d" in opts:
dbconfig = DBConfig.from_cfg(opts["-d"])
else:
dbconfig = DBConfig.from_yaml(opts["-c"])
#print("dbconfig: url:", dbconfig.DBURL, "schema:", dbconfig.Schema)
config = Config(opts["-c"])
stats = None if stats_file is None else Stats(stats_file)
if stats:
stats[stats_key] = {
"status":"started",
"version":Version,
"rse":rse_name,
"start_time":t0,
import falcon
import json
from config import DBConfig
from db import Database
db_config = DBConfig('db_config.yaml')
db = Database(
host=db_config.host,
username=db_config.username,
password=db_config.password,
port=db_config.port,
db_name=db_config.db_name
)
class ReadingsResource(object):
def on_get(self, req, resp, box_id, from_date, to_date):
result = db.get_reading(box_id, from_date, to_date)
resp.text = json.dumps(result, indent=4, sort_keys=True, default=str)
resp.status = falcon.HTTP_200
class AllResource(object):
def on_get(self, req, resp):
result = db.get_all()
resp.text = json.dumps(result, indent=4, sort_keys=True, default=str)
resp.status = falcon.HTTP_200
app = application = falcon.App()