def launch_optimizer():
args = parse_arguments()
config = opt_utils.read_yaml(args.config_path)
create_logger(config)
opt_utils.create_dirs(config.get('directories', ['data', 'log', 'outputs']).values())
opt_rest.init_service(config)
opt_rest.app.run(debug=True,
host=args.host,
port=args.port)
def get_advice():
global _last
_last = request.args.get('last', default = True)
logger.info('----------------------------------------------------------')
logger.info('------------------------ ADVISOR -------------------------')
logger.info(f'_last parameter variable set = {_last}')
logger.info('------------------------ ADVISOR -------------------------')
logger.info('----------------------------------------------------------')
logger.info('----------------------------------------------------------')
logger.info('------------------------ ADVISOR -------------------------')
logger.info('Advisor get_advice() called')
logger.info('------------------------ ADVISOR -------------------------')
logger.info('----------------------------------------------------------')
constants = opt_utils.read_yaml('data/constants.yaml')
logger.info('----------------------------------------------------------')
logger.info('-------------------------- YAML --------------------------')
logger.info(f'Constants received: {constants}')
logger.info('-------------------------- YAML --------------------------')
logger.info('----------------------------------------------------------')
logger.info('----------------------------------------------------------')
logger.info('------------------------ ADVISOR -------------------------')
logger.info('opt_advisor.init() called')
logger.info('------------------------ ADVISOR -------------------------')
logger.info('----------------------------------------------------------')
# logger.info('----------------------------------------------------------')
# logger.info('------------------------ ADVISOR -------------------------')
# logger.info('opt_utils.readCSV(config.nn_filename)')
# logger.info('------------------------ ADVISOR -------------------------')
# logger.info('----------------------------------------------------------')
# df = opt_utils.readCSV(config.nn_filename)
# logger.info('----------------------------------------------------------')
# logger.info('------------------------ ADVISOR -------------------------')
# logger.info(f'df.shape[0] = {df.shape[0]}')
# logger.info('------------------------ ADVISOR -------------------------')
# logger.info('----------------------------------------------------------')
# Tehát igazából abban sem vagyok biztos, hogy az Adviser API
# hívásánaál be kellene e olvasnaom a CSV-t
# lehet, hogy ezt megtehetné maga az advisor is
# Szerintem az kéne, hogy az adviser nem kap semmilyen előszürést
# hanem saját maga megvizsgálja, hogy van-e elég adat és ha van
# akkor ad javaslatot, ha nincs akkor a józsinak megfelelően
# visszadja, hogy False
# opt_advisor_return = opt_advisor.run(config.nn_filename, last = _last)
# Felmerült az a probléma, hogy néha nem kapunk elfogadható mintát
# a Sample API-ban, ezért két változót átadok az Advisornak
# akkor is ha ezek nem lesznek letárolva az adatok között
opt_advisor_return = opt_advisor.run(config.nn_filename, vm_number, target_variable, last = _last, training_result = training_result)
logger.info('----------------------------------------------------------')
logger.info('------------------ opt_advisor_return --------------------')
logger.info(f'opt_advisor_return with message: {opt_advisor_return}')
logger.info('------------------ opt_advisor_return --------------------')
logger.info('----------------------------------------------------------')
logger.info('Get Advice recieved and processed.')
return opt_advisor_return
def sample():
constants = opt_utils.read_yaml('data/constants.yaml')
logger.info('-------------------------- YAML --------------------------')
logger.info(f'Constants received: {constants}')
logger.info('-------------------------- YAML --------------------------')
sample_yaml = request.stream.read()
if not sample_yaml:
raise RequestException(400, 'Empty POST data')
else:
sample = yaml.safe_load(sample_yaml)
logger.info(f'New sample received: {sample}')
logger.info('')
logger.info('--------------------------------------------------------------')
logger.info(' Getting sample data... ')
logger.info('--------------------------------------------------------------')
input_metrics = [metric.get('value')
for metric in sample.get('sample').get('input_metrics')]
target_metrics = [metric.get('value')
for metric in sample.get('sample').get('target_metrics')]
global target_variable
target_variable = target_metrics
# Ez volt a jó megoldás, de Józsi valmiért a Polcy Keeperben a sample-n kívül küldi el a vm_number értéket
# ezért ezt most átírom, csak egy próba ereéig
global vm_number
vm_number = sample.get('sample').get('vm_number')
# vm_number = sample.get('vm_number')
timestamp_col = [sample.get('sample').get('timestamp')]
logger.info('')
logger.info('Sample data is going to be stored in corresponding variables.')
logger.info('--------------------------------------------------------------')
logger.info(f' input_metrics = {input_metrics}')
logger.info(f' target_metrics = {target_metrics}')
logger.info(f' vm_number = {vm_number}')
logger.info(f' timestamp_col = {timestamp_col}')
logger.info(f' target_variable = {target_variable}')
# logger.info(f' np.isnan(target_metrics[0]) = {np.isnan(target_metrics[0])}') # can cause error if array is empty or None
logger.info(' ----------------------- sample -----------------------')
logger.info(f' {sample.get("sample")}')
logger.info(' ----------------------- sample -----------------------')
logger.info('')
logger.info('--------------------------------------------------------------')
# if None not in timestamp_col+input_metrics+target_metrics+[vm_number]:
# if( len(input_metrics) != 0 and len(target_metrics) != 0 and None not in timestamp_col+input_metrics+target_metrics+[vm_number]):
if( len(input_metrics) == len(constants.get('input_metrics')) and len(target_metrics) != 0 and None not in timestamp_col+input_metrics+target_metrics+[vm_number] and np.isnan(target_metrics[0]) == False):
logger.info('----------------------------------------------')
logger.info('Sample accepted.')
logger.info('----------------------------------------------')
logger.info(f' (constants.get("target_metrics") = {constants.get("target_metrics")}')
logger.info(f' len(target_metrics) = {len(target_metrics)}')
logger.info(f' len(constants.get("target_metrics")) = {len(constants.get("target_metrics"))}')
logger.info('----------------------------------------------')
# itt csak beolvassa a csv fájlt és csinál belőle egy data framet
df = opt_utils.readCSV(config.nn_filename)
logger.info('----------------------------------------------')
logger.info(' df = opt_utils.readCSV(config.nn_filename) ')
logger.info(f' pandas dataframe df.columns = {df.columns}')
logger.info('----------------------------------------------')
# Hozzáfűzöm az új adatokat a beolvasott csv-ből csinált data framehez
tmp_df = df.append(pd.Series(timestamp_col+input_metrics+[vm_number]+target_metrics, index=df.columns ), ignore_index=True)
# Elmenteni ezt a tmp_df pandas dataframet ugyan abba a csv fájlba
tmp_df.to_csv(config.nn_filename, sep=',', encoding='utf-8', index=False)
logger.info('----------------------------------------------')
logger.info("tmp_df.to_csv(config.nn_filename, sep=',', encoding='utf-8', index=False)'")
logger.info('Data has been added and saved to csv file')
logger.info('----------------------------------------------')
# ## ------------------------------------------------------------------------------------------------------
# ## Start Training
# ## ------------------------------------------------------------------------------------------------------
# Ha egy megadott számnál hosszabb a dataframe akkor kezdje el a tanítást
logger.info(f'tmp_df.shape = {tmp_df.shape}')
logger.info(f'tmp_df.shape[0] = {tmp_df.shape[0]}')
_min_training = constants.get('training_samples_required')
#TODO:
#Itt van egy kis diszkrepancia
#Mivel a trainer maga végez egy kis adattisztitást, lehet, hogy itt 300-ra van állítva a df érték, de a trainer maga
#dobja a null sorokat, ezért leheet, hogy ott kisebb lesz a df szám amikor elkezd tanulni (mivel itt a df 300 ott a null
#dobása után mondjuk 243
logger.info('----------------------------------------------')
logger.info(f'Now we have rows = {tmp_df.shape[0]}')
logger.info(f'Minimum number when training start = {constants.get("training_samples_required")}')
logger.info(f'Minimum number when training start = {_min_training}')
logger.info('----------------------------------------------')
if( tmp_df.shape[0] >= _min_training ):
# TODO:
# Kivezetni hogy hány mintánként tanuljon
# Comment: Nehogy már minden körben tanítsuk
if( tmp_df.shape[0] % learning_round == 0 ):
logger.info('----------------------------------------------')
logger.info(f'Now we have rows = {tmp_df.shape[0]}')
logger.info('We have enough data to start learning')
logger.info('----------------------------------------------')
logger.info('----------------------------------------------')
logger.info('Learning NN and Linear Regression Phase')
logger.info('----------------------------------------------')
global training_result
logger.info('----------------------------------------------')
logger.info('opt_trainer.run(config.nn_filename)')
logger.info('----------------------------------------------')
training_result = opt_trainer.run(config.nn_filename, visualize = False)
logger.info(f'\n\nTraining result = {training_result}\n')
if( training_result[0] == 'No error' ):
global is_reportable
is_reportable = True
logger.info(f' is_reportable = {is_reportable}\n')
else:
logger.info('----------------------------------------------')
logger.info('There is not enough data for start learning')
logger.info('----------------------------------------------')
logger.info('----------------------------------------------')
logger.info('Samples received and processed.')
logger.info('----------------------------------------------')
else:
logger.info('----------------------------------------------')
logger.info('Sample ignored cause it contains empty value.')
logger.info('----------------------------------------------')
return jsonify('OK'), 200