def create_dataframe_from_src(path: str,
typ: str,
extension: str,
storage_infos: dict = None) -> pandas.DataFrame:
from apyml.context import Context
context = Context()
reader_opts = context.get_from_config('readers_opts')
handler = Handler(reader_opts, storage_infos)
return handler.read_from_src(path, typ, extension)
def set_dataframe_hash_to_context(dataframe: df):
columns = list(dataframe.columns.values)
target = Context().get_from_config('dataset')['target']
if target in columns:
columns.remove(target)
dataframe_hash = merkle_root(columns)
Context().set(os.getpid(), dataframe_hash)
return
def run_build_directive(dataframe: pandas.DataFrame, job: dict):
name = job['name']
func = job['build_directive']
dest = f'{Context().get_store_path()}/{name}/{func}'
try:
info(f'Building model {name}...')
res = getattr(importlib.import_module('apyml.directives.directives'),
func)(dataframe.copy())
dataframe_hash = Context().get(os.getpid())
if not os.path.exists(dest):
os.makedirs(dest)
if isinstance(res, types.GeneratorType):
i = 0
for r in res:
pickle.dump(
r,
open(f"{dest}/{dataframe_hash}__STEP__{chr(ord('A') + i)}",
'wb'))
i += 1
else:
pickle.dump(res, open(f'{dest}/{dataframe_hash}', 'wb'))
info(f'Model {name} built [{ColorStatus.SUCCESS}]')
except Exception:
fatal(f'Building model {name} [{ColorStatus.FAILURE}]')
raise
def main() -> int:
try:
from . import ExitStatus
exit_status = ExitStatus.OK
from apyml.cli import parser
args = vars(parser.parse_args())
from apyml.internal.logging import init_logger
init_logger()
from apyml.context import Context
Context()
info(f'Context creation [{ColorStatus.SUCCESS}]')
except KeyboardInterrupt:
info('Keyboard interruption (ctrl+c).')
exit_status = ExitStatus.CTRL_C
raise
except Exception as e:
fatal(f'Core initialization [{ColorStatus.FAILURE}]')
fatal(e)
exit_status = ExitStatus.ERROR
raise
else:
try:
program(args=args)
except KeyboardInterrupt:
info('Keyboard interruption (ctrl+c).')
exit_status = ExitStatus.CTRL_C
raise
except Exception as e:
fatal(e)
exit_status = ExitStatus.ERROR
raise
return exit_status
def run_predict_directive(dataframe: pandas.DataFrame,
job: dict) -> pandas.DataFrame:
func = job['predict_directive']
model = job['model_name']
name = job['name']
path = f'{Context().get_store_path()}/{name}/{model}'
final = pandas.DataFrame(
index=[Context().get_from_config('dataset')['index']],
columns=[
Context().get_from_config('dataset')['target'], 'predictions'
])
try:
info(f'Running model(s)...')
ret = getattr(importlib.import_module('apyml.directives.directives'),
func)(dataframe, path)
def predictions_to_dataframe(true: pandas.Series,
preds) -> pandas.DataFrame:
df = true.to_frame()
df['predictions'] = preds
return df.copy()
if isinstance(ret, types.GeneratorType):
for r in ret:
final = pandas.concat(
[final, predictions_to_dataframe(r[0], r[1])])
else:
final = pandas.concat(
[final, predictions_to_dataframe(ret[0], ret[1])])
except Exception:
fatal(f'Running model(s)... [{ColorStatus.FAILURE}]')
raise
return final.copy()
def run(self):
tmp = dict(self._infos)
jobs = Context().get_from_config(self._mode)[self._mode]
process_number = len(jobs)
path, typ, ext = tmp['path'], tmp['type'], tmp['extension']
for k in ['path', 'type', 'extension']:
tmp.pop(k, None)
try:
self._dataframe = create_dataframe_from_src(path,
typ,
ext,
storage_infos={**tmp})
info(f'Dataframe creation [{ColorStatus.SUCCESS}]')
except Exception:
fatal(f'Dataframe creation [{ColorStatus.FAILURE}]')
raise
return
pool = Pool(processes=process_number)
pool.starmap(self._worker, [(job, ) for job in jobs])
pool.close()
def wrapper(dataframe: df, path: str) -> object:
set_dataframe_hash_to_context(dataframe)
return func(
dataframe,
Context().get_available_models(path,
Context().get(os.getpid())))