def train(material_id_or_tag, settings, allowcompleted):
""" モデルを訓練する """
material_id = wfutils.get_material_id(material_id_or_tag)
mm = get_session().query(ModelMaterial).filter_by(id=material_id).first()
if mm is None:
raise Exception('material not found!')
if mm.is_completed() and (not allowcompleted):
raise Exception('already train completed.')
sobj = build_material(material_id, settings, True)
with session_scope() as session:
# notify processing
mm = session.query(ModelMaterial).filter_by(id=material_id).first()
mm.status = ModelMaterial.Status.PROCESSING.value
mm.updated_at = datetime.datetime.now()
with session_scope() as session:
# update meta
mm = session.query(ModelMaterial).filter_by(id=material_id).first()
mm.updated_at = datetime.datetime.now()
mm.status = ModelMaterial.Status.COMPLETED.value
# run
sobj.train_all()
logger.debug('train_all {} done.'.format(material_id))
def train(material_id_or_tag, settings, allowcompleted):
""" モデルを訓練する """
material_id = wfutils.get_material_id(material_id_or_tag)
mm = get_session().query(ModelMaterial).filter_by(id=material_id).first()
if mm is None:
raise Exception('material not found!')
if mm.is_completed() and (not allowcompleted):
raise Exception('already train completed.')
sobj = build_material(material_id, settings, True)
with session_scope() as session:
# notify processing
mm = session.query(ModelMaterial).filter_by(id=material_id).first()
mm.status = ModelMaterial.Status.PROCESSING.value
mm.updated_at = datetime.datetime.now()
with session_scope() as session:
# update meta
mm = session.query(ModelMaterial).filter_by(id=material_id).first()
mm.updated_at = datetime.datetime.now()
mm.status = ModelMaterial.Status.COMPLETED.value
# run
sobj.train_all()
logger.debug('train_all {} done.'.format(material_id))
def list_materials(scenario_name):
rosess = get_session()
ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
if ms is None:
return []
else:
return ms.materials
def materialize(scenario_name, scenario_path, mtpath):
os.makedirs(mtpath, exist_ok=True)
material_id = utils.get_hash(datetime.datetime.now().isoformat())
spath = os.path.join(mtpath, material_id)
if os.path.isfile(spath):
raise ValueError('{} already exists.'.format(material_id))
try:
# update materialized meta
rosess = get_session()
ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
with session_scope() as session:
if ms is None:
session.add(ModelScenario(name=scenario_name))
ms = session.query(ModelScenario).filter_by(
name=scenario_name).first()
session.add(
ModelMaterial(scenario_id=ms.id,
id=material_id,
tag=material_id))
logger.debug('meta information materialized done.')
# create and sync scenario_dir
shutil.copytree(scenario_path, spath, ignore=ignore_for_copytree)
logger.debug('scenario {} materialized done.'.format(spath))
return material_id
except:
shutil.rmtree(spath)
logger.debug('remove materialized scenario {} done.'.format(spath))
raise
def materialize(scenario_name, scenario_path, mtpath):
os.makedirs(mtpath, exist_ok=True)
material_id = utils.get_hash(datetime.datetime.now().isoformat())
spath = os.path.join(mtpath, material_id)
if os.path.isfile(spath):
raise ValueError('{} already exists.'.format(material_id))
try:
# update materialized meta
rosess = get_session()
ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
with session_scope() as session:
if ms is None:
session.add(ModelScenario(name=scenario_name))
ms = session.query(ModelScenario).filter_by(name=scenario_name).first()
session.add(ModelMaterial(scenario_id=ms.id, id=material_id, tag=material_id))
logger.debug('meta information materialized done.')
# create and sync scenario_dir
shutil.copytree(scenario_path, spath, ignore=ignore_for_copytree)
logger.debug('scenario {} materialized done.'.format(spath))
return material_id
except:
shutil.rmtree(spath)
logger.debug('remove materialized scenario {} done.'.format(spath))
raise
def list_materials(scenario_name):
rosess = get_session()
ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
if ms is None:
return []
else:
return ms.materials
def remove_scenarios(scenario_names, mtpath):
rosess = get_session()
targets = rosess.query(ModelScenario).filter(ModelScenario.name.in_(scenario_names))
material_ids = list(itertools.chain.from_iterable([[mm.id for mm in ms.materials] for ms in targets]))
with session_scope() as session:
# remove materials
remove_materials(material_ids, mtpath)
# delete from db
session.query(ModelScenario).filter(ModelScenario.name.in_(scenario_names)).delete(synchronize_session='fetch')
def execute(self, namespace):
if (not namespace.all) and namespace.target is None:
raise Exception('target or all must be activated!')
rosess = wfstate.get_session()
if namespace.all:
target_names = [t.name for t in rosess.query(wfstate.Scenario).all()]
else:
target_names = namespace.target.split(',')
settings_general = self.settings['general']
wfproject.remove_scenarios(target_names, settings_general['materialized_dir'])
def create_scenarios(base_path, scenario_module_name):
scenarios_path = os.path.join(base_path, scenario_module_name)
scenario_modules = [m for m in os.listdir(scenarios_path) if os.path.isdir(os.path.join(scenarios_path, m))]
rosess = get_session()
ms = rosess.query(ModelScenario).filter(ModelScenario.name.in_(scenario_modules))
s_exist = [m.name for m in ms]
valid_modules = [m for m in scenario_modules if m not in s_exist]
# create scenarios
if len(valid_modules) > 0:
with session_scope() as session:
session.add_all([ModelScenario(name=m) for m in valid_modules])
else:
logger.debug('all scenarios seem to be already created.')
def execute(self, namespace):
if (not namespace.all) and namespace.target is None:
raise Exception('target or all must be activated!')
rosess = wfstate.get_session()
if namespace.all:
target_names = [
t.name for t in rosess.query(wfstate.Scenario).all()
]
else:
target_names = namespace.target.split(',')
settings_general = self.settings['general']
wfproject.remove_scenarios(target_names,
settings_general['materialized_dir'])
def remove_scenarios(scenario_names, mtpath):
rosess = get_session()
targets = rosess.query(ModelScenario).filter(
ModelScenario.name.in_(scenario_names))
material_ids = list(
itertools.chain.from_iterable([[mm.id for mm in ms.materials]
for ms in targets]))
with session_scope() as session:
# remove materials
remove_materials(material_ids, mtpath)
# delete from db
session.query(ModelScenario).filter(
ModelScenario.name.in_(scenario_names)).delete(
synchronize_session='fetch')
def build_material(material_id, settings, load_enabled):
target_dir = os.path.join(settings['materialized_dir'], material_id)
if os.path.isdir(target_dir):
sys.path.append(settings['materialized_dir'])
os.environ['WATARU_MATERIAL_ID'] = material_id
smod = importlib.import_module(material_id + '.' + settings['scenario_entry_module_name'])
# check status
mm_all = get_session().query(ModelMaterial).filter_by(id=material_id)
sobj = None
if mm_all.count() > 0:
mm = mm_all[0]
sobj = getattr(smod, settings['scenario_entry_function_name'])(material_id, target_dir, mm.status).build_managers(load_enabled, True)
else:
raise Exception('unexpected error! .. material management system may be broken.')
return sobj
else:
raise Exception('material not found!')
def create_scenarios(base_path, scenario_module_name):
scenarios_path = os.path.join(base_path, scenario_module_name)
scenario_modules = [
m for m in os.listdir(scenarios_path)
if os.path.isdir(os.path.join(scenarios_path, m))
]
rosess = get_session()
ms = rosess.query(ModelScenario).filter(
ModelScenario.name.in_(scenario_modules))
s_exist = [m.name for m in ms]
valid_modules = [m for m in scenario_modules if m not in s_exist]
# create scenarios
if len(valid_modules) > 0:
with session_scope() as session:
session.add_all([ModelScenario(name=m) for m in valid_modules])
else:
logger.debug('all scenarios seem to be already created.')
def build_material(material_id, settings, load_enabled):
target_dir = os.path.join(settings['materialized_dir'], material_id)
if os.path.isdir(target_dir):
sys.path.append(settings['materialized_dir'])
os.environ['WATARU_MATERIAL_ID'] = material_id
smod = importlib.import_module(material_id + '.' +
settings['scenario_entry_module_name'])
# check status
mm_all = get_session().query(ModelMaterial).filter_by(id=material_id)
sobj = None
if mm_all.count() > 0:
mm = mm_all[0]
sobj = getattr(smod, settings['scenario_entry_function_name'])(
material_id, target_dir,
mm.status).build_managers(load_enabled, True)
else:
raise Exception(
'unexpected error! .. material management system may be broken.'
)
return sobj
else:
raise Exception('material not found!')
def list_scenarios():
rosess = get_session()
ms = rosess.query(ModelScenario).all()
return ms
def list_scenarios():
rosess = get_session()
ms = rosess.query(ModelScenario).all()
return ms