Exemplo n.º 1
0
def train(material_id_or_tag, settings, allowcompleted):
    """ モデルを訓練する """
    material_id = wfutils.get_material_id(material_id_or_tag)
    mm = get_session().query(ModelMaterial).filter_by(id=material_id).first()
    if mm is None:
        raise Exception('material not found!')

    if mm.is_completed() and (not allowcompleted):
        raise Exception('already train completed.')

    sobj = build_material(material_id, settings, True)

    with session_scope() as session:
        # notify processing
        mm = session.query(ModelMaterial).filter_by(id=material_id).first()
        mm.status = ModelMaterial.Status.PROCESSING.value
        mm.updated_at = datetime.datetime.now()

    with session_scope() as session:
        # update meta
        mm = session.query(ModelMaterial).filter_by(id=material_id).first()
        mm.updated_at = datetime.datetime.now()
        mm.status = ModelMaterial.Status.COMPLETED.value

        # run
        sobj.train_all()
    logger.debug('train_all {} done.'.format(material_id))
Exemplo n.º 2
0
def train(material_id_or_tag, settings, allowcompleted):
    """ モデルを訓練する """
    material_id = wfutils.get_material_id(material_id_or_tag)
    mm = get_session().query(ModelMaterial).filter_by(id=material_id).first()
    if mm is None:
        raise Exception('material not found!')
    
    if mm.is_completed() and (not allowcompleted):
        raise Exception('already train completed.')

    sobj = build_material(material_id, settings, True)

    with session_scope() as session:
        # notify processing
        mm = session.query(ModelMaterial).filter_by(id=material_id).first()
        mm.status = ModelMaterial.Status.PROCESSING.value
        mm.updated_at = datetime.datetime.now()

    with session_scope() as session:
        # update meta
        mm = session.query(ModelMaterial).filter_by(id=material_id).first()
        mm.updated_at = datetime.datetime.now()
        mm.status = ModelMaterial.Status.COMPLETED.value

        # run
        sobj.train_all()
    logger.debug('train_all {} done.'.format(material_id))
Exemplo n.º 3
0
def list_materials(scenario_name):
    rosess = get_session()
    ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
    if ms is None:
        return []
    else:
        return ms.materials
Exemplo n.º 4
0
def materialize(scenario_name, scenario_path, mtpath):
    os.makedirs(mtpath, exist_ok=True)

    material_id = utils.get_hash(datetime.datetime.now().isoformat())

    spath = os.path.join(mtpath, material_id)
    if os.path.isfile(spath):
        raise ValueError('{} already exists.'.format(material_id))

    try:
        # update materialized meta
        rosess = get_session()
        ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
        with session_scope() as session:
            if ms is None:
                session.add(ModelScenario(name=scenario_name))
                ms = session.query(ModelScenario).filter_by(
                    name=scenario_name).first()
            session.add(
                ModelMaterial(scenario_id=ms.id,
                              id=material_id,
                              tag=material_id))
            logger.debug('meta information materialized done.')

            # create and sync scenario_dir
            shutil.copytree(scenario_path, spath, ignore=ignore_for_copytree)
            logger.debug('scenario {} materialized done.'.format(spath))
        return material_id

    except:
        shutil.rmtree(spath)
        logger.debug('remove materialized scenario {} done.'.format(spath))
        raise
Exemplo n.º 5
0
def materialize(scenario_name, scenario_path, mtpath):
    os.makedirs(mtpath, exist_ok=True)

    material_id = utils.get_hash(datetime.datetime.now().isoformat())

    spath = os.path.join(mtpath, material_id)
    if os.path.isfile(spath):
        raise ValueError('{} already exists.'.format(material_id))

    try:
        # update materialized meta
        rosess = get_session()
        ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
        with session_scope() as session:
            if ms is None:
                session.add(ModelScenario(name=scenario_name))
                ms = session.query(ModelScenario).filter_by(name=scenario_name).first()
            session.add(ModelMaterial(scenario_id=ms.id, id=material_id, tag=material_id))
            logger.debug('meta information materialized done.')

            # create and sync scenario_dir
            shutil.copytree(scenario_path, spath, ignore=ignore_for_copytree)
            logger.debug('scenario {} materialized done.'.format(spath))
        return material_id

    except:
        shutil.rmtree(spath)
        logger.debug('remove materialized scenario {} done.'.format(spath))
        raise 
Exemplo n.º 6
0
def list_materials(scenario_name):
    rosess = get_session()
    ms = rosess.query(ModelScenario).filter_by(name=scenario_name).first()
    if ms is None:
        return []
    else:
        return ms.materials
Exemplo n.º 7
0
def remove_scenarios(scenario_names, mtpath):
    rosess = get_session()
    targets = rosess.query(ModelScenario).filter(ModelScenario.name.in_(scenario_names))
    material_ids = list(itertools.chain.from_iterable([[mm.id for mm in ms.materials] for ms in targets]))

    with session_scope() as session:
        # remove materials
        remove_materials(material_ids, mtpath)

        # delete from db 
        session.query(ModelScenario).filter(ModelScenario.name.in_(scenario_names)).delete(synchronize_session='fetch')
Exemplo n.º 8
0
    def execute(self, namespace):
        if (not namespace.all) and namespace.target is None:
            raise Exception('target or all must be activated!')

        rosess = wfstate.get_session()
        if namespace.all:
            target_names = [t.name for t in rosess.query(wfstate.Scenario).all()]
        else:
            target_names = namespace.target.split(',')

        settings_general = self.settings['general']
        wfproject.remove_scenarios(target_names, settings_general['materialized_dir'])
Exemplo n.º 9
0
def create_scenarios(base_path, scenario_module_name):
    scenarios_path = os.path.join(base_path, scenario_module_name)
    scenario_modules = [m for m in os.listdir(scenarios_path) if os.path.isdir(os.path.join(scenarios_path, m))]

    rosess = get_session()
    ms = rosess.query(ModelScenario).filter(ModelScenario.name.in_(scenario_modules))
    s_exist = [m.name for m in ms]
    valid_modules = [m for m in scenario_modules if m not in s_exist]

    # create scenarios
    if len(valid_modules) > 0:
        with session_scope() as session:
            session.add_all([ModelScenario(name=m) for m in valid_modules])
    else:
        logger.debug('all scenarios seem to be already created.')
Exemplo n.º 10
0
    def execute(self, namespace):
        if (not namespace.all) and namespace.target is None:
            raise Exception('target or all must be activated!')

        rosess = wfstate.get_session()
        if namespace.all:
            target_names = [
                t.name for t in rosess.query(wfstate.Scenario).all()
            ]
        else:
            target_names = namespace.target.split(',')

        settings_general = self.settings['general']
        wfproject.remove_scenarios(target_names,
                                   settings_general['materialized_dir'])
Exemplo n.º 11
0
def remove_scenarios(scenario_names, mtpath):
    rosess = get_session()
    targets = rosess.query(ModelScenario).filter(
        ModelScenario.name.in_(scenario_names))
    material_ids = list(
        itertools.chain.from_iterable([[mm.id for mm in ms.materials]
                                       for ms in targets]))

    with session_scope() as session:
        # remove materials
        remove_materials(material_ids, mtpath)

        # delete from db
        session.query(ModelScenario).filter(
            ModelScenario.name.in_(scenario_names)).delete(
                synchronize_session='fetch')
Exemplo n.º 12
0
def build_material(material_id, settings, load_enabled):
    target_dir = os.path.join(settings['materialized_dir'], material_id)
    if os.path.isdir(target_dir):
        sys.path.append(settings['materialized_dir'])
        os.environ['WATARU_MATERIAL_ID'] = material_id
        smod = importlib.import_module(material_id + '.' + settings['scenario_entry_module_name'])
        # check status
        mm_all = get_session().query(ModelMaterial).filter_by(id=material_id)
        sobj = None
        if mm_all.count() > 0:
            mm = mm_all[0]
            sobj = getattr(smod, settings['scenario_entry_function_name'])(material_id, target_dir, mm.status).build_managers(load_enabled, True)
        else:
            raise Exception('unexpected error! .. material management system may be broken.')
        return sobj
    else:
        raise Exception('material not found!')
Exemplo n.º 13
0
def create_scenarios(base_path, scenario_module_name):
    scenarios_path = os.path.join(base_path, scenario_module_name)
    scenario_modules = [
        m for m in os.listdir(scenarios_path)
        if os.path.isdir(os.path.join(scenarios_path, m))
    ]

    rosess = get_session()
    ms = rosess.query(ModelScenario).filter(
        ModelScenario.name.in_(scenario_modules))
    s_exist = [m.name for m in ms]
    valid_modules = [m for m in scenario_modules if m not in s_exist]

    # create scenarios
    if len(valid_modules) > 0:
        with session_scope() as session:
            session.add_all([ModelScenario(name=m) for m in valid_modules])
    else:
        logger.debug('all scenarios seem to be already created.')
Exemplo n.º 14
0
def build_material(material_id, settings, load_enabled):
    target_dir = os.path.join(settings['materialized_dir'], material_id)
    if os.path.isdir(target_dir):
        sys.path.append(settings['materialized_dir'])
        os.environ['WATARU_MATERIAL_ID'] = material_id
        smod = importlib.import_module(material_id + '.' +
                                       settings['scenario_entry_module_name'])
        # check status
        mm_all = get_session().query(ModelMaterial).filter_by(id=material_id)
        sobj = None
        if mm_all.count() > 0:
            mm = mm_all[0]
            sobj = getattr(smod, settings['scenario_entry_function_name'])(
                material_id, target_dir,
                mm.status).build_managers(load_enabled, True)
        else:
            raise Exception(
                'unexpected error! .. material management system may be broken.'
            )
        return sobj
    else:
        raise Exception('material not found!')
Exemplo n.º 15
0
def list_scenarios():
    rosess = get_session()
    ms = rosess.query(ModelScenario).all()
    return ms
Exemplo n.º 16
0
def list_scenarios():
    rosess = get_session()
    ms = rosess.query(ModelScenario).all()
    return ms