Ejemplo n.º 1
0
    adapt_nnet_spec = arguments['adapt_nnet_spec'];
    wdir = arguments['wdir']
    init_model_file = arguments['init_model']

    # parse network configuration from arguments, and initialize data reading
    cfg_si = NetworkConfig(); cfg_si.model_type = 'CNN'
    cfg_si.parse_config_cnn(arguments, '10:' + si_nnet_spec, si_conv_nnet_spec)
    cfg_si.init_data_reading(train_data_spec, valid_data_spec)

    # parse the structure of the i-vector network 
    cfg_adapt = NetworkConfig()
    net_split = adapt_nnet_spec.split(':')
    adapt_nnet_spec = ''
    for n in range(len(net_split) - 1):
        adapt_nnet_spec += net_split[n] + ':'
    cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
    log('> ... initializing the model')
    # setup up the model 
    dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt)
    # read the initial DNN  (the SI DNN which has been well trained)
#    _file2nnet(dnn.cnn_si.layers, filename = init_model_file)
    _file2nnet(dnn.cnn_si.layers, filename = 'BKUP/nnet.param.si')
    _file2nnet(dnn.dnn_adapt.layers, filename = 'BKUP/nnet.param.adapt')

    # get the training and  validation functions for adaptation network training
    dnn.params = dnn.dnn_adapt.params  # only update the parameters of the i-vector nnet
    dnn.delta_params = dnn.dnn_adapt.delta_params
    log('> ... getting the finetuning functions for iVecNN')
Ejemplo n.º 2
0
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg)
            exit(1)

    # mandatory arguments
    train_data_spec = arguments['train_data']
    valid_data_spec = arguments['valid_data']
    si_nnet_spec = arguments['si_nnet_spec']
    adapt_nnet_spec = arguments['adapt_nnet_spec']
    wdir = arguments['wdir']
    init_model_file = arguments['init_model']

    # parse network configuration from arguments, and initialize data reading
    cfg_si = NetworkConfig()
    cfg_si.parse_config_dnn(arguments, si_nnet_spec)
    cfg_si.init_data_reading(train_data_spec, valid_data_spec)
    # parse the structure of the i-vector network
    cfg_adapt = NetworkConfig()
    #    net_split = adapt_nnet_spec.split(':')
    #    adapt_nnet_spec = ''
    #    for n in xrange(len(net_split) - 1):
    #        adapt_nnet_spec += net_split[n] + ':'
    #    cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
    cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0')

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2**30))
    log('> ... initializing the model')
    # setup up the model
    dnn = DNN_SAT(numpy_rng=numpy_rng,
Ejemplo n.º 3
0
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg); exit(1)

    # mandatory arguments
    train_data_spec = arguments['train_data']
    valid_data_spec = arguments['valid_data']
    nnet_spec = arguments['nnet_spec']
    wdir = arguments['wdir']
    multi_label = arguments['multi_label']
    if multi_label=="true": multi_label = True
    else:   multi_label = False

    # parse network configuration from arguments, and initialize data reading
    cfg = NetworkConfig(multi_label)
    cfg.parse_config_dnn(arguments, nnet_spec)
    cfg.init_data_reading(train_data_spec, valid_data_spec)

    # parse pre-training options
    # pre-training files and layer number (how many layers are set to the pre-training parameters)
    ptr_layer_number = 0; ptr_file = ''
    if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
        ptr_file = arguments['ptr_file']
        ptr_layer_number = int(arguments['ptr_layer_number'])

    # check working dir to see whether it's resuming training
    resume_training = False
    if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
        resume_training = True
        cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
        log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
Ejemplo n.º 4
0
    
    # parse data specification
    train_data_spec_array = parse_data_spec_mtl(train_data_spec)
    valid_data_spec_array = parse_data_spec_mtl(valid_data_spec)
    if len(train_data_spec_array) != task_number or len(valid_data_spec_array) != task_number:
        print len(train_data_spec_array)
        print task_number
        print "Error: #datasets in data specification doesn't match #tasks"; exit(1)
    # split shared_spec ans indiv_spec into individual task's networks
    nnet_spec_array, shared_layers_num = parse_nnet_spec_mtl(shared_spec, indiv_spec)   
    if len(nnet_spec_array) != task_number:
        print "Error: #networks specified by --indiv-spec doesn't match #tasks"; exit(1)
    # parse network configuration from arguments, and initialize data reading
    for n in xrange(task_number):
        network_config = NetworkConfig()
        network_config.parse_config_dnn(arguments, nnet_spec_array[n])
        network_config.init_data_reading(train_data_spec_array[n], valid_data_spec_array[n]) 
        config_array.append(network_config) 

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
    resume_training = False; resume_tasks = []  # if we are resuming training, then MLT only operates on the terminated tasks
    for n in xrange(task_number):
        log('> ... building the model for task %d' % (n))
        cfg = config_array[n]
        # set up the model
        dnn_shared = None; shared_layers = []
        if n > 0:
            dnn_shared = dnn_array[0]; shared_layers = [m for m in xrange(shared_layers_num)]
        print shared_layers
        dnn = DNN_MTL(task_id=n,numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg,
Ejemplo n.º 5
0
    arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
    arguments = parse_arguments(arg_elements)
    required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg); exit(1)

    # mandatory arguments
    train_data_spec = arguments['train_data']
    valid_data_spec = arguments['valid_data']
    nnet_spec = arguments['nnet_spec']
    wdir = arguments['wdir']

    # parse network configuration from arguments, and initialize data reading
    cfg = NetworkConfig()
    cfg.parse_config_dnn(arguments, nnet_spec)
    cfg.init_data_reading(train_data_spec, valid_data_spec)

    # parse pre-training options
    # pre-training files and layer number (how many layers are set to the pre-training parameters)
    ptr_layer_number = 0; ptr_file = ''
    if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
        ptr_file = arguments['ptr_file']
        ptr_layer_number = int(arguments['ptr_layer_number'])

    # check working dir to see whether it's resuming training
    resume_training = False
    if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
        resume_training = True
        cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
        log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
Ejemplo n.º 6
0
    wdir = arguments['wdir']
    init_model_file = arguments['init_model']

    # parse network configuration from arguments, and initialize data reading
    cfg_si = NetworkConfig()
    cfg_si.model_type = 'CNN'
    cfg_si.parse_config_cnn(arguments, '10:' + si_nnet_spec, si_conv_nnet_spec)
    cfg_si.init_data_reading(train_data_spec, valid_data_spec)

    # parse the structure of the i-vector network
    cfg_adapt = NetworkConfig()
    net_split = adapt_nnet_spec.split(':')
    adapt_nnet_spec = ''
    for n in xrange(len(net_split) - 1):
        adapt_nnet_spec += net_split[n] + ':'
    cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2**30))
    log('> ... initializing the model')
    # setup up the model
    dnn = CNN_SAT(numpy_rng=numpy_rng,
                  theano_rng=theano_rng,
                  cfg_si=cfg_si,
                  cfg_adapt=cfg_adapt)
    # read the initial DNN  (the SI DNN which has been well trained)
    #    _file2nnet(dnn.cnn_si.layers, filename = init_model_file)
    _file2nnet(dnn.cnn_si.layers, filename='BKUP/nnet.param.si')
    _file2nnet(dnn.dnn_adapt.layers, filename='BKUP/nnet.param.adapt')

    # get the training and  validation functions for adaptation network training
Ejemplo n.º 7
0
    arguments = parse_arguments(arg_elements)
    required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'nnet_spec_tower1', 'nnet_spec_tower2', 'wdir']
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg); exit(1)

    # mandatory arguments
    train_data_spec = arguments['train_data']
    valid_data_spec = arguments['valid_data']
    nnet_spec = arguments['nnet_spec']
    nnet_spec_tower1 = arguments['nnet_spec_tower1']
    nnet_spec_tower2 = arguments['nnet_spec_tower2']
    wdir = arguments['wdir']

    # parse network configuration from arguments, and initialize data reading
    cfg_tower1 = NetworkConfig(); cfg_tower1.parse_config_dnn(arguments, nnet_spec_tower1 + ":0")
    cfg_tower2 = NetworkConfig(); cfg_tower2.parse_config_dnn(arguments, nnet_spec_tower2 + ":0")
    cfg = NetworkConfig(); cfg.parse_config_dnn(arguments, str(cfg_tower1.hidden_layers_sizes[-1] + cfg_tower2.hidden_layers_sizes[-1]) + ":" + nnet_spec)
    cfg.init_data_reading(train_data_spec, valid_data_spec)

    # parse pre-training options
    # pre-training files and layer number (how many layers are set to the pre-training parameters)
    ptr_layer_number = 0; ptr_file = ''
    if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
        ptr_file = arguments['ptr_file']
        ptr_layer_number = int(arguments['ptr_layer_number'])

    # check working dir to see whether it's resuming training
    resume_training = False
    if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
        resume_training = True
Ejemplo n.º 8
0
    arguments = parse_arguments(arg_elements)
    required_arguments = ['train_data', 'valid_data', 'si_nnet_spec', 'wdir', 'adapt_nnet_spec', 'init_model']
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg); exit(1)

    # mandatory arguments
    train_data_spec = arguments['train_data']; valid_data_spec = arguments['valid_data']
    si_nnet_spec = arguments['si_nnet_spec']
    adapt_nnet_spec = arguments['adapt_nnet_spec'];
    wdir = arguments['wdir']
    init_model_file = arguments['init_model']

    # parse network configuration from arguments, and initialize data reading
    cfg_si = NetworkConfig()
    cfg_si.parse_config_dnn(arguments, si_nnet_spec)
    cfg_si.init_data_reading(train_data_spec, valid_data_spec)   
    # parse the structure of the i-vector network 
    cfg_adapt = NetworkConfig()
#    net_split = adapt_nnet_spec.split(':')
#    adapt_nnet_spec = ''
#    for n in xrange(len(net_split) - 1):
#        adapt_nnet_spec += net_split[n] + ':'
#    cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
    cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0')

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
    log('> ... initializing the model')
    # setup up the model 
    dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt)
Ejemplo n.º 9
0
def dnn_run(arguments):
    
    required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg); exit(1)
    train_data_spec = arguments['train_data']
    valid_data_spec = arguments['valid_data']
    nnet_spec = arguments['nnet_spec']
    wdir = arguments['wdir']
    cfg = NetworkConfig()
    cfg.parse_config_dnn(arguments, nnet_spec)
    cfg.init_data_reading(train_data_spec, valid_data_spec)
    
    # parse pre-training options
    # pre-training files and layer number (how many layers are set to the pre-training parameters)
    ptr_layer_number = 0; ptr_file = ''
    if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
        ptr_file = arguments['ptr_file']
        ptr_layer_number = int(arguments['ptr_layer_number'])

    # check working dir to see whether it's resuming training
    resume_training = False
    if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
        resume_training = True
        cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
        log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
    log('> ... building the model')
    # setup model
    if cfg.do_dropout:
        dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
    else:
        dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)

    # initialize model parameters
    # if not resuming training, initialized from the specified pre-training file
    # if resuming training, initialized from the tmp model file
    if (ptr_layer_number > 0) and (resume_training is False):
        _file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file)
    if resume_training:
        _file2nnet(dnn.layers, filename = wdir + '/nnet.tmp')

    # get the training, validation and testing function for the model
    log('> ... getting the finetuning functions')
    train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
                batch_size=cfg.batch_size)

    log('> ... finetuning the model')
    while (cfg.lrate.get_rate() != 0):
        # one epoch of sgd training
        train_error = train_sgd(train_fn, cfg)
        log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) + '(%)')
        # validation
        valid_error = validate_by_minibatch(valid_fn, cfg)
        log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100*numpy.mean(valid_error)) + '(%)')
        cfg.lrate.get_next_rate(current_error = 100*numpy.mean(valid_error))
        # output nnet parameters and lrate, for training resume
        if cfg.lrate.epoch % cfg.model_save_step == 0:
            _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp')
            _lrate2file(cfg.lrate, wdir + '/training_state.tmp')

    # save the model and network configuration
    if cfg.param_output_file != '':
        _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor)
        log('> ... the final PDNN model parameter is ' + cfg.param_output_file)
    if cfg.cfg_output_file != '':
        _cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
        log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
Ejemplo n.º 10
0
def dnn_run(arguments):

    required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
    for arg in required_arguments:
        if arguments.has_key(arg) == False:
            print "Error: the argument %s has to be specified" % (arg)
            exit(1)
    train_data_spec = arguments['train_data']
    valid_data_spec = arguments['valid_data']
    nnet_spec = arguments['nnet_spec']
    wdir = arguments['wdir']
    cfg = NetworkConfig()
    cfg.parse_config_dnn(arguments, nnet_spec)
    cfg.init_data_reading(train_data_spec, valid_data_spec)

    # parse pre-training options
    # pre-training files and layer number (how many layers are set to the pre-training parameters)
    ptr_layer_number = 0
    ptr_file = ''
    if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
        ptr_file = arguments['ptr_file']
        ptr_layer_number = int(arguments['ptr_layer_number'])

    # check working dir to see whether it's resuming training
    resume_training = False
    if os.path.exists(wdir +
                      '/nnet.tmp') and os.path.exists(wdir +
                                                      '/training_state.tmp'):
        resume_training = True
        cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
        log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch '
            + str(cfg.lrate.epoch))

    numpy_rng = numpy.random.RandomState(89677)
    theano_rng = RandomStreams(numpy_rng.randint(2**30))
    log('> ... building the model')
    # setup model
    if cfg.do_dropout:
        dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
    else:
        dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)

    # initialize model parameters
    # if not resuming training, initialized from the specified pre-training file
    # if resuming training, initialized from the tmp model file
    if (ptr_layer_number > 0) and (resume_training is False):
        _file2nnet(dnn.layers,
                   set_layer_num=ptr_layer_number,
                   filename=ptr_file)
    if resume_training:
        _file2nnet(dnn.layers, filename=wdir + '/nnet.tmp')

    # get the training, validation and testing function for the model
    log('> ... getting the finetuning functions')
    train_fn, valid_fn = dnn.build_finetune_functions(
        (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
        batch_size=cfg.batch_size)

    log('> ... finetuning the model')
    while (cfg.lrate.get_rate() != 0):
        # one epoch of sgd training
        train_error = train_sgd(train_fn, cfg)
        log('> epoch %d, training error %f ' %
            (cfg.lrate.epoch, 100 * numpy.mean(train_error)) + '(%)')
        # validation
        valid_error = validate_by_minibatch(valid_fn, cfg)
        log('> epoch %d, lrate %f, validation error %f ' %
            (cfg.lrate.epoch, cfg.lrate.get_rate(),
             100 * numpy.mean(valid_error)) + '(%)')
        cfg.lrate.get_next_rate(current_error=100 * numpy.mean(valid_error))
        # output nnet parameters and lrate, for training resume
        if cfg.lrate.epoch % cfg.model_save_step == 0:
            _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp')
            _lrate2file(cfg.lrate, wdir + '/training_state.tmp')

    # save the model and network configuration
    if cfg.param_output_file != '':
        _nnet2file(dnn.layers,
                   filename=cfg.param_output_file,
                   input_factor=cfg.input_dropout_factor,
                   factor=cfg.dropout_factor)
        log('> ... the final PDNN model parameter is ' + cfg.param_output_file)
    if cfg.cfg_output_file != '':
        _cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
        log('> ... the final PDNN model config is ' + cfg.cfg_output_file)