def load_data(batch_size=128):
'''
Loads the dataset
'''
path_src = '../data/vocab_and_data_sub_europarl/binarized_sub_europarl-v7.fr-en.en.h5'
path_trg = '../data/vocab_and_data_sub_europarl/binarized_sub_europarl-v7.fr-en.fr.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size,
path_trg,
path_src,
use_infinite_loop=False,
queue_size=20)
valid = HomogenousData(batch_size,
path_trg,
path_src,
use_infinite_loop=False,
queue_size=20)
test = HomogenousData(batch_size,
path_trg,
path_src,
use_infinite_loop=False,
queue_size=20)
return train, valid, test
def load_data(batch_size=128):
'''
Loads the dataset
'''
train_path_src = '../data/vocab_and_data_small_europarl_v7_enfr/bin_train.en.h5'
train_path_trg = '../data/vocab_and_data_small_europarl_v7_enfr/bin_train.fr.h5'
valid_path_src = '../data/vocab_and_data_small_europarl_v7_enfr/bin_valid.en.h5'
valid_path_trg = '../data/vocab_and_data_small_europarl_v7_enfr/bin_valid.fr.h5'
test_path_src = '../data/vocab_and_data_small_europarl_v7_enfr/bin_test.en.h5'
test_path_trg = '../data/vocab_and_data_small_europarl_v7_enfr/bin_test.fr.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size,
train_path_trg,
train_path_src,
use_infinite_loop=False)
valid = HomogenousData(batch_size,
valid_path_trg,
valid_path_src,
use_infinite_loop=False)
test = HomogenousData(batch_size,
test_path_trg,
test_path_src,
use_infinite_loop=False)
return train, valid, test
def load_data(batch_size=128):
'''
Loads the dataset
'''
path_src = '../data/data_vocab_europarl_en_de_h5/bin_train_de.shuf.h5'
path_trg = '../data/data_vocab_europarl_en_de_h5/bin_train_en.shuf.h5'
path_src_valid = '../data/data_vocab_europarl_en_de_h5/bin_valid_de.h5'
path_trg_valid = '../data/data_vocab_europarl_en_de_h5/bin_valid_en.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size,
path_trg,
path_src,
use_infinite_loop=False)
valid = HomogenousData(batch_size,
path_trg_valid,
path_src_valid,
use_infinite_loop=False)
test = None
return train, valid, test
def load_data(batch_size=128):
'''
Loads the dataset
'''
train_path_src = '../../data/vocab_and_data_wmt_all_enfr/binarized_wmt_all.en.h5'
train_path_trg = '../../data/vocab_and_data_wmt_all_enfr/binarized_wmt_all.fr.h5'
valid_path_src = '../../data/vocab_and_data_wmt_all_enfr/binarized_ntst1213.en.h5'
valid_path_trg = '../../data/vocab_and_data_wmt_all_enfr/binarized_ntst1213.fr.h5'
test_path_src = '../../data/vocab_and_data_wmt_all_enfr/binarized_ntst14.en.h5'
test_path_trg = '../../data/vocab_and_data_wmt_all_enfr/binarized_ntst14.fr.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size,
train_path_trg,
train_path_src,
use_infinite_loop=False)
valid = HomogenousData(batch_size,
valid_path_trg,
valid_path_src,
use_infinite_loop=False)
test = HomogenousData(batch_size,
test_path_trg,
test_path_src,
use_infinite_loop=False)
return train, valid, test
def load_data(batch_size=128):
'''
Loads the dataset
'''
path_src = '/data/lisatmp3/chokyun/transliteration/TranslitDataset/binarized_text.en.shuf.h5'
path_trg = '/data/lisatmp3/chokyun/transliteration/TranslitDataset/binarized_text.hi.shuf.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size, path_trg, path_src, use_infinite_loop=True)
valid = None
test = None
return train, valid, test
def load_data(batch_size=128):
'''
Loads the dataset
'''
path_src = './openmt15/binarized_text.zh.shuf.h5'
path_trg = './openmt15/binarized_text.en.shuf.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size,
path_trg,
path_src,
use_infinite_loop=False)
valid = None
test = None
return train, valid, test
def load_data(batch_size=128):
'''
Loads the dataset
'''
path_src = '/data/lisatmp3/firatorh/nmt/zh-en_lm/trainedModels/union/binarized_text.zh.shuf.h5'
path_trg = '/data/lisatmp3/firatorh/nmt/zh-en_lm/trainedModels/union/binarized_text.en.shuf.h5'
#############
# LOAD DATA #
#############
print '... initializing data iterators'
train = HomogenousData(batch_size,
path_trg,
path_src,
use_infinite_loop=False)
valid = None
test = None
return train, valid, test