prebuilt_path_model = prebuilt_model_params.get("model", None)
model_args = prebuilt_model_params.get("args", {})
embedder = WordVectorEmbedder(embedder_model, model_fullpath=prebuilt_path_model, model_args=model_args)
# update embedder parameters
if prebuilt_path_model:
model_path_dir, model_path_filename, model_path_filext = WordVectorBuilder.filename_components(
prebuilt_path_model
)
embedder.model_subset = model_path_filename
# training data (custom or default)
if prebuilt_model_params.get("train", None):
prebuilt_path_train = prebuilt_model_params.get("train")
else:
prebuilt_path_train = WordVectorBuilder.filename_train(prebuilt_path_model)
with open(prebuilt_path_train, "rb") as f:
data_train = pickle.load(f)
# testing data (custom or default)
if prebuilt_model_params.get("test", None):
prebuilt_path_test = prebuilt_model_params.get("test")
else:
prebuilt_path_test = WordVectorBuilder.filename_test(prebuilt_path_model)
with open(prebuilt_path_test, "rb") as f:
data_test = pickle.load(f)
# initialize lists (will be converted later into numpy arrays)
values_train = []
labels_train = []
values_test = []
klass = data_params['class']
loader = klass(data_params['path'])
data_args = data_params['args']
data = loader.load_data()
# initialize lists (will be converted later into numpy arrays)
values = []
labels = []
# initialize vector embedder
prebuilt_model_path = data_args.get('models', {}).get(embedder_model, {}).get('prebuilt_model_path', None)
embedder = WordVectorEmbedder(embedder_model, prebuilt_model_path)
# load pre-sampled data from disk
if prebuilt_model_path:
with open(WordVectorBuilder.filename_train(prebuilt_model_path), 'rb') as f:
data = pickle.load(f)
else:
# get equal-sized subsets of each class
min_samples = data_args['min_samples'] if data_args.has_key('min_samples') else None
data_sampler = DataSampler(klass, file_path=data_params['path'], num_classes=2)
data = data_sampler.sample_balanced(min_samples)
# load dataset
logger.info("processing {} samples from {}...".format(len(data), data_params['path']))
profile_results = timed_dataload(data, data_args, values, labels)
# store loading time
seconds_loading = profile_results.timer.total_tt
prebuilt_path_model = None
if isinstance(embedder_model, dict):
embedder_model, prebuilt_model_params = embedder_model.items().pop()
prebuilt_path_model = prebuilt_model_params.get('model')
# initialize word vector embedder
embedder = WordVectorEmbedder(embedder_model, prebuilt_path_model)
# load pre-sampled data from disk
if prebuilt_path_model:
# training data (custom or default)
if prebuilt_model_params.get('train', None):
prebuilt_path_train = prebuilt_model_params.get('train')
else:
prebuilt_path_train = WordVectorBuilder.filename_train(prebuilt_path_model)
# testing data (custom or default)
if prebuilt_model_params.get('test', None):
prebuilt_path_test = prebuilt_model_params.get('test')
else:
prebuilt_path_test = WordVectorBuilder.filename_test(prebuilt_path_model)
# import pickled data
with open(prebuilt_path_train, 'rb') as f:
data_train = pickle.load(f)
with open(prebuilt_path_test, 'rb') as f:
data_test = pickle.load(f)
# update embedder parameters
model_path_dir, model_path_filename, model_path_filext = WordVectorBuilder.filename_components(prebuilt_path_model)
loader = klass(data_params['path'])
data_args = data_params['args']
data = loader.load_data()
# initialize lists (will be converted later into numpy arrays)
values = []
labels = []
# initialize vector embedder
prebuilt_model_path = data_args.get('models', {}).get(
embedder_model, {}).get('prebuilt_model_path', None)
embedder = WordVectorEmbedder(embedder_model, prebuilt_model_path)
# load pre-sampled data from disk
if prebuilt_model_path:
with open(WordVectorBuilder.filename_train(prebuilt_model_path),
'rb') as f:
data = pickle.load(f)
else:
# get equal-sized subsets of each class
min_samples = data_args['min_samples'] if data_args.has_key(
'min_samples') else None
data_sampler = DataSampler(klass,
file_path=data_params['path'],
num_classes=2)
data = data_sampler.sample_balanced(min_samples)
# load dataset
logger.info("processing {} samples from {}...".format(
len(data), data_params['path']))
