def get_encoder(model_path):
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return ImageOnnxEncoder(output_feature='mobilenetv20_features_relu1_fwd',
model_path=model_path,
metas=metas)
def test_metas_workspace_replica_peas(tmpdir, replica_id, pea_id):
metas = get_default_metas()
metas['workspace'] = str(tmpdir)
metas['name'] = 'test'
metas['replica_id'] = replica_id
metas['pea_id'] = pea_id
return metas
def get_encoder():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return BigTransferEncoder(model_path='pretrained',
channel_axis=1,
metas=metas)
def metas(tmpdir):
os.environ['TEST_WORKSPACE'] = str(tmpdir)
metas = get_default_metas()
metas['workspace'] = os.environ['TEST_WORKSPACE']
metas['name'] = 'faiss_idx'
yield metas
del os.environ['TEST_WORKSPACE']
def test_metas(tmpdir, random_workspace_name):
os.environ[random_workspace_name] = str(tmpdir)
metas = get_default_metas()
metas['workspace'] = os.environ[random_workspace_name]
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
yield metas
del os.environ[random_workspace_name]
def test_metas(tmpdir, random_workspace_name):
from jina.executors.metas import get_default_metas
os.environ[random_workspace_name] = str(tmpdir)
metas = get_default_metas()
metas['workspace'] = os.environ[random_workspace_name]
yield metas
del os.environ[random_workspace_name]
def encoder(tmpdir):
model_path = 'models/vision/classification/mobilenet/model/mobilenetv2-7.onnx'
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
metas['workspace'] = str(tmpdir)
return ImageOnnxEncoder(output_feature='mobilenetv20_features_relu1_fwd',
model_path=model_path,
metas=metas)
def get_encoder():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
path = tempfile.NamedTemporaryFile().name
model = ExampleNet()
torch.save(model, path)
return CustomImageTorchEncoder(model_path=path,
layer_name='conv1',
metas=metas)
def get_encoder():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
path = tempfile.NamedTemporaryFile().name
model = TestNet().create_model()
model.save(path)
return CustomKerasImageEncoder(channel_axis=1,
model_path=path,
layer_name='dense',
metas=metas)
def get_encoder(model_path_tmp_dir):
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
metas['workspace'] = model_path_tmp_dir
path = os.path.join(model_path_tmp_dir, 'model.pth')
model = ExampleNet()
torch.save(model, path)
return CustomImageTorchEncoder(model_path=path,
layer_name='conv1',
metas=metas)
def get_encoder(model_path_tmp_dir):
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
metas['workspace'] = model_path_tmp_dir
path = os.path.join(model_path_tmp_dir, 'model.pth')
model = TestNet().create_model()
model.save(path)
return CustomKerasImageEncoder(channel_axis=1,
model_path=path,
layer_name='dense',
metas=metas)
def test_encoding_results(tmpdir):
target_output_dim = 512
batch_size = 10
signal_length = 1024
test_data = np.random.randn(batch_size, signal_length).astype('f')
metas = get_default_metas()
metas['workspace'] = str(tmpdir)
encoder = Wav2VecSpeechEncoder(model_path='/tmp/wav2vec_large.pt',
input_sample_rate=16000,
metas=metas)
encoded_data = encoder.encode(test_data)
assert encoded_data.shape[0] == batch_size
assert encoded_data.shape[1] % target_output_dim == 0
def test_save_load_config(tmp_path):
transforms = ['VerticalFlip', {'Resize': {'width': 200, 'height': 300}}]
metas = get_default_metas()
metas['workspace'] = str(tmp_path)
orig_crafter = AC(transforms, metas=metas)
orig_crafter.save_config()
orig_trs = orig_crafter.transforms._to_dict()
load_crafter1 = BaseExecutor.load_config('tests/config.yaml')
load_crafter2 = BaseExecutor.load_config(orig_crafter.config_abspath)
assert orig_trs == load_crafter1.transforms._to_dict()
assert orig_trs == load_crafter2.transforms._to_dict()
def test_save_load_config(tmp_path):
from jina.executors import BaseExecutor
from jina.executors.metas import get_default_metas
transforms = [{'RandomVerticalFlip': dict(p=1.0)}]
metas = get_default_metas()
metas['workspace'] = str(tmp_path)
orig_crafter = ImageTorchTransformation(transforms, metas=metas)
orig_crafter.save_config()
orig_trs = orig_crafter.transforms_specification
load_crafter1 = BaseExecutor.load_config(
os.path.join(cur_dir, '../tests/config.yaml'))
load_crafter2 = BaseExecutor.load_config(orig_crafter.config_abspath)
assert orig_trs == load_crafter1.transforms_specification
assert orig_trs == load_crafter2.transforms_specification
def get_encoder(*args, **kwargs):
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return VideoTorchEncoder(metas=metas)
def test_bad_metas_workspace(tmpdir):
metas = get_default_metas()
return metas
def test_metas_workspace_simple(tmpdir):
metas = get_default_metas()
metas['workspace'] = str(tmpdir)
metas['name'] = 'test'
return metas
def get_metas():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return metas
def test_metas(tmp_path):
metas = get_default_metas()
metas['workspace'] = str(tmp_path)
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
yield metas
from jina.executors import BaseExecutor
from jina.executors.metas import get_default_metas
def rm_files(tmp_files):
for k in tmp_files:
if os.path.exists(k):
if os.path.isfile(k):
os.remove(k)
elif os.path.isdir(k):
shutil.rmtree(k, ignore_errors=False, onerror=None)
os.environ['TEST_WORKDIR'] = os.getcwd()
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
encoders = [
TransformerTorchEncoder(
pretrained_model_name_or_path='bert-base-uncased',
model_save_path='bert-base-uncased',
metas=metas),
TransformerTorchEncoder(
pooling_strategy='mean',
pretrained_model_name_or_path='bert-base-uncased',
model_save_path='bert-base-uncased-mean',
metas=metas),
TransformerTorchEncoder(
pooling_strategy='min',
def test_set_is_trained_meta():
metas = get_default_metas()
metas['is_trained'] = True
executor = BaseExecutor(metas=metas)
assert executor.is_trained
def get_encoder():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return VideoPaddleEncoder(metas=metas)
def metas(tmpdir):
metas = get_default_metas()
metas['workspace'] = tmpdir
metas['dump_path'] = os.path.join(tmpdir, 'dump')
yield metas
def test_set_batch_size():
batch_size = 325
metas = get_default_metas()
metas['batch_size'] = batch_size
indexer = NumpyIndexer(index_filename=f'test.gz', metas=metas)
assert indexer.batch_size == batch_size
def get_encoder():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return ImageKerasEncoder(channel_axis=1, metas=metas)
def test_metas(tmpdir, random_workspace_name):
os.environ[random_workspace_name] = str(tmpdir)
metas = get_default_metas()
metas['workspace'] = os.environ[random_workspace_name]
yield metas
del os.environ[random_workspace_name]
def get_encoder():
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
return ImageTorchEncoder(metas=metas)
def metas(tmpdir):
metas = get_default_metas()
metas['workspace'] = str(tmpdir)
yield metas
def metas(tmpdir):
metas = get_default_metas()
if 'JINA_TEST_GPU' in os.environ:
metas['on_gpu'] = True
metas['workspace'] = str(tmpdir)
yield metas
def test_set_dummy_meta():
dummy = 325
metas = get_default_metas()
metas['dummy'] = dummy
executor = BaseExecutor(metas=metas)
assert executor.dummy == dummy
