def test_provider_factory_static_compute():
# Static compute
test_config = {'value': 'test'}
provider = create_provider(ProviderClass.compute.value, 'static', 'compute test', test_config, None)
assert provider is not None
assert provider.config == test_config
with pytest.raises(errors.ValidationError):
provider.validate()
provider.config = {}
provider.validate() # Only empty config is valid for static
def post(self):
# Creating a new new provider
payload = self.request.json
validators.validate_data(payload, 'provider.json', 'input', 'POST')
provider = create_provider(class_=payload['provider_class'],
type_=payload['provider_type'],
label=payload['label'],
config=payload['config'],
creds=payload['creds'])
provider.origin = self.origin
provider_id = insert_provider(provider)
return {'_id': provider_id}
def update_provider(provider_id, doc):
"""Update the given provider instance, with fields from doc.
Args:
provider_id (ObjectId|str): The provider id
doc (dict): The update fields
Raises:
APINotFoundException: If the provider does not exist.
APIValidationException: If the update would result in an invalid provider
configuration, or an invalid field was specified
(e.g. attempt to change provider type)
"""
mapper = mappers.Providers()
current_provider = mapper.get(provider_id)
if not current_provider:
raise errors.ResourceNotFound(provider_id,
'Provider {path} not found!')
# NOTE: We do NOT permit updating provider class or type
if 'provider_class' in doc:
raise errors.ValidationError('Cannot change provider class!')
if 'provider_type' in doc:
raise errors.ValidationError('Cannot change provider type!')
if 'label' in doc:
current_provider.label = doc['label']
# If we do it this way we can only ever update keys not delete them
if 'config' in doc:
current_provider.config = doc['config']
#for key in doc['config']:
# current_provider.config[key] = doc['config'][key]
current_provider.modifed = datetime.datetime.utcnow()
current_provider.validate()
if 'creds' in doc:
# Do full validation if the creds are changed to confirm they are correct
provider = create_provider(current_provider.provider_class,
current_provider.provider_type,
current_provider.label,
current_provider.config, doc['creds'],
provider_id)
provider.validate_permissions()
mapper.patch(provider_id, current_provider)
def test_provider_factory_storage(mocker):
# spy_fs = mocker.spy(storage, 'create_flywheel_fs')
mock_is_signed = mocker.patch('api.storage.py_fs.py_fs_storage.PyFsStorage.is_signed_url', return_value=True)
mock_get_signed = mocker.patch('api.storage.py_fs.py_fs_storage.PyFsStorage.get_signed_url', return_value='url')
mock_get_info = mocker.patch('api.storage.py_fs.py_fs_storage.PyFsStorage.get_file_info', return_value={'filesize': 100})
test_config = {'path': local_fs_url}
provider = create_provider(ProviderClass.storage.value, 'local', 'local test', test_config, None)
# The call to create_flywheel_fs is different instances.
# We can assume it was called if the storage_plugins is created. But it might have been called 2 or more times
# TODO: find a way to mock an interface function with spy.. the count does not get recorded
#assert storage.create_flywheel_fs.call_count == 1
assert provider is not None
assert provider.storage_plugin is not None
assert isinstance(provider, LocalStorageProvider)
assert provider.config == test_config
assert isinstance(provider.storage_plugin, PyFsStorage)
def _load_provider(self, doc):
"""Loads a single item from a mongo dictionary"""
if doc is None:
return None
# Remove site key
doc.pop('_site', None)
provider = create_provider(
class_=doc['provider_class'],
type_=doc['provider_type'],
label=doc['label'],
config=doc['config'],
creds=doc.get('creds'), #Creds is not required in local
id_=doc['_id'])
provider.origin = doc['origin']
provider.modified = doc['modified']
provider.created = doc['created']
provider.validate()
return provider
def main(*argv):
date_format = '%Y-%m-%d %H:%M:%S'
log_format = '%(asctime)s %(levelname)6.6s %(message)s'
logging.basicConfig(datefmt=date_format, format=log_format, level='info')
if not os.environ.get('SCITRAN_PERSISTENT_FS_URL'):
log.error(
'You must have a SCITRAN_PERSISTENT_FS_URL environment variable defined'
)
exit(1)
scheme, bucket_name, path, params = parse_storage_url(
os.environ['SCITRAN_PERSISTENT_FS_URL'])
if scheme == 's3':
config_ = {
'bucket': bucket_name,
'path': path,
'region': params.get('region', "")
}
creds = {
# IF these are not found the error will happen in create_provider
'aws_access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
'aws_secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY')
}
type_ = 'aws'
elif scheme == 'gc':
with open(params['private_key'], 'rU') as f:
creds = json.load(f)
config_ = {
"path": path,
"bucket": bucket_name,
}
if params.get('region'):
config_['region'] = params['region']
type_ = 'gc'
else:
# Local is a special case that uses no creds
config_ = {"path": os.environ['SCITRAN_PERSISTENT_FS_URL']}
creds = None
type_ = 'local'
# Create a provider for a quick sanity check to verify the values in the env var work
_ = create_provider(ProviderClass.storage.value, type_, 'Primary Provider',
config_, creds)
storage = db.providers.insert_one({
"origin": {
"type": "system",
"id": "system"
},
"created": datetime.datetime.now(),
"config": config_,
"creds": creds,
"modified": datetime.datetime.now(),
"label": "Primary Storage",
"provider_class": "storage",
"provider_type": type_
})
# We know its storage becase we just made it that way
# validate_provider_class(args.storage, ProviderClass.storage.value)
log.info('Setting storage provider: %s', storage.inserted_id)
update = {'$set': {'providers.storage': storage.inserted_id}}
db.singletons.update({'_id': 'site'}, update)
log.info('Providers have been modified')
def get_local_storage(self):
""" Local storage is a storage plugin that supports get_fs. But it will not clean up automatically"""
return create_provider(ProviderClass.storage.value, 'local',
'temp_storage', {'path': config.local_fs_url},
None)
def test_provider_factory_error():
# Non-existent storage
with pytest.raises(errors.ValidationError):
provider = create_provider(ProviderClass.storage.value, 'garbage', 'garbage test', {}, {})