def __init__(self, uri, config_path=None):
parse_result = generous_parse_uri(uri)
self.bucket = parse_result.netloc
uuid = parse_result.path[1:]
self.uuid = uuid
ecs_endpoint = get_config_value("DTOOL_ECS_ENDPOINT_{}".format(
self.bucket))
ecs_access_key_id = get_config_value(
"DTOOL_ECS_ACCESS_KEY_ID_{}".format(self.bucket))
ecs_secret_access_key = get_config_value(
"DTOOL_ECS_SECRET_ACCESS_KEY_{}".format(self.bucket))
if not ecs_endpoint:
raise RuntimeError(
"No ECS endpoint specified for bucket '{bucket}', "
"please set DTOOL_ECS_ENDPOINT_{bucket}.".format(
bucket=self.bucket))
if not ecs_access_key_id:
raise RuntimeError(
"No ECS access key id specified for bucket '{bucket}', "
"please set DTOOL_ECS_ACCESS_KEY_ID_{bucket}.".format(
bucket=self.bucket))
if not ecs_secret_access_key:
raise RuntimeError(
"No ECS secret access key specified for bucket '{bucket}', "
"please set DTOOL_ECS_SECRET_ACCESS_KEY_{bucket}.".format(
bucket=self.bucket))
session = Session(aws_access_key_id=ecs_access_key_id,
aws_secret_access_key=ecs_secret_access_key)
self.s3resource = session.resource('s3',
endpoint_url=ecs_endpoint,
config=BOTO3_CONFIG)
self.s3client = session.client('s3',
endpoint_url=ecs_endpoint,
config=BOTO3_CONFIG)
self._structure_parameters = _ECS_STRUCTURE_PARAMETERS
self.dataset_registration_key = 'dtool-{}'.format(self.uuid)
self._structure_parameters[
"dataset_registration_key"] = self.dataset_registration_key # NOQA
self.data_key_prefix = self._generate_key_prefix("data_key_infix")
self.fragments_key_prefix = self._generate_key_prefix(
"fragment_key_infix")
self.overlays_key_prefix = self._generate_key_prefix(
"overlays_key_infix")
self.annotations_key_prefix = self._generate_key_prefix(
"annotations_key_infix")
self.http_manifest_key = self._generate_key("http_manifest_key")
self._s3_cache_abspath = get_config_value("DTOOL_CACHE_DIRECTORY",
config_path=config_path,
default=DEFAULT_CACHE_PATH)
def list_dataset_uris(cls, base_uri, config_path):
"""Return list containing URIs with base URI."""
uri_list = []
parse_result = generous_parse_uri(base_uri)
bucket = parse_result.netloc
ecs_endpoint = get_config_value("DTOOL_ECS_ENDPOINT_{}".format(bucket))
ecs_access_key_id = get_config_value(
"DTOOL_ECS_ACCESS_KEY_ID_{}".format(bucket))
ecs_secret_access_key = get_config_value(
"DTOOL_ECS_SECRET_ACCESS_KEY_{}".format(bucket))
session = Session(aws_access_key_id=ecs_access_key_id,
aws_secret_access_key=ecs_secret_access_key)
resource = session.resource('s3',
endpoint_url=ecs_endpoint,
config=BOTO3_CONFIG)
parse_result = generous_parse_uri(base_uri)
bucket_name = parse_result.netloc
bucket = resource.Bucket(bucket_name)
for obj in bucket.objects.filter(Prefix='dtool').all():
uuid = obj.key.split('-', 1)[1]
uri = cls.generate_uri(None, uuid, base_uri)
storage_broker = cls(uri, config_path)
if storage_broker.has_admin_metadata():
uri_list.append(uri)
return uri_list
def _get_resource_and_client(cls, bucket_name):
# Get S3 endpoint, access key and secret key. Can be left
# unconfigured, in which case the AWS configuration is used.
s3_endpoint = get_config_value(
"DTOOL_S3_ENDPOINT_{}".format(bucket_name))
s3_access_key_id = get_config_value(
"DTOOL_S3_ACCESS_KEY_ID_{}".format(bucket_name))
s3_secret_access_key = get_config_value(
"DTOOL_S3_SECRET_ACCESS_KEY_{}".format(bucket_name))
if (s3_endpoint is not None or s3_access_key_id is not None
or s3_secret_access_key is not None):
# We can only proceed if all three of endpoint, access key id and
# secret access key are configure
if (not s3_endpoint or not s3_access_key_id
or not s3_secret_access_key):
raise RuntimeError(
"If you want to configure your S3 endpoint for bucket "
"'{bucket}' via the dtool config file, please set "
"DTOOL_S3_ENDPOINT_{bucket}, "
"DTOOL_S3_ACCESS_KEY_ID_{bucket} and "
"DTOOL_S3_SECRET_ACCESS_KEY_{bucket}.".format(
bucket=bucket_name))
session = Session(aws_access_key_id=s3_access_key_id,
aws_secret_access_key=s3_secret_access_key)
s3resource = session.resource('s3', endpoint_url=s3_endpoint)
s3client = session.client('s3', endpoint_url=s3_endpoint)
else:
s3resource = boto3.resource('s3')
s3client = boto3.client('s3')
return s3resource, s3client
def __init__(self, uri, config_path=None):
parse_result = generous_parse_uri(uri)
self.bucket = parse_result.netloc
uuid = parse_result.path[1:]
self.dataset_prefix = get_config_value("DTOOL_S3_DATASET_PREFIX")
self.uuid = uuid
self.s3resource, self.s3client = \
self._get_resource_and_client(self.bucket)
self._structure_parameters = _STRUCTURE_PARAMETERS
self.dataset_registration_key = 'dtool-{}'.format(self.uuid)
self._structure_parameters[
"dataset_registration_key"] = self.dataset_registration_key # NOQA
self.data_key_prefix = self._generate_key_prefix("data_key_infix")
self.fragments_key_prefix = self._generate_key_prefix(
"fragment_key_infix")
self.overlays_key_prefix = self._generate_key_prefix(
"overlays_key_infix")
self.annotations_key_prefix = self._generate_key_prefix(
"annotations_key_infix")
self.tags_key_prefix = self._generate_key_prefix("tags_key_infix")
self.http_manifest_key = self._generate_key("http_manifest_key")
self._s3_cache_abspath = get_config_value("DTOOL_CACHE_DIRECTORY",
config_path=config_path,
default=DEFAULT_CACHE_PATH)
def test_get_config_value_from_file(tmp_dir_fixture): # NOQA
from dtoolcore.utils import get_config_value, get_config_value_from_file
config_path = os.path.join(tmp_dir_fixture, "my.conf")
# Test when config file is missing.
assert get_config_value_from_file(
key="MY_KEY",
config_path=config_path
) is None
config = {"MY_KEY": "from_file"}
with open(config_path, "w") as fh:
json.dump(config, fh)
# Test when config file exists.
assert "from_file" == get_config_value_from_file(
key="MY_KEY",
config_path=config_path
)
# Test use in get_config_value function.
value = get_config_value(
key="MY_KEY",
config_path=config_path,
default="hello"
)
assert value == "from_file"
def __init__(self, uri, config_path=None):
self._abspath = os.path.abspath(uri)
self._dtool_abspath = os.path.join(self._abspath, '.dtool')
self._admin_metadata_fpath = os.path.join(self._dtool_abspath, 'dtool')
self._data_abspath = os.path.join(self._abspath, 'data')
self._manifest_abspath = os.path.join(
self._dtool_abspath,
'manifest.json'
)
self._readme_abspath = os.path.join(
self._abspath,
'README.yml'
)
self._overlays_abspath = os.path.join(
self._dtool_abspath,
'overlays'
)
self._metadata_fragments_abspath = os.path.join(
self._dtool_abspath,
'tmp_fragments'
)
self._irods_cache_abspath = get_config_value(
"DTOOL_IRODS_CACHE_DIRECTORY",
config_path=config_path,
default=os.path.expanduser("~/.cache/dtool/irods")
)
# Cache for optimisation
self._use_cache = False
self._ls_abspath_cache = {}
self._metadata_cache = {}
self._size_and_timestamp_cache = {}
self._metadata_dir_exists_cache = None
def __init__(self, uri, config_path=None):
parse_result = generous_parse_uri(uri)
self.storage_account_name = parse_result.netloc
uuid = parse_result.path[1:]
self.uuid = uuid
self.fragments_key_prefix = self._generate_key('fragments_key_prefix')
self.overlays_key_prefix = self._generate_key('overlays_key_prefix')
self.annotations_key_prefix = self._generate_key(
'annotations_key_prefix')
self.tags_key_prefix = self._generate_key('tags_key_prefix')
self.http_manifest_key = self._generate_key("http_manifest_key")
self._azure_cache_abspath = get_config_value(
"DTOOL_CACHE_DIRECTORY",
config_path=config_path,
default=DEFAULT_CACHE_PATH)
self._blobservice = get_blob_service(self.storage_account_name,
config_path)
def http_enable(self):
logger.debug("HTTP enable {}".format(self))
expiry = get_config_value("DTOOL_S3_PUBLISH_EXPIRY")
if expiry is not None:
if expiry != "":
try:
expiry = int(expiry)
except ValueError:
logger.error(
"DTOOL_S3_PUBLISH_EXPIRY must be set to a value that can be converted to an integer"
) # NOQA
raise (RuntimeError())
http_manifest = self._generate_http_manifest(expiry)
manifest_url = self._write_http_manifest(http_manifest, expiry) # NOQA
manifest_presignature = None
if "?" in manifest_url:
_, manifest_presignature = manifest_url.split("?")
access_url = self._url(self._get_prefix() + self.uuid)
if manifest_presignature is not None:
access_url = access_url + "?" + manifest_presignature
return access_url
def test_get_config_value():
from dtoolcore.utils import get_config_value
value = get_config_value(
key="MY_KEY",
config_path=None,
default="hello"
)
assert value == "hello"
def test_get_config_value_from_file(tmp_dir_fixture): # NOQA
from dtoolcore.utils import get_config_value
config = {"MY_KEY": "from_file"}
config_path = os.path.join(tmp_dir_fixture, "my.conf")
with open(config_path, "w") as fh:
json.dump(config, fh)
value = get_config_value(
key="MY_KEY",
config_path=config_path,
default="hello"
)
assert value == "from_file"
def __init__(self, uri, config_path=None):
parse_result = generous_parse_uri(uri)
self.config_name = parse_result.netloc
uuid = parse_result.path[1:]
self.uuid = uuid
# Connect to SMB server.
self.conn, self.service_name, self.path = \
SMBStorageBroker._connect(uri, config_path)
# Define some other more abspaths.
self._data_path = self._generate_path("data_directory")
self._overlays_path = self._generate_path("overlays_directory")
self._annotations_path = self._generate_path(
"annotations_directory"
)
self._tags_path = self._generate_path(
"tags_directory"
)
self._metadata_fragments_path = self._generate_path(
"metadata_fragments_directory"
)
# Define some essential directories to be created.
self._essential_subdirectories = [
self._generate_path("dtool_directory"),
self._data_path,
self._overlays_path,
self._annotations_path,
self._tags_path,
]
# Cache for file hashes computed on upload
self._hash_cache = {}
self._smb_cache_abspath = get_config_value(
"DTOOL_CACHE_DIRECTORY",
config_path=config_path,
default=DEFAULT_CACHE_PATH
)
def _connect(cls, uri, config_path):
parse_result = generous_parse_uri(uri)
config_name = parse_result.netloc
username = get_config_value(
"DTOOL_SMB_USERNAME_{}".format(config_name),
config_path=config_path
)
server_name = get_config_value(
"DTOOL_SMB_SERVER_NAME_{}".format(config_name),
config_path=config_path
)
server_port = get_config_value(
"DTOOL_SMB_SERVER_PORT_{}".format(config_name),
config_path=config_path
)
domain = get_config_value(
"DTOOL_SMB_DOMAIN_{}".format(config_name),
config_path=config_path
)
service_name = get_config_value(
"DTOOL_SMB_SERVICE_NAME_{}".format(config_name),
config_path=config_path
)
path = get_config_value(
"DTOOL_SMB_PATH_{}".format(config_name),
config_path=config_path
)
if not username:
raise RuntimeError("No username specified for service '{name}', "
"please set DTOOL_SMB_USERNAME_{name}."
.format(name=config_name))
if not server_name:
raise RuntimeError("No server name specified for service '{name}', "
"please set DTOOL_SMB_SERVER_NAME_{name}."
.format(name=config_name))
if not server_port:
raise RuntimeError("No server port specified for service '{name}', "
"please set DTOOL_SMB_SERVER_PORT_{name}."
.format(name=config_name))
if not domain:
raise RuntimeError("No domain specified for service '{name}', "
"please set DTOOL_SMB_DOMAIN_{name}."
.format(name=config_name))
if not service_name:
raise RuntimeError("No service name specified for service '{name}', "
"please set DTOOL_SMB_SERVICE_NAME_{name}. "
"(The service name is the name of the 'share'.)"
.format(name=config_name))
if not path:
raise RuntimeError("No path specified for service '{name}', "
"please set DTOOL_SMB_PATH_{name}."
.format(name=config_name))
# server_port might be string, i.e. if specified via env vars
if not isinstance(server_port, int):
server_port = int(server_port)
server_ip = socket.gethostbyname(server_name)
host_name = socket.gethostname()
password = get_config_value(
"DTOOL_SMB_PASSWORD_{}".format(config_name),
config_path=config_path
)
if password is None:
if cls._connect.num_calls == 1:
password = getpass.getpass()
cls.password = password
else:
password = cls.password
conn = SMBConnection(username, password, host_name, server_name,
domain=domain, use_ntlm_v2=True, is_direct_tcp=True)
logger.info( ( "Connecting from '{host:s}' to "
"'smb://{user:s}@{ip:s}({server:s}):{port:d}', "
"DOMAIN '{domain:s}'").format(user=username,
ip=server_ip, server=server_name,
port=server_port, host=host_name,
domain=domain) )
# for testing, see types of arguments
logger.debug( ( "Types HOST '{host:s}', USER '{user:s}', IP '{ip:s}', "
"SERVER '{server:s}', PORT '{port:s}', DOMAIN '{domain:s}'").format(
user=type(username).__name__,
ip=type(server_ip).__name__,
server=type(server_name).__name__,
port=type(server_port).__name__,
host=type(host_name).__name__,
domain=type(domain).__name__))
conn.connect(server_ip, port=server_port)
return conn, service_name, path
def get_azure_account_key(account_name, config_path):
"""Return the Azure account key associated with the account name."""
account_key = get_config_value("DTOOL_AZURE_ACCOUNT_KEY_" + account_name,
config_path=config_path)
return account_key
