def _configure_backing_store(self): try: backing_stores = [] for bs in self.config['Backing Store']: if 'Type' in bs: for key, item in bs.items(): bs[key] = _get_from_env(item) if bs['Type'].lower() == 's3': backing_stores.append(S3FS( bs['Bucket'], strict=False, aws_access_key_id=bs.get('Key ID', None), aws_secret_access_key=bs.get('Secret Key', None), endpoint_url=bs.get('Endpoint URL', None) )) elif 'dav' in bs['Type'].lower(): if not webdav_available: raise exceptions.NoWebdav("no webdavfs module was found") if bs['Root'][0] != '/': bs['Root'] = '/' + bs['Root'] backing_stores.append(WebDAVFS( url=bs['Base URL'], login=bs['Username'], password=bs['Password'], root=bs['Root'] )) else: _config_error("Unknown filesystem type.") else: backing_stores.append(fs.open_fs(bs['URI'], create=True)) except (KeyError, OSError, CreateFailed) as err: _config_error(err) return backing_stores
def load_s3_filesystem(path, strict=False, config=None): """ Loads AWS s3 filesystem from a path :param path: A path to a folder on s3 bucket that will be the base folder in this filesystem :type path: str :param strict: If `True` the filesystem will be making additional checks to the s3. Default is `False`. :type strict: bool :param config: A configuration object with AWS credentials. By default is set to None and in this case the default configuration will be taken. :type config: SHConfig or None :return: A S3 filesystem object :rtype: fs_s3fs.S3FS """ if not path.startswith('s3://'): raise ValueError( "AWS path has to start with s3:// but found '{}'".format(path)) if config is None: config = SHConfig() path_chunks = path.split('/', 3)[2:] bucket_name = path_chunks[0] dir_path = path_chunks[1] if len(path_chunks) > 1 else '/' return S3FS(bucket_name=bucket_name, dir_path=dir_path, aws_access_key_id=config.aws_access_key_id if config.aws_access_key_id else None, aws_secret_access_key=config.aws_secret_access_key if config.aws_secret_access_key else None, strict=strict)
def test_upload_args(self): s3 = S3FS("foo", acl="acl", cache_control="cc") self.assertDictEqual( s3._get_upload_args("test.jpg"), { "ACL": "acl", "CacheControl": "cc", "ContentType": "image/jpeg" }, ) self.assertDictEqual( s3._get_upload_args("test.mp3"), { "ACL": "acl", "CacheControl": "cc", "ContentType": "audio/mpeg" }, ) self.assertDictEqual( s3._get_upload_args("test.json"), { "ACL": "acl", "CacheControl": "cc", "ContentType": "application/json" }, ) self.assertDictEqual( s3._get_upload_args("unknown.unknown"), { "ACL": "acl", "CacheControl": "cc", "ContentType": "binary/octet-stream" }, )
def get_services(**options): """Instantiate an S3 filesystem service for loading and saving files from the ETL.""" return { 'fs': S3FS(options["bucket"], aws_access_key_id=options["key"], aws_secret_access_key=options["secret_key"], endpoint_url=options["endpoint_url"],) }
def get_s3fs(namespace): """ Helper method to get_filesystem for a file system on S3 """ key_id = DJFS_SETTINGS.get('aws_access_key_id', None) key_secret = DJFS_SETTINGS.get('aws_secret_access_key', None) fullpath = namespace if 'prefix' in DJFS_SETTINGS: fullpath = os.path.join(DJFS_SETTINGS['prefix'], fullpath) s3fs = S3FS(DJFS_SETTINGS['bucket'], fullpath, aws_secret_access_key=key_secret, aws_access_key_id=key_id, acl=DJFS_SETTINGS.get('acl', None)) def get_s3_url(self, filename, timeout=60): """ Patch method to returns a signed S3 url for the given filename Note that this will return a url whether or not the requested file exsits. Arguments: self (obj): S3FS instance that this function has been patched onto filename (str): The name of the file we are retrieving a url for timeout (int): How long the url should be valid for; S3 enforces this limit Returns: str: A signed url to the requested file in S3 """ global S3CONN try: if not S3CONN: S3CONN = S3Connection(aws_access_key_id=key_id, aws_secret_access_key=key_secret) return S3CONN.generate_url(timeout, 'GET', bucket=DJFS_SETTINGS['bucket'], key=os.path.join(fullpath, filename)) except Exception: # pylint: disable=broad-except # Retry on error; typically, if the connection has timed out, but # the broad except covers all errors. S3CONN = S3Connection(aws_access_key_id=key_id, aws_secret_access_key=key_secret) return S3CONN.generate_url(timeout, 'GET', bucket=DJFS_SETTINGS['bucket'], key=os.path.join(fullpath, filename)) s3fs = patch_fs(s3fs, namespace, get_s3_url) return s3fs
def _resolve_neural_files_bom(neural_files_or_bom: list = None): """ This function is typically used internally by map_video_to_neural_and_sleep_state(...), Use save_neural_files_bom to create the CSV once. This function resolves a list of neural filenames, or a CSV bill of materials containing a list of the neural files with their sizes and ecube timestamps to a list of (ecube_time, file_size, neural_filename) :param neural_files_or_bom: a list of neural files (non-globs), or a list of a single CSV file which is the bill of materials (BOM) CSV file containing a list of all neural data files in format: ecube_time, file_size, neural_filename :return: list in the form [(ecube_time, file_size, neural_filename), (...), ...] """ assert neural_files_or_bom is not None and len( neural_files_or_bom) > 0, 'No neural files found.' uses_s3 = any([f.startswith('s3://') for f in neural_files_or_bom]) if uses_s3: _verify_s3_support() if len(neural_files_or_bom) == 1 and neural_files_or_bom[0].endswith( '.csv'): with open(neural_files_or_bom[0], 'r') as csv_file: csv_reader = csv.reader(csv_file) result = [tuple(row) for row in csv_reader] else: result = [] for nfile in neural_files_or_bom: if nfile.startswith('s3://'): o = urllib.parse.urlparse(nfile) bucket = o.netloc key = o.path with S3FS(bucket, endpoint_url=os.environ.get('ENDPOINT_URL', None), strict=False) as s3fs: s3f = s3fs.openbin(key) ecube_time = np.frombuffer(s3f.read(8), dtype=np.uint64)[0] file_size = s3f.size else: with open(nfile, 'rb') as f: ecube_time = np.fromfile(f, dtype=np.uint64, count=1)[0] file_size = os.fstat(f.fileno()).st_size filename = os.path.split(nfile)[-1] result.append((ecube_time, file_size, filename)) return result
def _create_new_s3_fs(): """Creates a new empty mocked s3 bucket. If one such bucket already exists it deletes it first.""" bucket_name = "mocked-test-bucket" s3resource = boto3.resource("s3", region_name="eu-central-1") bucket = s3resource.Bucket(bucket_name) if bucket.creation_date: # If bucket already exists for key in bucket.objects.all(): key.delete() bucket.delete() s3resource.create_bucket( Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "eu-central-1"}) return S3FS(bucket_name=bucket_name)
def _resolve_glob(file_glob): if file_glob.startswith('s3://'): _verify_s3_support() o = urllib.parse.urlparse(file_glob) bucket = o.netloc key = o.path s3fs = S3FS(bucket, endpoint_url=os.environ.get('ENDPOINT_URL', None), strict=False) result = [ 's3://{}{}'.format(bucket, match.path) for match in s3fs.glob(key) ] else: result = glob.glob(file_glob) return result
def __enter__(self) -> S3FS: AwsSessionHook.__enter__(self) self.bucket = self.conn_params.extra['bucket'] self.base_path = self.conn_params.extra.get('base_path') if self.conn_params.login and self.conn_params.password: kwargs = { 'aws_access_key_id': self.conn_params.login, 'aws_secret_access_key': self.conn_params.password } elif self.session: # Get session token client = self.session.client('sts') session_token = client.get_session_token() kwargs = {'aws_session_token': session_token} else: kwargs = {} self.conn = S3FS(self.bucket, dir_path=self.base_path, **kwargs) return self.conn
def _configure_backing_store(self): try: bs = self.config['Backing Store'] if 'Type' in bs: for key, item in bs.items(): bs[key] = _get_from_env(item) if bs['Type'].lower() == 's3': return S3FS( bs['Bucket'], strict=False, aws_access_key_id=bs.get('Key ID', None), aws_secret_access_key=bs.get('Secret Key', None), endpoint_url=bs.get('Endpoint URL', None) ) else: return fs.open_fs(bs['URI'], create=True) except (KeyError, OSError, CreateFailed) as err: _config_error(err)
def open_fs(fs_url, **kwargs): """Open a pyfs filesystem. Like fs.open_fs will simply return FS if an instance if given as the fs_url parameter. """ if isinstance(fs_url, fs.base.FS): return fs_url # Now assume a string that may be a path (no ://) or else a filesystem URL if "://" not in fs_url: # A path, assume this is not URI escaped which is what the OSFS(..) # creator assumes (as opposed to open_fs(..)) return OSFS(fs_url, **kwargs) # We have a URL, parse it parse_result = fs.opener.parse(fs_url) if parse_result.protocol == 's3': # And S3 URL, mostly repeat # https://github.com/PyFilesystem/s3fs/blob/master/fs_s3fs/opener.py # but adjust the handling of strict to default to strict=False bucket_name, _, dir_path = parse_result.resource.partition("/") if not bucket_name: raise fs.opener.errors.OpenerError( "invalid bucket name in '{}'".format(fs_url)) # Instead of allowing this to be turned on by a strict=1 in the # URL query params, allow it to be turned off by strict!=1 strict = (parse_result.params["strict"] == "1" if "strict" in parse_result.params else False) s3fs = S3FS(bucket_name, dir_path=dir_path or "/", aws_access_key_id=parse_result.username or None, aws_secret_access_key=parse_result.password or None, endpoint_url=parse_result.params.get("endpoint_url", None), acl=parse_result.params.get("acl", None), cache_control=parse_result.params.get( "cache_control", None), strict=strict) # Patch in version of getinfo method that doesn't check parent directory s3fs.getinfo = s3fs._getinfo # pylint: disable=protected-access return s3fs # Non-S3 URL return fs.open_fs(fs_url, **kwargs)
def ocfl_opendir(pyfs, dir, **kwargs): """Open directory while handling the case of S3 without directory objects. FIXME - DIRTY HACK """ if isinstance(pyfs, S3FS): # Hack for S3 because the standard opendir(..) fails when there # isn't a directory object (even with strict=False) new_dir_path = fs.path.join(pyfs.dir_path, dir) s3fs = S3FS( pyfs._bucket_name, # pylint: disable=protected-access dir_path=new_dir_path, aws_access_key_id=pyfs.aws_access_key_id, aws_secret_access_key=pyfs.aws_secret_access_key, endpoint_url=pyfs.endpoint_url, # acl=pyfs.acl, # cache_control=pyfs.cache_control), strict=pyfs.strict) # Patch in version of getinfo method that doesn't check parent directory s3fs.getinfo = s3fs._getinfo # pylint: disable=protected-access return s3fs # Not S3, just use regular opendir(..) return pyfs.opendir(dir, **kwargs)
def test_upload_args(self): s3 = S3FS('foo', acl='acl', cache_control='cc') self.assertDictEqual(s3._get_upload_args('test.jpg'), { 'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'image/jpeg' }) self.assertDictEqual(s3._get_upload_args('test.mp3'), { 'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'audio/mpeg' }) self.assertDictEqual(s3._get_upload_args('test.json'), { 'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'application/json' }) self.assertDictEqual( s3._get_upload_args('unknown.unknown'), { 'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'binary/octet-stream' })
def _open_fs(directory): if directory.startswith("s3://"): """Manually fetch the permissions from the environment Requires the following env variables: - S3_ACCESS_KEY - S3_SECRET_KEY - S3_URL """ from fs_s3fs import S3FS if not directory.endswith("/"): directory += "/" bucket, fpath = directory[len("s3://"):].split("/", 1) return S3FS(bucket, dir_path=fpath, aws_access_key_id=os.environ.get("S3_ACCESS_KEY", None), aws_secret_access_key=os.environ.get( 'S3_SECRET_KEY', None), strict=False, endpoint_url=os.environ.get('S3_URL', None)) else: return open_fs(directory)
def load_s3_filesystem(path: str, strict: bool = False, config: Optional[SHConfig] = None, aws_profile: Optional[str] = None) -> S3FS: """Loads AWS s3 filesystem from a path. :param path: A path to a folder on s3 bucket that will be the base folder in this filesystem :type path: str :param strict: If `True` the filesystem will be making additional checks to the s3. Default is `False`. :type strict: bool :param config: A configuration object with AWS credentials. By default is set to None and in this case the default configuration will be taken. :type config: SHConfig or None :param aws_profile: A name of AWS profile. If given, AWS credentials will be taken from there. :return: A S3 filesystem object :rtype: fs_s3fs.S3FS """ if not is_s3_path(path): raise ValueError( f"AWS path has to start with s3:// but found '{path}'.") config = config or SHConfig() if aws_profile: config = get_aws_credentials(aws_profile, config=config) path_chunks = path.split("/", 3)[2:] bucket_name = path_chunks[0] dir_path = path_chunks[1] if len(path_chunks) > 1 else "/" return S3FS( bucket_name=bucket_name, dir_path=dir_path, aws_access_key_id=config.aws_access_key_id or None, aws_secret_access_key=config.aws_secret_access_key or None, aws_session_token=config.aws_session_token or None, strict=strict, )
def test_path_to_key_subdir(self): s3 = S3FS("foo", "/dir") self.assertEqual(s3._path_to_key("foo.bar"), "dir/foo.bar") self.assertEqual(s3._path_to_key("foo/bar"), "dir/foo/bar")
def test_path_to_key_subdir(self): s3 = S3FS('foo', '/dir') self.assertEqual(s3._path_to_key('foo.bar'), 'dir/foo.bar') self.assertEqual(s3._path_to_key('foo/bar'), 'dir/foo/bar')
def test_path_to_key(self): s3 = S3FS('foo') self.assertEqual(s3._path_to_key('foo.bar'), 'foo.bar') self.assertEqual(s3._path_to_key('foo/bar'), 'foo/bar')
def make_fs(self): self._delete_bucket_contents() self.s3.Object(self.bucket_name, 'subdirectory').put() return S3FS(self.bucket_name, dir_path='subdirectory')
def make_fs(self): self._delete_bucket_contents() return S3FS(self.bucket_name)
config = get_aws_credentials("default", config=default_config) assert config.aws_access_key_id != default_config.aws_access_key_id assert config.aws_secret_access_key != default_config.aws_secret_access_key @pytest.mark.parametrize( argnames="path_parts, expected_path", ids=["local", "s3"], argvalues=[ (["/tmp", "folder", "xyz", "..", "file.json"], os.path.join("/tmp", "folder", "file.json")), (["s3://xx/", "/y/z", "a", "..", "b.json"], "s3://xx/y/z/b.json"), ], ) def test_join_path(path_parts, expected_path): assert join_path(*path_parts) == expected_path @pytest.mark.parametrize( "filesystem, path, expected_full_path", [ (OSFS("/tmp"), "my/folder", "/tmp/my/folder"), (S3FS(bucket_name="data", dir_path="/folder"), "/sub/folder", "s3://data/folder/sub/folder"), (S3FS(bucket_name="data"), "/sub/folder", "s3://data/sub/folder"), ], ) def test_get_full_path(filesystem, path, expected_full_path): full_path = get_full_path(filesystem, path) assert full_path == expected_full_path
def prepare_filesystem(config: BaseConfig) -> S3FS: return S3FS(bucket_name=config.bucket_name, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key, region=config.aws_region)
def _open_fs(self, user_context): props = self._serialization_props(user_context) handle = S3FS(**props) return handle
from fs_s3fs import S3FS s3fs = S3FS(u'fsexample') print(s3fs) with s3fs.openbin(u'test.bin', u'w') as f: f.write(b'a') f.write(b'b') f.write(b'c') print s3fs.getinfo(u'test.bin', namespaces=['s3']).raw import io f = io.BytesIO(b'Hello, World') s3fs.setbinfile(u'b', f) print(s3fs.geturl(u'b')) s3fs.makedir(u'foo', recreate=True) print(s3fs.geturl(u'/foo')) s3fs.settext(u'/foo/bar', u'Hello') s3fs = S3FS(u'fsexample', dir_path='foo') print(s3fs) print(s3fs._prefix) print(s3fs.listdir(u'/')) print(s3fs._path_to_dir_key(u'/')) print(s3fs._path_to_dir_key(u'')) print(s3fs._path_to_dir_key(u'bar')) print(s3fs._path_to_dir_key(u'/bar')) # f = s3fs.openbin(u'newfile', 'ab')
def test_path_to_key(self): s3 = S3FS("foo") self.assertEqual(s3._path_to_key("foo.bar"), "foo.bar") self.assertEqual(s3._path_to_key("foo/bar"), "foo/bar")
def _prepare_filesystem(sampling_config: SamplingConfig) -> S3FS: return S3FS(bucket_name=sampling_config.bucket_name, aws_access_key_id=sampling_config.aws_access_key_id, aws_secret_access_key=sampling_config.aws_secret_access_key, region=sampling_config.aws_region)
import os import json import datetime from fs_s3fs import S3FS from airtable import Airtable from airtable_local_backup import restore base = os.environ['ATDB'] table_name = os.environ['TABLE'] dokey = os.environ['DOKEY'] dosecret = os.environ['DOSECRET'] endpoint_url = os.environ['URL'] bucket = os.environ['BUCKET'] prefix = 'testrestore-{}/'.format(datetime.datetime.now()) table = Airtable(base_key=base, table_name=table_name) space = S3FS(bucket, endpoint_url=endpoint_url, aws_access_key_id=dokey, aws_secret_access_key=dosecret) with open('tests/lots_of_fields.json', 'r') as jsonfile: tabledata = json.load(jsonfile) records = restore.prepare_records(tabledata, s3fs=space, check_integrity=True, prefix=prefix) for rec in records: table.insert(rec)