def test_glob(s3_mock): s3 = boto3.resource('s3') s3.create_bucket(Bucket='test-bucket') object_summary = s3.ObjectSummary('test-bucket', 'directory/Test.test') object_summary.put(Body=b'test data') assert list(S3Path('/test-bucket/').glob('*.test')) == [] assert list(S3Path('/test-bucket/directory/').glob('*.test')) == [S3Path('/test-bucket/directory/Test.test')] assert list(S3Path('/test-bucket/').glob('**/*.test')) == [S3Path('/test-bucket/directory/Test.test')] object_summary = s3.ObjectSummary('test-bucket', 'pathlib.py') object_summary.put(Body=b'test data') object_summary = s3.ObjectSummary('test-bucket', 'setup.py') object_summary.put(Body=b'test data') object_summary = s3.ObjectSummary('test-bucket', 'test_pathlib.py') object_summary.put(Body=b'test data') object_summary = s3.ObjectSummary('test-bucket', 'docs/conf.py') object_summary.put(Body=b'test data') object_summary = s3.ObjectSummary('test-bucket', 'build/lib/pathlib.py') object_summary.put(Body=b'test data') assert sorted(S3Path.from_uri('s3://test-bucket/').glob('*.py')) == [ S3Path('/test-bucket/pathlib.py'), S3Path('/test-bucket/setup.py'), S3Path('/test-bucket/test_pathlib.py')] assert sorted(S3Path.from_uri('s3://test-bucket/').glob('*/*.py')) == [S3Path('/test-bucket/docs/conf.py')] assert sorted(S3Path.from_uri('s3://test-bucket/').glob('**/*.py')) == [ S3Path('/test-bucket/build/lib/pathlib.py'), S3Path('/test-bucket/docs/conf.py'), S3Path('/test-bucket/pathlib.py'), S3Path('/test-bucket/setup.py'), S3Path('/test-bucket/test_pathlib.py')] assert sorted(S3Path.from_uri('s3://test-bucket/').glob('*cs')) == [ S3Path('/test-bucket/docs/'), ]
def _route_path(*args, endpoint_url: str) -> Union[Path, S3Path]: """use S3Path or Path depending on the input. """ if str(args[0]).startswith("s3://"): s3path._s3_accessor.s3 = boto3.resource("s3", endpoint_url=endpoint_url) return S3Path.from_uri(*args) elif isinstance(args[0], S3Path): s3path._s3_accessor.s3 = boto3.resource("s3", endpoint_url=endpoint_url) return S3Path(*args) else: return Path(*args)
def extract_records(records): keys = [] prefixes = [] for record in records: handle = None if 'receiptHandle' in record: handle = record['receiptHandle'] logger.debug(f"message {record}") if record['eventSource'] == 'aws:sqs': # pipeline comes through record['body'] as text body = json.loads(record['body']) logger.debug(f"body {body}") message = json.loads(body['Message']) logger.debug(f"message {message}") bucket = message['bucket'] key = message['key'].strip('/') if record['eventSource'] == 'aws:sns': # pipeline comes through record['body'] as text message = json.loads(record['message']) bucket = message['bucket'] key = message['key'].strip('/') elif record['eventSource'] == 'aws:s3': # pipeline is the bucket/key bucket = record['s3']['bucket']['name'] key = record['s3']['object']['key'] key = key.lstrip('/') uri = os.path.join('s3://', bucket, key) key = S3Path.from_uri(uri) prefix = Prefix(key) if prefix not in prefixes: prefixes.append(prefix) return prefixes
def notify(event, context): info = get_job_info(context) event['info'] = info # keep this around for our event logger.debug(f'notifying for job {info}') # Notify Slack # Check if we have a slack-hook tag on our lambda slack_url = info['slack'] if slack_url: notify_slack(slack_url, event, context) message = event bucket = message['detail']['parameters']['bucketname'] collect = message['detail']['parameters']['collectname'] output = message['detail']['parameters']['outputname'] collect = collect.lstrip('/') uri = os.path.join('s3://', bucket, collect, 'settings.yaml') settings = S3Path.from_uri(uri) # if set have a settings file and there is an array # of email addresses in the notifications top-level key, send email logger.debug(f'checking prefix for job {settings}') if settings.is_file(): with settings.open() as f: config = yaml.load(f, Loader=yaml.FullLoader) logger.debug(f'fetched yaml config {config}') # if no notifications list set in the config, we skip try: config['notifications'] except KeyError: return notify_email(config['notifications'], event, context)
def __init__(self, uri): if isinstance(uri, S3Path): self.uri = uri.parent else: key = S3Path.from_uri(uri) self.uri = key.parent