def test_glob(s3_mock):
s3 = boto3.resource('s3')
s3.create_bucket(Bucket='test-bucket')
object_summary = s3.ObjectSummary('test-bucket', 'directory/Test.test')
object_summary.put(Body=b'test data')
assert list(S3Path('/test-bucket/').glob('*.test')) == []
assert list(S3Path('/test-bucket/directory/').glob('*.test')) == [S3Path('/test-bucket/directory/Test.test')]
assert list(S3Path('/test-bucket/').glob('**/*.test')) == [S3Path('/test-bucket/directory/Test.test')]
object_summary = s3.ObjectSummary('test-bucket', 'pathlib.py')
object_summary.put(Body=b'test data')
object_summary = s3.ObjectSummary('test-bucket', 'setup.py')
object_summary.put(Body=b'test data')
object_summary = s3.ObjectSummary('test-bucket', 'test_pathlib.py')
object_summary.put(Body=b'test data')
object_summary = s3.ObjectSummary('test-bucket', 'docs/conf.py')
object_summary.put(Body=b'test data')
object_summary = s3.ObjectSummary('test-bucket', 'build/lib/pathlib.py')
object_summary.put(Body=b'test data')
assert sorted(S3Path.from_uri('s3://test-bucket/').glob('*.py')) == [
S3Path('/test-bucket/pathlib.py'),
S3Path('/test-bucket/setup.py'),
S3Path('/test-bucket/test_pathlib.py')]
assert sorted(S3Path.from_uri('s3://test-bucket/').glob('*/*.py')) == [S3Path('/test-bucket/docs/conf.py')]
assert sorted(S3Path.from_uri('s3://test-bucket/').glob('**/*.py')) == [
S3Path('/test-bucket/build/lib/pathlib.py'),
S3Path('/test-bucket/docs/conf.py'),
S3Path('/test-bucket/pathlib.py'),
S3Path('/test-bucket/setup.py'),
S3Path('/test-bucket/test_pathlib.py')]
assert sorted(S3Path.from_uri('s3://test-bucket/').glob('*cs')) == [
S3Path('/test-bucket/docs/'),
]
def _route_path(*args, endpoint_url: str) -> Union[Path, S3Path]:
"""use S3Path or Path depending on the input.
"""
if str(args[0]).startswith("s3://"):
s3path._s3_accessor.s3 = boto3.resource("s3",
endpoint_url=endpoint_url)
return S3Path.from_uri(*args)
elif isinstance(args[0], S3Path):
s3path._s3_accessor.s3 = boto3.resource("s3",
endpoint_url=endpoint_url)
return S3Path(*args)
else:
return Path(*args)
def extract_records(records):
keys = []
prefixes = []
for record in records:
handle = None
if 'receiptHandle' in record:
handle = record['receiptHandle']
logger.debug(f"message {record}")
if record['eventSource'] == 'aws:sqs':
# pipeline comes through record['body'] as text
body = json.loads(record['body'])
logger.debug(f"body {body}")
message = json.loads(body['Message'])
logger.debug(f"message {message}")
bucket = message['bucket']
key = message['key'].strip('/')
if record['eventSource'] == 'aws:sns':
# pipeline comes through record['body'] as text
message = json.loads(record['message'])
bucket = message['bucket']
key = message['key'].strip('/')
elif record['eventSource'] == 'aws:s3':
# pipeline is the bucket/key
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
key = key.lstrip('/')
uri = os.path.join('s3://', bucket, key)
key = S3Path.from_uri(uri)
prefix = Prefix(key)
if prefix not in prefixes:
prefixes.append(prefix)
return prefixes
def notify(event, context):
info = get_job_info(context)
event['info'] = info # keep this around for our event
logger.debug(f'notifying for job {info}')
# Notify Slack
# Check if we have a slack-hook tag on our lambda
slack_url = info['slack']
if slack_url:
notify_slack(slack_url, event, context)
message = event
bucket = message['detail']['parameters']['bucketname']
collect = message['detail']['parameters']['collectname']
output = message['detail']['parameters']['outputname']
collect = collect.lstrip('/')
uri = os.path.join('s3://', bucket, collect, 'settings.yaml')
settings = S3Path.from_uri(uri)
# if set have a settings file and there is an array
# of email addresses in the notifications top-level key, send email
logger.debug(f'checking prefix for job {settings}')
if settings.is_file():
with settings.open() as f:
config = yaml.load(f, Loader=yaml.FullLoader)
logger.debug(f'fetched yaml config {config}')
# if no notifications list set in the config, we skip
try:
config['notifications']
except KeyError:
return
notify_email(config['notifications'], event, context)
def __init__(self, uri):
if isinstance(uri, S3Path):
self.uri = uri.parent
else:
key = S3Path.from_uri(uri)
self.uri = key.parent
