def __init__(self, loop=None, endpoint=None):
loop = loop if loop else asyncio.get_event_loop()
session = aiobotocore.get_session(loop=loop)
self._client = session.create_client('sqs', endpoint_url=endpoint)
self._queue_urls = {}
self._unfinished_messages = []
self._sigterm_recieved = None
def __init__(self, settings, loop=None):
self._aws_access_key = settings["aws_client_id"]
self._aws_secret_key = settings["aws_client_secret"]
opts = dict(
aws_secret_access_key=self._aws_secret_key,
aws_access_key_id=self._aws_access_key,
endpoint_url=settings.get("endpoint_url"),
verify=settings.get("verify_ssl"),
use_ssl=settings.get("ssl", True),
region_name=settings.get("region_name"),
config=aiobotocore.config.AioConfig(
None,
max_pool_connections=settings.get("max_pool_connections", 30)),
)
if loop is None:
loop = asyncio.get_event_loop()
self._loop = loop
self._s3aiosession = aiobotocore.get_session(loop=loop)
# This client is for downloads only
self._s3aioclient = self._s3aiosession.create_client("s3", **opts)
self._cached_buckets = []
self._bucket_name = settings["bucket"]
async def __aenter__(self):
session = aiobotocore.get_session()
self.client = session.create_client(
self.option.service_name,
region_name=self.option.region_name,
config=self.option.config)
return self.client
def go(loop):
bucket = 'dataintake'
filename = 'dummy.bin'
folder = 'aiobotocore'
key = '{}/{}'.format(folder, filename)
session = aiobotocore.get_session(loop=loop)
client = session.create_client('s3', region_name='us-west-2',
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY_ID)
# upload object to amazon s3
data = b'\x01'*1024
resp = yield from client.put_object(Bucket=bucket,
Key=key,
Body=data)
print(resp)
# getting s3 object properties of file we just uploaded
resp = yield from client.get_object_acl(Bucket=bucket, Key=key)
print(resp)
# delete object from s3
resp = yield from client.delete_object(Bucket=bucket, Key=key)
print(resp)
def main():
if len(sys.argv) == 1 or len(sys.argv) > 1 and (sys.argv[1] == "-h" or
sys.argv[1] == "--help"):
printUsage()
rootid = sys.argv[1]
if not isValidUuid(rootid):
print("Invalid root id!")
sys.exit(1)
if not isSchema2Id(rootid):
print("This tool can only be used with Schema v2 ids")
sys.exit(1)
# we need to setup a asyncio loop to query s3
loop = asyncio.get_event_loop()
app = {}
app["bucket_name"] = config.get("bucket_name")
app["loop"] = loop
session = get_session(loop=loop)
app["session"] = session
loop.run_until_complete(run_delete(app, rootid))
loop.close()
print("done!")
async def _run(self):
for i in range(self._concurrency):
proc = await asyncio.create_subprocess_exec(
sys.executable,
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"s3log_worker.py"),
self._script,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=sys.stdout,
)
await self._q.put(proc)
session = aiobotocore.get_session(loop=self._loop)
async with session.create_client(
"s3",
region_name=self._aws_region,
aws_secret_access_key=self._aws_secret_access_key,
aws_access_key_id=self._aws_access_key_id,
) as client:
if self._show_progress:
self._loop.create_task(self._status())
waiter = self._loop.create_task(self._wait())
# # list s3 objects using paginator
paginator = client.get_paginator("list_objects")
async for result in paginator.paginate(Bucket=self._bucket,
Prefix=self._prefix):
for c in result.get("Contents", []):
key = c["Key"]
print(f"Processing key: {key}", file=sys.stderr)
proc = await self._q.get()
await self._tasks_queue.put(
self._loop.create_task(self.feed(client, key, proc)))
await self._tasks_queue.put(None)
await waiter
def create_client(cls: Any, name: str, context: Dict) -> None:
logging.getLogger(
'botocore.vendored.requests.packages.urllib3.connectionpool'
).setLevel(logging.WARNING)
if not cls.clients:
cls.clients = {}
cls.clients_creation_time = {}
loop = asyncio.get_event_loop()
session = aiobotocore.get_session(loop=loop)
try:
if cls.clients_creation_time.get(
name
) and cls.clients_creation_time[name] + 30 > time.time():
return
cls.clients[name] = session.create_client(name,
region_name='eu-west-1')
cls.clients_creation_time[name] = time.time()
except (botocore.exceptions.PartialCredentialsError,
botocore.exceptions.NoRegionError) as e:
error_message = str(e)
logging.getLogger('transport.aws_sns_sqs').warning(
'Invalid credentials [{}] to AWS ({})'.format(
name, error_message))
raise AWSSNSSQSConnectionException(
error_message, log_level=context.get('log_level')) from e
async def async_main(login, password, command, src, dest, bucket, loop):
"""
Асинхронная функция, обрабатывающая команды
:param login: Логин
:param password: Пароль
:param command: Команда
:param src: /path/to/src
:param dest: /path/to/dest
:param bucket: Название бакета
:param loop: Асинхронный цикл
:return:
"""
session = aiobotocore.get_session(loop=loop)
async with session.create_client("s3",
endpoint_url=storage_url,
region_name='us-west-2',
aws_secret_access_key=password,
aws_access_key_id=login) as client:
if command == C_DOWNLOAD:
await download_bucket(bucket, client)
elif command == C_GET:
await download_one_file(src, dest, bucket, client)
elif command == C_LIST:
await print_bucket(bucket, client)
elif command == C_UPLOAD:
await upload(src, bucket, client)
else:
sys.exit("Unknown command {}".format(command))
async def go(loop):
# Boto should get credentials from ~/.aws/credentials or the environment
session = aiobotocore.get_session()
async with session.create_client('sqs') as client:
print('Pulling messages off the queue')
while True:
try:
# This loop wont spin really fast as there is
# essentially a sleep in the receive_message call
response = await client.receive_message(
QueueUrl=QUEUE_URL,
WaitTimeSeconds=20,
)
if 'Messages' in response:
for message in response['Messages']:
print("Message received:" + message["Body"])
asyncio.ensure_future(handle(session, client, message))
else:
print('No messages in queue')
except KeyboardInterrupt:
break
print('Finished')
async def go():
session = aiobotocore.get_session()
client = session.create_client('dynamodb', region_name='us-west-2')
# Create random table name
table_name = 'aiobotocore-' + str(uuid.uuid4())
print('Requesting table creation...')
await client.create_table(TableName=table_name,
AttributeDefinitions=[
{
'AttributeName': 'testKey',
'AttributeType': 'S'
},
],
KeySchema=[
{
'AttributeName': 'testKey',
'KeyType': 'HASH'
},
],
ProvisionedThroughput={
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
})
print("Waiting for table to be created...")
waiter = client.get_waiter('table_exists')
await waiter.wait(TableName=table_name)
print("Table {0} created".format(table_name))
await client.close()
async def go():
bucket = 'dataintake'
filename = 'dummy.bin'
folder = 'aiobotocore'
key = '{}/{}'.format(folder, filename)
session = aiobotocore.get_session()
async with session.create_client(
's3',
region_name='us-west-2',
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY_ID) as client:
# upload object to amazon s3
data = b'\x01' * 1024
resp = await client.put_object(Bucket=bucket, Key=key, Body=data)
print(resp)
# getting s3 object properties of file we just uploaded
resp = await client.get_object_acl(Bucket=bucket, Key=key)
print(resp)
# delete object from s3
resp = await client.delete_object(Bucket=bucket, Key=key)
print(resp)
async def test_s3_parameter_capture(loop, recorder):
segment = recorder.begin_segment('name')
bucket_name = 'mybucket'
key = 'mykey'
version_id = 'myversionid'
response = {
'ResponseMetadata': {
'RequestId': '1234',
'HTTPStatusCode': 200
}
}
session = aiobotocore.get_session()
async with session.create_client('s3', region_name='eu-west-2') as client:
with Stubber(client) as stubber:
stubber.add_response('get_object', response, {
'Bucket': bucket_name,
'Key': key,
'VersionId': version_id
})
await client.get_object(Bucket=bucket_name,
Key=key,
VersionId=version_id)
subsegment = segment.subsegments[0]
aws_meta = subsegment.aws
assert aws_meta['bucket_name'] == bucket_name
assert aws_meta['key'] == key
assert aws_meta['version_id'] == version_id
assert aws_meta['operation'] == 'GetObject'
async def test_map_parameter_grouping(loop, recorder):
"""
Test special parameters that have shape of map are recorded
as a list of keys based on `para_whitelist.json`
"""
segment = recorder.begin_segment('name')
response = {
'ResponseMetadata': {
'RequestId': '1234',
'HTTPStatusCode': 500,
}
}
session = aiobotocore.get_session()
async with session.create_client('dynamodb',
region_name='eu-west-2') as client:
with Stubber(client) as stubber:
stubber.add_response('batch_write_item', response,
{'RequestItems': ANY})
await client.batch_write_item(RequestItems={
'table1': [{}],
'table2': [{}]
})
subsegment = segment.subsegments[0]
assert subsegment.fault
assert subsegment.http['response']['status'] == 500
aws_meta = subsegment.aws
assert sorted(aws_meta['table_names']) == ['table1', 'table2']
def main():
if len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"):
printUsage()
if len(sys.argv) > 1 and sys.argv[1] == "--update":
do_update = True
else:
do_update = False
# we need to setup a asyncio loop to query s3
loop = asyncio.get_event_loop()
app = {}
app["bucket_name"] = config.get("bucket_name")
app["loop"] = loop
session = get_session(loop=loop)
app["session"] = session
loop.run_until_complete(run_scan(app, update=do_update))
loop.close()
results = app["bucket_scan"]
print("root_count:", results["root_count"])
print("info_count:", results["info_count"])
print("group_count", results["group_count"])
print("dataset_count:", results["dataset_count"])
print("datatype_count", results["datatype_count"])
print("chunk_count:", results["chunk_count"])
print('allocated_bytes:', results["allocated_bytes"])
print("metadata_bytes:", results["metadata_bytes"])
print("updated_count:", results["updated_count"])
print("done!")
def main():
if len(sys.argv) == 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
printUsage()
sys.exit(1)
chunk_id = sys.argv[-1]
if not isValidChunkId(chunk_id):
print("Invalid chunk id")
sys.exit(1)
# we need to setup a asyncio loop to query s3
loop = asyncio.get_event_loop()
session = get_session(loop=loop)
app = {}
app["session"] = session
app['bucket_name'] = config.get("bucket_name")
app['node_count'] = 1
app['node_number'] = 0
app['deleted_ids'] = set()
app['meta_cache'] = {}
app['pending_s3_read'] = {}
app['meta_cache'] = LruCache(mem_target=1024*1024, chunk_cache=False)
app['chunk_cache'] = LruCache(mem_target=64*1024*1024, chunk_cache=True)
domain = config.get("domain")
if not domain:
printUsage()
sys.exit(-1)
print("got domain:", domain)
loop.run_until_complete(printChunkValues(app, domain, chunk_id))
loop.close()
async def coro3(app):
queue_name = 'jreuter-edge-detection'
logging.info('C3 - About to call SQS')
loop = asyncio.get_event_loop()
session = aiobotocore.get_session(loop=loop)
async with session.create_client('sqs', region_name='us-east-1') as client:
try:
queue = await client.get_queue_url(QueueName=queue_name)
except ClientError as e:
if e.response['Error']['Code'] == "AWS.SimpleQueueService.NonExistentQueue":
logging.info(e.response['Error']['Message'])
logging.info("Creating SQS queue %s in region %s...", queue_name, 'us-east-1')
queue = await client.create_queue(QueueName=queue_name)
else:
raise
queue_url = queue.get('QueueUrl')
logging.info("C3 - recieving messages")
while True:
result = await client.receive_message(WaitTimeSeconds=20,
MaxNumberOfMessages=1,
QueueUrl=queue_url)
if 'Messages' in result:
logging.info('C3 - We got some messages')
for message in result['Messages']:
logging.info(message['Body'])
await client.delete_message(ReceiptHandle=str(message.get('ReceiptHandle')),
QueueUrl=queue_url)
else:
logging.info('C3 - We got no messages')
await asyncio.sleep(60)
async def test_list_parameter_counting(loop, recorder):
"""
Test special parameters that have shape of list are recorded
as count based on `para_whitelist.json`
"""
segment = recorder.begin_segment('name')
queue_urls = ['url1', 'url2']
queue_name_prefix = 'url'
response = {
'QueueUrls': queue_urls,
'ResponseMetadata': {
'RequestId': '1234',
'HTTPStatusCode': 200,
}
}
session = aiobotocore.get_session(loop=loop)
async with session.create_client('sqs', region_name='eu-west-2') as client:
with Stubber(client) as stubber:
stubber.add_response('list_queues', response,
{'QueueNamePrefix': queue_name_prefix})
await client.list_queues(QueueNamePrefix='url')
subsegment = segment.subsegments[0]
assert subsegment.http['response']['status'] == 200
aws_meta = subsegment.aws
assert aws_meta['queue_count'] == len(queue_urls)
# all whitelisted input parameters will be converted to snake case
# unless there is an explicit 'rename_to' attribute in json key
assert aws_meta['queue_name_prefix'] == queue_name_prefix
def _get_async_client(self, loop=None):
"""Return an asynchronous s3 client for the backend"""
return aiobotocore.get_session().create_client(
's3',
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
region_name=self.region)
async def publish_file_for_5_minutes_and_return_url(dataset: str,
filename: str) -> str:
"""
Little aux function to make the user happy.
When results are shown (if they are), near each record with filename
there is a `Download` button, so user can click it
and file will be open/downloaded.
The thing is files are located on a private S3 bucket, so there is a need
to expose them/make them public for some short time. So particular function
makes files (of datasets) public for a 5 minutes on a request.
:param str dataset: the name of dataset / text collection (as it on S3)
:param str filename: the name of file in the collection
:return: an URL to publicly accessible for a 5 minutes file
:rtype: str
:raises: possibly some aiobotocore exception. Code that calls this function
does exceptions handling.
"""
session = aiobotocore.get_session()
# create shareable url to download the file (valid for 300 seconds)
async with session.create_client("s3") as s3_client:
url: str = await s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket": S3_BUCKET,
"Key": f"static/{dataset}/{filename}",
},
ExpiresIn=300, # seconds
)
return url
async def main(loop, args):
session = get_session(loop=loop)
config = AioConfig(max_pool_connections=args.semaphore)
home = path.dirname(path.realpath(__file__))
keys = listdir('./files/')
paths = [home + '/files/' + keys[i] for i in range(0, len(keys))]
async with session.create_client(
's3',
config=config,
aws_access_key_id=args.key_id,
aws_secret_access_key=args.key) as s3_client:
async def put(key, f_path):
f = await aio_open(f_path, mode='r')
fl = await f.read()
await f.close()
response = await s3_client.put_object(Bucket=args.bucket,
Key=key,
Body=fl)
if response['ResponseMetadata']['HTTPStatusCode'] != 200:
return {key: 'Failed'}
else:
return {key: 'Uploaded'}
return await gather(
*[put(keys[i], paths[i]) for i in range(0, len(keys))])
async def main(args):
session = get_session()
config = AioConfig(max_pool_connections=args.semaphore)
with open(file_name) as fl:
files = [file[:-1] for file in fl.readlines()]
async with session.create_client(
's3',
config=config,
aws_access_key_id=args.key_id,
aws_secret_access_key=args.key) as s3_client:
async def get(key):
response = await s3_client.get_object(Bucket=args.bucket, Key=key)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
async with response['Body'] as stream:
data = await stream.read()
f = await aio_open(storage_path + '/' + key, mode='w')
await f.write(data.decode('UTF-8'))
await f.flush()
return {key: 'Downloaded'}
else:
return {key: 'Failed'}
return await gather(*[get(file) for file in files])
def __init__(self, replica):
self.access_key_id = replica['meta']['access_key_id']
self.secret_access_key = replica['meta']['secret_access_key']
loop = asyncio.get_running_loop()
self.session = aiobotocore.get_session(loop=loop)
self.bucket = 'openpacs'
self.region = replica['location']
async def go(loop):
session = aiobotocore.get_session(loop=loop)
client = session.create_client('dynamodb', region_name='us-west-2')
# Create random table name
table_name = 'aiobotocore-' + str(uuid.uuid4())
print('Requesting table creation...')
await client.create_table(
TableName=table_name,
AttributeDefinitions=[
{
'AttributeName': 'testKey',
'AttributeType': 'S'
},
],
KeySchema=[
{
'AttributeName': 'testKey',
'KeyType': 'HASH'
},
],
ProvisionedThroughput={
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
}
)
print("Waiting for table to be created...")
waiter = client.get_waiter('table_exists')
await waiter.wait(TableName=table_name)
print("Table {0} created".format(table_name))
await client.close()
async def s3_flusher(loop, opts, ev):
global _records
buffer = io.BytesIO()
part_count = 1
while True:
await ev.wait()
ev.clear()
print("s3: flushing {} entries...".format(len(_records)))
if opts["codec"] == "msgpack":
packer = umsgpack.Packer()
for rec in _records:
buffer.write(packer.pack(rec.data))
elif opts["codec"] == "text":
for rec in _records:
print(str(rec).encode("utf8"), file=buffer)
_records.clear() # must be cleared before any await
session = aiobotocore.get_session(loop=loop)
client = session.create_client(
"s3",
region_name=opts["region"],
aws_secret_access_key=opts["secret_key"],
aws_access_key_id=opts["access_key"],
)
now = datetime.now()
ts_month = now.strftime("%Y-%m")
ts_monthday = now.strftime("%Y-%m-%d")
ts_time = now.strftime("%Y-%m-%dT%H.%M.%S")
key = "{}/{}/{}/{}.part{}.msgpack".format(opts["key_prefix"], ts_month, ts_monthday, ts_time, part_count)
resp = await client.put_object(Bucket=opts["bucket"], Key=key, Body=buffer.getvalue(), ACL="private")
buffer.seek(0, io.SEEK_SET)
buffer.truncate(0)
part_count += 1
async def run(loop, input_filepath, s3bucket_name, batch_size, concurrency):
"""
Top level async task loop that coordinates with the get_versions (producer)
and delete_versions (consumer) coroutines.
Arguments:
loop {asyncio.AbstractEventLoop} -- asyncio event loop
input_filepath {str} -- path of the file holding keys to delete
s3bucket_name {str} -- bucket to delete from
batch_size {int} -- the desired number of object versions to batch together into a single delete_objects
request
concurrency {int} -- max number of concurrent asyncio actions
"""
# Setup asyncio objects
queue = asyncio.Queue(loop=loop)
sem = asyncio.Semaphore(concurrency)
session = aiobotocore.get_session(loop=loop)
# quickly find the total keys we expect to delete to setup the progress bar
total_keys = key_file_len(input_filepath)
async with session.create_client('s3') as s3client:
# wait until we have completed:
# - the extraction of all object versions
# - the deletions of all those object versions
await asyncio.gather(
get_versions(sem, queue, s3client, s3bucket_name, total_keys,
input_filepath),
delete_versions(sem,
queue,
s3client,
s3bucket_name,
total_keys,
batch_size=batch_size))
async def setup(s3_cfg):
session = aiobotocore.get_session()
s3_ctx = session.create_client("s3",
region_name=region_name,
config=s3_cfg)
s3 = await s3_ctx.__aenter__()
return (session, s3, s3_ctx)
async def session(self):
"""
Returns a valid aiobotocore session
"""
if getattr(self, '_session', None) is None:
self._session = aiobotocore.get_session()
return self._session
def __init__(self, settings, loop=None):
self._aws_access_key = settings['aws_client_id']
self._aws_secret_key = settings['aws_client_secret']
opts = dict(aws_secret_access_key=self._aws_secret_key,
aws_access_key_id=self._aws_access_key,
endpoint_url=settings.get('endpoint_url'),
verify=settings.get('verify_ssl'),
use_ssl=settings.get('ssl', True),
region_name=settings.get('region_name'))
if loop is None:
loop = asyncio.get_event_loop()
self._loop = loop
self._s3aiosession = aiobotocore.get_session(loop=loop)
# This client is for downloads only
self._s3aioclient = self._s3aiosession.create_client('s3', **opts)
self._cached_buckets = []
self._bucket_name = settings['bucket']
# right now, only used for upload_fileobj in executor
self._s3client = boto3.client('s3', **opts)
async def test_describe_table(loop, recorder):
segment = recorder.begin_segment('name')
req_id = '1234'
response = {
'ResponseMetadata': {
'RequestId': req_id,
'HTTPStatusCode': 403
}
}
session = aiobotocore.get_session()
async with session.create_client('dynamodb',
region_name='eu-west-2') as client:
with Stubber(client) as stubber:
stubber.add_response('describe_table', response,
{'TableName': 'mytable'})
await client.describe_table(TableName='mytable')
subsegment = segment.subsegments[0]
assert subsegment.error
assert subsegment.http['response']['status'] == 403
aws_meta = subsegment.aws
assert aws_meta['table_name'] == 'mytable'
assert aws_meta['request_id'] == req_id
assert aws_meta['region'] == 'eu-west-2'
assert aws_meta['operation'] == 'DescribeTable'
async def check_slack_file_quota(opsdroid, config, message):
try:
aws_access_key_id = config["aws_access_key_id"]
aws_secret_access_key = config["aws_secret_access_key"]
slack_api_token = config["slack_api_token"]
s3_region_name = config["s3_region_name"]
max_total_file_size = config["max_total_file_size"]
s3_bucket = config["s3_bucket"]
s3_prefix = config.get("s3_prefix", "")
file_size_buffer = config.get("file_size_buffer", 0)
except KeyError:
_LOGGER.error("Missing config item(s) in skill %s.",
config.get('name', 'aws-tag-compliance'))
return
if message is None:
message = Message("", None, config.get("room", connector.default_room),
opsdroid.default_connector)
else:
await message.respond("I'm on it!")
files_removed = 0
data_saved = 0
files = await get_file_list(slack_api_token)
size_threshold = max_total_file_size
while await count_total_file_size(files) > size_threshold:
if size_threshold == max_total_file_size:
size_threshold = max_total_file_size - file_size_buffer
session = aiobotocore.get_session()
async with session.create_client(
's3',
region_name=s3_region_name,
aws_secret_access_key=aws_secret_access_key,
aws_access_key_id=aws_access_key_id) as client:
data = await download_file(slack_api_token, files[-1])
if await upload_file(client, files[-1], data, s3_bucket,
s3_prefix):
if await cleanup_file(slack_api_token, files[-1]):
_LOGGER.debug("Uploaded %s to S3", files[-1]["name"])
files_removed = files_removed + 1
data_saved = data_saved + files[-1]["size"]
files.remove(files[-1])
else:
_LOGGER.debug(
"%s uploaded to S3 but failed to clean up on Slack",
files[-1]["name"])
else:
_LOGGER.debug("Upload of %s failed", files[-1]["name"])
if files_removed > 0:
await message.respond(
"You were getting close to your Slack file limit so I've moved {} files to the {} bucket on S3 saving {}."
.format(files_removed, s3_bucket, human_bytes(data_saved)))
else:
if message.regex:
await message.respond(
"Nothing to do, file size is {} and quota is {}".format(
human_bytes(await count_total_file_size(files)),
human_bytes(max_total_file_size)))
async def get_text(self, audio_data):
session = aiobotocore.get_session()
upload = session.create_client(
"s3",
region_name=self.REGION_NAME,
aws_secret_access_key=self.SECRET_ACCESS_KEY,
aws_access_key_id=self.ACCESS_KEY_ID,
)
transcribe = session.create_client(
"transcribe",
region_name=self.REGION_NAME,
aws_secret_access_key=self.SECRET_ACCESS_KEY,
aws_access_key_id=self.ACCESS_KEY_ID,
)
filename = f"{uuid4().hex}.mp3"
# Upload audio file to bucket
await upload.put_object(
Bucket=self.S3_BUCKET, Key=filename, Body=audio_data
)
job_name = uuid4().hex
job_uri = (
f"https://s3.{self.REGION_NAME}.amazonaws.com/{self.S3_BUCKET}/"
f"{filename}"
)
# Send audio file URI to Transcribe
await transcribe.start_transcription_job(
TranscriptionJobName=job_name,
Media={"MediaFileUri": job_uri},
MediaFormat="mp3",
LanguageCode="en-US",
)
# Wait 90 seconds for transcription
timeout = 90
while time.time() > timeout:
status = await transcribe.get_transcription_job(
TranscriptionJobName=job_name
)
if status["TranscriptionJob"]["TranscriptionJobStatus"] in [
"COMPLETED",
"FAILED",
]:
break
await asyncio.sleep(5)
# Delete audio file from bucket
await upload.delete_object(Bucket=self.S3_BUCKET, Key=filename)
if "TranscriptFileUri" in status["TranscriptionJob"]["Transcript"]:
transcript_uri = status["TranscriptionJob"]["Transcript"][
"TranscriptFileUri"
]
data = json.loads(await get_page(transcript_uri))
transcript = data["results"]["transcripts"][0]["transcript"]
return transcript
# Delete audio file
await upload.delete_object(Bucket=self.S3_BUCKET, Key=filename)
# Close clients
await upload._endpoint._aio_session.close()
await transcribe._endpoint._aio_session.close()
def setUp(self):
super().setUp()
self.session = aiobotocore.get_session(loop=self.loop)
self.region = 'us-east-1'
self.client = self.session.create_client('s3', region_name=self.region)
self.keys = []
self.addCleanup(self.client.close)
def setUp(self):
super().setUp()
self.session = aiobotocore.get_session(loop=self.loop)
self.region = 'us-east-1'
self.client = self.session.create_client('s3', region_name=self.region)
self.keys = []
self.addCleanup(self.client.close)
def __init__(self,
endpoint_url=None,
use_ssl=True,
dry_run=False,
**kwargs):
super().__init__(endpoint_url, use_ssl, dry_run, **kwargs)
self._session = aiobotocore.get_session()
self.aws_account_id = None
async def go(loop):
session = aiobotocore.get_session(loop=loop)
client = session.create_client('dynamodb', region_name='us-west-2')
table_name = 'test'
print('Writing to dynamo')
start = 0
while True:
# Loop adding 25 items to dynamo at a time
request_items = create_batch_write_structure(table_name, start, 25)
response = await client.batch_write_item(
RequestItems=request_items
)
if len(response['UnprocessedItems']) == 0:
print('Writted 25 items to dynamo')
else:
# Hit the provisioned write limit
print('Hit write limit, backing off then retrying')
await asyncio.sleep(5)
# Items left over that haven't been inserted
unprocessed_items = response['UnprocessedItems']
print('Resubmitting items')
# Loop until unprocessed items are written
while len(unprocessed_items) > 0:
response = await client.batch_write_item(
RequestItems=unprocessed_items
)
# If any items are still left over, add them to the
# list to be written
unprocessed_items = response['UnprocessedItems']
# If there are items left over, we could do with
# sleeping some more
if len(unprocessed_items) > 0:
print('Backing off for 5 seconds')
await asyncio.sleep(5)
# Inserted all the unprocessed items, exit loop
print('Unprocessed items successfully inserted')
break
start += 25
# See if DynamoDB has the last item we inserted
final_item = 'item' + str(start + 24)
print('Item "{0}" should exist'.format(final_item))
response = await client.get_item(
TableName=table_name,
Key={'pk': {'S': final_item}}
)
print('Response: ' + str(response['Item']))
await client.close()
async def get_available_regions(hass, service):
"""Get available regions for a service."""
import aiobotocore
session = aiobotocore.get_session()
# get_available_regions is not a coroutine since it does not perform
# network I/O. But it still perform file I/O heavily, so put it into
# an executor thread to unblock event loop
return await hass.async_add_executor_job(
session.get_available_regions, service
)
async def cache_package(project_name, url, package_path):
key = '/'.join((PIPSY_SIMPLE_ROOT, project_name, os.path.basename(package_path))).lstrip('/')
boto_session = aiobotocore.get_session()
async with boto_session.create_client('s3') as s3_client:
try:
release = await s3_client.get_object(Bucket=PIPSY_BUCKET, Key=key)
except s3_client.exceptions.NoSuchKey:
async with ClientSession() as session:
async with session.get(url) as index_response:
data = await index_response.content.read()
await s3_client.put_object(
Body=data,
Bucket=PIPSY_BUCKET,
Key=key,
)
def init():
global session
session = aiobotocore.get_session()
app = web.Application(middlewares=[heartbeat_middleware_factory])
aiogithubauth.add_github_auth_middleware(
app,
github_id=gh_id,
github_secret=gh_secret,
github_org=gh_org,
cookie_name=cookie_name,
cookie_key=cookie_key
)
app.router.add_route('GET', '/test', handle_auth)
app.router.add_route('GET', '/{tail:.*}', stream_file) # Everything else
return app
def setUp(self):
super().setUp()
self.session = aiobotocore.get_session(loop=self.loop)
self.client = self.session.create_client('s3', region_name='us-west-2')
self.bucket_name = 'aiobotocoretest%s-%s' % (
int(time.time()), random.randint(1, 1000))
self.loop.run_until_complete(self.client.create_bucket(
Bucket=self.bucket_name,
CreateBucketConfiguration={
'LocationConstraint': 'us-west-2',
}
))
self.addCleanup(self.loop.run_until_complete,
self.client.delete_bucket(Bucket=self.bucket_name))
async def fetch_releases_for_project_from_s3(self, project_name):
if PIPSY_SIMPLE_ROOT:
prefix = '/'.join([PIPSY_SIMPLE_ROOT, project_name])
else:
prefix = project_name
release_files = {}
session = aiobotocore.get_session()
async with session.create_client('s3') as client:
paginator = client.get_paginator('list_objects_v2')
async for result in paginator.paginate(Bucket=PIPSY_BUCKET, Prefix=prefix):
for k in result.get('Contents', []):
release_files[os.path.basename(k['Key'])] = {
"filename": os.path.basename(k['Key']),
"url": os.path.basename(k['Key']),
"md5_digest": k['ETag'].strip('"'),
}
return release_files
async def go(loop):
session = aiobotocore.get_session(loop=loop)
client = session.create_client('sqs', region_name='us-west-2')
print('Creating test_queue1')
response = await client.create_queue(QueueName='test_queue1')
queue_url = response['QueueUrl']
response = await client.list_queues()
print('Queue URLs:')
for queue_name in response.get('QueueUrls', []):
print(' ' + queue_name)
print('Deleting queue {0}'.format(queue_url))
await client.delete_queue(QueueUrl=queue_url)
print('Done')
await client.close()
def go(loop, task_id):
print('Start task {}'.format(task_id))
bucket = 'jharai'
filename = '{}-{}'.format(time.strftime("%H-%M-%S"), task_id)
folder = 'aiobotocore'
key = '{}/{}'.format(folder, filename)
session = aiobotocore.get_session(loop=loop)
client = session.create_client(
's3',
region_name='ap-northeast-1')
try:
# upload object to amazon s3
data = b'\x01' * 1024
resp = yield from client.put_object(Bucket=bucket, Key=key, Body=data)
print('Finish task {}'.format(task_id))
print(resp)
finally:
client.close()
async def get(self):
if not self.request.path.endswith('/'):
return web.HTTPFound(self.request.path + '/')
keys = []
session = aiobotocore.get_session()
async with session.create_client('s3') as client:
paginator = client.get_paginator('list_objects_v2')
async for result in paginator.paginate(Bucket=PIPSY_BUCKET, Prefix=PIPSY_SIMPLE_ROOT, Delimiter="/"):
for k in result.get('CommonPrefixes', []):
keys.append(k['Prefix'])
projects = sorted(list({normalize_project_name(k.rstrip('/')) for k in keys}))
body = [f" <a href='{project}/'>{project}</a><br/>" for project in projects]
body = SIMPLE_TEMPLATE.format(body="\n".join(body))
return web.Response(
status=200,
body=body,
headers={
'Content-Type': 'text/html; charset=utf-8',
}
)
async def stream_key_from_s3(self, key):
session = aiobotocore.get_session()
async with session.create_client('s3') as client:
try:
release = await client.get_object(Bucket=PIPSY_BUCKET, Key=key)
except client.exceptions.NoSuchKey:
raise KeyError
response = web.StreamResponse(
status=200,
headers={
"Content-Type": release['ContentType'],
"ETag": release['ETag'].strip('"'),
}
)
response.content_length = release['ContentLength']
await response.prepare(self.request)
while True:
data = await release['Body'].read(8192)
if not data:
await response.drain()
break
response.write(data)
return response
def __init__(self, name, *,
aws_region='us-west-2',
connector=None,
scheme='http',
boto_creds=None,
logger=None,
num_retries=6,
timeout=None,
loop=None):
"""
Bucket class used to access S3 buckets
@param name: name of bucket to
@param aws_region: AWS region to use for communication
@param connector:
@param scheme: http or https
@param boto_creds: botocore credential resolver
@param logger:
@param num_retries: number of retries for AWS operations
@param timeout: aiohttp timeout in seconds
@return: aios3 Bucket object
"""
if logger is None: logger = logging.getLogger('aio-s3')
self._name = name
self._connector = connector
self._num_retries = num_retries
self._num_requests = 0
self._aws_region = aws_region
self._boto_creds = boto_creds
self._timeout = timeout
self._logger = logger
self._loop = loop
self._presign_cache = dict() # (method, params) -> url
self._cache_hits = 0
self._cache_misses = 0
self._retry_handler = functools.partial(_RetryHandler, timeout=self._timeout)
self._scheme = scheme
self._aio_boto_session = None
# Virtual style host URL
# ----------------------
# endpoint: bucket.s3.amazonaws.com / bucket.s3-aws-region.amazonaws.com
# host: bucket.s3.amazonaws.com
#
# Path Style
# ----------
# endpoint: s3.amazonaws.com/bucket / s3-aws-region.amazonaws.com/bucket
# host: s3.amazonaws.com
#
# We use Path Style because the Amazon SSL wildcard cert will not match for virtual style with buckets
# that have '.'s: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
if aws_region == 'us-east-1':
self._host = "s3.amazonaws.com"
else:
self._host = "s3-" + aws_region + ".amazonaws.com"
self._endpoint = self._host + "/" + self._name
if self._connector is None:
kwargs = {}
if timeout:
kwargs['conn_timeout'] = timeout
self._connector = aiohttp.TCPConnector(force_close=False, keepalive_timeout=10, use_dns_cache=False, loop=self._loop, **kwargs)
use_ssl = self._scheme == 'https'
connector_args = {
'use_dns_cache': self._connector._use_dns_cache,
'force_close': self._connector._force_close,
'keepalive_timeout': self._connector._keepalive_timeout}
aio_config = aiobotocore.client.AioConfig(signature_version='s3v4', connector_args=connector_args)
self._aio_boto_session = aiobotocore.get_session(loop=self._loop)
self._aio_boto_client = self._aio_boto_session.create_client('s3', region_name=self._aws_region, config=aio_config, use_ssl=use_ssl)
self._parsers = dict() # OpName: (op_model, parser) map
if self._boto_creds is None:
self._boto_creds = botocore.credentials.create_credential_resolver(self._aio_boto_session).load_credentials()
self._session = aiohttp.ClientSession(connector=self._connector, loop=self._loop, response_class=aiobotocore.endpoint.ClientResponseProxy)
self._signer = botocore.auth.S3SigV4Auth(self._boto_creds, 's3', self._aws_region)
# stats support
self._concurrent = 0
self._last_stat_time = time.time()
self._request_times = []
def client(self):
session = aiobotocore.get_session(loop=self._loop)
return session.create_client(self.boto_service_name, **self._client_options)
def session(event_loop):
session = aiobotocore.get_session(loop=event_loop)
return session
def session(loop):
session = aiobotocore.get_session(loop=loop)
return session
def configure():
app = aiohttp.web.Application()
# Pull configuration out of the environment
app["settings"] = {
"endpoint": os.environ["CONVEYOR_ENDPOINT"],
"docs_bucket": os.environ["DOCS_BUCKET"],
}
# Setup a HTTP session for our clients to share connections with and
# register a shutdown callback to close the session.
app["http.session"] = aiohttp.ClientSession(
loop=asyncio.get_event_loop(),
headers={"User-Agent": "conveyor"},
)
app["boto.session"] = aiobotocore.get_session(
loop=asyncio.get_event_loop(),
)
app.on_shutdown.append(session_close)
app["tasks"] = []
app["redirects"] = {}
_fetch_redirects_task = asyncio.ensure_future(
redirects_refresh_task(app),
loop=asyncio.get_event_loop(),
)
app.on_shutdown.append(cancel_tasks)
# Allow cross-origin GETs by default
cors = aiohttp_cors.setup(app, defaults={
"*": aiohttp_cors.ResourceOptions(
allow_methods="GET",
)
})
# Add routes and views to our application
cors.add(app.router.add_route(
"GET",
"/packages/{python_version}/{project_l}/{project_name}/{filename}",
redirect,
))
app.router.add_route(
"HEAD",
"/packages/{python_version}/{project_l}/{project_name}/{filename}",
redirect,
)
cors.add(app.router.add_route(
"GET",
"/packages/{tail:.*}",
not_found,
))
cors.add(app.router.add_route(
"GET",
"/packages",
not_found,
))
app.router.add_route(
"GET",
"/_health/",
health,
)
app.router.add_route(
"GET",
"/_health",
health,
)
# Add Documentation routes
app.router.add_route(
"GET",
"/",
index,
)
app.router.add_route(
"HEAD",
"/",
index,
)
app.router.add_route(
"GET",
"/{project_name}/{path:.*}",
documentation,
)
app.router.add_route(
"GET",
"/{project_name}",
documentation_top,
)
return app
def create_sample_files(self, filenames, content):
loop = asyncio.get_event_loop()
session = aiobotocore.get_session(loop=loop)
s3 = session.create_client('s3', aws_secret_access_key=settings.AWS_SECRET,
aws_access_key_id=settings.AWS_KEY)
return loop.run_until_complete(asyncio.wait([asyncio.Task(self._create_sample_file(s3, fname, content)) for fname in filenames]))
def get_aioclient(loop=None):
session = aiobotocore.get_session(loop=loop)
return session.create_client('s3')
