async def remove_user(thalia_user_id: int): logger.info(f"Removing user {thalia_user_id} from DB.") async with aioboto3.resource("dynamodb") as dynamodb: users_table = await dynamodb.Table(USERS_TABLE) await users_table.delete_item( Key={"thalia_user_id": str(thalia_user_id)}) return
async def main(): global COUNT # pylint: disable=global-statement config = Config(signature_version=botocore.UNSIGNED, max_pool_connections=50, inject_host_prefix=False, parameter_validation=False) async with aioboto3.resource('dynamodb', endpoint_url='http://127.0.0.1:8080', config=config) as dynamo_resource: # pylint: disable=not-async-context-manager table = dynamo_resource.Table('usertable') while True: requests = [] start = time.time() for _ in range(10000): key = next(KEYS) np.random.seed(key) requests.append( table.put_item( Item={ 'p': key.to_bytes(10, byteorder='big'), 'C0': np.random.bytes(64) })) asyncio.gather(*requests) COUNT += len(requests) elapsed = time.time() - start elapsed_since_start = time.time() - START_TIME print(COUNT, elapsed_since_start, elapsed, COUNT / elapsed_since_start)
async def _initialize(self): """Initialize our async resources: aiohttp, aioboto3""" self.http = aiohttp.ClientSession() self.dynamo = aioboto3.resource('dynamodb', region_name=DYNAMO_REGION) self.ddb_user_table = self.dynamo.Table(DYNAMO_USER_TABLE) self.ddb_entity_table = self.dynamo.Table(DYNAMO_ENTITY_TABLE) log.info("DDB client initialized")
async def main(): async with aioboto3.resource('dynamodb') as dynamo: dynamo_table = await dynamo.Table(DATA_TABLE) connection = await connect_robust("amqp://*****:*****@localhost/") _channel = await connection.channel() _exchange = await _channel.declare_exchange('hcTest', ExchangeType.TOPIC) # Create the queues _queue_get = await _channel.declare_queue('hcTest-get-queue') _queue_save = await _channel.declare_queue('hcTest-save-queue') # Bind the queues await _queue_get.bind(_exchange, 'hcTest-get') await _queue_save.bind(_exchange, 'hcTest-save') topic_queues = [] topic_queues.append(MessageProcessor(_queue_get, process_get_event)) topic_queues.append(MessageProcessor(_queue_save, process_save_event)) # Call the functions for each event await asyncio.wait([ asyncio.create_task( proccess_events(message_process.queue, message_process.fn, dynamo_table)) for message_process in topic_queues ])
async def test_getting_resource(event_loop): """Simple getting of resource.""" aioboto3.DEFAULT_SESSION = None resource = aioboto3.resource('dynamodb', loop=event_loop, region_name='eu-central-1') assert isinstance(resource.meta.client, AioBaseClient) await resource.close()
async def get_and_send_styled_image(chat_id, styled_image_key, caption, resulted_notebook_key=None): async with aioboto3.resource( "s3", aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], region_name=os.environ["REGION_NAME"], ) as s3: image_obj = await s3.Object(bucket_name=S3_BUCKET_WITH_RESULTS_NAME, key=styled_image_key) image_resp = await image_obj.get() image_body = image_resp["Body"] image_bytes = await image_body.read() await bot.send_photo(chat_id, image_bytes, caption=caption) await image_obj.delete() if resulted_notebook_key is not None: try: resulted_notebook_obj = await s3.Object( bucket_name=S3_BUCKET_WITH_RESULTS_NAME, key=resulted_notebook_key) await resulted_notebook_obj.delete() except Exception as e: logging.error(e)
async def loop(self): with contextlib.suppress(Exception): async with aioboto3.resource( 'sqs', region_name='ru-central1', endpoint_url='https://message-queue.api.cloud.yandex.net/' ) as resource: queue = await resource.Queue(url=Config.SQS_URL) while True: with contextlib.suppress(Exception): async with session_scope() as session: async for position, ticket in self.get_tickets( session): with contextlib.suppress(Exception): await queue.send_message( MessageBody=ujson.dumps({ 'target': ticket.user.email, 'time_left': ticket.service.average_waiting_time * position, 'position': position, 'from': self.__class__.__name__ })) ticket.pushed = True await asyncio.sleep(self.SLEEP_INTERVAL)
async def _create_sqs_resource(app: web.Application) -> None: config = app['config'] app['sqs_resource'] = aioboto3.resource( 'sqs', loop=app.loop, region_name=config.aws_region_name, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
async def main(): async with aioboto3.resource("s3", verify=False) as s3: start_time = time.perf_counter() bucket = await s3.Bucket('superbai-test13') async for s3_object in bucket.objects.all(): print(s3_object) end_time = time.perf_counter() print(f"Took: {end_time-start_time}")
async def main(): async with aioboto3.resource('s3') as s3_resource: bucket = s3_resource.Bucket(name=envs.get("bucket_name")) tasks = [] async for obj in bucket.objects.all(): task = asyncio.create_task(download_file(s3_resource, bucket, obj)) tasks.append(task) await asyncio.gather(*tasks)
async def process_bucket(self): async with aioboto3.resource("s3") as s3: bucket = await s3.Bucket(self.__name) ( self.__number_of_files, self.__size, self.__most_recent_file, self.__last_modified, ) = await self.__process_files(bucket) print(self)
async def main(): # pylint: disable=not-async-context-manager async with aioboto3.resource( "s3", **self.resource_kwargs) as as3_resource: tasks = [] for file_name in file_names: bucket, key = self.bucket_key(file_name) object_acl = as3_resource.ObjectAcl(bucket, key) tasks.append(object_acl.put(ACL=ACLS[access])) await asyncio.gather(*tasks)
async def main(): # pylint: disable=not-async-context-manager async with aioboto3.resource( "s3", **self.resource_kwargs) as as3_resource: tasks = [] for file_name, datum in zip(file_names, data): bucket, key = self.bucket_key(file_name) object_ = as3_resource.Object(bucket, key) tasks.append(object_.download_fileobj(datum)) await asyncio.gather(*tasks, return_exceptions=True)
async def main(): # pylint: disable=not-async-context-manager async with aioboto3.resource( "s3", **self.resource_kwargs) as as3_resource: tasks = [] for file_name in file_names: bucket, key = self.bucket_key(file_name) object_ = as3_resource.Object(bucket, key) tasks.append(object_.content_length) return await asyncio.gather(*tasks, return_exceptions=True)
async def __get_dynamo_resource(self): """ Creates a connection to the table referenced by this instance. :return: The table to be used by this instance. """ async with aioboto3.resource( 'dynamodb', region_name=self.region_name, endpoint_url=self.endpoint_url) as dynamo_resource: logger.info('Establishing connection to DynamoDB') yield dynamo_resource
async def get_user_by_thalia_id(thalia_user_id: Union[str, int]) -> dict: async with aioboto3.resource("dynamodb") as dynamodb: users_table = await dynamodb.Table(USERS_TABLE) try: response = await users_table.query(KeyConditionExpression=Key( "thalia_user_id").eq(str(thalia_user_id))) except ClientError as e: logger.error(e.response["Error"]["Message"]) return None else: return response["Items"][0] if response["Count"] > 0 else None
async def s3_write(uri, txt): parsed = urlparse(uri) if parsed.scheme == 's3': bucket = parsed.netloc key = parsed.path[1:] async with aioboto3.resource("s3") as s3: obj = await s3.Object(bucket, key) await obj.put(Body=txt) #ogger.debug(f"Wrote {key}") else: await STAC_IO.default_write_text_method(uri, txt)
async def init_repository(): context_stack = contextlib.AsyncExitStack() _dynamodb["context_stack"] = context_stack _dynamodb["resource"] = await context_stack.enter_async_context( aioboto3.resource("dynamodb", **settings.DYNAMODB)) if settings.CREATE_REDIRECTS_TABLE: await _dynamodb["resource"].create_table( **settings.REDIRECTS_TABLE_CONFIG) _dynamodb["redirects_table"] = await _dynamodb["resource"].Table( settings.REDIRECTS_TABLE)
async def create(cls, id, loc): self = MockUser() self.id = id self.loc = loc # create the user in dynamodb async with aioboto3.resource("dynamodb", region_name="us-east-1") as dynamodb: table = await dynamodb.Table("user") await table.put_item(Item={"id": id, "last_location": loc}) return self
async def write_user(thalia_user_id: Union[str, int], discord_user_id: Union[str, int]): async with aioboto3.resource("dynamodb") as dynamodb: users_table = await dynamodb.Table(USERS_TABLE) response = await users_table.update_item( Key={"thalia_user_id": str(thalia_user_id)}, UpdateExpression="SET discord_user_id = :v", ExpressionAttributeValues={ ":v": str(discord_user_id), }, ) return response
async def upload(): async with aioboto3.resource( 's3', endpoint_url=config.s3.endpoint_url, aws_access_key_id=config.s3.access_key_id, aws_secret_access_key=config.s3.secret_access_key. get_secret_value(), config=AioConfig( s3={'addressing_style': config.s3.addressing_style })) as s3: bucket = s3.Bucket(config.s3.bucket) await bucket.upload_file(filename, Path(filename).name)
def resource(self) -> 'AioBotoS3ResourceContext': """Provide a context manager for accessing the S3 resources. async with handler.resource as s3: pass If the bare client is needed: async with handler.resource as s3: client = s3.meta.client """ return aioboto3.resource('s3', **self.__conn_options)
async def __get_dynamo_table(self): """ Creates a connection to the table referenced by this instance. :return: The table to be used by this instance. """ async with aioboto3.resource('dynamodb', region_name='eu-west-2', endpoint_url=config.get_config( 'DYNAMODB_ENDPOINT_URL', None)) as dynamo_resource: logger.info('008', 'Establishing connection to {table_name}', {'table_name': self.table_name}) yield dynamo_resource.Table(self.table_name)
async def run_notifier(): """Run processing SQS telegram notifications.""" async with aioboto3.resource("sqs") as sqs, aiohttp.ClientSession() as aio_session: queue = await sqs.get_queue_by_name(QueueName=config.SQS_QUEUE_NAME) while True: messages = await queue.receive_messages({"MaxNumberOfMessages": config.SQS_MAX_MESSAGES}) delete_tasks = [] notifications_tasks = [] for message in messages: delete_tasks.append(message.delete()) try: message_body = json.loads(await message.body) except json.JSONDecodeError as err: LOGGER.error("Could not deserialize message body. Error: %s", err) continue try: text = message_body["text"] except KeyError: LOGGER.error("Missed text field in message body: %s.", message_body) continue telegram_id = message_body.get("telegram_id") user_id = message_body.get("user_id") if not user_id and telegram_id is None: LOGGER.error("Missed user identifiers in message body: %s.", message_body) continue if not telegram_id: try: telegram_id = await User.get_telegram_id(1) except DatabaseError: continue parse_mode = message_body.get("parse_mode", "markdown") disable_notification = message_body.get("disable_notification", True) notifications_tasks.append(send_notification( aio_session=aio_session, telegram_id=telegram_id, text=text, parse_mode=parse_mode, disable_notification=disable_notification )) await asyncio.gather(*notifications_tasks, *delete_tasks) await asyncio.sleep(10)
async def _get_s3_files(self, path: str): new_cache = {} start = time.perf_counter() async with aioboto3.resource("s3") as s3: bucket = await s3.Bucket(self._bucket) object_key = S3Observer._get_object_key(path) async for file in bucket.objects.filter(Prefix=object_key): new_cache[ f"s3://{file.bucket_name}/{file.key}"] = await file.last_modified end = time.perf_counter() duration = end - start return new_cache
async def main(loop): connection = await aio_pika.connect_robust( "amqp://*****:*****@127.0.0.1/", loop=loop ) queue_name = "test_queue" other_queue = "test_queue2" print('MAIN --') # return await core.run_exe(connection, queue_name, other_queue) async with aioboto3.resource('dynamodb') as dynamo: dynamo_table = await dynamo.Table(RULES_TABLE) proc = RouterOrchestrator(connection, queue_name, other_queue, dynamo_table) await proc.run(dynamo_table)
async def _download_s3_file(url: str): parsed_url = parse.urlparse(url) logger.info( "Downloading S3 file from bucket '{}' with key '{}'".format( parsed_url.hostname, parsed_url.path[1:])) async with aioboto3.resource("s3") as s3: obj = await s3.Object(bucket_name=parsed_url.hostname, key=parsed_url.path[1:]) response = await obj.get() data = await response['Body'].read() logger.info("Finished downloading S3 file.") fp = tempfile.NamedTemporaryFile() fp.write(data) logger.info("Saved downloaded file to {}.".format(fp.name)) return fp
async def writeEndorserRegistrationLog(entry, status, reason, isotimestamp): if AWS_ENV: # Write an item to the trust_anchor_registration table in DynamoDB async with aioboto3.resource('dynamodb', region_name='us-west-2', endpoint_url="https://dynamodb.us-west-2.amazonaws.com") as dynamo_resource: table = dynamo_resource.Table('trust_anchor_registration') await table.put_item( Item={ 'did': entry['DID'], 'timestamp': isotimestamp, 'paymentaddr': entry['paymentaddr'], 'sourceIP': 'unknown', 'verkey': entry['verkey'], 'status': status, 'reason': reason } )
async def get_discord_users_by_thalia_ids( thalia_user_ids: List[str]) -> List[dict]: async with aioboto3.resource("dynamodb") as dynamodb: users_table = await dynamodb.Table(USERS_TABLE) try: response = await users_table.scan() data = response["Items"] while "LastEvaluatedKey" in response: response = await users_table.scan( ExclusiveStartKey=response["LastEvaluatedKey"]) data.extend(response["Items"]) except ClientError as e: logger.error(e.response["Error"]["Message"]) return None else: data = [ member for member in data if member["thalia_user_id"] in thalia_user_ids ] return data if len(data) > 0 else []
async def list_objects( self, prefix: str = "", limit: int = 100, as_url: bool = False, simple: bool = False ) -> List: objects = [] async with aioboto3.resource("s3", endpoint_url=self.endpoint) as s3: bucket = await s3.Bucket(self.bucket) async for obj in bucket.objects.filter(MaxKeys=limit, Prefix=prefix): if simple and as_url: raise RuntimeError(f"'simple' and 'as_url' args can't be passed together.") elif simple: obj = f"{obj.bucket_name}/{obj.key}" elif as_url: obj = f"{self.endpoint}/{obj.bucket_name}/{obj.key}" objects.append(obj) return objects