async def remove_user(thalia_user_id: int):
logger.info(f"Removing user {thalia_user_id} from DB.")
async with aioboto3.resource("dynamodb") as dynamodb:
users_table = await dynamodb.Table(USERS_TABLE)
await users_table.delete_item(
Key={"thalia_user_id": str(thalia_user_id)})
return
async def main():
global COUNT # pylint: disable=global-statement
config = Config(signature_version=botocore.UNSIGNED,
max_pool_connections=50,
inject_host_prefix=False,
parameter_validation=False)
async with aioboto3.resource('dynamodb',
endpoint_url='http://127.0.0.1:8080',
config=config) as dynamo_resource: # pylint: disable=not-async-context-manager
table = dynamo_resource.Table('usertable')
while True:
requests = []
start = time.time()
for _ in range(10000):
key = next(KEYS)
np.random.seed(key)
requests.append(
table.put_item(
Item={
'p': key.to_bytes(10, byteorder='big'),
'C0': np.random.bytes(64)
}))
asyncio.gather(*requests)
COUNT += len(requests)
elapsed = time.time() - start
elapsed_since_start = time.time() - START_TIME
print(COUNT, elapsed_since_start, elapsed,
COUNT / elapsed_since_start)
async def _initialize(self):
"""Initialize our async resources: aiohttp, aioboto3"""
self.http = aiohttp.ClientSession()
self.dynamo = aioboto3.resource('dynamodb', region_name=DYNAMO_REGION)
self.ddb_user_table = self.dynamo.Table(DYNAMO_USER_TABLE)
self.ddb_entity_table = self.dynamo.Table(DYNAMO_ENTITY_TABLE)
log.info("DDB client initialized")
async def main():
async with aioboto3.resource('dynamodb') as dynamo:
dynamo_table = await dynamo.Table(DATA_TABLE)
connection = await connect_robust("amqp://*****:*****@localhost/")
_channel = await connection.channel()
_exchange = await _channel.declare_exchange('hcTest',
ExchangeType.TOPIC)
# Create the queues
_queue_get = await _channel.declare_queue('hcTest-get-queue')
_queue_save = await _channel.declare_queue('hcTest-save-queue')
# Bind the queues
await _queue_get.bind(_exchange, 'hcTest-get')
await _queue_save.bind(_exchange, 'hcTest-save')
topic_queues = []
topic_queues.append(MessageProcessor(_queue_get, process_get_event))
topic_queues.append(MessageProcessor(_queue_save, process_save_event))
# Call the functions for each event
await asyncio.wait([
asyncio.create_task(
proccess_events(message_process.queue, message_process.fn,
dynamo_table))
for message_process in topic_queues
])
async def test_getting_resource(event_loop):
"""Simple getting of resource."""
aioboto3.DEFAULT_SESSION = None
resource = aioboto3.resource('dynamodb', loop=event_loop, region_name='eu-central-1')
assert isinstance(resource.meta.client, AioBaseClient)
await resource.close()
async def get_and_send_styled_image(chat_id,
styled_image_key,
caption,
resulted_notebook_key=None):
async with aioboto3.resource(
"s3",
aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
region_name=os.environ["REGION_NAME"],
) as s3:
image_obj = await s3.Object(bucket_name=S3_BUCKET_WITH_RESULTS_NAME,
key=styled_image_key)
image_resp = await image_obj.get()
image_body = image_resp["Body"]
image_bytes = await image_body.read()
await bot.send_photo(chat_id, image_bytes, caption=caption)
await image_obj.delete()
if resulted_notebook_key is not None:
try:
resulted_notebook_obj = await s3.Object(
bucket_name=S3_BUCKET_WITH_RESULTS_NAME,
key=resulted_notebook_key)
await resulted_notebook_obj.delete()
except Exception as e:
logging.error(e)
async def loop(self):
with contextlib.suppress(Exception):
async with aioboto3.resource(
'sqs',
region_name='ru-central1',
endpoint_url='https://message-queue.api.cloud.yandex.net/'
) as resource:
queue = await resource.Queue(url=Config.SQS_URL)
while True:
with contextlib.suppress(Exception):
async with session_scope() as session:
async for position, ticket in self.get_tickets(
session):
with contextlib.suppress(Exception):
await queue.send_message(
MessageBody=ujson.dumps({
'target':
ticket.user.email,
'time_left':
ticket.service.average_waiting_time
* position,
'position':
position,
'from':
self.__class__.__name__
}))
ticket.pushed = True
await asyncio.sleep(self.SLEEP_INTERVAL)
async def _create_sqs_resource(app: web.Application) -> None:
config = app['config']
app['sqs_resource'] = aioboto3.resource(
'sqs',
loop=app.loop,
region_name=config.aws_region_name,
aws_access_key_id=config.aws_access_key_id,
aws_secret_access_key=config.aws_secret_access_key)
async def main():
async with aioboto3.resource("s3", verify=False) as s3:
start_time = time.perf_counter()
bucket = await s3.Bucket('superbai-test13')
async for s3_object in bucket.objects.all():
print(s3_object)
end_time = time.perf_counter()
print(f"Took: {end_time-start_time}")
async def main():
async with aioboto3.resource('s3') as s3_resource:
bucket = s3_resource.Bucket(name=envs.get("bucket_name"))
tasks = []
async for obj in bucket.objects.all():
task = asyncio.create_task(download_file(s3_resource, bucket, obj))
tasks.append(task)
await asyncio.gather(*tasks)
async def process_bucket(self):
async with aioboto3.resource("s3") as s3:
bucket = await s3.Bucket(self.__name)
(
self.__number_of_files,
self.__size,
self.__most_recent_file,
self.__last_modified,
) = await self.__process_files(bucket)
print(self)
async def main():
# pylint: disable=not-async-context-manager
async with aioboto3.resource(
"s3", **self.resource_kwargs) as as3_resource:
tasks = []
for file_name in file_names:
bucket, key = self.bucket_key(file_name)
object_acl = as3_resource.ObjectAcl(bucket, key)
tasks.append(object_acl.put(ACL=ACLS[access]))
await asyncio.gather(*tasks)
async def main():
# pylint: disable=not-async-context-manager
async with aioboto3.resource(
"s3", **self.resource_kwargs) as as3_resource:
tasks = []
for file_name, datum in zip(file_names, data):
bucket, key = self.bucket_key(file_name)
object_ = as3_resource.Object(bucket, key)
tasks.append(object_.download_fileobj(datum))
await asyncio.gather(*tasks, return_exceptions=True)
async def main():
# pylint: disable=not-async-context-manager
async with aioboto3.resource(
"s3", **self.resource_kwargs) as as3_resource:
tasks = []
for file_name in file_names:
bucket, key = self.bucket_key(file_name)
object_ = as3_resource.Object(bucket, key)
tasks.append(object_.content_length)
return await asyncio.gather(*tasks, return_exceptions=True)
async def __get_dynamo_resource(self):
"""
Creates a connection to the table referenced by this instance.
:return: The table to be used by this instance.
"""
async with aioboto3.resource(
'dynamodb',
region_name=self.region_name,
endpoint_url=self.endpoint_url) as dynamo_resource:
logger.info('Establishing connection to DynamoDB')
yield dynamo_resource
async def get_user_by_thalia_id(thalia_user_id: Union[str, int]) -> dict:
async with aioboto3.resource("dynamodb") as dynamodb:
users_table = await dynamodb.Table(USERS_TABLE)
try:
response = await users_table.query(KeyConditionExpression=Key(
"thalia_user_id").eq(str(thalia_user_id)))
except ClientError as e:
logger.error(e.response["Error"]["Message"])
return None
else:
return response["Items"][0] if response["Count"] > 0 else None
async def s3_write(uri, txt):
parsed = urlparse(uri)
if parsed.scheme == 's3':
bucket = parsed.netloc
key = parsed.path[1:]
async with aioboto3.resource("s3") as s3:
obj = await s3.Object(bucket, key)
await obj.put(Body=txt)
#ogger.debug(f"Wrote {key}")
else:
await STAC_IO.default_write_text_method(uri, txt)
async def init_repository():
context_stack = contextlib.AsyncExitStack()
_dynamodb["context_stack"] = context_stack
_dynamodb["resource"] = await context_stack.enter_async_context(
aioboto3.resource("dynamodb", **settings.DYNAMODB))
if settings.CREATE_REDIRECTS_TABLE:
await _dynamodb["resource"].create_table(
**settings.REDIRECTS_TABLE_CONFIG)
_dynamodb["redirects_table"] = await _dynamodb["resource"].Table(
settings.REDIRECTS_TABLE)
async def create(cls, id, loc):
self = MockUser()
self.id = id
self.loc = loc
# create the user in dynamodb
async with aioboto3.resource("dynamodb",
region_name="us-east-1") as dynamodb:
table = await dynamodb.Table("user")
await table.put_item(Item={"id": id, "last_location": loc})
return self
async def write_user(thalia_user_id: Union[str, int],
discord_user_id: Union[str, int]):
async with aioboto3.resource("dynamodb") as dynamodb:
users_table = await dynamodb.Table(USERS_TABLE)
response = await users_table.update_item(
Key={"thalia_user_id": str(thalia_user_id)},
UpdateExpression="SET discord_user_id = :v",
ExpressionAttributeValues={
":v": str(discord_user_id),
},
)
return response
async def upload():
async with aioboto3.resource(
's3',
endpoint_url=config.s3.endpoint_url,
aws_access_key_id=config.s3.access_key_id,
aws_secret_access_key=config.s3.secret_access_key.
get_secret_value(),
config=AioConfig(
s3={'addressing_style': config.s3.addressing_style
})) as s3:
bucket = s3.Bucket(config.s3.bucket)
await bucket.upload_file(filename, Path(filename).name)
def resource(self) -> 'AioBotoS3ResourceContext':
"""Provide a context manager for accessing the S3 resources.
async with handler.resource as s3:
pass
If the bare client is needed:
async with handler.resource as s3:
client = s3.meta.client
"""
return aioboto3.resource('s3', **self.__conn_options)
async def __get_dynamo_table(self):
"""
Creates a connection to the table referenced by this instance.
:return: The table to be used by this instance.
"""
async with aioboto3.resource('dynamodb',
region_name='eu-west-2',
endpoint_url=config.get_config(
'DYNAMODB_ENDPOINT_URL',
None)) as dynamo_resource:
logger.info('008', 'Establishing connection to {table_name}',
{'table_name': self.table_name})
yield dynamo_resource.Table(self.table_name)
async def run_notifier():
"""Run processing SQS telegram notifications."""
async with aioboto3.resource("sqs") as sqs, aiohttp.ClientSession() as aio_session:
queue = await sqs.get_queue_by_name(QueueName=config.SQS_QUEUE_NAME)
while True:
messages = await queue.receive_messages({"MaxNumberOfMessages": config.SQS_MAX_MESSAGES})
delete_tasks = []
notifications_tasks = []
for message in messages:
delete_tasks.append(message.delete())
try:
message_body = json.loads(await message.body)
except json.JSONDecodeError as err:
LOGGER.error("Could not deserialize message body. Error: %s", err)
continue
try:
text = message_body["text"]
except KeyError:
LOGGER.error("Missed text field in message body: %s.", message_body)
continue
telegram_id = message_body.get("telegram_id")
user_id = message_body.get("user_id")
if not user_id and telegram_id is None:
LOGGER.error("Missed user identifiers in message body: %s.", message_body)
continue
if not telegram_id:
try:
telegram_id = await User.get_telegram_id(1)
except DatabaseError:
continue
parse_mode = message_body.get("parse_mode", "markdown")
disable_notification = message_body.get("disable_notification", True)
notifications_tasks.append(send_notification(
aio_session=aio_session,
telegram_id=telegram_id,
text=text,
parse_mode=parse_mode,
disable_notification=disable_notification
))
await asyncio.gather(*notifications_tasks, *delete_tasks)
await asyncio.sleep(10)
async def _get_s3_files(self, path: str):
new_cache = {}
start = time.perf_counter()
async with aioboto3.resource("s3") as s3:
bucket = await s3.Bucket(self._bucket)
object_key = S3Observer._get_object_key(path)
async for file in bucket.objects.filter(Prefix=object_key):
new_cache[
f"s3://{file.bucket_name}/{file.key}"] = await file.last_modified
end = time.perf_counter()
duration = end - start
return new_cache
async def main(loop):
connection = await aio_pika.connect_robust(
"amqp://*****:*****@127.0.0.1/", loop=loop
)
queue_name = "test_queue"
other_queue = "test_queue2"
print('MAIN --')
# return await core.run_exe(connection, queue_name, other_queue)
async with aioboto3.resource('dynamodb') as dynamo:
dynamo_table = await dynamo.Table(RULES_TABLE)
proc = RouterOrchestrator(connection, queue_name, other_queue, dynamo_table)
await proc.run(dynamo_table)
async def _download_s3_file(url: str):
parsed_url = parse.urlparse(url)
logger.info(
"Downloading S3 file from bucket '{}' with key '{}'".format(
parsed_url.hostname, parsed_url.path[1:]))
async with aioboto3.resource("s3") as s3:
obj = await s3.Object(bucket_name=parsed_url.hostname,
key=parsed_url.path[1:])
response = await obj.get()
data = await response['Body'].read()
logger.info("Finished downloading S3 file.")
fp = tempfile.NamedTemporaryFile()
fp.write(data)
logger.info("Saved downloaded file to {}.".format(fp.name))
return fp
async def writeEndorserRegistrationLog(entry, status, reason, isotimestamp):
if AWS_ENV:
# Write an item to the trust_anchor_registration table in DynamoDB
async with aioboto3.resource('dynamodb', region_name='us-west-2', endpoint_url="https://dynamodb.us-west-2.amazonaws.com") as dynamo_resource:
table = dynamo_resource.Table('trust_anchor_registration')
await table.put_item(
Item={
'did': entry['DID'],
'timestamp': isotimestamp,
'paymentaddr': entry['paymentaddr'],
'sourceIP': 'unknown',
'verkey': entry['verkey'],
'status': status,
'reason': reason
}
)
async def get_discord_users_by_thalia_ids(
thalia_user_ids: List[str]) -> List[dict]:
async with aioboto3.resource("dynamodb") as dynamodb:
users_table = await dynamodb.Table(USERS_TABLE)
try:
response = await users_table.scan()
data = response["Items"]
while "LastEvaluatedKey" in response:
response = await users_table.scan(
ExclusiveStartKey=response["LastEvaluatedKey"])
data.extend(response["Items"])
except ClientError as e:
logger.error(e.response["Error"]["Message"])
return None
else:
data = [
member for member in data
if member["thalia_user_id"] in thalia_user_ids
]
return data if len(data) > 0 else []
async def list_objects(
self,
prefix: str = "",
limit: int = 100,
as_url: bool = False,
simple: bool = False
) -> List:
objects = []
async with aioboto3.resource("s3", endpoint_url=self.endpoint) as s3:
bucket = await s3.Bucket(self.bucket)
async for obj in bucket.objects.filter(MaxKeys=limit, Prefix=prefix):
if simple and as_url:
raise RuntimeError(f"'simple' and 'as_url' args can't be passed together.")
elif simple:
obj = f"{obj.bucket_name}/{obj.key}"
elif as_url:
obj = f"{self.endpoint}/{obj.bucket_name}/{obj.key}"
objects.append(obj)
return objects
