def test_pagination_produces_correct_number_of_pages(self):
bank_id = self._create_200_members()
api = TApp(
get_bank_api(
self.get_table(),
"irrelevant_s3_bucket_for_this_test",
"irrelevant_sqs_queue",
get_default_signal_type_mapping(),
)
)
running_count = 0
continuation_token = None
unique_member_ids = set()
while True:
if continuation_token:
response = json.loads(
api.get(
f"/get-members/{bank_id}?content_type=photo&continuation_token={continuation_token}"
).body
)
else:
response = json.loads(
api.get(f"/get-members/{bank_id}?content_type=photo").body
)
running_count += len(response["bank_members"])
continuation_token = response["continuation_token"]
unique_member_ids.update(
map(lambda member: member["bank_member_id"], response["bank_members"])
)
if continuation_token == None:
# Last page should not have any continuation_token
break
# Checks for total number of items received. Should work with any page size.
assert running_count == 200
# Checks that the number of unique member ids is equal to the expected
# value (ie. no repeats)
assert len(unique_member_ids) == 200
def bottle_init_once() -> t.Tuple[bottle.AppStack, t.Callable[
[t.Dict[str, t.Any], t.Any], t.Dict[str, t.Any]]]:
"""
Meant to be called once per lambda instance. Returns a bottle app and an
api_wsgi_handler that can be plugged into a lambda handler.
The method also serves as a closure for all dependencies that need to be
resolved at startup.
"""
app = bottle.default_app()
# Initialize hmaconfig at module level. Mounted SubApps need not initialize
# their own HMAConfigs.
HMAConfig.initialize(HMA_CONFIG_TABLE)
functionality_mapping = get_pytx_functionality_mapping()
@app.get("/root/")
def root():
"""
root endpoint to make sure the API is live and check when it was last updated
"""
context = bottle.request.environ.get("apig_wsgi.context")
invoked_function_arn = context.invoked_function_arn
client = boto3.client("lambda")
last_modified = client.get_function_configuration(
FunctionName=invoked_function_arn)["LastModified"]
return {
"message": "Welcome to the HMA API!",
"last_modified": last_modified,
}
app.mount(
"/action-rules/",
get_action_rules_api(hma_config_table=HMA_CONFIG_TABLE),
)
app.mount(
"/matches/",
get_matches_api(
datastore_table=dynamodb.Table(DYNAMODB_TABLE),
hma_config_table=HMA_CONFIG_TABLE,
indexes_bucket_name=INDEXES_BUCKET_NAME,
writeback_queue_url=WRITEBACK_QUEUE_URL,
bank_table=dynamodb.Table(BANKS_TABLE),
signal_type_mapping=functionality_mapping.signal_and_content,
),
)
app.mount(
"/content/",
get_content_api(
dynamodb_table=dynamodb.Table(DYNAMODB_TABLE),
image_bucket=IMAGE_BUCKET_NAME,
image_prefix=IMAGE_PREFIX,
signal_type_mapping=functionality_mapping.signal_and_content,
),
)
app.mount(
"/submit/",
get_submit_api(
dynamodb_table=dynamodb.Table(DYNAMODB_TABLE),
image_bucket=IMAGE_BUCKET_NAME,
image_prefix=IMAGE_PREFIX,
submissions_queue_url=SUBMISSIONS_QUEUE_URL,
hash_queue_url=HASHES_QUEUE_URL,
signal_type_mapping=functionality_mapping.signal_and_content,
),
)
app.mount(
"/datasets/",
get_datasets_api(
hma_config_table=HMA_CONFIG_TABLE,
datastore_table=dynamodb.Table(DYNAMODB_TABLE),
threat_exchange_data_bucket_name=THREAT_EXCHANGE_DATA_BUCKET_NAME,
threat_exchange_data_folder=THREAT_EXCHANGE_DATA_FOLDER,
),
)
app.mount("/stats/",
get_stats_api(counts_table=dynamodb.Table(COUNTS_TABLE_NAME)))
app.mount(
"/actions/",
get_actions_api(hma_config_table=HMA_CONFIG_TABLE),
)
app.mount(
"/banks/",
get_bank_api(
bank_table=dynamodb.Table(BANKS_TABLE),
bank_user_media_bucket=BANKS_MEDIA_BUCKET_NAME,
submissions_queue_url=SUBMISSIONS_QUEUE_URL,
signal_type_mapping=functionality_mapping.signal_and_content,
),
)
app.mount(
"/indexes/",
get_indexes_api(
indexes_bucket_name=INDEXES_BUCKET_NAME,
indexer_function_name=INDEXER_FUNCTION_NAME,
),
)
app.mount(
"/lcc/",
get_lcc_api(
storage_path=LCC_DURABLE_FS_PATH,
signal_type_mapping=functionality_mapping.signal_and_content,
),
)
apig_wsgi_handler = make_lambda_handler(app)
return (app, apig_wsgi_handler)
image_bucket=IMAGE_BUCKET_NAME,
image_prefix=IMAGE_PREFIX,
submissions_queue_url=SUBMISSIONS_QUEUE_URL,
hash_queue_url=HASHES_QUEUE_URL,
),
)
app.mount(
"/datasets/",
get_datasets_api(
hma_config_table=HMA_CONFIG_TABLE,
datastore_table=dynamodb.Table(DYNAMODB_TABLE),
threat_exchange_data_bucket_name=THREAT_EXCHANGE_DATA_BUCKET_NAME,
threat_exchange_data_folder=THREAT_EXCHANGE_DATA_FOLDER,
threat_exchange_pdq_file_extension=THREAT_EXCHANGE_PDQ_FILE_EXTENSION,
),
)
app.mount("/stats/",
get_stats_api(dynamodb_table=dynamodb.Table(DYNAMODB_TABLE)))
app.mount(
"/actions/",
get_actions_api(hma_config_table=HMA_CONFIG_TABLE),
)
app.mount("/banks/", get_bank_api(dynamodb.Table(BANKS_TABLE)))
if __name__ == "__main__":
app.run()
),
)
app.mount("/stats/",
get_stats_api(counts_table=dynamodb.Table(COUNTS_TABLE_NAME)))
app.mount(
"/actions/",
get_actions_api(hma_config_table=HMA_CONFIG_TABLE),
)
app.mount(
"/banks/",
get_bank_api(
bank_table=dynamodb.Table(BANKS_TABLE),
bank_user_media_bucket=BANKS_MEDIA_BUCKET_NAME,
submissions_queue_url=SUBMISSIONS_QUEUE_URL,
),
)
app.mount(
"/indexes/",
get_indexes_api(
indexes_bucket_name=INDEXES_BUCKET_NAME,
indexer_function_name=INDEXER_FUNCTION_NAME,
),
)
if __name__ == "__main__":
app.run()