import utils.grid_error_logger as errlog
from utils.dynamodb_common import make_partition_key_4_state
from api.in_out_manager import in_out_manager
region = os.environ["REGION"]
sqs = boto3.resource('sqs', endpoint_url=f'https://sqs.{region}.amazonaws.com')
queue = sqs.get_queue_by_name(QueueName=os.environ['TASKS_QUEUE_NAME'])
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(os.environ['TASKS_STATUS_TABLE_NAME'])
perf_tracker = performance_tracker_initializer(
os.environ["METRICS_ARE_ENABLED"],
os.environ["METRICS_SUBMIT_TASKS_LAMBDA_CONNECTION_STRING"],
os.environ["METRICS_GRAFANA_PRIVATE_IP"])
task_input_passed_via_external_storage = os.environ["TASK_INPUT_PASSED_VIA_EXTERNAL_STORAGE"]
stdin_iom = in_out_manager(os.environ['GRID_STORAGE_SERVICE'], os.environ['S3_BUCKET'], os.environ['REDIS_URL'])
def write_to_dynamodb(task_json, batch):
"""
Args:
task_json:
batch:
Returns:
# TODO: We are using two retry logics for accessing DynamoDB config, and config_cc (for congestion control)
# Revisit this code and unify the logic.
config = Config(retries={'max_attempts': 5, 'mode': 'standard'})
dynamodb = boto3.resource('dynamodb', region_name=region, config=config)
status_table = dynamodb.Table(agent_config_data['ddb_status_table'])
config_cc = Config(retries={'max_attempts': 10, 'mode': 'adaptive'})
dynamodb_cc = boto3.resource('dynamodb', region_name=region, config=config_cc)
status_table_cc = dynamodb_cc.Table(agent_config_data['ddb_status_table'])
stdout_iom = in_out_manager(agent_config_data['grid_storage_service'],
agent_config_data['s3_bucket'],
agent_config_data['redis_url'])
perf_tracker_pre = performance_tracker_initializer(
agent_config_data["metrics_are_enabled"],
agent_config_data["metrics_pre_agent_connection_string"],
agent_config_data["metrics_grafana_private_ip"])
event_counter_pre = EventsCounter([
"agent_no_messages_in_tasks_queue", "agent_failed_to_claim_ddb_task",
"agent_successful_acquire_a_task", "agent_auto_throtling_event",
"rc_cubic_decrease_event"
])
perf_tracker_post = performance_tracker_initializer(
agent_config_data["metrics_are_enabled"],
agent_config_data["metrics_post_agent_connection_string"],
agent_config_data["metrics_grafana_private_ip"])
event_counter_post = EventsCounter([
"ddb_set_task_finished_failed", "ddb_set_task_finished_succeeded",
"counter_update_ttl", "counter_update_ttl_failed",
"counter_user_code_ret_code_failed", "bootstrap_failure",