Пример #1
0
import utils.grid_error_logger as errlog
from utils.dynamodb_common import make_partition_key_4_state

from api.in_out_manager import in_out_manager

region = os.environ["REGION"]

sqs = boto3.resource('sqs', endpoint_url=f'https://sqs.{region}.amazonaws.com')
queue = sqs.get_queue_by_name(QueueName=os.environ['TASKS_QUEUE_NAME'])

dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(os.environ['TASKS_STATUS_TABLE_NAME'])

perf_tracker = performance_tracker_initializer(
    os.environ["METRICS_ARE_ENABLED"],
    os.environ["METRICS_SUBMIT_TASKS_LAMBDA_CONNECTION_STRING"],
    os.environ["METRICS_GRAFANA_PRIVATE_IP"])

task_input_passed_via_external_storage = os.environ["TASK_INPUT_PASSED_VIA_EXTERNAL_STORAGE"]
stdin_iom = in_out_manager(os.environ['GRID_STORAGE_SERVICE'], os.environ['S3_BUCKET'], os.environ['REDIS_URL'])


def write_to_dynamodb(task_json, batch):
    """

    Args:
      task_json:
      batch:

    Returns:
Пример #2
0
# TODO: We are using two retry logics for accessing DynamoDB config, and config_cc (for congestion control)
# Revisit this code and unify the logic.
config = Config(retries={'max_attempts': 5, 'mode': 'standard'})
dynamodb = boto3.resource('dynamodb', region_name=region, config=config)
status_table = dynamodb.Table(agent_config_data['ddb_status_table'])

config_cc = Config(retries={'max_attempts': 10, 'mode': 'adaptive'})
dynamodb_cc = boto3.resource('dynamodb', region_name=region, config=config_cc)
status_table_cc = dynamodb_cc.Table(agent_config_data['ddb_status_table'])

stdout_iom = in_out_manager(agent_config_data['grid_storage_service'],
                            agent_config_data['s3_bucket'],
                            agent_config_data['redis_url'])

perf_tracker_pre = performance_tracker_initializer(
    agent_config_data["metrics_are_enabled"],
    agent_config_data["metrics_pre_agent_connection_string"],
    agent_config_data["metrics_grafana_private_ip"])
event_counter_pre = EventsCounter([
    "agent_no_messages_in_tasks_queue", "agent_failed_to_claim_ddb_task",
    "agent_successful_acquire_a_task", "agent_auto_throtling_event",
    "rc_cubic_decrease_event"
])

perf_tracker_post = performance_tracker_initializer(
    agent_config_data["metrics_are_enabled"],
    agent_config_data["metrics_post_agent_connection_string"],
    agent_config_data["metrics_grafana_private_ip"])
event_counter_post = EventsCounter([
    "ddb_set_task_finished_failed", "ddb_set_task_finished_succeeded",
    "counter_update_ttl", "counter_update_ttl_failed",
    "counter_user_code_ret_code_failed", "bootstrap_failure",