import boto3 import logging import os import shutil import sys from botocore.config import Config from time import sleep from logging.handlers import TimedRotatingFileHandler AWS_CONFIG = Config(retries=dict(max_attempts=100)) # optional schedule class in schedule.py to add logic when to send files # default is to send all pictures to AWS try: from schedule import check_schedule except ImportError: class Schedule(): def check_schedule(self, image_file_name): # optional logic return True def make_dir_if_not_exists(dir): if not os.path.exists(dir): os.makedirs(dir) def detect_hooman(image_file): client = boto3.client('rekognition', config=AWS_CONFIG)
def s3_etag(url, proxies=None): """Check ETag on S3 object.""" s3_resource = boto3.resource("s3", config=Config(proxies=proxies)) bucket_name, s3_path = split_s3_path(url) s3_object = s3_resource.Object(bucket_name, s3_path) return s3_object.e_tag
def create_client(self): client_config = Config(signature_version='s3v4') return self.session.create_client('s3', self.region, config=client_config)
def get_cloudformation_client(session): config = Config(retries=dict(max_attempts=MAX_ATTEMPTS)) return session.client('cloudformation', config=config)
def route_53_client(self): if hasattr(self, '_route_53_client') == False: self._route_53_client = self.account_ctx.get_aws_client( 'route53', client_config=Config(retries={'max_attempts': 10})) return self._route_53_client
def createClient(self): myConfig = Config(region_name='us-west-2', ) self.client = boto3.client('s3', config=myConfig)
def test_sigv2_presign(self): config = Config(signature_version='s3') client = self.session.create_client('s3', self.region, config=config) url = client.generate_presigned_url(ClientMethod='list_buckets') self.assertNotIn('Algorithm=AWS4-HMAC-SHA256', url)
def sagemaker_runtime_client(boto_session): return boto_session.client("runtime.sagemaker", config=Config(retries={"max_attempts": 10}))
from mephisto.abstractions.providers.mturk.mturk_utils import setup_aws_credentials from mephisto.abstractions.architects.router import build_router from botocore import client # type: ignore from botocore.exceptions import ClientError, ProfileNotFound # type: ignore from botocore.config import Config # type: ignore from mephisto.utils.logger_core import get_logger logger = get_logger(name=__name__) if TYPE_CHECKING: from omegaconf import DictConfig # type: ignore botoconfig = Config(region_name="us-east-2", retries={ "max_attempts": 10, "mode": "standard" }) DEFAULT_AMI_ID = "ami-0f19d220602031aed" AMI_DEFAULT_USER = "******" DEFAULT_INSTANCE_TYPE = "m2.micro" FALLBACK_INSTANCE_TYPE = "t2.nano" MY_DIR = os.path.abspath(os.path.dirname(__file__)) DEFAULT_KEY_PAIR_DIRECTORY = os.path.join(MY_DIR, "keypairs") DEFAULT_SERVER_DETAIL_LOCATION = os.path.join(MY_DIR, "servers") SCRIPTS_DIRECTORY = os.path.join(MY_DIR, "run_scripts") DEFAULT_FALLBACK_FILE = os.path.join(DEFAULT_SERVER_DETAIL_LOCATION, "fallback.json") FALLBACK_SERVER_LOC = os.path.join(MY_DIR, "fallback_server") KNOWN_HOST_PATH = os.path.expanduser("~/.ssh/known_hosts")
def test_ns_bucket_unsigned_access(self, mcg_obj, bucket_factory, namespace_store_factory): """ Test anonymous(unsigned) access of S3 operations are denied on Namespace bucket. """ sample_data = "Sample string content to write to a S3 object" object_key = "ObjKey-" + str(uuid.uuid4().hex) # Create the namespace bucket nss_tup = ("oc", {"aws": [(1, self.DEFAULT_REGION)]}) ns_store = namespace_store_factory(*nss_tup)[0] bucketclass_dict = { "interface": "OC", "namespace_policy_dict": { "type": "Single", "namespacestores": [ns_store], }, } ns_bucket = bucket_factory( amount=1, interface=bucketclass_dict["interface"], bucketclass=bucketclass_dict, )[0].name # Put and Get object operations done with s3 credentials logger.info(f"Put and Get object operations on {ns_bucket}") assert bucket_utils.s3_put_object( s3_obj=mcg_obj, bucketname=ns_bucket, object_key=object_key, data=sample_data, ), "Failed: PutObject" assert bucket_utils.s3_get_object( s3_obj=mcg_obj, bucketname=ns_bucket, object_key=object_key), "Failed: GetObject" # Boto3 client with signing disabled anon_s3_client = boto3.client( "s3", verify=retrieve_verification_mode(), endpoint_url=mcg_obj.s3_endpoint, config=Config(signature_version=UNSIGNED), ) logger.info( f"Verifying anonymous access is blocked on namespace bucket: {ns_bucket}" ) try: anon_s3_client.get_object(Bucket=ns_bucket, Key=object_key) except boto3exception.ClientError as e: response = HttpResponseParser(e.response) assert (response.error["Code"] == "AccessDenied" ), f"Invalid error code:{response.error['Code']}" assert (response.status_code == 403 ), f"Invalid status code:{response.status_code}" assert (response.error["Message"] == "Access Denied" ), f"Invalid error message:{response.error['Message']}" else: assert ( False ), "GetObject operation has been granted access, when it should have been blocked"
logger.addHandler(handler) logger.addHandler(f_handler) logger.info( f"Path {folder_path}, credentials: {ACCESS_KEY}, {SECRET_KEY}, {BUCKET}, {ENDPOINT}" ) session = boto3.session.Session( aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, ) s3 = session.client( "s3", endpoint_url=ENDPOINT, use_ssl=True, config=Config(max_pool_connections=200), ) def download_file(bucket, k, dest_pathname, count, total): session_s = boto3.session.Session( aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, ) client = session_s.client( "s3", endpoint_url=ENDPOINT, use_ssl=True, config=Config(max_pool_connections=200), ) if Path(dest_pathname).is_file() and SKIP_EXISTING:
def get_credentials(short_term_name, lt_key_id, lt_access_key, args, config): if args.token: logger.debug("Received token as argument") mfa_token = '%s' % (args.token) else: console_input = prompter() mfa_token = console_input('Enter AWS MFA code for device [%s] ' '(renewing for %s seconds):' % (args.device, args.duration)) if args.real_proxies: client = boto3.client( 'sts', aws_access_key_id=lt_key_id, aws_secret_access_key=lt_access_key, config=Config(proxies=args.real_proxies) ) else: client = boto3.client( 'sts', aws_access_key_id=lt_key_id, aws_secret_access_key=lt_access_key ) if args.assume_role: logger.info("Assuming Role - Profile: %s, Role: %s, Duration: %s", short_term_name, args.assume_role, args.duration) if args.role_session_name is None: log_error_and_exit(logger, "You must specify a role session name " "via --role-session-name") try: response = client.assume_role( RoleArn=args.assume_role, RoleSessionName=args.role_session_name, DurationSeconds=args.duration, SerialNumber=args.device, TokenCode=mfa_token ) except ClientError as e: log_error_and_exit(logger, "An error occured while calling " "assume role: {}".format(e)) except ParamValidationError: log_error_and_exit(logger, "Token must be six digits") config.set( short_term_name, 'assumed_role', 'True', ) config.set( short_term_name, 'assumed_role_arn', args.assume_role, ) else: logger.info("Fetching Credentials - Profile: %s, Duration: %s", short_term_name, args.duration) try: response = client.get_session_token( DurationSeconds=args.duration, SerialNumber=args.device, TokenCode=mfa_token ) except ClientError as e: log_error_and_exit( logger, "An error occured while calling assume role: {}".format(e)) except ParamValidationError: log_error_and_exit( logger, "Token must be six digits") config.set( short_term_name, 'assumed_role', 'False', ) config.remove_option(short_term_name, 'assumed_role_arn') # aws_session_token and aws_security_token are both added # to support boto and boto3 options = [ ('aws_access_key_id', 'AccessKeyId'), ('aws_secret_access_key', 'SecretAccessKey'), ('aws_session_token', 'SessionToken'), ('aws_security_token', 'SessionToken'), ] for option, value in options: config.set( short_term_name, option, response['Credentials'][value] ) # Save expiration individiually, so it can be manipulated config.set( short_term_name, 'expiration', response['Credentials']['Expiration'].strftime('%Y-%m-%d %H:%M:%S') ) with open(AWS_CREDS_PATH, 'w') as configfile: config.write(configfile) logger.info( "Success! Your credentials will expire in %s seconds at: %s" % (args.duration, response['Credentials']['Expiration'])) sys.exit(0)
# or in the "license" file accompanying this file. This file is distributed # # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express # # or implied. See the License for the specific language governing permis- # # sions and limitations under the License. # ############################################################################### # # Imports import re import json import inspect import os import boto3 from botocore.config import Config from lib.metrics import Metrics boto_config = Config(retries={'max_attempts': 10}) # Get AWS region from Lambda environment. If not present then we're not # running under lambda, so defaulting to us-east-1 AWS_REGION = os.getenv('AWS_DEFAULT_REGION', 'us-east-1') securityhub = boto3.client('securityhub', config=boto_config, region_name=AWS_REGION) # Classes class InvalidFindingJson(Exception): pass
from datetime import datetime, date # Disable insecure warnings urllib3.disable_warnings() AWS_DEFAULT_REGION = demisto.params()['defaultRegion'] AWS_ROLE_ARN = demisto.params()['roleArn'] AWS_ROLE_SESSION_NAME = demisto.params()['roleSessionName'] AWS_ROLE_SESSION_DURATION = demisto.params()['sessionDuration'] AWS_ROLE_POLICY = None AWS_ACCESS_KEY_ID = demisto.params().get('access_key') AWS_SECRET_ACCESS_KEY = demisto.params().get('secret_key') VERIFY_CERTIFICATE = not demisto.params().get('insecure', True) proxies = handle_proxy(proxy_param_name='proxy', checkbox_default_value=False) config = Config(connect_timeout=1, retries=dict(max_attempts=5), proxies=proxies) """HELPER FUNCTIONS""" def aws_session(service='cloudtrail', region=None, roleArn=None, roleSessionName=None, roleSessionDuration=None, rolePolicy=None): kwargs = {} if roleArn and roleSessionName is not None: kwargs.update({ 'RoleArn': roleArn, 'RoleSessionName': roleSessionName,
ssm_parameter_credentials = os.environ['ssm_parameter_credentials'] checkip_url = os.environ['checkip_url'] # 内部参数 JobType = "PUT" MaxRetry = 10 # 最大请求重试次数 MaxThread = 50 # 最大线程数 MaxParallelFile = 1 # Lambda 中暂时没用到 JobTimeout = 900 ResumableThreshold = 5 * 1024 * 1024 # Accelerate to ignore small file CleanUnfinishedUpload = False # For debug ChunkSize = 5 * 1024 * 1024 # For debug, will be auto-change ifVerifyMD5Twice = False # For debug s3_config = Config(max_pool_connections=30) # 最大连接数 # Set environment logger = logging.getLogger() logger.setLevel(logging.INFO) dynamodb = boto3.resource('dynamodb') table = dynamodb.Table(table_queue_name) # 取另一个Account的credentials ssm = boto3.client('ssm') logger.info(f'Get ssm_parameter_credentials: {ssm_parameter_credentials}') credentials = json.loads( ssm.get_parameter(Name=ssm_parameter_credentials, WithDecryption=True)['Parameter']['Value']) credentials_session = boto3.session.Session(
def download(profile, output, include_non_default_policy_versions): """ Runs aws iam get-authorization-details on all accounts specified in the aws credentials file, and stores them in account-alias.json """ default_region = "us-east-1" session_data = {"region_name": default_region} if profile: session_data["profile_name"] = profile output_filename = os.path.join(output, f"{profile}.json") else: output_filename = "default.json" session = boto3.Session(**session_data) config = Config(connect_timeout=5, retries={"max_attempts": 10}) iam_client = session.client("iam", config=config) results = { "UserDetailList": [], "GroupDetailList": [], "RoleDetailList": [], "Policies": [], } paginator = iam_client.get_paginator("get_account_authorization_details") for page in paginator.paginate(Filter=["User"]): # Always add inline user policies results["UserDetailList"].extend(page["UserDetailList"]) for page in paginator.paginate(Filter=["Group"]): results["GroupDetailList"].extend(page["GroupDetailList"]) for page in paginator.paginate(Filter=["Role"]): results["RoleDetailList"].extend(page["RoleDetailList"]) # Ignore Service Linked Roles for policy in page["Policies"]: if policy["Path"] != "/service-role/": results["RoleDetailList"].append(policy) for page in paginator.paginate(Filter=["LocalManagedPolicy"]): # Add customer-managed policies IF they are attached to IAM principals for policy in page["Policies"]: if policy["AttachmentCount"] > 0: results["Policies"].append(policy) for page in paginator.paginate(Filter=["AWSManagedPolicy"]): # Add customer-managed policies IF they are attached to IAM principals for policy in page["Policies"]: if policy["AttachmentCount"] > 0: if include_non_default_policy_versions: results["Policies"].append(policy) else: policy_version_list = [] for policy_version in policy.get("PolicyVersionList"): if policy_version.get("VersionId") == policy.get( "DefaultVersionId"): policy_version_list.append(policy_version) break entry = { "PolicyName": policy.get("PolicyName"), "PolicyId": policy.get("PolicyId"), "Arn": policy.get("Arn"), "Path": policy.get("Path"), "DefaultVersionId": policy.get("DefaultVersionId"), "AttachmentCount": policy.get("AttachmentCount"), "PermissionsBoundaryUsageCount": policy.get("PermissionsBoundaryUsageCount"), "IsAttachable": policy.get("IsAttachable"), "CreateDate": policy.get("CreateDate"), "UpdateDate": policy.get("UpdateDate"), "PolicyVersionList": policy_version_list, } results["Policies"].append(entry) if os.path.exists(output_filename): os.remove(output_filename) with open(output_filename, "w") as file: json.dump(results, file, indent=4, default=str) print(f"Saved results to {output_filename}") return 1
import argparse import re from contextlib import contextmanager from functools import partial from time import time import boto3 from botocore.config import Config import s3mothball.commands from smart_open import open, parse_uri # set up boto clients config = Config(retries={'max_attempts': 10, 'mode': 'standard'}) s3_client = boto3.client('s3', config=config) s3_resource = boto3.resource('s3') # helpers def parse_s3_url(url): source_path_parsed = parse_uri(url) return source_path_parsed.bucket_id, source_path_parsed.key_id def delete_file(url): bucket, key = parse_s3_url(url) s3_client.delete_object(Bucket=bucket, Key=key) def iter_prefixes(url): bucket, key = parse_s3_url(url)
requests.put('http://169.254.169.254/latest/api/token', headers={ "X-aws-ec2-metadata-token-ttl-seconds": "21600" }).content, 'utf-8') instance_id = str( requests.get('http://169.254.169.254/latest/meta-data/instance-id', headers={ "X-aws-ec2-metadata-token": token }).content, 'utf-8') region = str( requests.get('http://169.254.169.254/latest/meta-data/placement/region', headers={ "X-aws-ec2-metadata-token": token }).content, 'utf-8') my_config = Config(region_name=region, ) ec2 = boto3.client('ec2', config=my_config) instances = ec2.describe_instances(Filters=[{ 'Name': 'tag:App', 'Values': ['tags-web-server'] }]) ips = [] for r in instances['Reservations']: for i in r['Instances']: if i.get('PrivateIpAddress') is not None: ips.append(i.get('PrivateIpAddress'))
class Organizations: """Class used for modeling Organizations """ _config = Config(retries=dict(max_attempts=30)) def __init__(self, role, account_id=None): self.client = role.client('organizations', config=Organizations._config) self.account_id = account_id self.account_ids = [] def get_parent_info(self): response = self.list_parents(self.account_id) return { "ou_parent_id": response.get('Id'), "ou_parent_type": response.get('Type') } def get_account_ids(self): for account in paginator(self.client.list_accounts): if account.get('Status') == 'ACTIVE': self.account_ids.append(account['Id']) return self.account_ids def get_organization_info(self): response = self.client.describe_organization() return { "organization_master_account_id": response.get('Organization').get('MasterAccountId'), "organization_id": response.get('Organization').get('Id') } def describe_ou_name(self, ou_id): try: response = self.client.describe_organizational_unit( OrganizationalUnitId=ou_id) return response['OrganizationalUnit']['Name'] except ClientError: raise RootOUIDError("OU is the Root of the Organization") @staticmethod def determine_ou_path(ou_path, ou_child_name): return '{0}/{1}'.format(ou_path, ou_child_name) if ou_path else ou_child_name def list_parents(self, ou_id): return self.client.list_parents(ChildId=ou_id).get('Parents')[0] def get_accounts_for_parent(self, parent_id): return paginator(self.client.list_accounts_for_parent, ParentId=parent_id) def get_child_ous(self, parent_id): return paginator(self.client.list_organizational_units_for_parent, ParentId=parent_id) def dir_to_ou(self, path): p = path.split('/')[1:] ou_id = self.client.list_roots().get('Roots')[0].get('Id') while p: for ou in self.get_child_ous(ou_id): if ou['Name'] == p[0]: p.pop(0) ou_id = ou['Id'] break else: raise Exception( "Path {} failed to return a child OU at '{}'".format( path, p[0])) else: # pylint: disable=W0120 return self.get_accounts_for_parent(ou_id) def build_account_path(self, ou_id, account_path, cache): """Builds a path tree to the account from the root of the Organization """ current = self.list_parents(ou_id) # While not at the root of the Organization while current.get('Type') != "ROOT": # check cache for ou name of id if not cache.check(current.get('Id')): cache.add(current.get('Id'), self.describe_ou_name(current.get('Id'))) ou_name = cache.check(current.get('Id')) account_path.append(ou_name) return self.build_account_path(current.get('Id'), account_path, cache) return Organizations.determine_ou_path( '/'.join(list(reversed(account_path))), self.describe_ou_name(self.get_parent_info().get("ou_parent_id")))
class InvalidInputError(ValueError): pass # Custom encoder for datetime objects class MyEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): return int(mktime(obj.timetuple())) return json.JSONEncoder.default(self, obj) # limit the number of retries submitted by boto3 because Step Functions will handle the exponential retries more efficiently config = Config(retries=dict(max_attempts=2)) client = boto3.client('transcribe', config=config) # Entrypoint for lambda funciton def lambda_handler(event, context): session = boto3.session.Session() region = session.region_name # Default to unsuccessful isSuccessful = "FALSE" # Create a random name for the transcription job jobname = id_generator()
def test_presign_unsigned(self): config = Config(signature_version=botocore.UNSIGNED) client = self.session.create_client('s3', 'us-east-2', config=config) url = client.generate_presigned_url(ClientMethod='list_buckets') self.assertEqual('https://s3.us-east-2.amazonaws.com/', url)
def __init__(self, bucket="congo8khz-pnnn"): self.bucket = bucket self.s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
# The partition size of each multipart chunk multipart_chunksize = 5 * MB # Whether use threads use_threads = True # Max retry times retries = 100 # target bucket and key name region = 'ap-northeast-1' bucket = 'multipart--upload-andrew-test' key = 'test/file' # to upload file name to_upload = 'bigfile' # Get the service client s3 = boto3.client('s3', region, config=Config(retries={'max_attempts': retries})) # Group things into config config = TransferConfig(multipart_threshold=multipart_threshold, max_concurrency=max_concurrency, multipart_chunksize=multipart_chunksize, use_threads=use_threads) # Upload tmp.txt to bucket-name at key-name before = time.time() s3.upload_file(to_upload, bucket, key, Config=config) after = time.time() print(after - before)
# initialise LOGGERs LOG_LEVEL = os.getenv('log_level', 'info') LOGGER = Logger(loglevel=LOG_LEVEL) APPLOGGER = LogHandler(PLAYBOOK) # application LOGGER for CW Logs # Get AWS region from Lambda environment. If not present then we're not # running under lambda, so defaulting to us-east-1 AWS_REGION = os.getenv('AWS_DEFAULT_REGION', 'us-east-1') AWS_PARTITION = os.getenv('AWS_PARTITION', 'aws') # Append region name to LAMBDA_ROLE LAMBDA_ROLE += '_' + AWS_REGION BOTO_CONFIG = Config( retries={ 'max_attempts': 10 }, region_name=AWS_REGION ) AWS = AWSClient(AWS_PARTITION, AWS_REGION) #------------------------------------------------------------------------------ # HANDLER #------------------------------------------------------------------------------ def lambda_handler(event, context): LOGGER.debug(event) metrics = Metrics(event) try: for finding_rec in event['detail']['findings']: finding = Finding(finding_rec) LOGGER.info('FINDING_ID: ' + str(finding.details.get('Id')))
def __init__( # type: ignore self, agent_config_id: str = None, name: str = None, labels: Iterable[str] = None, env_vars: dict = None, max_polls: int = None, agent_address: str = None, no_cloud_logs: bool = None, launch_type: str = "FARGATE", aws_access_key_id: str = None, aws_secret_access_key: str = None, aws_session_token: str = None, region_name: str = None, botocore_config: dict = None, enable_task_revisions: bool = False, use_external_kwargs: bool = False, external_kwargs_s3_bucket: str = None, external_kwargs_s3_key: str = None, **kwargs, ) -> None: super().__init__( agent_config_id=agent_config_id, name=name, labels=labels, env_vars=env_vars, max_polls=max_polls, agent_address=agent_address, no_cloud_logs=no_cloud_logs, ) if not kwargs.pop("_called_from_cli", False): warnings.warn( "`FargateAgent` is deprecated, please transition to using `ECSAgent` instead" ) from boto3 import client as boto3_client from boto3 import resource as boto3_resource from botocore.config import Config # Config used for boto3 client initialization aws_access_key_id = aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID") aws_secret_access_key = aws_secret_access_key or os.getenv( "AWS_SECRET_ACCESS_KEY" ) aws_session_token = aws_session_token or os.getenv("AWS_SESSION_TOKEN") region_name = region_name or os.getenv("REGION_NAME") botocore_config = botocore_config or {} # revisions and kwargs configurations self.enable_task_revisions = enable_task_revisions self.use_external_kwargs = use_external_kwargs self.external_kwargs_s3_bucket = external_kwargs_s3_bucket self.external_kwargs_s3_key = external_kwargs_s3_key self.launch_type = launch_type # Parse accepted kwargs for task definition, run, and container definitions key of task # definition ( self.task_definition_kwargs, self.task_run_kwargs, self.container_definitions_kwargs, ) = self._parse_kwargs(kwargs, True) # Client initialization self.boto3_client = boto3_client( "ecs", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, config=Config(**botocore_config), ) # fetch external kwargs from s3 if needed if self.use_external_kwargs: self.logger.info("Use of external S3 kwargs enabled.") self.s3_resource = boto3_resource( "s3", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, ) # get boto3 client for resource groups tagging api if self.enable_task_revisions: self.logger.info("Native ECS task revisions enabled.") self.boto3_client_tags = boto3_client( "resourcegroupstaggingapi", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, config=Config(**botocore_config), ) self.logger.debug(f"Launch type: {self.launch_type}") self.logger.debug(f"Enable task revisions: {self.enable_task_revisions}") self.logger.debug(f"Use external kwargs: {self.use_external_kwargs}") self.logger.debug( f"External kwargs S3 bucket: {self.external_kwargs_s3_bucket}" ) self.logger.debug(f"External kwargs S3 key: {self.external_kwargs_s3_key}")
def _resource(name, config): boto_config = Config(retries={'max_attempts': BOTO_MAX_RETRIES}) return boto3.resource( name, config["provider"]["region"], config=boto_config)
def s3_get(url, temp_file, proxies=None): """Pull a file directly from S3.""" s3_resource = boto3.resource("s3", config=Config(proxies=proxies)) bucket_name, s3_path = split_s3_path(url) s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file)
def jobs_compare(args): job_source = args['job-name-source'] job_target = args['job-name-target'] region = args['region'] profile = args['profile'] boto3.setup_default_session(profile_name=profile) region_config = Config(region_name=region) glue = boto3.client('glue', config=region_config) response_source = glue.get_job(JobName=job_source) response_source = response_source['Job'] del response_source['Name'] del response_source['CreatedOn'] del response_source['LastModifiedOn'] response_target = glue.get_job(JobName=job_target) response_target = response_target['Job'] del response_target['Name'] del response_target['CreatedOn'] del response_target['LastModifiedOn'] diff = {} for key in response_source: value = response_source[key] if isinstance(value, dict): diff_dict = {} for key_dict in value: value_dict = value[key_dict] if key_dict in response_target[key]: if str(value_dict) != str(response_target[key][key_dict]): diff_dict[key_dict] = "{} <> {}".format( str(value_dict), str(response_target[key][key_dict])) else: diff_dict[key_dict] = "{} <> {}".format( str(value_dict), "") if len(diff_dict) > 0: diff[key] = diff_dict else: if key in response_target: if str(value) != str(response_target[key]): diff[key] = "{} <> {}".format(str(value), str(response_target[key])) else: diff[key] = "{} <> {}".format(str(value), "") for key in response_target: value = response_target[key] if isinstance(value, dict): diff_dict = diff[key] if key in diff else {} for key_dict in value: value_dict = value[key_dict] if not key_dict in response_source[key]: diff_dict[key_dict] = "{} <> {}".format( "", str(value_dict)) if len(diff_dict) > 0: diff[key] = diff_dict else: if not key in response_source: diff[key] = "{} <> {}".format("", str(value)) if len(diff) > 0: print(json.dumps(diff, indent=4)) else: print("Não existem diferenças")
def setUp(self): super(TestS3SigV4Client, self).setUp() self.client = self.session.create_client( 's3', self.region, config=Config(signature_version='s3v4'))
def __init__(self, ec2=None, autoscaling=None): """ Initializes aws sdk clients """ self.ec2 = ec2 or boto3.client( 'ec2', config=Config(retries={'max_attempts': BOTO3_RETRIES})) self.asg = autoscaling or boto3.client('autoscaling')