def run_ebs_deploy(command, args, parser=None): """ the main """ init_logging(args.use_logging) # make sure we have an archive or a directory if not args.config_file or not os.path.exists(args.config_file): out("Config file not found: "+args.config_file) parser and parser.print_help() return -1 # make sure that if we have a role to assume, that we also have a role name to display if (args.role_arn and not args.role_name) or (args.role_name and not args.role_arn): out("You must use and --role-arn and --role-name together") parser and parser.print_help() return -1 # enable logging if args.verbose: from boto import set_stream_logger set_stream_logger('boto') config = load_config(args.config_file) aws = make_aws_credentials(config, args.role_arn, args.role_name) helper = EbsHelper(aws, app_name=get(config, 'app.app_name'), wait_time_secs=args.wait_time) # execute the command return command.execute(helper, config, args)
def MANUAL_test_redeliver_bad(self): sqs_backend.SAFE_MAX_VISIBILITY_TIMEOUT = sqs_backend.AWS_MAX_VISIBILITY_TIMEOUT - 1 with self.backend.open(self.test_queue) as queue: print 'PURGE' queue.purge() boto.set_stream_logger('tellme') print '-' * 80 print 'Add Message' print '-' * 80 queue.put('hello9') time.sleep(1) print '-' * 80 print 'Get Message' print '-' * 80 message_body, message_id = queue.get() self.assertEqual(message_body, 'hello9') print '-' * 80 print 'Redeliver Message %d seconds' % sqs_backend.SAFE_MAX_VISIBILITY_TIMEOUT print '-' * 80 queue.redeliver((sqs_backend.SAFE_MAX_VISIBILITY_TIMEOUT, message_id)) time.sleep(5) print '-' * 80 print 'Redeliver Message %d seconds' % sqs_backend.SAFE_MAX_VISIBILITY_TIMEOUT print '-' * 80 queue.redeliver((sqs_backend.SAFE_MAX_VISIBILITY_TIMEOUT, message_id))
def __init__(self, hostname, aws_access_key, aws_secret_key, path='/services/Properties', port=8773, is_secure=False, boto_debug_level=0, err_method=None, logger=None, **kwargs): """ Primary Admin/Properties Query interface for a Eucalyptus Cloud :param hostname: service endpoint, hostname, ip, etc.. :param access_key: cloud user access key to auth this connection :param secret_key: cloud user secret key to auth this connection :param port: remote port to be used for this connection :param path: service path for this connection :param is_secure: bool :param boto_debug_level: int 0-2 :param debug_method: method to be used for logging debug information :param err_method: method to be used for logging error information :param kwargs: Additional kwargs to be passed to connection init :raise ValueError: Upon or invalid params """ # Note: aws_access and secret_key get assigned to self during super().__init__() self.host = hostname if not isinstance(self.host, basestring) or \ not isinstance(aws_access_key, basestring) or \ not isinstance(aws_secret_key, basestring): raise ValueError('Missing or invalid type for required arg. host:"{0}", ' 'aws_access_key_id:"{1}", aws_secret_access_key:"{2}"' .format(self.host, aws_access_key, aws_secret_key)) self.is_secure = is_secure self.port = port self.path = path # debug is an int representation of the debug level. Use log.debug() for # logging debug information self.debug = boto_debug_level if self.debug: boto.set_stream_logger('boto') if not logger: logger = eulogger.Eulogger(identifier=self.__repr__()) self.log = logger self._show_method = self.log.info self.debug_method = self.log.debug if err_method: self.err_method = err_method super(PropertiesConnection, self).__init__(path=self.path, aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key, port=self.port, is_secure=self.is_secure, host=self.host, debug=self.debug, **kwargs)
def send(self, verb='GET', **args): self.args.update(args) if 'debug' in self.args and self.args['debug'] >= 2: boto.set_stream_logger(self.name()) self.process_args() self.process_filters() conn = self.get_connection(**self.args) self.http_response = conn.make_request(self.name(), self.request_params, verb=verb) self.body = self.http_response.read() boto.log.debug(self.body) if self.http_response.status == 200: self.aws_response = boto.jsonresponse.Element(list_marker=self.list_markers, item_marker=self.item_markers) h = boto.jsonresponse.XmlHandler(self.aws_response, self) h.parse(self.body) return self.aws_response else: boto.log.error('%s %s' % (self.http_response.status, self.http_response.reason)) boto.log.error('%s' % self.body) raise conn.ResponseError(self.http_response.status, self.http_response.reason, self.body)
def set_verbose(self): import logging import boto self.verbose = True logging.getLogger('boto').setLevel(logging.DEBUG) logging.getLogger('botocore').setLevel(logging.DEBUG) boto.set_stream_logger('boto', level=logging.DEBUG)
def sns_send_push_notification_to_user(endpoint=None, message=None, data=None, user=None, **kwargs): if not user: return boto.set_stream_logger('boto') sns = boto.sns.connect_to_region('us-east-1') apns_dict = {'aps':{'alert':message,'sound':'default'}} if data: apns_dict['data'] = data apns_string = json.dumps(apns_dict, ensure_ascii=False) message = {'default':message, settings.AWS_SNS_APNS_PLATFORM:apns_string} messageJSON = json.dumps(message, ensure_ascii=False) try: for device in user.installations.all(): apns_endpoint = None apns = device.push_endpoints.get('apns', None) if apns: apns_endpoint = apns.get(endpoint, None) if apns_endpoint: msg_id = sns.publish(topic=None, message = messageJSON, target_arn=apns_endpoint, message_structure='json') except BotoServerError as e: logger.error("utils.aws.sns_send_push_notification_to_user. Boto error {} to user {}".format(e.code, user)) except: logger.error("utils.aws.sns_send_push_notification_to_user. Unknown error {} {}".format(sys.exc_info()[0], sys.exc_info()[1]))
def enable_connection_debug(self, level=DEBUG, format_string=None): try: self.connection.debug = 2 level = Eulogger.format_log_level(level, 'DEBUG') set_stream_logger('boto', level=level, format_string=None) except: self.log.error('Could not enable debug for: "{0}"'.format(self)) raise
def disable_connection_debug(self, level=NOTSET): try: self.connection.debug = 0 level = Eulogger.format_log_level(level, 'NOTSET') set_stream_logger('boto', level=level, format_string=None) except: self.log.error('Could not disable debug for: "{0}"'.format(self)) raise
def S3Test(): boto.set_stream_logger('boto') bucket = s3.get_bucket('latest-snapshot', validate=False) exists = s3.lookup('latest-snapshot') for bucket in s3: for key in bucket: print(key.name) key = s3.get_bucket('latest-snapshot').get_key('knapp.JPG') key.set_contents_from_filename('/home/pi/OSecuritySnapshots/latestSnapshot.JPG')
def __init__(self, config): # init config and database self._config = config self._mbs_db_client = None self._type_bindings = self._get_type_bindings() # make the maker self._maker = Maker(type_bindings=self._type_bindings) # notifications self._notifications = None # init object collections self._backup_collection = None self._plan_collection = None self._deleted_plan_collection = None self._audit_collection = None self._restore_collection = None # load backup system/engines lazily self._backup_system = None self._api_server = None # listens for backup events coming through rest self._backup_event_listener = None self._api_client = None self._engines = None # init the encryptor self._encryptor = self._get_encryptor() # self._backup_source_builder = None self._default_backup_assistant = None self._temp_dir = resolve_path(DEFAULT_BACKUP_TEMP_DIR_ROOT) self._event_colllection = None self._event_listener_collection = None self._event_queue = None # allow boto debug to be configuable if config.get("enableBotoDebug"): import boto boto.set_stream_logger('boto') # read log path if config.get("logPath"): import mbs_config mbs_config.MBS_LOG_PATH = config.get("logPath")
def connect(self, profile=None): """create or reuse a boto s3 connection. An option aws profile may be given. """ if profile: self.profile = profile if boto.config.getint('Boto', 'debug', 0) > 0: boto.set_stream_logger('boto') if not self.conn: self.conn = self.new_connection(profile) return self.conn
def main(): parser = argparse.ArgumentParser(description="List S3 contents (with retry) to a file") parser.add_argument("--output-file", type=argparse.FileType('w')) parser.add_argument("--bucket", default="telemetry-published-v2") parser.add_argument("--prefix", default="") parser.add_argument("--verbose", action="store_true") parser.add_argument("--debug", action="store_true") args = parser.parse_args() if args.debug: boto.set_stream_logger('boto') list_files(args.bucket, args.output_file, prefix=args.prefix)
def parse(*args, **kwargs): options, args = fn(*args, **kwargs) if options.debug: boto.set_stream_logger("boto") global DEBUG DEBUG = True if options.verbose: global VERBOSE VERBOSE = True return options, args
def addimage(user_id): if(db.isvaliduser(user_id)): upload = request.files.get('content') name, ext = os.path.splitext(upload.filename) if ext not in ('.png','.jpg','.jpeg'): return 'File extension not allowed.' print ext #here check if the directory is existing #save_path = '/Users/poojasrinivas/Desktop/275/save' save_path = 'C:\\tmp' if not os.path.exists(save_path): os.makedirs(save_path) upload.save(save_path) addedPinPath = save_path + '\\' + name +ext #addedPinPath = save_path + '/' + name +ext boto.set_stream_logger('boto') #TODO removed s3 line s3 = S3Connection('AKIAIZGBRZFQJ6ZO7O6Q', 'y5zskvZW+yyH0OuUlbOuq6alNQvQzalhLarp2MAv') #s3 = boto.connect_s3() pin._pinid = db.returnPinid() print 'Connected to S3' bucket = s3.get_bucket('bucket275') #create key k=Key(bucket) k.key=pin._pinid # TODO set auto generated pin id here print addedPinPath k.set_contents_from_filename(addedPinPath) time.sleep(2) #pin._pinid = '3' #TODO generate the pin id pin._pinname = name pin._pinurl='https://bucket275.s3.amazonaws.com/'+ str(pin._pinid) pin._boardid = '' pindetails = db.insertPin(pin) os.remove(addedPinPath) response.set_header("content-type","application/json") response.body = pindetails print "***Response returned is:\n" print response.status print response.body print "\n***" return response else: return erroruser()
def process_args(self, **args): """ Responsible for walking through Params defined for the request and: * Matching them with keyword parameters passed to the request constructor or via the command line. * Checking to see if all required parameters have been specified and raising an exception, if not. * Encoding each value into the set of request parameters that will be sent in the request to the AWS service. """ self.args.update(args) self.connection_args = copy.copy(self.args) if 'debug' in self.args and self.args['debug'] >= 2: boto.set_stream_logger(self.name()) required = [p.name for p in self.Params+self.Args if not p.optional] for param in self.Params+self.Args: if param.long_name: python_name = param.long_name.replace('-', '_') else: python_name = boto.utils.pythonize_name(param.name, '_') value = None if python_name in self.args: value = self.args[python_name] if value is None: value = param.default if value is not None: if param.name in required: required.remove(param.name) if param.request_param: if param.encoder: param.encoder(param, self.request_params, value) else: Encoder.encode(param, self.request_params, value) if python_name in self.args: del self.connection_args[python_name] if required: l = [] for p in self.Params+self.Args: if p.name in required: if p.short_name and p.long_name: l.append('(%s, %s)' % (p.optparse_short_name, p.optparse_long_name)) elif p.short_name: l.append('(%s)' % p.optparse_short_name) else: l.append('(%s)' % p.optparse_long_name) raise RequiredParamError(','.join(l)) boto.log.debug('request_params: %s' % self.request_params) self.process_markers(self.Response)
def __init__(self, clc_host, access_id, secret_key, token): boto.set_stream_logger('foo') path='/services/elb' port=8773 if clc_host[len(clc_host)-13:] == 'amazonaws.com': clc_host = clc_host.replace('ec2', 'elasticloadbalancing', 1) path = '/' reg = None port=443 reg = RegionInfo(name='eucalyptus', endpoint=clc_host) self.conn = ELBConnection(access_id, secret_key, region=reg, port=port, path=path, validate_certs=False, is_secure=True, security_token=token, debug=2) self.conn.http_connection_kwargs['timeout'] = 30
def __init__(self, host, access_key, secret_key, admin_path='/admin', aws_signature='AWS4', timeout=30, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=False, https_connection_factory=None, security_token=None, validate_certs=True): """Constructor.""" self._admin_path = admin_path if debug: boto.set_stream_logger('boto') debug_boto = 10 else: debug_boto = 0 # AWS4 and AWS2 signature support # see boto.auth.S3HmacAuthV4Handler # see boto.auth.HmacAuthV1Handler if aws_signature == 'AWS4': # AWS4 signature self._signature_algo = ['hmac-v4-s3'] else: # old style AWS2 signature algo self._signature_algo = ['hmac-v1'] # init AWS connection boto.connection.AWSAuthConnection.__init__(self, host=host, aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, debug=debug_boto, https_connection_factory=https_connection_factory, path=self._admin_path, provider='aws', security_token=security_token, suppress_consec_slashes=True, validate_certs=validate_certs) # set http_socket_timeout self.http_connection_kwargs['timeout'] = timeout if aws_signature == 'AWS4': self._set_auth_region_name('s3')
def __init__(self, clc_host, access_id, secret_key, token): boto.set_stream_logger('foo') path='/services/Walrus' port=8773 try: # this call is just here to check if the ec2 feature is config-ed host = eucaconsole.config.get('test', 'ec2.endpoint') path = '/' port=443 except ConfigParser.Error: pass self.conn = S3Connection(access_id, secret_key, host=clc_host, port=port, path=path, is_secure=True, security_token=token, debug=2) self.conn.http_connection_kwargs['timeout'] = 30
def __init__(self, index_path, bucket_name, num_processes=2, log_file=None, log_level=logging.INFO): self.index_path = index_path self.bucket_name = bucket_name self.num_processes = num_processes if log_file: boto.set_file_logger('pynas-uploader', log_file, log_level) else: boto.set_stream_logger('pynas-uploader', log_level) self.task_queue = multiprocessing.JoinableQueue() self.status_queue = multiprocessing.Queue() self.s3 = boto.connect_s3() self.bucket = self.s3.lookup(self.bucket_name) self.index = pynas.index.Index(index_path) self.n_tasks = 0
def update(self, app_obj): """Update a given pattern.""" if app_obj.pargs.debug is True: boto.set_stream_logger("aws_provider") context = self.scenario.context raw_template = self.scenario.template template_dict = parse_cf_json(raw_template) # Minimize JSON to save space template_json = json.dumps(template_dict, separators=(",", ":")) params = construct_params(context["parameters"], template_dict["Parameters"].keys()) print " - Determining owner information" try: iam_user = self.iam_conn.get_user() except: print colored("Error: ", "red") + "Unable to get IAM username" exit(1) # Archive payload directory and send to S3 payload_url = self._send_payload(context, timeout=3600) params.append(("PayloadURL", payload_url)) try: print " - Updating CloudFormation stack" stack_id = self.connection.update_stack( self.stack_name, template_body=template_json, parameters=params, capabilities=["CAPABILITY_IAM"], tags={"OwnerId": iam_user.user_id, "OwnerName": iam_user.user_name, "CreatedBy": "nepho"}, disable_rollback=True, ) try: self._show_status(self.stack_name) except exc.CaughtSignal: exit() except boto.exception.BotoServerError as e: print colored("Error: ", "red") + "Problem communicating with CloudFormation" # Use e.message instead of e.body as per: https://github.com/boto/boto/issues/1658 msg = literal_eval(e.message) print "(%s) %s " % (msg["Error"]["Code"], msg["Error"]["Message"]) exit(1) return stack_id
def __init__( self, aws_id=None, aws_key=None, debug=0): if aws_id is None: self.aws_access_key = None sys.exit("Please provide AWS access key and secret!!") else: self.aws_access_key = aws_id self.aws_secret_access_key = aws_key self.debug = debug if debug > 0: boto.set_stream_logger('default.log') self.s3 = None self.autoscale = None self.ec2 = None self.cf = None self.elb = None self.r53 = None
def set_endpoint(self, endpoint, debug=0): if debug > 0: boto.set_stream_logger('clc') reg = RegionInfo(name='eucalyptus', endpoint=endpoint) path = '/services/Eucalyptus' port = 8773 if endpoint[len(endpoint)-13:] == 'amazonaws.com': path = '/' reg = RegionInfo(endpoint=endpoint) port = 443 self.conn = EC2Connection(self.access_id, self.secret_key, region=reg, port=port, path=path, is_secure=True, security_token=self.token, debug=debug) logging.info("set new ec2 connection for host : "+endpoint) self.conn.APIVersion = '2012-12-01' self.conn.https_validate_certificates = False self.conn.http_connection_kwargs['timeout'] = 30
def run_query(connection, table_name, query_value, output_json): """ Run the given query against DynamoDB. :param connection: a DynamoDB connection object to use :param table_name: the table name to query :param query_value: the query to issue to DynamoDB, should be in JSON format :param output_json: whether we should try to output the result in JSON instead of shell :return: content for the given query, if it exists """ if DYNQ_DEBUG: boto.set_stream_logger('boto') items = Table(table_name, connection=connection).get_item(**query_value) if output_json: click.echo(json.dumps(items._data)) else: for field, value in items.items(): click.echo(field + "=" + value)
def __init__(self, creds=None, host=None, aws_access_key=None, aws_secret_key=None, is_secure=None, port=None, path=None, logger=None, boto_debug=0, **kwargs): self.debug = boto_debug if self.debug: set_stream_logger('boto') if creds and not isinstance(creds, Eucarc): credsattr = getattr(creds, 'creds', None) if not isinstance(credsattr, Eucarc): raise ValueError('Unknown type passed for creds arg: "{0}/{1}"' .format(creds, type(creds))) creds = credsattr self._eucarc = creds self.account_id = None self.user_id = None if not logger: logger = Eulogger(identifier=self.__class__.__name__) self.log = logger if creds: self.user_id = creds.ec2_user_id self.account_id = creds.ec2_account_number assert isinstance(creds, Eucarc), 'UserAdmin. eucarc not type Eucarc(), got:"{0}/{1}"'\ .format(creds, type(creds)) urlp = urlparse(creds.euare_url) host = host or getattr(urlp, 'hostname', None) port = port or getattr(urlp, 'port', 8773) path = path or getattr(urlp, 'path', '/services/Euare') if is_secure is None and urlp.scheme == 'https': is_secure = True aws_secret_key = aws_secret_key or creds.aws_secret_key aws_access_key = aws_access_key or creds.aws_access_key is_secure = is_secure or False ac_kwargs = {'host': host, 'aws_access_key_id': aws_access_key, 'aws_secret_access_key': aws_secret_key, 'is_secure': is_secure, 'port': port, 'path': path} ac_kwargs.update(kwargs) try: super(AccessConnection, self).__init__(**ac_kwargs) except: self.log.error('Failed to create AccessConnection with kwargs:"{0}"'.format(ac_kwargs)) raise
def process_cli_args(self): ''' Process CLI args to fill in missing parts of self.args and enable debugging if necessary. ''' cli_args = self._cli_parser.parse_args().__dict__ for (key, val) in cli_args.iteritems(): self.args.setdefault(key, val) if self.args.get('debug'): boto.set_stream_logger(self.name()) if self.args.get('debugger'): sys.excepthook = _requestbuilder_except_hook( self.args.get('debugger', False), self.args.get('debug', False)) if '_filters' in self.args: self.args['Filter'] = _process_filters(cli_args.pop('_filters')) self._arg_routes.setdefault(self.DefaultRoute, []) self._arg_routes[self.DefaultRoute].append('Filter')
def addimage(user_id): if(db.isvaliduser(user_id)): upload = request.files.get('content') name, ext = os.path.splitext(upload.filename) if ext not in ('.png','.jpg','.jpeg'): return 'File extension not allowed.' #here check if the directory is existing save_path = 'C:\\tmp' upload.save(save_path) addedPinPath = save_path + '/' + name +'.jpg' boto.set_stream_logger('boto') #TODO removed s3 line s3 = S3Connection('', '') #s3 = boto.connect_s3() print 'Connected to S3' bucket = s3.get_bucket('bucket275') #create key k=Key(bucket) k.key='3' # TODO set auto generated pin id here k.set_contents_from_filename(addedPinPath) time.sleep(2) pin._pinid = '3' #TODO generate the pin id pin._pinname = name pin._pinurl='https://bucket275.s3.amazonaws.com/'+pin._pinid pin._boardid = '' pindetails = db.insertPin(pin) os.remove(addedPinPath) response.set_header("content-type","application/json") response.body = pindetails print "***Response returned is:\n" print response.status print response.body print "\n***" return response else: return erroruser()
def start_server(configs): """ The main entry point for the service, regardless of how it's used. Takes any number of filename or dictionary objects suitable for cherrypy.config.update. """ importlib.import_module('.agency', __package__) aws.set_connection_environment() server = JobServer() if hasattr(cherrypy.engine, "signal_handler"): cherrypy.engine.signal_handler.subscribe() if hasattr(cherrypy.engine, "console_control_handler"): cherrypy.engine.console_control_handler.subscribe() app = cherrypy.tree.mount(server, '/') server._app = app list(map(app.merge, configs)) admin_app = cherrypy.tree.mount(Admin(server), '/admin') devel_configs = list(configs) + [{ '/': { 'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'RecaptureDocs admin', 'tools.auth_basic.checkpassword': cherrypy.lib.auth_basic.checkpassword_dict(dict( admin='g0tch4-h4x0r', )), }, }] list(map(admin_app.merge, devel_configs)) cherrypy.tree.mount(GGCServer(), '/ggc') if not cherrypy.config.get('server.production', False): boto.set_stream_logger('recapturedocs') aws.ConnectionFactory.production = False server.send_notice( "RecaptureDocs {version} server starting on {hostname}" .format( hostname=socket.getfqdn(), version=pkg_resources.require('recapturedocs')[0].version)) cherrypy.engine.start() yield server cherrypy.engine.exit()
def get_configurator(settings, enable_auth=True): connection_debug = asbool(settings.get('connection.debug')) boto.set_stream_logger('boto', level=(logging.DEBUG if connection_debug else logging.CRITICAL)) ensure_session_keys(settings) check_types() config = Configurator(root_factory=SiteRootFactory, settings=settings) if enable_auth: authn_policy = SessionAuthenticationPolicy(callback=groupfinder) authz_policy = ACLAuthorizationPolicy() config.set_authentication_policy(authn_policy) config.set_authorization_policy(authz_policy) config.set_default_permission('view') config.add_request_method(User.get_auth_user, 'user', reify=True) cache_duration = int(settings.get('static.cache.duration', 43200)) config.add_static_view(name='static/' + __version__, path='static', cache_max_age=cache_duration) config.add_layout('eucaconsole.layout.MasterLayout', 'eucaconsole.layout:templates/master_layout.pt') for route in urls: config.add_route(route.name, route.pattern) setup_tweens(config) setup_exts(config) config.scan() return config
def Main(args): """Main, where the job dispatching happens and the result is printed. Args: args (dict): The result of the command-line argument parsing. """ global debug, info, warn, error s3statsLogging = Logging() s3statsLogging.Configure(args) debug, info, warn, error = s3statsLogging.GetLoggers() if args.verbose: boto.set_stream_logger('boto') s3 = boto.connect_s3(args.awsKeyId, args.awsSecret) buckets = ApplyFilters(args.bucket, s3.get_all_buckets(), args.bucketRe) results = ParallelStats(buckets, args.threads) if args.format == "json": print json.dumps(results, sort_keys = True, indent = 4) else: PrintResults(args, results)
def do_cli(self, cli_args=None): if not self.parser: self.build_cli_parser() options, args = self.parser.parse_args(cli_args) if hasattr(options, 'help_filters') and options.help_filters: print 'Available filters:' for filter in self.filters: print '%s\t%s' % (filter['name'], filter['doc']) sys.exit(0) d = {} for param in self.params: if param.cli_option: p_name = param.cli_option[-1] d[p_name] = getattr(options, p_name.replace('-', '_')) else: p_name = boto.utils.pythonize_name(param.name) d[p_name] = args try: self.process_args(d) except ValueError as ve: print ve.message sys.exit(1) if hasattr(options, 'filter') and options.filter: d = {} for filter in options.filter: name, value = filter.split('=') d[name] = value self.process_filters(d) try: if options.debug: boto.set_stream_logger(self.name) self.args['debug'] = 2 self.send() self.cli_output_formatter() except self.connection.ResponseError as err: print 'Error(%s): %s' % (err.error_code, err.error_message)
privateFileName = 'nodes-private' if not dryRun: fpublic = open(publicFileName, 'w') fprivate = open(privateFileName, 'w') print 'Using boto version', boto.Version if True: botoVersionArr = boto.Version.split(".") if (botoVersionArr[0] != 2): botoVersionMismatch if (botoVersionArr[1] < 13): botoVersionMismatch if (debug): boto.set_stream_logger('h2o-ec2') ec2 = boto.ec2.connect_to_region(regionName, debug=debug) print 'Launching', numInstancesToLaunch, 'instances.' reservation = ec2.run_instances(image_id=amiId, min_count=numInstancesToLaunch, max_count=numInstancesToLaunch, key_name=keyName, instance_type=instanceType, security_groups=[securityGroupName], instance_profile_arn=iam_profile_resource_name, instance_profile_name=iam_profile_name, dry_run=dryRun) for i in range(numInstancesToLaunch):
from functools import wraps from c2client.compat import get_connection from c2client.utils import prettify_xml # Nasty hack to workaround default ascii codec if sys.version_info[0] < 3: sys.stdout = codecs.getwriter('utf8')(sys.stdout) sys.stderr = codecs.getwriter('utf8')(sys.stderr) if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context if os.environ.get("DEBUG"): boto.set_stream_logger("c2") class EnvironmentVariableError(Exception): def __init__(self, name): super(EnvironmentVariableError, self).__init__( "Environment variable '{0}' not found.".format(name.upper())) def configure_boto(): """Configure boto runtime for CROC Cloud""" if not boto.config.has_section("Boto"): boto.config.add_section("Boto") boto.config.set("Boto", "is_secure", "True") boto.config.set("Boto", "num_retries", "0")
def main(): """Main function""" global options # DB instance classes as listed on # http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.DBInstanceClass.html db_classes = { 'db.t1.micro': 0.615, 'db.m1.small': 1.7, 'db.m1.medium': 3.75, 'db.m1.large': 7.5, 'db.m1.xlarge': 15, 'db.m4.large': 8, 'db.m4.xlarge': 16, 'db.m4.2xlarge': 32, 'db.m4.4xlarge': 64, 'db.m4.10xlarge': 160, 'db.r3.large': 15, 'db.r3.xlarge': 30.5, 'db.r3.2xlarge': 61, 'db.r3.4xlarge': 122, 'db.r3.8xlarge': 244, 'db.t2.micro': 1, 'db.t2.small': 2, 'db.t2.medium': 4, 'db.t2.large': 8, 'db.m3.medium': 3.75, 'db.m3.large': 7.5, 'db.m3.xlarge': 15, 'db.m3.2xlarge': 30, 'db.m2.xlarge': 17.1, 'db.m2.2xlarge': 34.2, 'db.m2.4xlarge': 68.4, 'db.cr1.8xlarge': 244, } # RDS metrics http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/rds-metricscollected.html metrics = { 'BinLogDiskUsage': 'binlog_disk_usage', # The amount of disk space occupied by binary logs on the master. Units: Bytes 'CPUUtilization': 'utilization', # The percentage of CPU utilization. Units: Percent 'DatabaseConnections': 'connections', # The number of database connections in use. Units: Count 'DiskQueueDepth': 'disk_queue_depth', # The number of outstanding IOs (read/write requests) waiting to access the disk. Units: Count 'ReplicaLag': 'replica_lag', # The amount of time a Read Replica DB Instance lags behind the source DB Instance. Units: Seconds 'SwapUsage': 'swap_usage', # The amount of swap space used on the DB Instance. Units: Bytes 'FreeableMemory': 'used_memory', # The amount of available random access memory. Units: Bytes 'FreeStorageSpace': 'used_space', # The amount of available storage space. Units: Bytes 'ReadIOPS': 'read_iops', # The average number of disk I/O operations per second. Units: Count/Second 'WriteIOPS': 'write_iops', # The average number of disk I/O operations per second. Units: Count/Second 'ReadLatency': 'read_latency', # The average amount of time taken per disk I/O operation. Units: Seconds 'WriteLatency': 'write_latency', # The average amount of time taken per disk I/O operation. Units: Seconds 'ReadThroughput': 'read_throughput', # The average number of bytes read from disk per second. Units: Bytes/Second 'WriteThroughput': 'write_throughput', # The average number of bytes written to disk per second. Units: Bytes/Second } # Parse options parser = optparse.OptionParser() parser.add_option('-l', '--list', help='list DB instances', action='store_true', default=False, dest='db_list') parser.add_option( '-n', '--profile', default=None, help= 'AWS profile from ~/.boto or /etc/boto.cfg. Default: None, fallbacks to "[Credentials]".' ) parser.add_option( '-r', '--region', default='us-east-1', help= 'AWS region. Default: us-east-1. If set to "all", we try to detect the instance region ' 'across all of them, note this will be slower than if you specify the region explicitly.' ) parser.add_option('-i', '--ident', help='DB instance identifier') parser.add_option( '-p', '--print', help='print status and other details for a given DB instance', action='store_true', default=False, dest='printinfo') parser.add_option('-m', '--metric', help='metrics to retrive separated by comma: [%s]' % ', '.join(metrics.keys())) parser.add_option('-d', '--debug', help='enable debugging', action='store_true', default=False) options, _ = parser.parse_args() # Strip a prefix _ which is sent by Cacti, so an empty argument is interpreted correctly. # Than set defaults if argument is supposed to be empty. options.region = options.region.lstrip('_') options.profile = options.profile.lstrip('_') if not options.region: options.region = 'us-east-1' if not options.profile: options.profile = None if options.debug: boto.set_stream_logger('boto') rds = RDS(region=options.region, profile=options.profile, identifier=options.ident) # Check args if len(sys.argv) == 1: parser.print_help() sys.exit() elif options.db_list: info = rds.get_list() print 'List of all DB instances in %s region(s):' % (options.region, ) pprint.pprint(info) sys.exit() elif not options.ident: parser.print_help() parser.error('DB identifier is not set.') elif options.printinfo: info = rds.get_info() pprint.pprint(vars(info)) sys.exit() elif not options.metric: parser.print_help() parser.error('Metric is not set.') selected_metrics = options.metric.split(',') for metric in selected_metrics: if metric not in metrics.keys(): parser.print_help() parser.error('Invalid metric.') # Do not remove the empty lines in the start and end of this docstring perl_magic_vars = """ # Define the variables to output. I use shortened variable names so maybe # it'll all fit in 1024 bytes for Cactid and Spine's benefit. Strings must # have some non-hex characters (non a-f0-9) to avoid a Cacti bug. This list # must come right after the word MAGIC_VARS_DEFINITIONS. The Perl script # parses it and uses it as a Perl variable. $keys = array( 'binlog_disk_usage' => 'gg', 'utilization' => 'gh', 'connections' => 'gi', 'disk_queue_depth' => 'gj', 'replica_lag' => 'gk', 'swap_usage' => 'gl', 'used_memory' => 'gm', 'total_memory' => 'gn', 'used_space' => 'go', 'total_space' => 'gp', 'read_iops' => 'gq', 'write_iops' => 'gr', 'read_latency' => 'gs', 'write_latency' => 'gt', 'read_throughput' => 'gu', 'write_throughput' => 'gv', ); """ output = dict() for row in perl_magic_vars.split('\n'): if row.find('=>') >= 0: k = row.split(' => ')[0].strip().replace("'", '') v = row.split(' => ')[1].strip().replace("'", '').replace(',', '') output[k] = v debug('Perl magic vars: %s' % output) debug('Metric associations: %s' % dict( (k, output[v]) for (k, v) in metrics.iteritems())) # Handle metrics results = [] for metric in selected_metrics: stats = rds.get_metric(metric) if metric == 'FreeableMemory': info = rds.get_info() try: memory = db_classes[info.instance_class] * 1024**3 except IndexError: print 'Unknown DB instance class "%s"' % info.instance_class sys.exit(1) results.append('%s:%.0f' % (output['used_memory'], memory - stats)) results.append('%s:%.0f' % (output['total_memory'], memory)) elif metric == 'FreeStorageSpace': info = rds.get_info() storage = float(info.allocated_storage) * 1024**3 results.append('%s:%.0f' % (output['used_space'], storage - stats)) results.append('%s:%.0f' % (output['total_space'], storage)) else: short_var = output.get(metrics[metric]) if not short_var: print 'Chosen metric does not have a correspondent entry in perl magic vars' sys.exit(1) results.append('%s:%s' % (short_var, stats)) print ' '.join(results)
proxyHost = None proxyPort = None # Local proxy setting determination myAddress = socket.gethostbyaddr(socket.gethostname()) if string.find(myAddress[0], "s3group") > 0: if string.find(myAddress[-1][0], "193.120") == 0: proxyHost = 'w3proxy.s3group.com' proxyPort = 3128 # Turn on verbose output: debugging aid boto.set_stream_logger('mylog') ########################################################################### # Stage #1: Load balancer ########################################################################### # Takes credentials from the environment : assumes they are defined in shell variables in one of the supported ways # If they aren't this will fail authentication reg_con = boto.ec2.connect_to_region(region_name=thisRegion, proxy=proxyHost, proxy_port=proxyPort) # We will allow all availability zones in the chosen region. # This is supposed to allow a better chance of the site staying up (although I am not sure it makes any difference) allZones = reg_con.get_all_zones() zoneStrings = [] for zone in allZones:
from __future__ import division from future import standard_library standard_library.install_aliases() import logging import re import fnmatch import configparser import math import os from urllib.parse import urlparse import warnings import boto from boto.s3.connection import S3Connection from boto.sts import STSConnection boto.set_stream_logger('boto') logging.getLogger("boto").setLevel(logging.INFO) from airflow.exceptions import AirflowException from airflow.hooks.base_hook import BaseHook def _parse_s3_config(config_file_name, config_format='boto', profile=None): """ Parses a config file for s3 credentials. Can currently parse boto, s3cmd.conf and AWS SDK config formats :param config_file_name: path to the config file :type config_file_name: str :param config_format: config type. One of "boto", "s3cmd" or "aws". Defaults to "boto"
# This will cause full debug output to go to the console >>> import boto >>> boto.set_stream_logger('foo') >>> ec2 = boto.connect_ec2(debug=2)
# Via Fine-uploader's server examples. # MIT License https://github.com/Widen/fine-uploader-server/blob/master/license.txt from django.conf import settings from django.http import HttpResponse from django.shortcuts import render from django.views.decorators.csrf import csrf_exempt import base64, hmac, hashlib, json if hasattr(settings, "AWS_UPLOAD_CLIENT_KEY") and hasattr( settings, "AWS_UPLOAD_CLIENT_SECRET_KEY"): try: import boto import logging from boto.s3.connection import Key, S3Connection boto.set_stream_logger('boto', level=logging.INFO) S3 = S3Connection(settings.AWS_UPLOAD_CLIENT_KEY, settings.AWS_UPLOAD_CLIENT_SECRET_KEY) except ImportError as e: print("Could not import boto, the Amazon SDK for Python.") print("Deleting files will not work.") print("Install boto with") print("$ pip install boto") def home(request): """ The 'home' page. Returns an HTML page with Fine Uploader code ready to upload to S3. """ return render(request, "index.html")
def set_debug(self, debug=False): if debug: boto.set_stream_logger('euca2ools') self.debug = 2
def main(): """Main function""" global options short_status = { OK: 'OK', WARNING: 'WARN', CRITICAL: 'CRIT', UNKNOWN: 'UNK' } # DB instance classes as listed on # http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.DBInstanceClass.html db_classes = { 'db.t1.micro': 0.615, 'db.m1.small': 1.7, 'db.m1.medium': 3.75, 'db.m1.large': 7.5, 'db.m1.xlarge': 15, 'db.m4.large': 8, 'db.m4.xlarge': 16, 'db.m4.2xlarge': 32, 'db.m4.4xlarge': 64, 'db.m4.10xlarge': 160, 'db.r3.large': 15, 'db.r3.xlarge': 30.5, 'db.r3.2xlarge': 61, 'db.r3.4xlarge': 122, 'db.r3.8xlarge': 244, 'db.t2.micro': 1, 'db.t2.small': 2, 'db.t2.medium': 4, 'db.t2.large': 8, 'db.m3.medium': 3.75, 'db.m3.large': 7.5, 'db.m3.xlarge': 15, 'db.m3.2xlarge': 30, 'db.m2.xlarge': 17.1, 'db.m2.2xlarge': 34.2, 'db.m2.4xlarge': 68.4, 'db.cr1.8xlarge': 244, } # RDS metrics http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/rds-metricscollected.html metrics = { 'status': 'RDS availability', 'load': 'CPUUtilization', 'memory': 'FreeableMemory', 'storage': 'FreeStorageSpace' } units = ('percent', 'GB') # Parse options parser = optparse.OptionParser() parser.add_option('-l', '--list', help='list of all DB instances', action='store_true', default=False, dest='db_list') parser.add_option( '-n', '--profile', default=None, help= 'AWS profile from ~/.boto or /etc/boto.cfg. Default: None, fallbacks to "[Credentials]".' ) parser.add_option( '-r', '--region', default='us-east-1', help= 'AWS region. Default: us-east-1. If set to "all", we try to detect the instance region ' 'across all of them, note this will be slower than if you specify the region explicitly.' ) parser.add_option('-i', '--ident', help='DB instance identifier') parser.add_option( '-p', '--print', help='print status and other details for a given DB instance', action='store_true', default=False, dest='printinfo') parser.add_option('-m', '--metric', help='metric to check: [%s]' % ', '.join(metrics.keys())) parser.add_option('-w', '--warn', help='warning threshold') parser.add_option('-c', '--crit', help='critical threshold') parser.add_option( '-u', '--unit', help='unit of thresholds for "storage" and "memory" metrics: [%s].' 'Default: percent' % ', '.join(units), default='percent') parser.add_option('-t', '--time', help='time period in minutes to query. Default: 5', type='int', default=5) parser.add_option('-a', '--avg', help='time average in minutes to request. Default: 1', type='int', default=1) parser.add_option( '-f', '--forceunknown', help='force alerts on unknown status. This prevents issues related to ' 'AWS Cloudwatch throttling limits Default: False', action='store_true', default=False) parser.add_option('-d', '--debug', help='enable debug output', action='store_true', default=False) options, _ = parser.parse_args() if options.debug: boto.set_stream_logger('boto') rds = RDS(region=options.region, profile=options.profile, identifier=options.ident) # Check args if len(sys.argv) == 1: parser.print_help() sys.exit() elif options.db_list: info = rds.get_list() print 'List of all DB instances in %s region(s):' % (options.region, ) pprint.pprint(info) sys.exit() elif not options.ident: parser.print_help() parser.error('DB identifier is not set.') elif options.printinfo: info = rds.get_info() if info: pprint.pprint(vars(info)) else: print 'No DB instance "%s" found on your AWS account and %s region(s).' % ( options.ident, options.region) sys.exit() elif not options.metric or options.metric not in metrics.keys(): parser.print_help() parser.error('Metric is not set or not valid.') elif not options.warn and options.metric != 'status': parser.print_help() parser.error('Warning threshold is not set.') elif not options.crit and options.metric != 'status': parser.print_help() parser.error('Critical threshold is not set.') elif options.avg <= 0 and options.metric != 'status': parser.print_help() parser.error('Average must be greater than zero.') elif options.time <= 0 and options.metric != 'status': parser.print_help() parser.error('Time must be greater than zero.') now = datetime.datetime.utcnow() status = None note = '' perf_data = None # RDS Status if options.metric == 'status': info = rds.get_info() if not info: status = UNKNOWN note = 'Unable to get RDS instance' else: status = OK try: version = info.EngineVersion except: version = info.engine_version note = '%s %s. Status: %s' % (info.engine, version, info.status) # RDS Load Average elif options.metric == 'load': # Check thresholds try: warns = [float(x) for x in options.warn.split(',')] crits = [float(x) for x in options.crit.split(',')] fail = len(warns) + len(crits) except: fail = 0 if fail != 6: parser.error( 'Warning and critical thresholds should be 3 comma separated numbers, e.g. 20,15,10' ) loads = [] fail = False j = 0 perf_data = [] for i in [1, 5, 15]: if i == 1: # Some stats are delaying to update on CloudWatch. # Let's pick a few points for 1-min load avg and get the last point. points = 5 else: points = i load = rds.get_metric( metrics[options.metric], now - datetime.timedelta(seconds=points * 60), now, i * 60) if not load: status = UNKNOWN note = 'Unable to get RDS statistics' perf_data = None break loads.append(str(load)) perf_data.append('load%s=%s;%s;%s;0;100' % (i, load, warns[j], crits[j])) # Compare thresholds if not fail: if warns[j] > crits[j]: parser.error( 'Parameter inconsistency: warning threshold is greater than critical.' ) elif load >= crits[j]: status = CRITICAL fail = True elif load >= warns[j]: status = WARNING j = j + 1 if status != UNKNOWN: if status is None: status = OK note = 'Load average: %s%%' % '%, '.join(loads) perf_data = ' '.join(perf_data) # RDS Free Storage # RDS Free Memory elif options.metric in ['storage', 'memory']: # Check thresholds try: warn = float(options.warn) crit = float(options.crit) except: parser.error('Warning and critical thresholds should be integers.') if crit > warn: parser.error( 'Parameter inconsistency: critical threshold is greater than warning.' ) if options.unit not in units: parser.print_help() parser.error('Unit is not valid.') info = rds.get_info() free = rds.get_metric( metrics[options.metric], now - datetime.timedelta(seconds=options.time * 60), now, options.avg * 60) if not info or not free: status = UNKNOWN note = 'Unable to get RDS details and statistics' else: if options.metric == 'storage': storage = float(info.allocated_storage) elif options.metric == 'memory': try: storage = db_classes[info.instance_class] except: print 'Unknown DB instance class "%s"' % info.instance_class sys.exit(CRITICAL) free = '%.2f' % (free / 1024**3) free_pct = '%.2f' % (float(free) / storage * 100) if options.unit == 'percent': val = float(free_pct) val_max = 100 elif options.unit == 'GB': val = float(free) val_max = storage # Compare thresholds if val <= crit: status = CRITICAL elif val <= warn: status = WARNING if status is None: status = OK note = 'Free %s: %s GB (%.0f%%) of %s GB' % ( options.metric, free, float(free_pct), storage) perf_data = 'free_%s=%s;%s;%s;0;%s' % (options.metric, val, warn, crit, val_max) # Final output if status != UNKNOWN and perf_data: print '%s %s | %s' % (short_status[status], note, perf_data) elif status == UNKNOWN and not options.forceunknown: print '%s %s | null' % ('OK', note) sys.exit(0) else: print '%s %s' % (short_status[status], note) sys.exit(status)
check_lzop, check_pv, compressed_pipe) DEFAULT_CONCURRENCY = max(multiprocessing.cpu_count() - 1, 1) BUFFER_SIZE = 64 # Default bufsize is 64M MBFACTOR = float(1 << 20) MAX_RETRY_COUNT = 4 SLEEP_TIME = 2 SLEEP_MULTIPLIER = 3 UPLOAD_TIMEOUT = 600 DEFAULT_REDUCED_REDUNDANCY = False logging_helper.configure(format='%(name)-12s %(levelname)-8s %(message)s') logger = logging_helper.CassandraSnapshotterLogger( 'cassandra_snapshotter.agent') boto.set_stream_logger('boto', logging.WARNING) def get_bucket(s3_bucket, aws_access_key_id, aws_secret_access_key, s3_connection_host): connection = S3Connection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, host=s3_connection_host) return connection.get_bucket(s3_bucket, validate=False) def destination_path(s3_base_path, file_path, compressed=True): suffix = compressed and '.lzo' or '' dest_path = "{}{}{}".format(s3_base_path, file_path, suffix) return dest_path