def __init__(self, queue_name, region_name, aws_access_key_id, aws_secret_access_key, auto_creation=False, endpoint_url=None ): """ :param queue_name: """ settings = {'region_name': region_name, 'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key, } if endpoint_url: settings.update( endpoint_url=endpoint_url ) self.client = SQSClient(**settings) self.queue_url = self.client.get_queue_url(queue_name) if not self.queue_url and auto_creation: self.client.create_queue(queue_name) self.queue_url = self.client.get_queue_url(queue_name) if hasattr(self.client, 'settings'): self.instance = boto3.resource( 'sqs', **self.client.settings ).Queue(self.queue_url) else: self.instance = boto3.resource('sqs').Queue(self.queue_url)
def create_annotation_job_request(): auth.require(fail_redirect='/') formdata = request.query key = formdata["key"] file_name = key.split("/")[-1] job_id = file_name.split("~")[0] sqs = boto3.resource('sqs') data = {"job_id": str(job_id), "username": auth.current_user.username, "s3_inputs_bucket": "gas-inputs", "s3_key_input_files": key, "input_file_name": file_name.split("~")[1], "submit_time": int(time.time()), "job_status": "PENDING" } dynamodb = boto3.resource('dynamodb') ann_table = dynamodb.Table('songty_annotations') ann_table.put_item(Item=data) #http://boto3.readthedocs.io/en/latest/reference/services/sqs.html#queue queue = sqs.get_queue_by_name(QueueName='songty_job_requests') queue.send_message(MessageBody=json.dumps(data)) return template(request.app.config['mpcs.env.templates'] + 'upload_confirm', auth=auth,job_id=job_id)
def test_vpc_peering_connections_cross_region_accept_wrong_region(): # create vpc in us-west-1 and ap-northeast-1 ec2_usw1 = boto3.resource('ec2', region_name='us-west-1') vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16') ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1') vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16') # create peering vpc_pcx_usw1 = ec2_usw1.create_vpc_peering_connection( VpcId=vpc_usw1.id, PeerVpcId=vpc_apn1.id, PeerRegion='ap-northeast-1', ) # accept wrong peering from us-west-1 which will raise error ec2_apn1 = boto3.client('ec2', region_name='ap-northeast-1') ec2_usw1 = boto3.client('ec2', region_name='us-west-1') with assert_raises(ClientError) as cm: ec2_usw1.accept_vpc_peering_connection( VpcPeeringConnectionId=vpc_pcx_usw1.id ) cm.exception.response['Error']['Code'].should.equal('OperationNotPermitted') exp_msg = 'Incorrect region ({0}) specified for this request.VPC ' \ 'peering connection {1} must be ' \ 'accepted in region {2}'.format('us-west-1', vpc_pcx_usw1.id, 'ap-northeast-1') cm.exception.response['Error']['Message'].should.equal(exp_msg)
def test_vpc_peering_connections_cross_region_delete(): # create vpc in us-west-1 and ap-northeast-1 ec2_usw1 = boto3.resource('ec2', region_name='us-west-1') vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16') ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1') vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16') # create peering vpc_pcx_usw1 = ec2_usw1.create_vpc_peering_connection( VpcId=vpc_usw1.id, PeerVpcId=vpc_apn1.id, PeerRegion='ap-northeast-1', ) # reject peering from ap-northeast-1 ec2_apn1 = boto3.client('ec2', region_name='ap-northeast-1') ec2_usw1 = boto3.client('ec2', region_name='us-west-1') del_pcx_apn1 = ec2_apn1.delete_vpc_peering_connection( VpcPeeringConnectionId=vpc_pcx_usw1.id ) des_pcx_apn1 = ec2_usw1.describe_vpc_peering_connections( VpcPeeringConnectionIds=[vpc_pcx_usw1.id] ) des_pcx_usw1 = ec2_usw1.describe_vpc_peering_connections( VpcPeeringConnectionIds=[vpc_pcx_usw1.id] ) del_pcx_apn1['Return'].should.equal(True) des_pcx_apn1['VpcPeeringConnections'][0]['Status']['Code'].should.equal('deleted') des_pcx_usw1['VpcPeeringConnections'][0]['Status']['Code'].should.equal('deleted')
def load_config(config): keys = load_yaml(abs_path("default")) keys["credentials"] = {} if os.path.exists(abs_path("credentials")): keys["credentials"] = load_yaml(abs_path("credentials")) if config != "default": keys.update(load_yaml(abs_path(config))) if "aws_access_key" in keys["credentials"]: keys["s3"] = boto3.resource( "s3", region_name=keys["region"], aws_access_key_id=keys["credentials"]["aws_access_key"], aws_secret_access_key=keys["credentials"]["aws_access_secret"], ) keys["s3_client"] = boto3.client( "s3", region_name=keys["region"], aws_access_key_id=keys["credentials"]["aws_access_key"], aws_secret_access_key=keys["credentials"]["aws_access_secret"], ) else: keys["s3"] = boto3.resource("s3", region_name=keys["region"]) keys["s3_client"] = boto3.client("s3", region_name=keys["region"]) return AttrDict(keys)
def main(event, context): logger.info('START') try: collector = conf.get('Collector') if collector is None: logger.info('No Collector') return 'No Collector' # 対象のセットアップ _init_fabric(_get_key()) # 事前処理を実行 for command in collector.get('CreateCommands', []): logger.info(command) fab.run(command) # 事後処理(ログファイルをここのローカルにコピーしてS3にアップ) s3info = collector.get('S3') for remotefile in collector.get('TargetFiles', []): filename = os.path.basename(remotefile) localfile = '/tmp/{0}.log'.format(uuid.uuid4()) fab.get(remote_path=remotefile, local_path=localfile) boto3.resource('s3').meta.client.upload_file(localfile, s3info.get('Bucket'), _gen_s3key(s3info, filename)) finally: _finalize()
def get_runners(region_name=REGION_NAME): '''Get a list of valid runner ID's A valid runner ID is an ID with an active Amazon SQS Message Queue and Amazon S3 Bucket. Parameters ---------- region_name : str Amazon region identifier Returns ------- list List of valid runner ID's See Also -------- get_workers ''' sqs = boto3.resource('sqs', region_name=region_name) s3 = boto3.resource('s3', region_name=region_name) queues = [os.path.split(q.url)[1] for q in sqs.queues.all()] runners = [] for bucket in s3.buckets.all(): if bucket.name in queues: runners.append(bucket.name) return runners
def process(runner_id, workingdir='.', region_name=REGION_NAME, stop_on_empty=False): '''Start processing loop for specific runner Each worker needs to start this processing procedure. It polls the queue, processes jobs and uploads the result. Parameters ---------- runner_id : str Runner ID workingdir : str Working dir at processing node region_name : str, optional Amazon region identifier stop_on_empty : bool, optional Flag to quit the processing loop if no messages are left [default: False] See Also -------- process_job ''' s3 = boto3.resource('s3', region_name=region_name) sqs = boto3.resource('sqs', region_name=region_name) while True: if not process_job(sqs, s3, runner_id, workingdir='.'): if stop_on_empty: logger.info('No jobs left. Stop') break
def _get_passwords_table(): """ This is a workaround to use both locally and on aws. :return: """ if STAGE != 'local': return boto3.resource('dynamodb', region_name='eu-central-1').Table(PASSWORDS_TABLE) db = boto3.resource( 'dynamodb', endpoint_url='http://localhost:8000', region_name='eu-central-1' ) try: db.meta.client.describe_table(TableName=PASSWORDS_TABLE) except ClientError: return db.create_table( TableName=PASSWORDS_TABLE, AttributeDefinitions=[{ 'AttributeName': 'id', 'AttributeType': 'S', }], KeySchema=[{ 'AttributeName': 'id', 'KeyType': 'HASH', }], ProvisionedThroughput={ 'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1, } ) return db.Table(PASSWORDS_TABLE)
def test_resource_passes_through_arguments(self): boto3.DEFAULT_SESSION = self.Session() boto3.resource('sqs', region_name='us-west-2', verify=False) boto3.DEFAULT_SESSION.resource.assert_called_with( 'sqs', region_name='us-west-2', verify=False)
def sign_in_by_email(event, context): User = boto3.resource('dynamodb').Table('FmSsoUsers') Auth = boto3.resource('dynamodb').Table('FmSsoAuths') email = event['email'] password = event['password'] auth_query = Auth.scan( FilterExpression=Attr('email').eq(email) ) if(auth_query['Count'] == 0): # Auth with given email not found raise Exception('Unauthorized') auth_rcd = auth_query['Items'][0] encrypted_password_key = 'fmsso' # TODO: Change to conf table value if validate_password(password, auth_rcd, encrypted_password_key): # TODO: find user record from auth user = User.get_item( Key={'id': auth_rcd['user_id']} ) # TODO: refresh token return { 'access_token': user['atoken'], 'refresh_token': user['rtoken'], } else: raise Exception('Unauthorized')
def setUp(self): """Setup for testing""" app.config.from_object('meerkat_auth.config.Testing') app.config.from_envvar('MEERKAT_AUTH_SETTINGS') User.DB = boto3.resource( 'dynamodb', endpoint_url=app.config['DB_URL'], region_name='eu-west-1' ) Role.DB = boto3.resource( 'dynamodb', endpoint_url=app.config['DB_URL'], region_name='eu-west-1' ) logging.warning(app.config['USERS']) logging.warning(app.config['ROLES']) logging.warning(app.config['DB_URL']) # The database should have the following objects already in it. roles = [ Role('demo', 'registered', 'Registered.', []), Role('demo', 'personal', 'Personal.', ['registered']), Role('demo', 'shared', 'Shared.', ['registered']), Role('demo', 'manager', 'Manager.', ['personal', 'shared']), Role('jordan', 'registered', 'Registered.', []), Role('jordan', 'personal', 'Personal.', ['registered']) ] # Update the objects in case something else spuriously changed them. for role in roles: role.to_db()
def link(): amazonId = request.args.get('amazonId') bigoven_username = request.args.get('bigoven_username') if app.debug: resource = boto3.resource('dynamodb', endpoint_url='http://localhost:8000', region_name="fake_region", aws_access_key_id="fake_id", aws_secret_access_key="fake_key") else: resource = boto3.resource('dynamodb', region_name='us-east-1', aws_access_key_id=os.environ['AWS_KEY'], aws_secret_access_key=os.environ['AWS_SECRET']) table = resource.Table('my_cookbook_users') key = { "userId": amazonId } updateExpr = 'SET bigoven_username=:b' exprAttributeValues = {':b': bigoven_username} condExpr = boto3.dynamodb.conditions.Attr('invocations').exists() try: response = table.update_item(Key=key, UpdateExpression=updateExpr, ExpressionAttributeValues=exprAttributeValues, ConditionExpression=condExpr) except ClientError as ce: print "invalid id: %s" % amazonId return render_template('error.html') return render_template('link.html')
def __init__(self, app): """Initializes a storage manager instance. """ self.app = app self.config = app.config['FILE_STORAGE'] self.secur = app.config['SECURITY_MANAGEMENT']['content'] if self.config['type'] == 's3': # Boto s3 instance if self.config['id'] != '' and self.config['key'] != '': self.s3 = boto3.resource('s3', aws_access_key_id=self.config['id'], aws_secret_access_key=self.config['key'], region_name=self.config['location']) else: self.s3 = boto3.resource('s3') # S3 bucket location try: self.bucket = self.config['name'] except: self.bucket = "" elif self.config['type'] == 'filesystem': self.storage_path = '{0}{1}'.format(self.config['location'], self.config['name']) if not os.path.exists(self.storage_path): os.makedirs(self.storage_path) os.makedirs('{0}/corr-bundles'.format(self.storage_path)) os.makedirs('{0}/corr-files'.format(self.storage_path)) os.makedirs('{0}/corr-logos'.format(self.storage_path)) os.makedirs('{0}/corr-outputs'.format(self.storage_path)) os.makedirs('{0}/corr-pictures'.format(self.storage_path)) os.makedirs('{0}/corr-resources'.format(self.storage_path)) elif self.config['type'] == 'mdcs': # TODO pass elif self.config['type'] == 'ftp' or self.config['type'] == 'sftp' : # TODO pass
def stopec2(): ec2 = boto3.resource('ec2'); ret=[] count = 0; for region in ec2regions: target=[] ec2 = boto3.resource('ec2',region); instances = ec2.instances.all() for instance in instances: ret.append({"Region": region, "id": instance.id}) if instance.state['Name'] != 'running': continue if instance.tags is not None and any(tag['Key'] == 'Name' for tag in instance.tags): nametag = (tag['Value'] for tag in instance.tags if tag['Key'] == 'Name').next() if not nametag in nametags: count+=1 target.append(instance) else: count+=1 target.append(instance) for instance in target: result = instance.stop() print (result) # stop all running instances we found print (count, "instances found") print (ret) return ret
def sign_up_by_email(event, context): User = boto3.resource('dynamodb').Table('FmSsoUsers') Auth = boto3.resource('dynamodb').Table('FmSsoAuths') email = event['email'] password = event['password'] auth_rcds = Auth.scan( FilterExpression=Attr('email').eq(email) ) user_rcds = User.scan( FilterExpression=Attr('email').eq(email) ) if(auth_rcds['Count'] != 0): # User exists, raise error raise Exception('User already registered!') elif(user_rcds['Count'] != 0): raise Exception( 'User already registered! Please login with original account and associate in it') user = create_auth(Auth, User, email, password) expires = time.time() + conf['cookie_expiry'] return { 'uid': user['uid'], 'email': user['email'], 'atoken': user['atoken'], 'atoken_expiry': user['atoken_expiry'], 'rtoken': user['rtoken'], 'rtoken_expiry': user['rtoken_expiry'], 'cookie': 'atoken={}; domain=.amazonaws.com; expires={}; path=/'.format(user['atoken'], time.strftime("%a, %d-%b-%Y %T GMT", time.gmtime(expires))) }
def handle_event(event, context): # parse the recipe id from the parameters try: recipe_id = event['params']['querystring']['id'] try: id_num = int(recipe_id) if id_num < 0: return {"code": -1, "data": "recipe_id (%d) most be positive." % id_num} if stage.PROD: resource = boto3.resource("dynamodb") else: resource = boto3.resource( "dynamodb", endpoint_url="http://localhost:8000", aws_access_key_id="fake", aws_secret_access_key="fake", region_name="us-east-1") table = resource.Table('my_cookbook_recipes') response = table.get_item(Key={'id': id_num}) if 'Item' in response: return {"code": 0, "data": response['Item']} else: return {"code": 1, "data": "No recipe ith id %d" % id_num} except ValueError: return {"code": -1, "data": "recipe_id must be an integer"} except KeyError: return {"code": -1, "data": "Failed to parse query prameter 'id'"}
def share_fits(uid, args): """ Create a map of the hash of eve uid and fitting id to crest json. Map is stored in a dynamodb table. param uid: string slack user id that sent the request param args: {'id': [fitting_id, ...]} :returns: [(ship, name, API endpoint url), ...] """ user = get_user(uid, args) fittings = boto3.resource('dynamodb').Table('fittings') shared_fits = boto3.resource('dynamodb').Table('shared_fits') eve_user_id = user['eve_user_id'] fittings_req = fittings.query(KeyConditionExpression=Key('eve_user_id').eq(eve_user_id)) shared = [] for fit in fittings_req['Items']: if fit['fitting_id'] not in args['id']: continue fit_hash = str(hash((eve_user_id, fit['fitting_id']))) shared_fits.put_item(Item={'fitting_hash': fit_hash, 'fit': fit['fit']}) shared.append((fit['ship'], fit['name'], settings.SHARE_ENDPOINT.format(fit_hash))) return shared
def test_close(self): self.s3_task_handler.set_context(self.ti) self.assertTrue(self.s3_task_handler.upload_on_close) self.s3_task_handler.close() # Should not raise boto3.resource('s3').Object('bucket', self.remote_log_key).get()
def __init__(self, repository_name, github_username, github_token, aws_key, aws_secret, aws_region, events, callbacks=None, **kwargs): """Instantiates a RepositoryListener. Additionally: * Creates or connects to a AWS SQS queue named for the repository * Creates or connects to a AWS SNS topic named for the repository * Connects the AWS SNS topic to the AWS SQS queue * Configures the Github repository to push hooks to the SNS topic Args: repository_name (str): name of a Github repository, like "tdsmith/homebrew-pypi-poet" github_username (str): Github username github_token (str): Github authentication token from https://github.com/settings/tokens/new with admin:org_hook privileges aws_key (str): AWS key aws_secret (str): AWS secret aws_region (str): AWS region (e.g. 'us-west-2') events (list<str>): List of Github webhook events to monitor for activity, from https://developer.github.com/webhooks/#events. callbacks (list<function(str event_type, Object event_payload)>): functions to call with a decoded Github JSON payload when a webhook event lands. You can register these after instantiation with register_callback. """ self.repository_name = repository_name self.github_username = github_username self.github_token = github_token self.aws_key = aws_key self.aws_secret = aws_secret self.aws_region = aws_region # create or reuse sqs queue sqs_resource = boto3.resource("sqs", region_name=self.aws_region) self.sqs_queue = sqs_resource.create_queue( QueueName="snooze__{}".format(self._to_topic(repository_name)) ) # create or reuse sns topic sns_resource = boto3.resource("sns", region_name=self.aws_region) sns_topic = sns_resource.create_topic( Name=self._to_topic(repository_name) ) sns_topic.subscribe( Protocol='sqs', Endpoint=self.sqs_queue.attributes["QueueArn"] ) # configure repository to push to the sns topic connect_github_to_sns(aws_key, aws_secret, aws_region, github_username, github_token, repository_name, sns_topic.arn, events) # register callbacks self._callbacks = [] if callbacks: [self.register_callback(f) for f in callbacks]
def test_has_good_error_message_when_no_resource(self): bad_resource_name = 'doesnotexist' err_regex = ( '%s.*resource does not exist.' % bad_resource_name ) with self.assertRaisesRegexp(ResourceNotExistsError, err_regex): boto3.resource(bad_resource_name)
def initialize(): # If you don't feel like setting environment variables for boto3 on your machine, # you can hard code them here for testing. #boto3.setup_default_session( #aws_access_key_id = XXXXX, #aws_secret_access_key = YYYYY # ) if Dynamo.dynamodb is None: try: if Dynamo.local: Dynamo.dynamodb = boto3.resource( 'dynamodb', endpoint_url='http://localhost:8000' # change if required ) else : Dynamo.dynamodb = boto3.resource( 'dynamodb', region_name='us-west-2', endpoint_url="https://dynamodb.us-west-2.amazonaws.com" ) except: if Dynamo.local: print('DB Connection Error: Unable to connect to local database. Check if database is ' 'running locally and ensure port number is correct.') else: print('DB Connection Error: Unable to connect to AWS Hosted DynamoDB server. Check if access ' 'credentials are properly configured.') raise return Dynamo
def redirect_from_upload(): check_connection(auth) # Get bucket name and key from the S3 redirect URL key = request.query.key bucket = request.query.bucket filename = key.split('/')[1] print key # get the unique job ID temp = filename.split('~') jobid = temp[0] # Create a job item and persist it to the annotations database username = auth.current_user.username usermail = auth.current_user.email_addr data = {"job_id":jobid, "username":username,"email":usermail, "submit_time":int(time.time()), "s3_inputs_bucket":bucket, "s3_key_input_file":key, "input_file_name":filename, "status":"PENDING"} dynamodb = boto3.resource('dynamodb') ann_table = dynamodb.Table(dynamo_table) ann_table.put_item(Item=data) #publish a notification to SNS topic #reference: http://boto3.readthedocs.io/en/latest/reference/services/sns.html#topic sns = boto3.resource('sns') topic = sns.Topic(job_arn) response = topic.publish( Message=json.dumps(data), ) return template(request.app.config['mpcs.env.templates'] + 'upload_confirm.tpl', job_id=jobid, auth=auth)
def deleteThumbnails(): bucket = boto3.resource('s3').Bucket(bucketname) for key in bucket.objects.filter(Prefix="thumbnail/"): if not key.__eq__(boto3.resource('s3').ObjectSummary(bucketname, unicode("thumbnail/dummy.jpg"))): key.delete() for key in bucket.objects.filter(Prefix="src/"): if not key.__eq__(boto3.resource('s3').ObjectSummary(bucketname, unicode("src/dummy.jpg"))): key.delete()
def get_table(table_name): """ Gets a Table resource by name""" # Get table resource if DYNAMO_LOCAL: dynamodb = boto3.resource('dynamodb', region_name='us-east-1', endpoint_url="http://localhost:4569") else: dynamodb = boto3.resource('dynamodb', region_name='us-east-1') return dynamodb.Table(table_name)
def amazon_s3_file_validator_no_delete(self, bucket): key_name = 'test-unvalidated-uploads/images/hello.jpg' boto3.resource('s3').Object(bucket.name, key_name).put(Body='test') return AmazonS3FileValidator( key_name=key_name, bucket=bucket, delete_unvalidated_file=False )
def save_file(key, symlink=False): output_file = os.path.join(dest_dir, key) if not os.path.exists(output_file): if symlink: os.symlink(os.path.join(s3_mount_dir, key), output_file) else: boto3.resource('s3').Bucket(bucket_name).download_file(key, output_file) return output_file
def amazon_s3_file_validator_with_acl(self, bucket): key_name = 'test-unvalidated-uploads/images/hello.jpg' boto3.resource('s3').Object(bucket.name, key_name).put(Body='test') return AmazonS3FileValidator( key_name=key_name, bucket=bucket, new_file_acl='private', )
def failing_amazon_s3_file_validator(self, bucket): key_name = 'images/fail.jpg' boto3.resource('s3').Object(bucket.name, key_name).put(Body='test') return AmazonS3FileValidator( key_name=key_name, bucket=bucket, validators=[CustomValidator()] )
def test_close_no_upload(self): self.ti.raw = True self.s3_task_handler.set_context(self.ti) self.assertFalse(self.s3_task_handler.upload_on_close) self.s3_task_handler.close() with self.assertRaises(self.conn.exceptions.NoSuchKey): boto3.resource('s3').Object('bucket', self.remote_log_key).get()
# http://boto3.readthedocs.io/en/latest/reference/services/sns.html # http://boto3.readthedocs.io/en/latest/reference/services/ses.html import os import boto3 import time from datetime import datetime from botocore.client import Config from boto3.dynamodb.conditions import Key import json import hardcode hardcodes = hardcode.Config try: sqs = boto3.resource('sqs', region_name=hardcodes.AWS_REGION_NAME) glacier_queue = sqs.get_queue_by_name( QueueName=hardcodes.AWS_SQS_GLACIER_ARCHIVE) except Exception as e: print(e) while True: messages = glacier_queue.receive_messages(WaitTimeSeconds=5) if not messages: print('Waiting for message...for archive restore') else: for message in messages: try: if message:
def collect_and_download(out_dir, less_than=0, greater_than=0, sex='', handedness='', sessions=SESSIONS, scans=SCANS, series=SERIES_MAP.keys(), derivatives=False, dryrun=False): ''' Function to collect and download images from the Rockland sample directory on FCP-INDI's S3 bucket Parameters ---------- out_dir : string filepath to a local directory to save files to less_than : float upper age (years) threshold for participants of interest greater_than : float lower age (years) threshold for participants of interest sex : string 'M' or 'F' to indicate whether to download male or female data handedness : string 'R' or 'L' to indicate whether to download right-handed or left-handed participants sessions : list the session names (e.g.,'CLG5','NFB3') scan : list the scan types to download. Can be 'anat','func','dwi' or 'fmap'. series : list the series to download (for functional scans) derivatives : boolean whether or not to download data derivatives for functional scans dryrun : boolean whether or not to perform a dry run (i.e., no actual downloads, just listing files that would be downloaded) Returns ------- boolean Returns true if the download was successful, false otherwise. ''' # Import packages import pandas import boto3 import botocore # For anonymous access to the bucket. from botocore import UNSIGNED from botocore.client import Config from botocore.handlers import disable_signing # Init variables s3_bucket_name = 'fcp-indi' s3_prefix = 'data/Projects/RocklandSample/RawDataBIDS' # Fetch bucket s3 = boto3.resource('s3') s3.meta.client.meta.events.register('choose-signer.s3.*', disable_signing) s3_bucket = s3.Bucket(s3_bucket_name) # Remove series that aren't in the series map keys. series = [s for s in series if s in SERIES_MAP.keys()] # If output path doesn't exist, create it if not os.path.exists(out_dir) and not dryrun: print 'Could not find %s, creating now...' % out_dir os.makedirs(out_dir) # Load the participants.tsv file from S3 s3_client = boto3.client('s3', config=Config(signature_version=UNSIGNED)) participants_obj = s3_client.get_object( Bucket=s3_bucket_name, Key='/'.join([s3_prefix, 'participants.tsv'])) participants_df = pandas.read_csv(participants_obj['Body'], delimiter='\t', na_values=['n/a']) # Init a list to store paths. print 'Collecting images of interest...' s3_keys = s3_bucket.objects.filter(Prefix=s3_prefix) s3_keylist = [key.key for key in s3_keys] # Remove the participants for whom age range, handedness and sex do not conform to the criteria. if less_than: participants_df = participants_df[participants_df['age'] < less_than] if greater_than: participants_df = participants_df[ participants_df['age'] > greater_than] if sex == 'M': participants_df = participants_df[participants_df['sex'] == 'MALE'] elif sex == 'F': participants_df = participants_df[participants_df['sex'] == 'FEMALE'] if handedness == 'R': participants_df = participants_df[participants_df['handedness'] == 'RIGHT'] elif handedness == 'L': participants_df = participants_df[participants_df['handedness'] == 'LEFT'] if len(participants_df) == 0: print 'No participants meet the criteria given. No download will be initiated.' return # Generate a list of participants to filter on. participants_filt = [ 'sub-' + label + '/' for label in participants_df['participant_id'].tolist() ] # Generate a list of sessions to filter on. sessions_filt = ['ses-' + session + '/' for session in sessions] # Generate a list of series to filter on. series_filt = [SERIES_MAP[s] for s in series] # Fetch top-level JSONs first. json_keylist = [ key for key in s3_keylist for s in series_filt if s in key and 'json' in key and 'sub' not in key ] # Applying filters. s3_keylist = [ key for key in s3_keylist for p in participants_filt if p in key ] s3_keylist = [key for key in s3_keylist for s in sessions_filt if s in key] s3_keylist = [key for key in s3_keylist for s in scans if s in key] s3_keylist = [ key for key in s3_keylist for s in series_filt if (s in key) or ('func' not in key) ] # Add back top-level files s3_keylist.extend(json_keylist) s3_keylist.append('/'.join([s3_prefix, 'CHANGES'])) s3_keylist.append('/'.join([s3_prefix, 'README'])) s3_keylist.append('/'.join([s3_prefix, 'dataset_description.json'])) # And download the items total_num_files = len(s3_keylist) files_downloaded = len(s3_keylist) for path_idx, s3_path in enumerate(s3_keylist): rel_path = s3_path.replace(s3_prefix, '') rel_path = rel_path.lstrip('/') download_file = os.path.join(out_dir, rel_path) download_dir = os.path.dirname(download_file) if not os.path.exists(download_dir) and not dryrun: os.makedirs(download_dir) try: if not os.path.exists(download_file): if dryrun: print 'Would download to: %s' % download_file else: print 'Downloading to: %s' % download_file with open(download_file, 'wb') as f: s3_client.download_fileobj(s3_bucket_name, s3_path, f) print '%.3f%% percent complete' % \ (100*(float(path_idx+1)/total_num_files)) else: print 'File %s already exists, skipping...' % download_file files_downloaded -= 1 except Exception as exc: print 'There was a problem downloading %s.\n'\ 'Check input arguments and try again.' % s3_path print exc # Print all done if dryrun: print '%d files would be downloaded for %d participant(s).' % ( files_downloaded, len(participants_df)) else: print '%d files downloaded for %d participant(s).' % ( files_downloaded, len(participants_df)) if not dryrun: print 'Saving out revised participants.tsv and session tsv files.' # Save out revised participants.tsv to output directory, if a participants.tsv already exists, open it and append it to the new one. if os.path.isfile(os.path.join(out_dir, 'participants.tsv')): old_participants_df = pandas.read_csv(os.path.join( out_dir, 'participants.tsv'), delimiter='\t', na_values=['n/a', 'N/A']) participants_df = participants_df.append(old_participants_df, ignore_index=True) participants_df.drop_duplicates(inplace=True) os.remove(os.path.join(out_dir, 'participants.tsv')) participants_df.to_csv(os.path.join(out_dir, 'participants.tsv'), sep="\t", na_rep="n/a", index=False) # Separate list for sessions TSVs. session_keylist = [ key.key for key in s3_keys if 'sessions.tsv' in key.key ] session_keylist = [ key for key in session_keylist for p in participants_filt if p in key ] # Save out revised session tsvs to output directory; if already exists, open it and merge with the new one. for session_key in session_keylist: participant = session_key.split('/')[-2] sessions_obj = s3_client.get_object(Bucket=s3_bucket_name, Key=session_key) sessions_df = pandas.read_csv(sessions_obj['Body'], delimiter='\t', na_values=['n/a']) # Drop all sessions not in specified. sessions_df = sessions_df[sessions_df['session_id'].isin( sessions_filt)] # Save out revised sessions tsv to output directory, if a sessions tsv already exists, open it and append it to the new one. if os.path.isfile( os.path.join(out_dir, participant, participant + '_sessions.tsv')): old_sessions_df = pandas.read_csv(os.path.join( out_dir, participant, participant + '_sessions.tsv'), delimiter='\t', na_values=['n/a', 'N/A']) sessions_df = sessions_df.append(old_sessions_df, ignore_index=True) sessions_df.drop_duplicates(inplace=True) os.remove( os.path.join(out_dir, participant, participant + '_sessions.tsv')) sessions_df.to_csv(os.path.join(out_dir, participant, participant + '_sessions.tsv'), sep="\t", na_rep="n/a", index=False) print 'Done!'
if not exit_node(): return 0 return db_backup.s3_upload_db(local_datadir_,s3alias) def s3_download_db(s3alias): # such as 'db-user1-node-190616' if not __LocalDbg__: return 0 if not db_backup: return 0 if not exit_node(): return 0 return db_backup.s3_download_db(local_datadir_,s3alias) if cloud_factory_ == 'AWS': import boto3 port_succ = False if region_name and instance_id: try: ec2 = boto3.resource('ec2',region_name=region_name) instance_ = ec2.Instance(instance_id) secur_groups = instance_.security_groups if secur_groups: exist30303 = False; exist30302 = False secur_group = ec2.SecurityGroup(secur_groups[0]['GroupId']) permis = secur_group.ip_permissions for item in permis: if item['FromPort'] == 30303: exist30303 = item['IpProtocol'] == 'tcp' elif item['FromPort'] == 30302: exist30302 = item['IpProtocol'] == 'udp' if not exist30303 or not exist30302: permit = []
(DWH_DB_USER, DWH_DB_PASSWORD, DWH_DB) pd.DataFrame({ 'Param': [ 'DWH_CLUSTER_TYPE', 'DWH_NUM_NODES', 'DWH_NODE_TYPE', 'DWH_CLUSTER_IDENTIFIER', 'DWH_DB', 'DWH_DB_USER', 'DWH_DB_PASSWORD', 'DWH_PORT', 'DWH_IAM_ROLE_NAME' ], 'Value': [ DWH_CLUSTER_TYPE, DWH_NUM_NODES, DWH_NODE_TYPE, DWH_CLUSTER_IDENTIFIER, DWH_DB, DWH_DB_USER, DWH_DB_PASSWORD, DWH_PORT, DWH_IAM_ROLE_NAME ] }) ec2 = boto3.resource('ec2', region_name="eu-central-1", aws_access_key_id=KEY, aws_secret_access_key=SECRET) s3 = boto3.resource('s3', region_name="eu-central-1", aws_access_key_id=KEY, aws_secret_access_key=SECRET) iam = boto3.client('iam', aws_access_key_id=KEY, aws_secret_access_key=SECRET, region_name='eu-central-1') redshift = boto3.client('redshift', region_name="eu-central-1", aws_access_key_id=KEY,
def setUp(self): self.mock = mock_s3() self.mock.start() conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket=BUCKET) self.lg = local.LocalGateway(app, Config())
def lambda_handler(event, context): '''Demonstrates a simple HTTP endpoint using API Gateway. You have full access to the request and response payload, including headers and status code. To scan a DynamoDB table, make a GET request with the TableName as a query string parameter. To put, update, or delete an item, make a POST, PUT, or DELETE request respectively, passing in the payload to the DynamoDB API as a JSON body. ''' s3 = boto3.resource('s3') # s3 = boto3.client('s3') print("Received event: " + json.dumps(event, indent=2)) operations = { 'DELETE': lambda dynamo, x: dynamo.delete_item(**x), 'GET': lambda dynamo, x: dynamo.scan(**x), 'POST': lambda dynamo, x: dynamo.put_item(**x), 'PUT': lambda dynamo, x: dynamo.update_item(**x), } operation = event['httpMethod'] if operation in operations: if operation == 'GET': bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus( event['Records'][0]['s3']['object']['key'].encode('utf8')) obj = s3.Object(bucket, key) content = obj.get()['Body'].read() contentString = base64.b64encode(content) # Since we don't do integration test, our jpg file content is binary # contentString = base64.b64encode(content) result = {"ImageName": key, "Content": contentString} return respond(None, result) elif operation == 'POST': bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus( event['Records'][0]['s3']['object']['key'].encode('utf8')) data = event['Records'][0]['s3']['object']['data'] obj = s3.Object(bucket, key) # f = open('test1.jpg', 'rb') # with f as myfile: # data = myfile.read() # f.close() dataAray = base64.b64decode(data) # Here we need to use Body=data since the arguement is a json object obj.put(Body=dataAray) print("Object is " + str(obj)) result = { "ImageName": key, "Result": "POST Success", "DataStream": base64.b64encode(dataAray) } return respond(None, result) elif operation == "DELETE": bucketName = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus( event['Records'][0]['s3']['object']['key'].encode('utf8')) if key == "": bucket = s3.Bucket(bucketName) bucket.objects.delete() else: obj = s3.Object(bucketName, key) obj.delete() result = {"Result": "DELETE all the objects in the bucket"} return respond(None, result) else: return respond(ValueError( 'Unsupported PUT "{}"'.format(operation))) else: return respond(ValueError('Unsupported method "{}"'.format(operation)))
#!/usr/bin/python import boto3 import json import time #QUEUE = 'https://sqs.us-west-2.amazonaws.com/384845588177/cloudstack-unused-instances' QUEUE = 'cloudstack-unused-instances' sqs = boto3.resource('sqs') queue = sqs.get_queue_by_name(QueueName=QUEUE) while True: for message in queue.receive_messages(): m = json.loads(message.body) m1 = json.loads(m['Message'].replace("'", "\"")) dimensions = m1['Trigger']['Dimensions'] instance_details = {d['name']:d['value'] for d in dimensions} for k,v in instance_details.iteritems(): print "Key= %s , Value=%s" % (k,v) message.delete() time.sleep(60)
def s3_etag(url): """Check ETag on S3 object.""" s3_resource = boto3.resource("s3") bucket_name, s3_path = split_s3_path(url) s3_object = s3_resource.Object(bucket_name, s3_path) return s3_object.e_tag
def __init__(self, table_name): self.dynamodb = boto3.resource('dynamodb') self.table = self.dynamodb.Table(table_name)
# -*- coding: utf-8 -*- from typing import Any, Optional import logging import boto3 import requests from trawsate import travis IamUser = Any AccessKeyPair = Any logger = logging.getLogger(__name__) iam = boto3.resource('iam') class AccessKeyStateError(Exception): """ Raised when a user's access keys are in a state we cannot handle and/or require that they are not in. """ pass def _handle_existing_keys(user: IamUser) -> Optional[IamUser]: """ Get a user's keys in a state where we can safely create a new key. :param user: The user whose keys to scan. :return: The user to deactivate after creating a new key, if any. """
from __future__ import unicode_literals import boto3 import json from datetime import datetime from collections import defaultdict import base64 import time import decimal dynamodb = boto3.resource('dynamodb') kinesis = boto3.client("kinesis") shard_id = 'shardId-000000000000' #only one shard shard_it = kinesis.get_shard_iterator( StreamName="twitter-stream", ShardId=shard_id, ShardIteratorType="LATEST")["ShardIterator"] event = kinesis.get_records(ShardIterator=shard_it, Limit=100) def update_dynamo_event_counter(tableName, event_name, longi, dynamodb=boto3.resource( service_name='dynamodb', region_name='us-west-2')): table = dynamodb.Table(tableName) longil = longi[0] latd = longi[1] print(longil) print(latd) if event_name: checkItemExists = table.get_item(Key={'full_name': event_name}) if 'Item' in checkItemExists:
def amibkp(region, days_del, slack_req, slack_channel, slack_webhook): """ This function is the crucial function, fetches all the instances which has tag Key:AMIBACKUPON, Value:yes and creates AMI in a loop, along with propogating all the tags from instance to AMI to EBS Snapshots. Also, it deletes all the AMI's which was created through this script and older than number of days you provide as an argument. Parameters ---------- region: string AWS region code. days: integer Number of days to keep AMI's before deleting. slack: string Optional argument. Passing this parameter as "true" will post the execption to slack if any. slack_channel: String Slack channel to where exceptions has to be posted Depends on the previous parameter "slack", this is required if slack is true. webhookurl: string Slack webhookurl to identify to which slack team exeception has to be posted? Depends on the previous parameter "slack", this is required if slack is true. Returns ------- list Returns list of AMI's/Snapshots deleted and newly created AMI's/Snapshots. """ client = boto3.client('ec2', region_name=region) ec2 = boto3.resource('ec2', region_name=region) image_response = client.describe_images(Filters=[ { 'Name': 'tag:DELETEON', 'Values': [ 'yes', ] }, ]) delete_ami(image_response, days_del, slack_req, slack_channel, slack_webhook, ec2, region) instance_response = client.describe_instances(Filters=[{ 'Name': 'tag:AMIBACKUPON', #Tag used to identify list of Instances to be backed up. 'Values': [ 'yes', ] }]) create_ami(instance_response, slack_req, slack_channel, slack_webhook, ec2, region) new_image_response = client.describe_images(Filters=[ { 'Name': 'tag:Snapshottag', 'Values': [ 'yes', ] }, ]) tag_snapshots(new_image_response, slack_req, slack_channel, slack_webhook, client, region)
" repository name. e.g. the React library's repo would be `facebook/react`" ) ERR_SETUP_CHANNEL_REPO_EXISTS = "It looks as though that repo is already setup in this channel" ERR_SETUP_REPO_EXISTS = "That repo is already setup in a different channel. Only one channel per repo." ERR_SETUP_CHANNEL_EXISTS = "This channel is already setup for a different repo. Only one repo per channel." ERR_RESET_FAILURE = "No config found for this channel" ERR_USER_BUTTON_FALLBACK = "You are unable to introduce yourself for some reason. Sorry." ERR_GOODBYE_NO_USER = "******" ERR_DEPLOY_ARGS = "You must specify what and where to deploy, as `deploy <branch-or-tag> to <environment>`" ERR_DEPLOY_VALID_ENV = "I don't recognise that environment name" ERR_DEPLOY_REFENV_VALIDATION = "That environment doesn't get built from that ref according to the supported workflow. " ERR_DEPLOY_REMOTE = "Something went wrong: " SLACK_SIGNING_SECRET_VERSION = "v0" dynamodb = boto3.resource('dynamodb', region_name=os.environ['SLS_AWS_REGION']) lambda_client = boto3.client('lambda', region_name=os.environ['SLS_AWS_REGION']) logger = logging.getLogger() if logger.handlers: for handler in logger.handlers: logger.removeHandler(handler) logging.basicConfig(level=logging.INFO) logger.setLevel(logging.INFO) # # # Callable user-facing functions # #
def lambda_handler(event, context): #logger.info('Event: ' + str(event)) print('Received event: ' + json.dumps(event, indent=2)) ids = [] try: region = event['region'] detail = event['detail'] eventname = detail['eventName'] arn = detail['userIdentity']['arn'] principal = detail['userIdentity']['principalId'] userType = detail['userIdentity']['type'] creationDate = detail['eventTime'] print( 'Assinged region, detail, eventname, arn, principal, userType, creationDate' ) if userType == 'IAMUser': user = detail['userIdentity']['userName'] else: user = principal.split(':')[1] print('User ' + user) logger.info('principalId: ' + str(principal)) logger.info('region: ' + str(region)) logger.info('eventName: ' + str(eventname)) logger.info('detail: ' + str(detail)) print('Check Response Elements') if not (detail['responseElements'] or eventname == 'CreateBucket'): logger.warning('No responseElements found') if detail['errorCode']: logger.error('errorCode: ' + detail['errorCode']) if detail['errorMessage']: logger.error('errorMessage: ' + detail['errorMessage']) return False common_tags = [{ 'Key': 'Requestor', 'Value': user }, { 'Key': 'PrincipalId', 'Value': principal }, { 'Key': 'creationDate', 'Value': creationDate }] allTags = defineTags(user, common_tags) ec2 = boto3.resource('ec2') print('Check Event Match') if eventname == 'CreateVolume': ids.append(detail['responseElements']['volumeId']) logger.info(ids) elif eventname == 'RunInstances': items = detail['responseElements']['instancesSet']['items'] for item in items: ids.append(item['instanceId']) logger.info(ids) logger.info('number of instances: ' + str(len(ids))) base = ec2.instances.filter(InstanceIds=ids) #loop through the instances for instance in base: for vol in instance.volumes.all(): ids.append(vol.id) for eni in instance.network_interfaces: ids.append(eni.id) elif eventname == 'CreateImage': ids.append(detail['responseElements']['imageId']) logger.info(ids) elif eventname == 'CreateSnapshot': ids.append(detail['responseElements']['snapshotId']) logger.info(ids) elif eventname == 'CreateBucket': s3 = boto3.resource('s3') s3_bucketName = detail['requestParameters']['bucketName'] if s3_bucketName: bucket_tagging = s3.BucketTagging(s3_bucketName) #print (ids) #allTags = defineTags(user,common_tags) #print (type(allTags)) s3_resp = bucket_tagging.put(Tagging={'TagSet': allTags}) print(s3_resp) #print ('Tagged S3 bucket') elif eventname == 'CreateDBInstance': print('CreateDBInstance') rds = boto3.client('rds') rds_arn = detail['responseElements']['dBInstanceArn'] if rds_arn: print('Tagging resource ' + rds_arn) #allTags = defineTags(user,common_tags) rds_resp = rds.add_tags_to_resource(ResourceName=rds_arn, Tags=allTags) print(rds_resp) elif eventname == 'CreateDBSnapshot': print('CreateDBSnapshot') rds_snp = boto3.client('rds') rds_snp_arn = detail['responseElements']['dBSnapshotArn'] if rds_snp_arn: print('Tagging resource ' + rds_snp_arn) #allTags = defineTags(user,common_tags) rds_snp_resp = rds_snp.add_tags_to_resource( ResourceName=rds_snp_arn, Tags=allTags) print(rds_snp_resp) elif eventname == 'CreateFileSystem': print('CreateFileSystem') efs = boto3.client('efs') efs_fsid = detail['responseElements']['fileSystemId'] if efs_fsid: print('Tagging resource ' + efs_fsid) efs_resp = efs.create_tags(FileSystemId=efs_fsid, Tags=allTags) print(efs_resp) elif eventname == 'CreateTable': print('CreateTable') dyn = boto3.client('dynamodb') dyn_arn = detail['responseElements']['tableDescription'][ 'tableArn'] if dyn_arn: print('Tagging resource ' + dyn_arn) #allTags = defineTags(user,common_tags) table_status = detail['responseElements']['tableDescription'][ 'tableStatus'] table_name = detail['responseElements']['tableDescription'][ 'tableName'] while (table_status == 'CREATING'): print('Table is being created') table_check = dyn.describe_table(TableName=table_name) print(table_check) table_status = table_check['Table']['TableStatus'] print(table_status) if (table_status == 'ACTIVE'): dyn_resp = dyn.tag_resource(ResourceArn=dyn_arn, Tags=allTags) print(dyn_resp) else: logger.warning('Not supported action') if ids: for resourceid in ids: print('Tagging resource ' + resourceid) #allTags = defineTags(user,common_tags) ec2.create_tags(Resources=ids, Tags=allTags) logger.info(' Remaining time (ms): ' + str(context.get_remaining_time_in_millis()) + '\n') return True except Exception as e: logger.error('Something went wrong: ' + str(e)) return False
import numpy as np import matplotlib.pyplot as plt import pandas as pd import boto3 from matplotlib.ticker import PercentFormatter # === Step 1. Download data === dynamodb = boto3.resource("dynamodb", region_name='ap-southeast-1', aws_access_key_id='...........', # erased out for safety reason aws_secret_access_key='.............' # erased out for safety reason ) table = dynamodb.Table('EE5111_A0039875M_JetEngine') response = table.scan() data = response['Items'] while 'LastEvaluatedKey' in response: response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey']) data.extend(response['Items']) data = pd.DataFrame(data) # === Step 2. Plot Graph === # Plot single engine single sensor data1Sensor2 = data[data['id'].str.contains('FD001')].loc[:, ['id','cycle','sensor2']] data1Sensor2['id'] = list(map(lambda x: int(x.split('_')[-1]), data1Sensor2['id'])) data1Sensor2.sort_values(by=['id','cycle'], inplace=True) plt.figure() plt.plot(range(len(data1Sensor2)), data1Sensor2['sensor2'], linewidth=0.1) plt.title('Engine 1 Sensor 2 in 99 Cycles') plt.figure() plt.plot(range(len(data1Sensor2[data1Sensor2['id']==1])), data1Sensor2[data1Sensor2['id']==1]['sensor2']) plt.title('Engine 1 Sensor 2 in First Cycle')
def s3_get(url, temp_file): """Pull a file directly from S3.""" s3_resource = boto3.resource("s3") bucket_name, s3_path = split_s3_path(url) s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file)
def main(): #retrieve argument args = parse_arguments() main_directory = args.directory class1 = args.class1 class2 = args.class2 force_by_user = args.force if args.verbose: lg.basicConfig(level=lg.INFO) #Variables declaration result = [] directory_feature = os.path.join(main_directory, "features", "*.json") nb_training_data = args.nb_training_data iteration_model = args.iteration_model min_partition = args.min_partition s3 = boto3.resource('s3') bucket = s3.Bucket('oc-calculdistribues-sberton') result_file = class1 + '_' + class2 + '_' + time.strftime( "%Y%m%d%H%M%S") + '.json' model_file = 'model_' + class1 + '_' + class2 + '_' + str( nb_training_data) + '_' + str(iteration_model) model_pathname = os.path.join(main_directory, "models", model_file) #Searching existing model and store existence in is_model boolean key = 'distributed_learning/models/' + model_file objs = list(bucket.objects.filter(Prefix=key)) is_model = len(objs) > 0 and objs[0].key.startswith(key + '/') start_time = time.time() lg.info( '#################### Starting pet-classification ######################' ) lg.info('Class 1 is %s', class1) lg.info('Class 2 is %s', class2) lg.info('Number of training datas is %s', nb_training_data) lg.info('Number of iterations model is %s', iteration_model) #persist a common rdd which is using by both training and testing datas common_rdd = sc.textFile(directory_feature, minPartitions=min_partition)\ .filter(lambda line: line.split(', ')[0] in (class1, class2) or class2 == 'All')\ .persist() #Loading model if exists if is_model and not force_by_user: model = SVMModel.load(sc, model_pathname) lg.info('Found and load recorded model %s', model_file) else: lg.info('No recorded model found') #create training rdd and train model if no model found or force train_data_rdd = common_rdd.filter(lambda line: int(line.split(',')[1]) <= nb_training_data)\ .map(lambda line: Row(label=0.0, features=line.split(', ')[2:]) if line.split(', ')[0] == class1 else Row(label=1.0, features=line.split(', ')[2:]))\ .map(lambda line: LabeledPoint(line.label, line.features)) lg.info('%s features for training datas', train_data_rdd.count()) lg.info('Start to training model') model = SVMWithSGD.train(train_data_rdd, iterations=iteration_model) lg.info('Training model terminated') training_time = time.time() training_duration = training_time - start_time #Create testing rdd test_data_rdd = common_rdd.filter(lambda line: int(line.split(', ')[1]) > nb_training_data)\ .map(lambda line: Row(label=0.0, features=line.split(', ')[2:]) if line.split(', ')[0] == class1 else Row(label=1.0, features=line.split(', ')[2:]))\ .map(lambda row: LabeledPoint(row.label, row.features)) lg.info('%s features for test datas', test_data_rdd.count()) # Evaluating the model on training data predictions = test_data_rdd.map( lambda row: (row.label, float(model.predict(row.features)))) train_error = predictions.filter(lambda lp: lp[0] != lp[1]).count() \ / float(predictions.count()) lg.info('Test Error : %s', str(train_error)) end_time = time.time() duration = end_time - start_time lg.info('Duration %s', str(duration)) prediction_duration = end_time - training_time # #Save and dump result on S3 result.append({ "class1": class1, "class2": class2, "iteration_model": iteration_model, "nb_training_data": nb_training_data, "total_duration": duration, "train_duration": training_duration, "predict_duration": prediction_duration, "error": train_error }) s3object = s3.Object('oc-calculdistribues-sberton', result_file) s3object.put(Body=(bytes(json.dumps(result, indent=2).encode('UTF-8')))) #Save model if not exists if not is_model: lg.info('Saving model at %s', model_file) model.save(sc, model_pathname) lg.info( '#################### Ending pet-classification ######################' )
def get_most_recent_ride_info(self, user_id=None, cookies=None, save=False): workout_ids = PelotonConnection.__get_workouts__( self, user_id, cookies) for workout_id in workout_ids: workout_url = f"https://api.onepeloton.com/api/workout/{workout_id}" workout = self.get(workout_url, cookies) created_at = workout.get("created_at") # Then get the ride_id for that workout ride_id = workout.get("ride").get("id") ride_id_details_url = f"https://api.onepeloton.com/api/ride/{ride_id}/details" ride_id_details = self.get(ride_id_details_url, cookies) # In the event you did one of those non-workout rides try: instructor = ride_id_details.get('ride').get('instructor').get( 'name') except Exception: instructor = None if instructor is not None: table = boto3.resource('dynamodb').Table('peloton_course_data') if save is True: table.put_item( Item={ "created_at": str(created_at), "difficulty": str( ride_id_details.get('ride').get( 'difficulty_rating_avg')), "instructor": instructor, "length": str( time.strftime( "%H:%M:%S", time.gmtime( ride_id_details.get('ride').get( 'duration')))), "name": ride_id_details.get('ride').get('title') }) # Also people wanted the music if instructor is not None: song_list = [ song for song in ride_id_details.get("playlist").get("songs") ] set_list = [ f"{f.get('title')} by {f.get('artists')[0].get('artist_name')}" for f in song_list ] table = boto3.resource('dynamodb').Table('peloton_music_sets') if save is True: table.put_item(Item={ "created_at": str(created_at), "set_list": set_list })
import sys import json import boto3 import logging import uuid from boto3.dynamodb.conditions import Key, Attr logger = logging.getLogger() logger.setLevel(logging.INFO) #DynamodDB table name and region DYNAMODB_REGION = "eu-west-2" DYNAMODB_TABLE_NAME = "kong" os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" dynamodb = boto3.resource('dynamodb', DYNAMODB_REGION) #Bucket s3 BUCKET_NAME = 'my-bucket' # image bucket name KEY = 'my_image_in_s3.jpg' # r object key s3 = boto3.resource('s3') def lambda_handler(event, context): return scan_table(DYNAMODB_TABLE_NAME) def scan_table(table_name, filter_key=None, filter_value=None): table = dynamodb.Table(table_name) response = table.scan()
def get_most_recent_ride_details(self, user_id=None, cookies=None, save=False): # Get the most recent workout ID workout_ids = PelotonConnection.__get_workouts__( self, user_id, cookies) for workout_id in workout_ids: workout_url = f"https://api.onepeloton.com/api/workout/{workout_id}" # Get the workout info workout = self.get(workout_url, cookies) created_at = workout.get("created_at") achievements_url = f"https://api.onepeloton.com/api/user/{user_id}/achievements" achievements = self.get(achievements_url, cookies) achievements = [ f for f in [ a.get("achievements") for a in achievements.get("categories") ] ] total_achievements = sum([ val.get("count") for sublist in achievements for val in sublist ]) # Performance Graph For that workout/ride performance_url = f"https://api.onepeloton.com/api/workout/{workout_id}/performance_graph?every_n=5" performance_res = self.get(performance_url, cookies) results = {} # Each of the averages (Cadence, Speed, Distance, Etc) are in the different summaries # So just loop over and grab out the data # There are some dupes like heart_rate/achievements_etc but wasn't sure where to put it averages = performance_res.get("average_summaries") for average in averages: heart_rate = [ f for f in performance_res.get("metrics") if f.get("display_name") == "Heart Rate" ] or None result = { 'name': average.get('display_name'), 'unit': average.get('display_unit'), 'value': average.get('value'), 'distance': [ f for f in performance_res.get("summaries") if f.get("display_name") == 'Distance' ][0].get("value"), 'heart_rate': heart_rate[0].get("average_value") if heart_rate is not None else None, 'total_achievements': total_achievements, 'miles_ridden': [ f for f in performance_res.get("summaries") if f.get("display_name") == "Distance" ][0].get("value") } results[average.get('display_name')] = result # At some point it would behove me to purge the dynamo db and move the dupes out of results # But for now, we will leave it. Also, account for no heart rate monitor my_json_record = { "Avg Cadence": results.get("Avg Cadence"), "Avg Output": results.get("Avg Output"), "Avg Resistance": results.get("Avg Resistance"), "Avg Speed": results.get("Avg Speed"), 'heart_rate': heart_rate[0].get("average_value") if heart_rate is not None else None, 'total_achievements': total_achievements, 'miles_ridden': [ f for f in performance_res.get("summaries") if f.get("display_name") == "Distance" ][0].get("value"), "ride_Id": str(created_at) } table = boto3.resource('dynamodb').Table('peloton_ride_data') # The info comes in as a float and Dynamo gets mad so just parse it out and make it a json obj ddb_data = json.loads(json.dumps(my_json_record), parse_float=Decimal) # Toss the json into Dynamo if save is True: table.put_item(Item=ddb_data)
def display_buckets(): s3 = boto3.resource('s3') for buckets in s3.buckets.all(): print(buckets.name)
from botocore.exceptions import ClientError # Helper class to convert a DynamoDB item to JSON. class DecimalEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, decimal.Decimal): if o % 1 > 0: return float(o) else: return int(o) return super(DecimalEncoder, self).default(o) dynamodb = boto3.resource('dynamodb', region_name='us-west-2', endpoint_url="http://localhost:8000") table = dynamodb.Table('Movies') title = "The Big New Movie" year = 2015 # Conditional update (will fail) print("Attempting conditional update...") try: response = table.update_item( Key={ 'year': year, 'title': title
#!/usr/bin/env python import boto3 import sys #it reads the command line as arguments ec2 = boto3.resource('ec2') for instance_id in sys.argv[1:]: instance = ec2.Instance(instance_id) response = instance.terminate() print response
import os import time import logging import boto3 import decimal import dateutil.parser import datetime from boto3.dynamodb.conditions import Key, Attr # Logger logger = logging.getLogger() logger.setLevel(logging.DEBUG) # DynamoDb table decaration dynamodb = boto3.resource('dynamodb', region_name='us-east-1') # Client for Comprehend client = boto3.client('comprehend') """ Helper function for Lex """ def get_slots(intent_request): return intent_request['currentIntent']['slots'] def safeDecimal(n): """ Safely convert n value to int. """ if n is not None: return decimal.Decimal(n) return (n) def safeString(n):
import json import os import boto3 import numpy as np import pandas as pd import csv s3_resource = boto3.resource("s3") ## Function to count the number of datasets in a given job. Right now, counts based ## on the number of generated Dataset status files, we could think of alternatives. def count_datasets(bucket, jobpath): """ Assumes that the given path points to the job path of a given analysis. Looks within the log path of that same analysis, and counts the number of dataset files generated for it. Inputs: bucket: (boto3 obj): an s3 boto3 resource object declaring the bucket this comes from. jobpath: (str) the path of the job directory for a particular analysis. """ logsdir = os.path.join(jobpath, "logs/") ## TODO: make this reference the environment variable indicating the right log directory. ## Now look within for datasets with a certain prefix: all_logs = [i.key for i in bucket.objects.filter(Prefix=logsdir)] ## Filter this by the DATASET_NAME prefix: dataset_logs = [ l for l in all_logs if l.split(logsdir)[-1].startswith("DATASET_NAME") ] return dataset_logs
import boto3 import os import datetime import pandas as pd import botocore import zipfile import shutil import rds_config import pymysql import logging import csv logging.basicConfig() chaqerObject = Chaqer() s3 = boto3.resource('s3') client = boto3.client('s3') srcBucket = str(os.environ.get('BUCKET')) srcKey = str(os.environ.get('FILE')) rds_host = rds_config.db_endpoint name = rds_config.db_username password = rds_config.db_password db_name = rds_config.db_name port = 3306 try: cnx = pymysql.connect(rds_host, user=name, passwd=password, db=db_name, connect_timeout=10)
sudo yum -y update sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 sudo yum -y install ntp sudo yum -y install telnet sudo yum -y install mtr sudo yum -y install tree sudo systemctl start ntpd sudo systemctl enable ntpd.service echo "* hard nofile 65535" | sudo tee -a /etc/security/limits.conf echo "* soft nofile 65535" | sudo tee -a /etc/security/limits.conf echo "net.ipv6.conf.all.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf echo "net.ipv6.conf.default.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf echo "net.ipv6.conf.lo.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf""" ec2AmiResource = boto3.resource('ec2') ec2InstanceAmi = ec2AmiResource.create_instances(ImageId=amiId, InstanceType='t2.micro', KeyName='AfterPaySSHKey', MinCount=1, MaxCount=1, UserData=userDataScriptAmi) logging.warning( 'Updatinge packages, installing services and configuring services in progress for customized AMI for AfterPay ...' ) checkStatusClient = boto3.client('ec2') ec2AmiStatus = checkStatusClient.describe_instance_status(InstanceIds=[ ec2InstanceAmi[0].instance_id, ])
import libraries.morflessLibs as libs import boto3 import json import logging, os settings_file = libs.constants.PCOM_REQ_FILE_SETTINGS sourcebucket= os.environ['SOURCE_BUCKET'] targetbucket= os.environ['TARGET_BUCKET'] listbucket = os.environ["LIST_BUCKET"] s3resource = boto3.resource('s3') s3client = boto3.client('s3') # process and output postlist constants def process_postlists_info(postlists_info,postlist,settings,list_meta,log): if postlists_info: for info in postlists_info: js_name = info['name'] list_info = info['postlists'] fileroot = info['fileroot'] js_constant,processed = libs.second_processes.pcom_process_postlist(list_info,postlist,settings,list_meta,fileroot) if processed: log['postlists_processed'].append(js_name) log = write_js_to_buckets(js_name,js_constant,log)
os.remove(directory_file + '.vcf.count.log') os.rmdir(task_dir) print('Delete task files successfully.') if __name__ == '__main__': config = configparser.ConfigParser() config.read(os.path.abspath('./configini.ini')) aws_config = config['ann'] if len(sys.argv) > 3: with Timer(): driver.run(sys.argv[1], 'vcf') print('Finish annotation successfully.') # upload results and logs try: s3 = boto3.resource('s3', region_name=aws_config['AWS_REGION_NAME']) task_file = sys.argv[1] directory_file = task_file.replace('.vcf', '') key = sys.argv[2].replace('.vcf', '') upload_file(s3, directory_file, key) except Exception as e: print(e) print('Cannot upload files.') # remove files on AnnTools instance try: task_file = sys.argv[1] directory_file = task_file.replace('.vcf', '') task_dir = os.path.dirname(task_file) remove_file(task_file, directory_file, task_dir) except Exception as e: print(e)
def notification_mailer(event, context): ddb = boto3.resource("dynamodb", region_name='us-east-1').Table(os.environ["BUILDABLES_TABLE_NAME"]) mm_helper = MessageMultiplexerHelper(endpoint=get_mm_endpoint(os.environ["MM_STAGE"]), own_account_number=os.environ["ACCOUNT_ID"]) notification_mailer_internal(event, context, ddb, mm_helper)