def __init__(self, queue_name,
region_name,
aws_access_key_id,
aws_secret_access_key,
auto_creation=False,
endpoint_url=None
):
"""
:param queue_name:
"""
settings = {'region_name': region_name,
'aws_access_key_id': aws_access_key_id,
'aws_secret_access_key': aws_secret_access_key,
}
if endpoint_url:
settings.update(
endpoint_url=endpoint_url
)
self.client = SQSClient(**settings)
self.queue_url = self.client.get_queue_url(queue_name)
if not self.queue_url and auto_creation:
self.client.create_queue(queue_name)
self.queue_url = self.client.get_queue_url(queue_name)
if hasattr(self.client, 'settings'):
self.instance = boto3.resource(
'sqs', **self.client.settings
).Queue(self.queue_url)
else:
self.instance = boto3.resource('sqs').Queue(self.queue_url)
def create_annotation_job_request():
auth.require(fail_redirect='/')
formdata = request.query
key = formdata["key"]
file_name = key.split("/")[-1]
job_id = file_name.split("~")[0]
sqs = boto3.resource('sqs')
data = {"job_id": str(job_id),
"username": auth.current_user.username,
"s3_inputs_bucket": "gas-inputs",
"s3_key_input_files": key,
"input_file_name": file_name.split("~")[1],
"submit_time": int(time.time()),
"job_status": "PENDING"
}
dynamodb = boto3.resource('dynamodb')
ann_table = dynamodb.Table('songty_annotations')
ann_table.put_item(Item=data)
#http://boto3.readthedocs.io/en/latest/reference/services/sqs.html#queue
queue = sqs.get_queue_by_name(QueueName='songty_job_requests')
queue.send_message(MessageBody=json.dumps(data))
return template(request.app.config['mpcs.env.templates'] + 'upload_confirm',
auth=auth,job_id=job_id)
def test_vpc_peering_connections_cross_region_accept_wrong_region():
# create vpc in us-west-1 and ap-northeast-1
ec2_usw1 = boto3.resource('ec2', region_name='us-west-1')
vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16')
ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1')
vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16')
# create peering
vpc_pcx_usw1 = ec2_usw1.create_vpc_peering_connection(
VpcId=vpc_usw1.id,
PeerVpcId=vpc_apn1.id,
PeerRegion='ap-northeast-1',
)
# accept wrong peering from us-west-1 which will raise error
ec2_apn1 = boto3.client('ec2', region_name='ap-northeast-1')
ec2_usw1 = boto3.client('ec2', region_name='us-west-1')
with assert_raises(ClientError) as cm:
ec2_usw1.accept_vpc_peering_connection(
VpcPeeringConnectionId=vpc_pcx_usw1.id
)
cm.exception.response['Error']['Code'].should.equal('OperationNotPermitted')
exp_msg = 'Incorrect region ({0}) specified for this request.VPC ' \
'peering connection {1} must be ' \
'accepted in region {2}'.format('us-west-1', vpc_pcx_usw1.id, 'ap-northeast-1')
cm.exception.response['Error']['Message'].should.equal(exp_msg)
def test_vpc_peering_connections_cross_region_delete():
# create vpc in us-west-1 and ap-northeast-1
ec2_usw1 = boto3.resource('ec2', region_name='us-west-1')
vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16')
ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1')
vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16')
# create peering
vpc_pcx_usw1 = ec2_usw1.create_vpc_peering_connection(
VpcId=vpc_usw1.id,
PeerVpcId=vpc_apn1.id,
PeerRegion='ap-northeast-1',
)
# reject peering from ap-northeast-1
ec2_apn1 = boto3.client('ec2', region_name='ap-northeast-1')
ec2_usw1 = boto3.client('ec2', region_name='us-west-1')
del_pcx_apn1 = ec2_apn1.delete_vpc_peering_connection(
VpcPeeringConnectionId=vpc_pcx_usw1.id
)
des_pcx_apn1 = ec2_usw1.describe_vpc_peering_connections(
VpcPeeringConnectionIds=[vpc_pcx_usw1.id]
)
des_pcx_usw1 = ec2_usw1.describe_vpc_peering_connections(
VpcPeeringConnectionIds=[vpc_pcx_usw1.id]
)
del_pcx_apn1['Return'].should.equal(True)
des_pcx_apn1['VpcPeeringConnections'][0]['Status']['Code'].should.equal('deleted')
des_pcx_usw1['VpcPeeringConnections'][0]['Status']['Code'].should.equal('deleted')
def load_config(config):
keys = load_yaml(abs_path("default"))
keys["credentials"] = {}
if os.path.exists(abs_path("credentials")):
keys["credentials"] = load_yaml(abs_path("credentials"))
if config != "default":
keys.update(load_yaml(abs_path(config)))
if "aws_access_key" in keys["credentials"]:
keys["s3"] = boto3.resource(
"s3",
region_name=keys["region"],
aws_access_key_id=keys["credentials"]["aws_access_key"],
aws_secret_access_key=keys["credentials"]["aws_access_secret"],
)
keys["s3_client"] = boto3.client(
"s3",
region_name=keys["region"],
aws_access_key_id=keys["credentials"]["aws_access_key"],
aws_secret_access_key=keys["credentials"]["aws_access_secret"],
)
else:
keys["s3"] = boto3.resource("s3", region_name=keys["region"])
keys["s3_client"] = boto3.client("s3", region_name=keys["region"])
return AttrDict(keys)
def main(event, context):
logger.info('START')
try:
collector = conf.get('Collector')
if collector is None:
logger.info('No Collector')
return 'No Collector'
# 対象のセットアップ
_init_fabric(_get_key())
# 事前処理を実行
for command in collector.get('CreateCommands', []):
logger.info(command)
fab.run(command)
# 事後処理(ログファイルをここのローカルにコピーしてS3にアップ)
s3info = collector.get('S3')
for remotefile in collector.get('TargetFiles', []):
filename = os.path.basename(remotefile)
localfile = '/tmp/{0}.log'.format(uuid.uuid4())
fab.get(remote_path=remotefile, local_path=localfile)
boto3.resource('s3').meta.client.upload_file(localfile, s3info.get('Bucket'), _gen_s3key(s3info, filename))
finally:
_finalize()
def get_runners(region_name=REGION_NAME):
'''Get a list of valid runner ID's
A valid runner ID is an ID with an active Amazon SQS Message Queue
and Amazon S3 Bucket.
Parameters
----------
region_name : str
Amazon region identifier
Returns
-------
list
List of valid runner ID's
See Also
--------
get_workers
'''
sqs = boto3.resource('sqs', region_name=region_name)
s3 = boto3.resource('s3', region_name=region_name)
queues = [os.path.split(q.url)[1] for q in sqs.queues.all()]
runners = []
for bucket in s3.buckets.all():
if bucket.name in queues:
runners.append(bucket.name)
return runners
def process(runner_id, workingdir='.', region_name=REGION_NAME, stop_on_empty=False):
'''Start processing loop for specific runner
Each worker needs to start this processing procedure. It polls the
queue, processes jobs and uploads the result.
Parameters
----------
runner_id : str
Runner ID
workingdir : str
Working dir at processing node
region_name : str, optional
Amazon region identifier
stop_on_empty : bool, optional
Flag to quit the processing loop if no messages are left
[default: False]
See Also
--------
process_job
'''
s3 = boto3.resource('s3', region_name=region_name)
sqs = boto3.resource('sqs', region_name=region_name)
while True:
if not process_job(sqs, s3, runner_id, workingdir='.'):
if stop_on_empty:
logger.info('No jobs left. Stop')
break
def _get_passwords_table():
"""
This is a workaround to use both locally and on aws.
:return:
"""
if STAGE != 'local':
return boto3.resource('dynamodb', region_name='eu-central-1').Table(PASSWORDS_TABLE)
db = boto3.resource(
'dynamodb',
endpoint_url='http://localhost:8000',
region_name='eu-central-1'
)
try:
db.meta.client.describe_table(TableName=PASSWORDS_TABLE)
except ClientError:
return db.create_table(
TableName=PASSWORDS_TABLE,
AttributeDefinitions=[{
'AttributeName': 'id',
'AttributeType': 'S',
}],
KeySchema=[{
'AttributeName': 'id',
'KeyType': 'HASH',
}],
ProvisionedThroughput={
'ReadCapacityUnits': 1,
'WriteCapacityUnits': 1,
}
)
return db.Table(PASSWORDS_TABLE)
def test_resource_passes_through_arguments(self):
boto3.DEFAULT_SESSION = self.Session()
boto3.resource('sqs', region_name='us-west-2', verify=False)
boto3.DEFAULT_SESSION.resource.assert_called_with(
'sqs', region_name='us-west-2', verify=False)
def sign_in_by_email(event, context):
User = boto3.resource('dynamodb').Table('FmSsoUsers')
Auth = boto3.resource('dynamodb').Table('FmSsoAuths')
email = event['email']
password = event['password']
auth_query = Auth.scan(
FilterExpression=Attr('email').eq(email)
)
if(auth_query['Count'] == 0):
# Auth with given email not found
raise Exception('Unauthorized')
auth_rcd = auth_query['Items'][0]
encrypted_password_key = 'fmsso' # TODO: Change to conf table value
if validate_password(password, auth_rcd, encrypted_password_key):
# TODO: find user record from auth
user = User.get_item(
Key={'id': auth_rcd['user_id']}
)
# TODO: refresh token
return {
'access_token': user['atoken'],
'refresh_token': user['rtoken'],
}
else:
raise Exception('Unauthorized')
def setUp(self):
"""Setup for testing"""
app.config.from_object('meerkat_auth.config.Testing')
app.config.from_envvar('MEERKAT_AUTH_SETTINGS')
User.DB = boto3.resource(
'dynamodb',
endpoint_url=app.config['DB_URL'],
region_name='eu-west-1'
)
Role.DB = boto3.resource(
'dynamodb',
endpoint_url=app.config['DB_URL'],
region_name='eu-west-1'
)
logging.warning(app.config['USERS'])
logging.warning(app.config['ROLES'])
logging.warning(app.config['DB_URL'])
# The database should have the following objects already in it.
roles = [
Role('demo', 'registered', 'Registered.', []),
Role('demo', 'personal', 'Personal.', ['registered']),
Role('demo', 'shared', 'Shared.', ['registered']),
Role('demo', 'manager', 'Manager.', ['personal', 'shared']),
Role('jordan', 'registered', 'Registered.', []),
Role('jordan', 'personal', 'Personal.', ['registered'])
]
# Update the objects in case something else spuriously changed them.
for role in roles:
role.to_db()
def link():
amazonId = request.args.get('amazonId')
bigoven_username = request.args.get('bigoven_username')
if app.debug:
resource = boto3.resource('dynamodb',
endpoint_url='http://localhost:8000',
region_name="fake_region",
aws_access_key_id="fake_id",
aws_secret_access_key="fake_key")
else:
resource = boto3.resource('dynamodb',
region_name='us-east-1',
aws_access_key_id=os.environ['AWS_KEY'],
aws_secret_access_key=os.environ['AWS_SECRET'])
table = resource.Table('my_cookbook_users')
key = {
"userId": amazonId
}
updateExpr = 'SET bigoven_username=:b'
exprAttributeValues = {':b': bigoven_username}
condExpr = boto3.dynamodb.conditions.Attr('invocations').exists()
try:
response = table.update_item(Key=key,
UpdateExpression=updateExpr,
ExpressionAttributeValues=exprAttributeValues,
ConditionExpression=condExpr)
except ClientError as ce:
print "invalid id: %s" % amazonId
return render_template('error.html')
return render_template('link.html')
def __init__(self, app):
"""Initializes a storage manager instance.
"""
self.app = app
self.config = app.config['FILE_STORAGE']
self.secur = app.config['SECURITY_MANAGEMENT']['content']
if self.config['type'] == 's3':
# Boto s3 instance
if self.config['id'] != '' and self.config['key'] != '':
self.s3 = boto3.resource('s3', aws_access_key_id=self.config['id'], aws_secret_access_key=self.config['key'], region_name=self.config['location'])
else:
self.s3 = boto3.resource('s3')
# S3 bucket location
try:
self.bucket = self.config['name']
except:
self.bucket = ""
elif self.config['type'] == 'filesystem':
self.storage_path = '{0}{1}'.format(self.config['location'], self.config['name'])
if not os.path.exists(self.storage_path):
os.makedirs(self.storage_path)
os.makedirs('{0}/corr-bundles'.format(self.storage_path))
os.makedirs('{0}/corr-files'.format(self.storage_path))
os.makedirs('{0}/corr-logos'.format(self.storage_path))
os.makedirs('{0}/corr-outputs'.format(self.storage_path))
os.makedirs('{0}/corr-pictures'.format(self.storage_path))
os.makedirs('{0}/corr-resources'.format(self.storage_path))
elif self.config['type'] == 'mdcs': # TODO
pass
elif self.config['type'] == 'ftp' or self.config['type'] == 'sftp' : # TODO
pass
def stopec2():
ec2 = boto3.resource('ec2');
ret=[]
count = 0;
for region in ec2regions:
target=[]
ec2 = boto3.resource('ec2',region);
instances = ec2.instances.all()
for instance in instances:
ret.append({"Region": region, "id": instance.id})
if instance.state['Name'] != 'running': continue
if instance.tags is not None and any(tag['Key'] == 'Name' for tag in instance.tags):
nametag = (tag['Value'] for tag in instance.tags if tag['Key'] == 'Name').next()
if not nametag in nametags:
count+=1
target.append(instance)
else:
count+=1
target.append(instance)
for instance in target:
result = instance.stop()
print (result)
# stop all running instances we found
print (count, "instances found")
print (ret)
return ret
def sign_up_by_email(event, context):
User = boto3.resource('dynamodb').Table('FmSsoUsers')
Auth = boto3.resource('dynamodb').Table('FmSsoAuths')
email = event['email']
password = event['password']
auth_rcds = Auth.scan(
FilterExpression=Attr('email').eq(email)
)
user_rcds = User.scan(
FilterExpression=Attr('email').eq(email)
)
if(auth_rcds['Count'] != 0):
# User exists, raise error
raise Exception('User already registered!')
elif(user_rcds['Count'] != 0):
raise Exception(
'User already registered! Please login with original account and associate in it')
user = create_auth(Auth, User, email, password)
expires = time.time() + conf['cookie_expiry']
return {
'uid': user['uid'],
'email': user['email'],
'atoken': user['atoken'],
'atoken_expiry': user['atoken_expiry'],
'rtoken': user['rtoken'],
'rtoken_expiry': user['rtoken_expiry'],
'cookie': 'atoken={}; domain=.amazonaws.com; expires={}; path=/'.format(user['atoken'], time.strftime("%a, %d-%b-%Y %T GMT", time.gmtime(expires)))
}
def handle_event(event, context):
# parse the recipe id from the parameters
try:
recipe_id = event['params']['querystring']['id']
try:
id_num = int(recipe_id)
if id_num < 0:
return {"code": -1,
"data": "recipe_id (%d) most be positive." % id_num}
if stage.PROD:
resource = boto3.resource("dynamodb")
else:
resource = boto3.resource(
"dynamodb",
endpoint_url="http://localhost:8000",
aws_access_key_id="fake",
aws_secret_access_key="fake",
region_name="us-east-1")
table = resource.Table('my_cookbook_recipes')
response = table.get_item(Key={'id': id_num})
if 'Item' in response:
return {"code": 0, "data": response['Item']}
else:
return {"code": 1, "data": "No recipe ith id %d" % id_num}
except ValueError:
return {"code": -1, "data": "recipe_id must be an integer"}
except KeyError:
return {"code": -1, "data": "Failed to parse query prameter 'id'"}
def share_fits(uid, args):
"""
Create a map of the hash of eve uid and fitting id to crest json. Map is stored
in a dynamodb table.
param uid: string slack user id that sent the request
param args: {'id': [fitting_id, ...]}
:returns: [(ship, name, API endpoint url), ...]
"""
user = get_user(uid, args)
fittings = boto3.resource('dynamodb').Table('fittings')
shared_fits = boto3.resource('dynamodb').Table('shared_fits')
eve_user_id = user['eve_user_id']
fittings_req = fittings.query(KeyConditionExpression=Key('eve_user_id').eq(eve_user_id))
shared = []
for fit in fittings_req['Items']:
if fit['fitting_id'] not in args['id']: continue
fit_hash = str(hash((eve_user_id, fit['fitting_id'])))
shared_fits.put_item(Item={'fitting_hash': fit_hash,
'fit': fit['fit']})
shared.append((fit['ship'], fit['name'], settings.SHARE_ENDPOINT.format(fit_hash)))
return shared
def test_close(self):
self.s3_task_handler.set_context(self.ti)
self.assertTrue(self.s3_task_handler.upload_on_close)
self.s3_task_handler.close()
# Should not raise
boto3.resource('s3').Object('bucket', self.remote_log_key).get()
def __init__(self, repository_name,
github_username, github_token,
aws_key, aws_secret, aws_region,
events, callbacks=None, **kwargs):
"""Instantiates a RepositoryListener.
Additionally:
* Creates or connects to a AWS SQS queue named for the repository
* Creates or connects to a AWS SNS topic named for the repository
* Connects the AWS SNS topic to the AWS SQS queue
* Configures the Github repository to push hooks to the SNS topic
Args:
repository_name (str): name of a Github repository, like
"tdsmith/homebrew-pypi-poet"
github_username (str): Github username
github_token (str): Github authentication token from
https://github.com/settings/tokens/new with admin:org_hook
privileges
aws_key (str): AWS key
aws_secret (str): AWS secret
aws_region (str): AWS region (e.g. 'us-west-2')
events (list<str>): List of Github webhook events to monitor for
activity, from https://developer.github.com/webhooks/#events.
callbacks (list<function(str event_type, Object event_payload)>):
functions to call with a decoded Github JSON payload when a
webhook event lands. You can register these after instantiation
with register_callback.
"""
self.repository_name = repository_name
self.github_username = github_username
self.github_token = github_token
self.aws_key = aws_key
self.aws_secret = aws_secret
self.aws_region = aws_region
# create or reuse sqs queue
sqs_resource = boto3.resource("sqs", region_name=self.aws_region)
self.sqs_queue = sqs_resource.create_queue(
QueueName="snooze__{}".format(self._to_topic(repository_name))
)
# create or reuse sns topic
sns_resource = boto3.resource("sns", region_name=self.aws_region)
sns_topic = sns_resource.create_topic(
Name=self._to_topic(repository_name)
)
sns_topic.subscribe(
Protocol='sqs',
Endpoint=self.sqs_queue.attributes["QueueArn"]
)
# configure repository to push to the sns topic
connect_github_to_sns(aws_key, aws_secret, aws_region,
github_username, github_token, repository_name,
sns_topic.arn, events)
# register callbacks
self._callbacks = []
if callbacks:
[self.register_callback(f) for f in callbacks]
def test_has_good_error_message_when_no_resource(self):
bad_resource_name = 'doesnotexist'
err_regex = (
'%s.*resource does not exist.' % bad_resource_name
)
with self.assertRaisesRegexp(ResourceNotExistsError, err_regex):
boto3.resource(bad_resource_name)
def initialize():
# If you don't feel like setting environment variables for boto3 on your machine,
# you can hard code them here for testing.
#boto3.setup_default_session(
#aws_access_key_id = XXXXX,
#aws_secret_access_key = YYYYY
# )
if Dynamo.dynamodb is None:
try:
if Dynamo.local:
Dynamo.dynamodb = boto3.resource(
'dynamodb',
endpoint_url='http://localhost:8000' # change if required
)
else :
Dynamo.dynamodb = boto3.resource(
'dynamodb',
region_name='us-west-2',
endpoint_url="https://dynamodb.us-west-2.amazonaws.com"
)
except:
if Dynamo.local:
print('DB Connection Error: Unable to connect to local database. Check if database is '
'running locally and ensure port number is correct.')
else:
print('DB Connection Error: Unable to connect to AWS Hosted DynamoDB server. Check if access '
'credentials are properly configured.')
raise
return Dynamo
def redirect_from_upload():
check_connection(auth)
# Get bucket name and key from the S3 redirect URL
key = request.query.key
bucket = request.query.bucket
filename = key.split('/')[1]
print key
# get the unique job ID
temp = filename.split('~')
jobid = temp[0]
# Create a job item and persist it to the annotations database
username = auth.current_user.username
usermail = auth.current_user.email_addr
data = {"job_id":jobid, "username":username,"email":usermail, "submit_time":int(time.time()), "s3_inputs_bucket":bucket, "s3_key_input_file":key, "input_file_name":filename, "status":"PENDING"}
dynamodb = boto3.resource('dynamodb')
ann_table = dynamodb.Table(dynamo_table)
ann_table.put_item(Item=data)
#publish a notification to SNS topic
#reference: http://boto3.readthedocs.io/en/latest/reference/services/sns.html#topic
sns = boto3.resource('sns')
topic = sns.Topic(job_arn)
response = topic.publish(
Message=json.dumps(data),
)
return template(request.app.config['mpcs.env.templates'] + 'upload_confirm.tpl', job_id=jobid, auth=auth)
def deleteThumbnails():
bucket = boto3.resource('s3').Bucket(bucketname)
for key in bucket.objects.filter(Prefix="thumbnail/"):
if not key.__eq__(boto3.resource('s3').ObjectSummary(bucketname, unicode("thumbnail/dummy.jpg"))):
key.delete()
for key in bucket.objects.filter(Prefix="src/"):
if not key.__eq__(boto3.resource('s3').ObjectSummary(bucketname, unicode("src/dummy.jpg"))):
key.delete()
def get_table(table_name):
""" Gets a Table resource by name"""
# Get table resource
if DYNAMO_LOCAL:
dynamodb = boto3.resource('dynamodb', region_name='us-east-1', endpoint_url="http://localhost:4569")
else:
dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
return dynamodb.Table(table_name)
def amazon_s3_file_validator_no_delete(self, bucket):
key_name = 'test-unvalidated-uploads/images/hello.jpg'
boto3.resource('s3').Object(bucket.name, key_name).put(Body='test')
return AmazonS3FileValidator(
key_name=key_name,
bucket=bucket,
delete_unvalidated_file=False
)
def save_file(key, symlink=False):
output_file = os.path.join(dest_dir, key)
if not os.path.exists(output_file):
if symlink:
os.symlink(os.path.join(s3_mount_dir, key), output_file)
else:
boto3.resource('s3').Bucket(bucket_name).download_file(key, output_file)
return output_file
def amazon_s3_file_validator_with_acl(self, bucket):
key_name = 'test-unvalidated-uploads/images/hello.jpg'
boto3.resource('s3').Object(bucket.name, key_name).put(Body='test')
return AmazonS3FileValidator(
key_name=key_name,
bucket=bucket,
new_file_acl='private',
)
def failing_amazon_s3_file_validator(self, bucket):
key_name = 'images/fail.jpg'
boto3.resource('s3').Object(bucket.name, key_name).put(Body='test')
return AmazonS3FileValidator(
key_name=key_name,
bucket=bucket,
validators=[CustomValidator()]
)
def test_close_no_upload(self):
self.ti.raw = True
self.s3_task_handler.set_context(self.ti)
self.assertFalse(self.s3_task_handler.upload_on_close)
self.s3_task_handler.close()
with self.assertRaises(self.conn.exceptions.NoSuchKey):
boto3.resource('s3').Object('bucket', self.remote_log_key).get()
# http://boto3.readthedocs.io/en/latest/reference/services/sns.html
# http://boto3.readthedocs.io/en/latest/reference/services/ses.html
import os
import boto3
import time
from datetime import datetime
from botocore.client import Config
from boto3.dynamodb.conditions import Key
import json
import hardcode
hardcodes = hardcode.Config
try:
sqs = boto3.resource('sqs', region_name=hardcodes.AWS_REGION_NAME)
glacier_queue = sqs.get_queue_by_name(
QueueName=hardcodes.AWS_SQS_GLACIER_ARCHIVE)
except Exception as e:
print(e)
while True:
messages = glacier_queue.receive_messages(WaitTimeSeconds=5)
if not messages:
print('Waiting for message...for archive restore')
else:
for message in messages:
try:
if message:
def collect_and_download(out_dir,
less_than=0,
greater_than=0,
sex='',
handedness='',
sessions=SESSIONS,
scans=SCANS,
series=SERIES_MAP.keys(),
derivatives=False,
dryrun=False):
'''
Function to collect and download images from the Rockland sample
directory on FCP-INDI's S3 bucket
Parameters
----------
out_dir : string
filepath to a local directory to save files to
less_than : float
upper age (years) threshold for participants of interest
greater_than : float
lower age (years) threshold for participants of interest
sex : string
'M' or 'F' to indicate whether to download male or female data
handedness : string
'R' or 'L' to indicate whether to download right-handed or
left-handed participants
sessions : list
the session names (e.g.,'CLG5','NFB3')
scan : list
the scan types to download. Can be 'anat','func','dwi' or 'fmap'.
series : list
the series to download (for functional scans)
derivatives : boolean
whether or not to download data derivatives for functional scans
dryrun : boolean
whether or not to perform a dry run (i.e., no actual downloads,
just listing files that would be downloaded)
Returns
-------
boolean
Returns true if the download was successful, false otherwise.
'''
# Import packages
import pandas
import boto3
import botocore
# For anonymous access to the bucket.
from botocore import UNSIGNED
from botocore.client import Config
from botocore.handlers import disable_signing
# Init variables
s3_bucket_name = 'fcp-indi'
s3_prefix = 'data/Projects/RocklandSample/RawDataBIDS'
# Fetch bucket
s3 = boto3.resource('s3')
s3.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)
s3_bucket = s3.Bucket(s3_bucket_name)
# Remove series that aren't in the series map keys.
series = [s for s in series if s in SERIES_MAP.keys()]
# If output path doesn't exist, create it
if not os.path.exists(out_dir) and not dryrun:
print 'Could not find %s, creating now...' % out_dir
os.makedirs(out_dir)
# Load the participants.tsv file from S3
s3_client = boto3.client('s3', config=Config(signature_version=UNSIGNED))
participants_obj = s3_client.get_object(
Bucket=s3_bucket_name, Key='/'.join([s3_prefix, 'participants.tsv']))
participants_df = pandas.read_csv(participants_obj['Body'],
delimiter='\t',
na_values=['n/a'])
# Init a list to store paths.
print 'Collecting images of interest...'
s3_keys = s3_bucket.objects.filter(Prefix=s3_prefix)
s3_keylist = [key.key for key in s3_keys]
# Remove the participants for whom age range, handedness and sex do not conform to the criteria.
if less_than:
participants_df = participants_df[participants_df['age'] < less_than]
if greater_than:
participants_df = participants_df[
participants_df['age'] > greater_than]
if sex == 'M':
participants_df = participants_df[participants_df['sex'] == 'MALE']
elif sex == 'F':
participants_df = participants_df[participants_df['sex'] == 'FEMALE']
if handedness == 'R':
participants_df = participants_df[participants_df['handedness'] ==
'RIGHT']
elif handedness == 'L':
participants_df = participants_df[participants_df['handedness'] ==
'LEFT']
if len(participants_df) == 0:
print 'No participants meet the criteria given. No download will be initiated.'
return
# Generate a list of participants to filter on.
participants_filt = [
'sub-' + label + '/'
for label in participants_df['participant_id'].tolist()
]
# Generate a list of sessions to filter on.
sessions_filt = ['ses-' + session + '/' for session in sessions]
# Generate a list of series to filter on.
series_filt = [SERIES_MAP[s] for s in series]
# Fetch top-level JSONs first.
json_keylist = [
key for key in s3_keylist for s in series_filt
if s in key and 'json' in key and 'sub' not in key
]
# Applying filters.
s3_keylist = [
key for key in s3_keylist for p in participants_filt if p in key
]
s3_keylist = [key for key in s3_keylist for s in sessions_filt if s in key]
s3_keylist = [key for key in s3_keylist for s in scans if s in key]
s3_keylist = [
key for key in s3_keylist for s in series_filt
if (s in key) or ('func' not in key)
]
# Add back top-level files
s3_keylist.extend(json_keylist)
s3_keylist.append('/'.join([s3_prefix, 'CHANGES']))
s3_keylist.append('/'.join([s3_prefix, 'README']))
s3_keylist.append('/'.join([s3_prefix, 'dataset_description.json']))
# And download the items
total_num_files = len(s3_keylist)
files_downloaded = len(s3_keylist)
for path_idx, s3_path in enumerate(s3_keylist):
rel_path = s3_path.replace(s3_prefix, '')
rel_path = rel_path.lstrip('/')
download_file = os.path.join(out_dir, rel_path)
download_dir = os.path.dirname(download_file)
if not os.path.exists(download_dir) and not dryrun:
os.makedirs(download_dir)
try:
if not os.path.exists(download_file):
if dryrun:
print 'Would download to: %s' % download_file
else:
print 'Downloading to: %s' % download_file
with open(download_file, 'wb') as f:
s3_client.download_fileobj(s3_bucket_name, s3_path, f)
print '%.3f%% percent complete' % \
(100*(float(path_idx+1)/total_num_files))
else:
print 'File %s already exists, skipping...' % download_file
files_downloaded -= 1
except Exception as exc:
print 'There was a problem downloading %s.\n'\
'Check input arguments and try again.' % s3_path
print exc
# Print all done
if dryrun:
print '%d files would be downloaded for %d participant(s).' % (
files_downloaded, len(participants_df))
else:
print '%d files downloaded for %d participant(s).' % (
files_downloaded, len(participants_df))
if not dryrun:
print 'Saving out revised participants.tsv and session tsv files.'
# Save out revised participants.tsv to output directory, if a participants.tsv already exists, open it and append it to the new one.
if os.path.isfile(os.path.join(out_dir, 'participants.tsv')):
old_participants_df = pandas.read_csv(os.path.join(
out_dir, 'participants.tsv'),
delimiter='\t',
na_values=['n/a', 'N/A'])
participants_df = participants_df.append(old_participants_df,
ignore_index=True)
participants_df.drop_duplicates(inplace=True)
os.remove(os.path.join(out_dir, 'participants.tsv'))
participants_df.to_csv(os.path.join(out_dir, 'participants.tsv'),
sep="\t",
na_rep="n/a",
index=False)
# Separate list for sessions TSVs.
session_keylist = [
key.key for key in s3_keys if 'sessions.tsv' in key.key
]
session_keylist = [
key for key in session_keylist for p in participants_filt
if p in key
]
# Save out revised session tsvs to output directory; if already exists, open it and merge with the new one.
for session_key in session_keylist:
participant = session_key.split('/')[-2]
sessions_obj = s3_client.get_object(Bucket=s3_bucket_name,
Key=session_key)
sessions_df = pandas.read_csv(sessions_obj['Body'],
delimiter='\t',
na_values=['n/a'])
# Drop all sessions not in specified.
sessions_df = sessions_df[sessions_df['session_id'].isin(
sessions_filt)]
# Save out revised sessions tsv to output directory, if a sessions tsv already exists, open it and append it to the new one.
if os.path.isfile(
os.path.join(out_dir, participant,
participant + '_sessions.tsv')):
old_sessions_df = pandas.read_csv(os.path.join(
out_dir, participant, participant + '_sessions.tsv'),
delimiter='\t',
na_values=['n/a', 'N/A'])
sessions_df = sessions_df.append(old_sessions_df,
ignore_index=True)
sessions_df.drop_duplicates(inplace=True)
os.remove(
os.path.join(out_dir, participant,
participant + '_sessions.tsv'))
sessions_df.to_csv(os.path.join(out_dir, participant,
participant + '_sessions.tsv'),
sep="\t",
na_rep="n/a",
index=False)
print 'Done!'
if not exit_node(): return 0
return db_backup.s3_upload_db(local_datadir_,s3alias)
def s3_download_db(s3alias): # such as 'db-user1-node-190616'
if not __LocalDbg__: return 0
if not db_backup: return 0
if not exit_node(): return 0
return db_backup.s3_download_db(local_datadir_,s3alias)
if cloud_factory_ == 'AWS':
import boto3
port_succ = False
if region_name and instance_id:
try:
ec2 = boto3.resource('ec2',region_name=region_name)
instance_ = ec2.Instance(instance_id)
secur_groups = instance_.security_groups
if secur_groups:
exist30303 = False; exist30302 = False
secur_group = ec2.SecurityGroup(secur_groups[0]['GroupId'])
permis = secur_group.ip_permissions
for item in permis:
if item['FromPort'] == 30303:
exist30303 = item['IpProtocol'] == 'tcp'
elif item['FromPort'] == 30302:
exist30302 = item['IpProtocol'] == 'udp'
if not exist30303 or not exist30302:
permit = []
(DWH_DB_USER, DWH_DB_PASSWORD, DWH_DB)
pd.DataFrame({
'Param': [
'DWH_CLUSTER_TYPE', 'DWH_NUM_NODES', 'DWH_NODE_TYPE',
'DWH_CLUSTER_IDENTIFIER', 'DWH_DB', 'DWH_DB_USER', 'DWH_DB_PASSWORD',
'DWH_PORT', 'DWH_IAM_ROLE_NAME'
],
'Value': [
DWH_CLUSTER_TYPE, DWH_NUM_NODES, DWH_NODE_TYPE, DWH_CLUSTER_IDENTIFIER,
DWH_DB, DWH_DB_USER, DWH_DB_PASSWORD, DWH_PORT, DWH_IAM_ROLE_NAME
]
})
ec2 = boto3.resource('ec2',
region_name="eu-central-1",
aws_access_key_id=KEY,
aws_secret_access_key=SECRET)
s3 = boto3.resource('s3',
region_name="eu-central-1",
aws_access_key_id=KEY,
aws_secret_access_key=SECRET)
iam = boto3.client('iam',
aws_access_key_id=KEY,
aws_secret_access_key=SECRET,
region_name='eu-central-1')
redshift = boto3.client('redshift',
region_name="eu-central-1",
aws_access_key_id=KEY,
def setUp(self):
self.mock = mock_s3()
self.mock.start()
conn = boto3.resource('s3', region_name='us-east-1')
conn.create_bucket(Bucket=BUCKET)
self.lg = local.LocalGateway(app, Config())
def lambda_handler(event, context):
'''Demonstrates a simple HTTP endpoint using API Gateway. You have full
access to the request and response payload, including headers and
status code.
To scan a DynamoDB table, make a GET request with the TableName as a
query string parameter. To put, update, or delete an item, make a POST,
PUT, or DELETE request respectively, passing in the payload to the
DynamoDB API as a JSON body.
'''
s3 = boto3.resource('s3')
# s3 = boto3.client('s3')
print("Received event: " + json.dumps(event, indent=2))
operations = {
'DELETE': lambda dynamo, x: dynamo.delete_item(**x),
'GET': lambda dynamo, x: dynamo.scan(**x),
'POST': lambda dynamo, x: dynamo.put_item(**x),
'PUT': lambda dynamo, x: dynamo.update_item(**x),
}
operation = event['httpMethod']
if operation in operations:
if operation == 'GET':
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
obj = s3.Object(bucket, key)
content = obj.get()['Body'].read()
contentString = base64.b64encode(content)
# Since we don't do integration test, our jpg file content is binary
# contentString = base64.b64encode(content)
result = {"ImageName": key, "Content": contentString}
return respond(None, result)
elif operation == 'POST':
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
data = event['Records'][0]['s3']['object']['data']
obj = s3.Object(bucket, key)
# f = open('test1.jpg', 'rb')
# with f as myfile:
# data = myfile.read()
# f.close()
dataAray = base64.b64decode(data)
# Here we need to use Body=data since the arguement is a json object
obj.put(Body=dataAray)
print("Object is " + str(obj))
result = {
"ImageName": key,
"Result": "POST Success",
"DataStream": base64.b64encode(dataAray)
}
return respond(None, result)
elif operation == "DELETE":
bucketName = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
if key == "":
bucket = s3.Bucket(bucketName)
bucket.objects.delete()
else:
obj = s3.Object(bucketName, key)
obj.delete()
result = {"Result": "DELETE all the objects in the bucket"}
return respond(None, result)
else:
return respond(ValueError(
'Unsupported PUT "{}"'.format(operation)))
else:
return respond(ValueError('Unsupported method "{}"'.format(operation)))
#!/usr/bin/python
import boto3
import json
import time
#QUEUE = 'https://sqs.us-west-2.amazonaws.com/384845588177/cloudstack-unused-instances'
QUEUE = 'cloudstack-unused-instances'
sqs = boto3.resource('sqs')
queue = sqs.get_queue_by_name(QueueName=QUEUE)
while True:
for message in queue.receive_messages():
m = json.loads(message.body)
m1 = json.loads(m['Message'].replace("'", "\""))
dimensions = m1['Trigger']['Dimensions']
instance_details = {d['name']:d['value'] for d in dimensions}
for k,v in instance_details.iteritems():
print "Key= %s , Value=%s" % (k,v)
message.delete()
time.sleep(60)
def s3_etag(url):
"""Check ETag on S3 object."""
s3_resource = boto3.resource("s3")
bucket_name, s3_path = split_s3_path(url)
s3_object = s3_resource.Object(bucket_name, s3_path)
return s3_object.e_tag
def __init__(self, table_name):
self.dynamodb = boto3.resource('dynamodb')
self.table = self.dynamodb.Table(table_name)
# -*- coding: utf-8 -*-
from typing import Any, Optional
import logging
import boto3
import requests
from trawsate import travis
IamUser = Any
AccessKeyPair = Any
logger = logging.getLogger(__name__)
iam = boto3.resource('iam')
class AccessKeyStateError(Exception):
"""
Raised when a user's access keys are in a state we cannot handle and/or
require that they are not in.
"""
pass
def _handle_existing_keys(user: IamUser) -> Optional[IamUser]:
"""
Get a user's keys in a state where we can safely create a new key.
:param user: The user whose keys to scan.
:return: The user to deactivate after creating a new key, if any.
"""
from __future__ import unicode_literals
import boto3
import json
from datetime import datetime
from collections import defaultdict
import base64
import time
import decimal
dynamodb = boto3.resource('dynamodb')
kinesis = boto3.client("kinesis")
shard_id = 'shardId-000000000000' #only one shard
shard_it = kinesis.get_shard_iterator(
StreamName="twitter-stream", ShardId=shard_id,
ShardIteratorType="LATEST")["ShardIterator"]
event = kinesis.get_records(ShardIterator=shard_it, Limit=100)
def update_dynamo_event_counter(tableName,
event_name,
longi,
dynamodb=boto3.resource(
service_name='dynamodb',
region_name='us-west-2')):
table = dynamodb.Table(tableName)
longil = longi[0]
latd = longi[1]
print(longil)
print(latd)
if event_name:
checkItemExists = table.get_item(Key={'full_name': event_name})
if 'Item' in checkItemExists:
def amibkp(region, days_del, slack_req, slack_channel, slack_webhook):
"""
This function is the crucial function,
fetches all the instances which has tag Key:AMIBACKUPON, Value:yes and creates AMI in a loop,
along with propogating all the tags from instance to AMI to EBS Snapshots.
Also, it deletes all the AMI's which was created through this script
and older than number of days you provide as an argument.
Parameters
----------
region: string
AWS region code.
days: integer
Number of days to keep AMI's before deleting.
slack: string
Optional argument.
Passing this parameter as "true" will post the execption to slack if any.
slack_channel: String
Slack channel to where exceptions has to be posted
Depends on the previous parameter "slack", this is required if slack is true.
webhookurl: string
Slack webhookurl to identify to which slack team exeception has to be posted?
Depends on the previous parameter "slack", this is required if slack is true.
Returns
-------
list
Returns list of AMI's/Snapshots deleted and newly created AMI's/Snapshots.
"""
client = boto3.client('ec2', region_name=region)
ec2 = boto3.resource('ec2', region_name=region)
image_response = client.describe_images(Filters=[
{
'Name': 'tag:DELETEON',
'Values': [
'yes',
]
},
])
delete_ami(image_response, days_del, slack_req, slack_channel,
slack_webhook, ec2, region)
instance_response = client.describe_instances(Filters=[{
'Name':
'tag:AMIBACKUPON', #Tag used to identify list of Instances to be backed up.
'Values': [
'yes',
]
}])
create_ami(instance_response, slack_req, slack_channel, slack_webhook, ec2,
region)
new_image_response = client.describe_images(Filters=[
{
'Name': 'tag:Snapshottag',
'Values': [
'yes',
]
},
])
tag_snapshots(new_image_response, slack_req, slack_channel, slack_webhook,
client, region)
" repository name. e.g. the React library's repo would be `facebook/react`"
)
ERR_SETUP_CHANNEL_REPO_EXISTS = "It looks as though that repo is already setup in this channel"
ERR_SETUP_REPO_EXISTS = "That repo is already setup in a different channel. Only one channel per repo."
ERR_SETUP_CHANNEL_EXISTS = "This channel is already setup for a different repo. Only one repo per channel."
ERR_RESET_FAILURE = "No config found for this channel"
ERR_USER_BUTTON_FALLBACK = "You are unable to introduce yourself for some reason. Sorry."
ERR_GOODBYE_NO_USER = "******"
ERR_DEPLOY_ARGS = "You must specify what and where to deploy, as `deploy <branch-or-tag> to <environment>`"
ERR_DEPLOY_VALID_ENV = "I don't recognise that environment name"
ERR_DEPLOY_REFENV_VALIDATION = "That environment doesn't get built from that ref according to the supported workflow. "
ERR_DEPLOY_REMOTE = "Something went wrong: "
SLACK_SIGNING_SECRET_VERSION = "v0"
dynamodb = boto3.resource('dynamodb', region_name=os.environ['SLS_AWS_REGION'])
lambda_client = boto3.client('lambda',
region_name=os.environ['SLS_AWS_REGION'])
logger = logging.getLogger()
if logger.handlers:
for handler in logger.handlers:
logger.removeHandler(handler)
logging.basicConfig(level=logging.INFO)
logger.setLevel(logging.INFO)
#
#
# Callable user-facing functions
#
#
def lambda_handler(event, context):
#logger.info('Event: ' + str(event))
print('Received event: ' + json.dumps(event, indent=2))
ids = []
try:
region = event['region']
detail = event['detail']
eventname = detail['eventName']
arn = detail['userIdentity']['arn']
principal = detail['userIdentity']['principalId']
userType = detail['userIdentity']['type']
creationDate = detail['eventTime']
print(
'Assinged region, detail, eventname, arn, principal, userType, creationDate'
)
if userType == 'IAMUser':
user = detail['userIdentity']['userName']
else:
user = principal.split(':')[1]
print('User ' + user)
logger.info('principalId: ' + str(principal))
logger.info('region: ' + str(region))
logger.info('eventName: ' + str(eventname))
logger.info('detail: ' + str(detail))
print('Check Response Elements')
if not (detail['responseElements'] or eventname == 'CreateBucket'):
logger.warning('No responseElements found')
if detail['errorCode']:
logger.error('errorCode: ' + detail['errorCode'])
if detail['errorMessage']:
logger.error('errorMessage: ' + detail['errorMessage'])
return False
common_tags = [{
'Key': 'Requestor',
'Value': user
}, {
'Key': 'PrincipalId',
'Value': principal
}, {
'Key': 'creationDate',
'Value': creationDate
}]
allTags = defineTags(user, common_tags)
ec2 = boto3.resource('ec2')
print('Check Event Match')
if eventname == 'CreateVolume':
ids.append(detail['responseElements']['volumeId'])
logger.info(ids)
elif eventname == 'RunInstances':
items = detail['responseElements']['instancesSet']['items']
for item in items:
ids.append(item['instanceId'])
logger.info(ids)
logger.info('number of instances: ' + str(len(ids)))
base = ec2.instances.filter(InstanceIds=ids)
#loop through the instances
for instance in base:
for vol in instance.volumes.all():
ids.append(vol.id)
for eni in instance.network_interfaces:
ids.append(eni.id)
elif eventname == 'CreateImage':
ids.append(detail['responseElements']['imageId'])
logger.info(ids)
elif eventname == 'CreateSnapshot':
ids.append(detail['responseElements']['snapshotId'])
logger.info(ids)
elif eventname == 'CreateBucket':
s3 = boto3.resource('s3')
s3_bucketName = detail['requestParameters']['bucketName']
if s3_bucketName:
bucket_tagging = s3.BucketTagging(s3_bucketName)
#print (ids)
#allTags = defineTags(user,common_tags)
#print (type(allTags))
s3_resp = bucket_tagging.put(Tagging={'TagSet': allTags})
print(s3_resp)
#print ('Tagged S3 bucket')
elif eventname == 'CreateDBInstance':
print('CreateDBInstance')
rds = boto3.client('rds')
rds_arn = detail['responseElements']['dBInstanceArn']
if rds_arn:
print('Tagging resource ' + rds_arn)
#allTags = defineTags(user,common_tags)
rds_resp = rds.add_tags_to_resource(ResourceName=rds_arn,
Tags=allTags)
print(rds_resp)
elif eventname == 'CreateDBSnapshot':
print('CreateDBSnapshot')
rds_snp = boto3.client('rds')
rds_snp_arn = detail['responseElements']['dBSnapshotArn']
if rds_snp_arn:
print('Tagging resource ' + rds_snp_arn)
#allTags = defineTags(user,common_tags)
rds_snp_resp = rds_snp.add_tags_to_resource(
ResourceName=rds_snp_arn, Tags=allTags)
print(rds_snp_resp)
elif eventname == 'CreateFileSystem':
print('CreateFileSystem')
efs = boto3.client('efs')
efs_fsid = detail['responseElements']['fileSystemId']
if efs_fsid:
print('Tagging resource ' + efs_fsid)
efs_resp = efs.create_tags(FileSystemId=efs_fsid, Tags=allTags)
print(efs_resp)
elif eventname == 'CreateTable':
print('CreateTable')
dyn = boto3.client('dynamodb')
dyn_arn = detail['responseElements']['tableDescription'][
'tableArn']
if dyn_arn:
print('Tagging resource ' + dyn_arn)
#allTags = defineTags(user,common_tags)
table_status = detail['responseElements']['tableDescription'][
'tableStatus']
table_name = detail['responseElements']['tableDescription'][
'tableName']
while (table_status == 'CREATING'):
print('Table is being created')
table_check = dyn.describe_table(TableName=table_name)
print(table_check)
table_status = table_check['Table']['TableStatus']
print(table_status)
if (table_status == 'ACTIVE'):
dyn_resp = dyn.tag_resource(ResourceArn=dyn_arn,
Tags=allTags)
print(dyn_resp)
else:
logger.warning('Not supported action')
if ids:
for resourceid in ids:
print('Tagging resource ' + resourceid)
#allTags = defineTags(user,common_tags)
ec2.create_tags(Resources=ids, Tags=allTags)
logger.info(' Remaining time (ms): ' +
str(context.get_remaining_time_in_millis()) + '\n')
return True
except Exception as e:
logger.error('Something went wrong: ' + str(e))
return False
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import boto3
from matplotlib.ticker import PercentFormatter
# === Step 1. Download data ===
dynamodb = boto3.resource("dynamodb",
region_name='ap-southeast-1',
aws_access_key_id='...........', # erased out for safety reason
aws_secret_access_key='.............' # erased out for safety reason
)
table = dynamodb.Table('EE5111_A0039875M_JetEngine')
response = table.scan()
data = response['Items']
while 'LastEvaluatedKey' in response:
response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(response['Items'])
data = pd.DataFrame(data)
# === Step 2. Plot Graph ===
# Plot single engine single sensor
data1Sensor2 = data[data['id'].str.contains('FD001')].loc[:, ['id','cycle','sensor2']]
data1Sensor2['id'] = list(map(lambda x: int(x.split('_')[-1]), data1Sensor2['id']))
data1Sensor2.sort_values(by=['id','cycle'], inplace=True)
plt.figure()
plt.plot(range(len(data1Sensor2)), data1Sensor2['sensor2'], linewidth=0.1)
plt.title('Engine 1 Sensor 2 in 99 Cycles')
plt.figure()
plt.plot(range(len(data1Sensor2[data1Sensor2['id']==1])), data1Sensor2[data1Sensor2['id']==1]['sensor2'])
plt.title('Engine 1 Sensor 2 in First Cycle')
def s3_get(url, temp_file):
"""Pull a file directly from S3."""
s3_resource = boto3.resource("s3")
bucket_name, s3_path = split_s3_path(url)
s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file)
def main():
#retrieve argument
args = parse_arguments()
main_directory = args.directory
class1 = args.class1
class2 = args.class2
force_by_user = args.force
if args.verbose:
lg.basicConfig(level=lg.INFO)
#Variables declaration
result = []
directory_feature = os.path.join(main_directory, "features", "*.json")
nb_training_data = args.nb_training_data
iteration_model = args.iteration_model
min_partition = args.min_partition
s3 = boto3.resource('s3')
bucket = s3.Bucket('oc-calculdistribues-sberton')
result_file = class1 + '_' + class2 + '_' + time.strftime(
"%Y%m%d%H%M%S") + '.json'
model_file = 'model_' + class1 + '_' + class2 + '_' + str(
nb_training_data) + '_' + str(iteration_model)
model_pathname = os.path.join(main_directory, "models", model_file)
#Searching existing model and store existence in is_model boolean
key = 'distributed_learning/models/' + model_file
objs = list(bucket.objects.filter(Prefix=key))
is_model = len(objs) > 0 and objs[0].key.startswith(key + '/')
start_time = time.time()
lg.info(
'#################### Starting pet-classification ######################'
)
lg.info('Class 1 is %s', class1)
lg.info('Class 2 is %s', class2)
lg.info('Number of training datas is %s', nb_training_data)
lg.info('Number of iterations model is %s', iteration_model)
#persist a common rdd which is using by both training and testing datas
common_rdd = sc.textFile(directory_feature, minPartitions=min_partition)\
.filter(lambda line: line.split(', ')[0] in (class1, class2) or class2 == 'All')\
.persist()
#Loading model if exists
if is_model and not force_by_user:
model = SVMModel.load(sc, model_pathname)
lg.info('Found and load recorded model %s', model_file)
else:
lg.info('No recorded model found')
#create training rdd and train model if no model found or force
train_data_rdd = common_rdd.filter(lambda line: int(line.split(',')[1]) <= nb_training_data)\
.map(lambda line: Row(label=0.0, features=line.split(', ')[2:])
if line.split(', ')[0] == class1
else Row(label=1.0, features=line.split(', ')[2:]))\
.map(lambda line: LabeledPoint(line.label, line.features))
lg.info('%s features for training datas', train_data_rdd.count())
lg.info('Start to training model')
model = SVMWithSGD.train(train_data_rdd, iterations=iteration_model)
lg.info('Training model terminated')
training_time = time.time()
training_duration = training_time - start_time
#Create testing rdd
test_data_rdd = common_rdd.filter(lambda line: int(line.split(', ')[1]) > nb_training_data)\
.map(lambda line: Row(label=0.0, features=line.split(', ')[2:])
if line.split(', ')[0] == class1
else Row(label=1.0, features=line.split(', ')[2:]))\
.map(lambda row: LabeledPoint(row.label, row.features))
lg.info('%s features for test datas', test_data_rdd.count())
# Evaluating the model on training data
predictions = test_data_rdd.map(
lambda row: (row.label, float(model.predict(row.features))))
train_error = predictions.filter(lambda lp: lp[0] != lp[1]).count() \
/ float(predictions.count())
lg.info('Test Error : %s', str(train_error))
end_time = time.time()
duration = end_time - start_time
lg.info('Duration %s', str(duration))
prediction_duration = end_time - training_time
# #Save and dump result on S3
result.append({
"class1": class1,
"class2": class2,
"iteration_model": iteration_model,
"nb_training_data": nb_training_data,
"total_duration": duration,
"train_duration": training_duration,
"predict_duration": prediction_duration,
"error": train_error
})
s3object = s3.Object('oc-calculdistribues-sberton', result_file)
s3object.put(Body=(bytes(json.dumps(result, indent=2).encode('UTF-8'))))
#Save model if not exists
if not is_model:
lg.info('Saving model at %s', model_file)
model.save(sc, model_pathname)
lg.info(
'#################### Ending pet-classification ######################'
)
def get_most_recent_ride_info(self,
user_id=None,
cookies=None,
save=False):
workout_ids = PelotonConnection.__get_workouts__(
self, user_id, cookies)
for workout_id in workout_ids:
workout_url = f"https://api.onepeloton.com/api/workout/{workout_id}"
workout = self.get(workout_url, cookies)
created_at = workout.get("created_at")
# Then get the ride_id for that workout
ride_id = workout.get("ride").get("id")
ride_id_details_url = f"https://api.onepeloton.com/api/ride/{ride_id}/details"
ride_id_details = self.get(ride_id_details_url, cookies)
# In the event you did one of those non-workout rides
try:
instructor = ride_id_details.get('ride').get('instructor').get(
'name')
except Exception:
instructor = None
if instructor is not None:
table = boto3.resource('dynamodb').Table('peloton_course_data')
if save is True:
table.put_item(
Item={
"created_at":
str(created_at),
"difficulty":
str(
ride_id_details.get('ride').get(
'difficulty_rating_avg')),
"instructor":
instructor,
"length":
str(
time.strftime(
"%H:%M:%S",
time.gmtime(
ride_id_details.get('ride').get(
'duration')))),
"name":
ride_id_details.get('ride').get('title')
})
# Also people wanted the music
if instructor is not None:
song_list = [
song
for song in ride_id_details.get("playlist").get("songs")
]
set_list = [
f"{f.get('title')} by {f.get('artists')[0].get('artist_name')}"
for f in song_list
]
table = boto3.resource('dynamodb').Table('peloton_music_sets')
if save is True:
table.put_item(Item={
"created_at": str(created_at),
"set_list": set_list
})
import sys
import json
import boto3
import logging
import uuid
from boto3.dynamodb.conditions import Key, Attr
logger = logging.getLogger()
logger.setLevel(logging.INFO)
#DynamodDB table name and region
DYNAMODB_REGION = "eu-west-2"
DYNAMODB_TABLE_NAME = "kong"
os.environ["AWS_DEFAULT_REGION"] = "eu-west-2"
dynamodb = boto3.resource('dynamodb', DYNAMODB_REGION)
#Bucket s3
BUCKET_NAME = 'my-bucket' # image bucket name
KEY = 'my_image_in_s3.jpg' # r object key
s3 = boto3.resource('s3')
def lambda_handler(event, context):
return scan_table(DYNAMODB_TABLE_NAME)
def scan_table(table_name, filter_key=None, filter_value=None):
table = dynamodb.Table(table_name)
response = table.scan()
def get_most_recent_ride_details(self,
user_id=None,
cookies=None,
save=False):
# Get the most recent workout ID
workout_ids = PelotonConnection.__get_workouts__(
self, user_id, cookies)
for workout_id in workout_ids:
workout_url = f"https://api.onepeloton.com/api/workout/{workout_id}"
# Get the workout info
workout = self.get(workout_url, cookies)
created_at = workout.get("created_at")
achievements_url = f"https://api.onepeloton.com/api/user/{user_id}/achievements"
achievements = self.get(achievements_url, cookies)
achievements = [
f for f in [
a.get("achievements")
for a in achievements.get("categories")
]
]
total_achievements = sum([
val.get("count") for sublist in achievements for val in sublist
])
# Performance Graph For that workout/ride
performance_url = f"https://api.onepeloton.com/api/workout/{workout_id}/performance_graph?every_n=5"
performance_res = self.get(performance_url, cookies)
results = {}
# Each of the averages (Cadence, Speed, Distance, Etc) are in the different summaries
# So just loop over and grab out the data
# There are some dupes like heart_rate/achievements_etc but wasn't sure where to put it
averages = performance_res.get("average_summaries")
for average in averages:
heart_rate = [
f for f in performance_res.get("metrics")
if f.get("display_name") == "Heart Rate"
] or None
result = {
'name':
average.get('display_name'),
'unit':
average.get('display_unit'),
'value':
average.get('value'),
'distance': [
f for f in performance_res.get("summaries")
if f.get("display_name") == 'Distance'
][0].get("value"),
'heart_rate':
heart_rate[0].get("average_value")
if heart_rate is not None else None,
'total_achievements':
total_achievements,
'miles_ridden': [
f for f in performance_res.get("summaries")
if f.get("display_name") == "Distance"
][0].get("value")
}
results[average.get('display_name')] = result
# At some point it would behove me to purge the dynamo db and move the dupes out of results
# But for now, we will leave it. Also, account for no heart rate monitor
my_json_record = {
"Avg Cadence":
results.get("Avg Cadence"),
"Avg Output":
results.get("Avg Output"),
"Avg Resistance":
results.get("Avg Resistance"),
"Avg Speed":
results.get("Avg Speed"),
'heart_rate':
heart_rate[0].get("average_value")
if heart_rate is not None else None,
'total_achievements':
total_achievements,
'miles_ridden': [
f for f in performance_res.get("summaries")
if f.get("display_name") == "Distance"
][0].get("value"),
"ride_Id":
str(created_at)
}
table = boto3.resource('dynamodb').Table('peloton_ride_data')
# The info comes in as a float and Dynamo gets mad so just parse it out and make it a json obj
ddb_data = json.loads(json.dumps(my_json_record),
parse_float=Decimal)
# Toss the json into Dynamo
if save is True:
table.put_item(Item=ddb_data)
def display_buckets():
s3 = boto3.resource('s3')
for buckets in s3.buckets.all():
print(buckets.name)
from botocore.exceptions import ClientError
# Helper class to convert a DynamoDB item to JSON.
class DecimalEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, decimal.Decimal):
if o % 1 > 0:
return float(o)
else:
return int(o)
return super(DecimalEncoder, self).default(o)
dynamodb = boto3.resource('dynamodb',
region_name='us-west-2',
endpoint_url="http://localhost:8000")
table = dynamodb.Table('Movies')
title = "The Big New Movie"
year = 2015
# Conditional update (will fail)
print("Attempting conditional update...")
try:
response = table.update_item(
Key={
'year': year,
'title': title
#!/usr/bin/env python
import boto3
import sys #it reads the command line as arguments
ec2 = boto3.resource('ec2')
for instance_id in sys.argv[1:]:
instance = ec2.Instance(instance_id)
response = instance.terminate()
print response
import os
import time
import logging
import boto3
import decimal
import dateutil.parser
import datetime
from boto3.dynamodb.conditions import Key, Attr
# Logger
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# DynamoDb table decaration
dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
# Client for Comprehend
client = boto3.client('comprehend')
"""
Helper function for Lex
"""
def get_slots(intent_request):
return intent_request['currentIntent']['slots']
def safeDecimal(n):
"""
Safely convert n value to int.
"""
if n is not None:
return decimal.Decimal(n)
return (n)
def safeString(n):
import json
import os
import boto3
import numpy as np
import pandas as pd
import csv
s3_resource = boto3.resource("s3")
## Function to count the number of datasets in a given job. Right now, counts based
## on the number of generated Dataset status files, we could think of alternatives.
def count_datasets(bucket, jobpath):
"""
Assumes that the given path points to the job path of a given analysis.
Looks within the log path of that same analysis, and counts the number of dataset files
generated for it.
Inputs:
bucket: (boto3 obj): an s3 boto3 resource object declaring the bucket this comes from.
jobpath: (str) the path of the job directory for a particular analysis.
"""
logsdir = os.path.join(jobpath, "logs/")
## TODO: make this reference the environment variable indicating the right log directory.
## Now look within for datasets with a certain prefix:
all_logs = [i.key for i in bucket.objects.filter(Prefix=logsdir)]
## Filter this by the DATASET_NAME prefix:
dataset_logs = [
l for l in all_logs if l.split(logsdir)[-1].startswith("DATASET_NAME")
]
return dataset_logs
import boto3
import os
import datetime
import pandas as pd
import botocore
import zipfile
import shutil
import rds_config
import pymysql
import logging
import csv
logging.basicConfig()
chaqerObject = Chaqer()
s3 = boto3.resource('s3')
client = boto3.client('s3')
srcBucket = str(os.environ.get('BUCKET'))
srcKey = str(os.environ.get('FILE'))
rds_host = rds_config.db_endpoint
name = rds_config.db_username
password = rds_config.db_password
db_name = rds_config.db_name
port = 3306
try:
cnx = pymysql.connect(rds_host,
user=name,
passwd=password,
db=db_name,
connect_timeout=10)
sudo yum -y update
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
sudo yum -y install ntp
sudo yum -y install telnet
sudo yum -y install mtr
sudo yum -y install tree
sudo systemctl start ntpd
sudo systemctl enable ntpd.service
echo "* hard nofile 65535" | sudo tee -a /etc/security/limits.conf
echo "* soft nofile 65535" | sudo tee -a /etc/security/limits.conf
echo "net.ipv6.conf.all.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf
echo "net.ipv6.conf.default.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf
echo "net.ipv6.conf.lo.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf"""
ec2AmiResource = boto3.resource('ec2')
ec2InstanceAmi = ec2AmiResource.create_instances(ImageId=amiId,
InstanceType='t2.micro',
KeyName='AfterPaySSHKey',
MinCount=1,
MaxCount=1,
UserData=userDataScriptAmi)
logging.warning(
'Updatinge packages, installing services and configuring services in progress for customized AMI for AfterPay ...'
)
checkStatusClient = boto3.client('ec2')
ec2AmiStatus = checkStatusClient.describe_instance_status(InstanceIds=[
ec2InstanceAmi[0].instance_id,
])
import libraries.morflessLibs as libs
import boto3
import json
import logging, os
settings_file = libs.constants.PCOM_REQ_FILE_SETTINGS
sourcebucket= os.environ['SOURCE_BUCKET']
targetbucket= os.environ['TARGET_BUCKET']
listbucket = os.environ["LIST_BUCKET"]
s3resource = boto3.resource('s3')
s3client = boto3.client('s3')
# process and output postlist constants
def process_postlists_info(postlists_info,postlist,settings,list_meta,log):
if postlists_info:
for info in postlists_info:
js_name = info['name']
list_info = info['postlists']
fileroot = info['fileroot']
js_constant,processed = libs.second_processes.pcom_process_postlist(list_info,postlist,settings,list_meta,fileroot)
if processed:
log['postlists_processed'].append(js_name)
log = write_js_to_buckets(js_name,js_constant,log)
os.remove(directory_file + '.vcf.count.log')
os.rmdir(task_dir)
print('Delete task files successfully.')
if __name__ == '__main__':
config = configparser.ConfigParser()
config.read(os.path.abspath('./configini.ini'))
aws_config = config['ann']
if len(sys.argv) > 3:
with Timer():
driver.run(sys.argv[1], 'vcf')
print('Finish annotation successfully.')
# upload results and logs
try:
s3 = boto3.resource('s3',
region_name=aws_config['AWS_REGION_NAME'])
task_file = sys.argv[1]
directory_file = task_file.replace('.vcf', '')
key = sys.argv[2].replace('.vcf', '')
upload_file(s3, directory_file, key)
except Exception as e:
print(e)
print('Cannot upload files.')
# remove files on AnnTools instance
try:
task_file = sys.argv[1]
directory_file = task_file.replace('.vcf', '')
task_dir = os.path.dirname(task_file)
remove_file(task_file, directory_file, task_dir)
except Exception as e:
print(e)
def notification_mailer(event, context):
ddb = boto3.resource("dynamodb", region_name='us-east-1').Table(os.environ["BUILDABLES_TABLE_NAME"])
mm_helper = MessageMultiplexerHelper(endpoint=get_mm_endpoint(os.environ["MM_STAGE"]),
own_account_number=os.environ["ACCOUNT_ID"])
notification_mailer_internal(event, context, ddb, mm_helper)
