def publish_msg_to_sns_topic(region, aws_access_key, aws_secret_key, topic, message, subject):
connect_to_region(region)
conn = SNSConnection(aws_access_key, aws_secret_key)
conn.publish(topic, message, subject)
def notify_usage( function ):
try:
from boto.sns import SNSConnection
c = SNSConnection(*s3accounts.accounts.get(s3accounts.preferred_account))
message = "Using sc2s3 at {0}".format( datetime.now().isoformat() )
c.publish(s3accounts.arn_sns, message )
except:
pass
return
def create_comment(self, **kw):
try:
#check if this user can create a comment in this topic
#for now every member of the topic can create a comment
creator = DBSession.query(MemberTopic).\
filter(MemberTopic.topic_id==kw['topic_id'],
MemberTopic.member_id==request.identity['user'].user_id).\
first()
if not creator:
#TODO: this should never happen, log this event and return only False
return dict(success=False, error_code=ErrorCode.UNAUTHORIZED)
comment = Comment()
comment.subject_id = kw['subject_id']
comment.creator_id = request.identity['user'].user_id
comment.content = kw['content']
DBSession.add(comment)
DBSession.flush()
#TODO: should not query then update, should only update
topic = DBSession.query(Topic).\
filter(Topic.uid==kw['topic_id']).first()
#TODO: need to rethink this time var, how about subject update_time
topic.update_time = comment.time
log.info("User %s creates comment %s"%(comment.creator_id, comment))
subject_title = DBSession.query(Subject.title).\
filter(Subject.uid==kw['subject_id']).first()[0]
members = DBSession.query(User.email_address).\
filter(MemberTopic.topic_id==kw['topic_id'],
MemberTopic.member_id!=creator.member_id,
MemberTopic.deleted==False,
MemberSubject.subject_id==kw['subject_id'],
MemberSubject.muted==False).\
join(MemberTopic,MemberSubject).\
all()
member_emails = [member[0] for member in members]
#send notification to users
notif=dict(type='new_comment',topic=topic.title,subject=subject_title,comment=comment.content,
registered_users=member_emails,user_name=request.identity['user'].user_name)
log.info('Sending SNS notification: %s'%notif)
sns=SNSConnection(config.get('AWS_ACCESS_KEY_ID'), config.get("AWS_SECRET_ACCESS_KEY"))
sns.publish(config.get('sqs_user_notification'),js.dumps(notif))
return dict(success=True, comment=dict(uid=comment.uid, time=comment.time))
except Exception as e:
log.exception('Got exception %s'%e)
#traceback.print_exc(file=sys.stdout)
return dict(success=False)
def delete_sns_topic():
"""
Deletes the SNS topic. Subscriptions must be deleted prior to that.
"""
con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY,
region=RegionInfo(name=REGION,
endpoint='sns.%s.amazonaws.com' % REGION))
topic_arn = get_topic_arn()
print "Deleting topic : %s" % topic_arn
con.delete_topic(topic_arn)
con.close()
def create_sns_topic():
"""
Creates the SNS topic and returns the topic arn string
"""
con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY,
region=RegionInfo(name=REGION,
endpoint='sns.%s.amazonaws.com' % REGION))
topic_arn = con.create_topic(SNS_TOPIC_NAME)
topic_arn = topic_arn['CreateTopicResponse']['CreateTopicResult']['TopicArn']
print "Topic created, arn is : %s" % topic_arn
con.close()
return topic_arn
def create_email_topic_subscription():
"""
Creates the email subscription for our topic. Topic must be created prior to that.
returns nothing.
"""
con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY,
region=RegionInfo(name=REGION,
endpoint='sns.%s.amazonaws.com' % REGION))
topic_arn = get_topic_arn()
subscription = con.subscribe(topic_arn, "email", SNS_EMAIL)
print "Subscribed email : %s to SNS notifications" % SNS_EMAIL
print subscription
print "Please make sure to check your inbox and confirm your subscription"
con.close()
class SNSLogHandler(logging.Handler):
def __init__(self, topic, subject, instance_id=None):
logging.Handler.__init__(self)
self.sns_conn = SNSConnection()
self.topic = topic
self.subject = subject
self.instance_id = instance_id
def emit(self, record):
if self.instance_id is None:
msg = record.message
else:
msg = "[from: %s] %s" % (self.instance_id, record.message)
self.sns_conn.publish(self.topic, msg,
subject=self.subject)
def get_sns_alert_function(self):
"""Get a lamdda function for SNS alert publishing
"""
if self.sns_topic_arn is None:
return None
return lambda message, subject: \
SNSConnection().publish(self.sns_topic_arn, message, subject)
def updateActions(self, actions):
'''Update the actions on this account based on the supplied actions. Actions
should a dictionary of Amazon Simple Notification Service topic names, and
their associated subscriptions.'''
# First, we need a SNS Connection to make this changes
conn = SNSConnection(**self.kwargs)
# Now make sure each subscription is registered to the topic
for name, subscriptions in actions.items():
logger.info('Creating topic %s' % name)
# Try to make a topic
try:
arn = conn.create_topic(name)['CreateTopicResponse']['CreateTopicResult']['TopicArn']
self.actions[name] = arn
except KeyError:
raise EmitterException('Bad response creating topic %s' % name)
if len(subscriptions) == 0:
raise EmitterException('No subscriptions for action %s' % name)
# Now try to arrange for subscriptions
# Oddly enough, calling create_topic doesn't have any effect
# if the topic already exists, but calling subscribe() for an
# existing subscription causes a second subscription to be added
# So, we have to get a list of current subscriptions, and then
# make sure to only add the subscription if it's currently there
logger.info('Getting a list of current subscriptions...')
current = conn.get_all_subscriptions_by_topic(arn)
current = current['ListSubscriptionsByTopicResponse']
current = current['ListSubscriptionsByTopicResult']
current = current['Subscriptions']
current = set(s['Endpoint'] for s in current)
# For all desired subscriptions not present, subscribe
for s in subscriptions:
if s['endpoint'] not in current:
logger.info('Adding %s to action %s' % (s['endpoint'], name))
conn.subscribe(arn, s.get('protocol', 'email'), s['endpoint'])
else:
logger.info('%s already subscribed to action' % s['endpoint'])
# Check for subscriptions that are active, but not listed...
activeUnlisted = set(current) - set([s['endpoint'] for s in subscriptions])
for s in activeUnlisted:
logger.warn('Subscript "%s" active, but not listed in config' % s)
def connect_sns(aws_access_key_id=None, aws_secret_access_key=None, **kwargs):
"""
:type aws_access_key_id: string
:param aws_access_key_id: Your AWS Access Key ID
:type aws_secret_access_key: string
:param aws_secret_access_key: Your AWS Secret Access Key
:rtype: :class:`boto.sns.SNSConnection`
:return: A connection to Amazon's SNS
"""
from boto.sns import SNSConnection
return SNSConnection(aws_access_key_id, aws_secret_access_key, **kwargs)
def updateActions(self, actions):
'''Update the actions on this account based on the supplied actions. Actions
should a dictionary of Amazon Simple Notification Service topic names, and
their associated subscriptions.'''
# First, we need a SNS Connection to make this changes
conn = SNSConnection(**self.kwargs)
# Now make sure each subscription is registered to the topic
for name, subscriptions in actions.items():
logger.info('Creating topic %s' % name)
# Try to make a topic
try:
arn = conn.create_topic(name)['CreateTopicResponse'][
'CreateTopicResult']['TopicArn']
self.actions[name] = arn
except KeyError:
raise EmitterException('Bad response creating topic %s' % name)
if len(subscriptions) == 0:
raise EmitterException('No subscriptions for action %s' % name)
# Now try to arrange for subscriptions
# Oddly enough, calling create_topic doesn't have any effect
# if the topic already exists, but calling subscribe() for an
# existing subscription causes a second subscription to be added
# So, we have to get a list of current subscriptions, and then
# make sure to only add the subscription if it's currently there
logger.info('Getting a list of current subscriptions...')
current = conn.get_all_subscriptions_by_topic(arn)
current = current['ListSubscriptionsByTopicResponse']
current = current['ListSubscriptionsByTopicResult']
current = current['Subscriptions']
current = set(s['Endpoint'] for s in current)
# For all desired subscriptions not present, subscribe
for s in subscriptions:
if s['endpoint'] not in current:
logger.info('Adding %s to action %s' %
(s['endpoint'], name))
conn.subscribe(arn, s.get('protocol', 'email'),
s['endpoint'])
else:
logger.info('%s already subscribed to action' %
s['endpoint'])
# Check for subscriptions that are active, but not listed...
activeUnlisted = set(current) - set(
[s['endpoint'] for s in subscriptions])
for s in activeUnlisted:
logger.warn('Subscript "%s" active, but not listed in config' %
s)
def get_topic_arn(topicname = SNS_TOPIC_NAME):
"""
Returns the topic arn, using our default SNS_TOPIC_NAME or a given parameter
"""
con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY,
region=RegionInfo(name=REGION,
endpoint='sns.%s.amazonaws.com' % REGION))
for t in con.get_all_topics()['ListTopicsResponse']['ListTopicsResult']['Topics']:
topicarn = t['TopicArn']
if SNS_TOPIC_NAME in topicarn:
con.close()
return topicarn
con.close()
return None
def delete_sns_subscription():
"""
Deletes the SNS email subscription
"""
con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_access_key_id=AWS_ACCESS_KEY,
region=RegionInfo(name=REGION,
endpoint='sns.%s.amazonaws.com' % REGION))
topic_arn = get_topic_arn()
subscriptions = con.get_all_subscriptions_by_topic(topic_arn)['ListSubscriptionsByTopicResponse']\
['ListSubscriptionsByTopicResult']\
['Subscriptions']
for s in subscriptions:
try:
print "Unsubscribing %s" % s
con.unsubscribe(s['SubscriptionArn'])
except:
print "Could not unsubscribe %s" % s
con.close()
def dependency_check():
parser = argparse.ArgumentParser()
parser.add_argument('--dependencies', type=str, nargs='+', default=[])
parser.add_argument('--dependencies_ok_to_fail',
type=str,
nargs='+',
default=[])
parser.add_argument('--pipeline_name', dest='pipeline_name')
parser.add_argument('--refresh_rate', dest='refresh_rate', default='900')
parser.add_argument('--start_date', dest='start_date')
parser.add_argument('--sns_topic_arn', dest="sns_topic_arn")
args = parser.parse_args()
# Exit if there are no dependencies
if not args.dependencies and not args.dependencies_ok_to_fail:
sys.exit()
# Create mapping from pipeline name to id
pipeline_name_to_id = dict(
(pipeline['name'], pipeline['id']) for pipeline in list_pipelines())
# Remove whitespace from dependency lists
dependencies = map(str.strip, args.dependencies)
dependencies_to_ignore = map(str.strip, args.dependencies_ok_to_fail)
# Add the dependencies which can fail to the list of dependencies
dependencies.extend(dependencies_to_ignore)
# Check if all dependencies are valid pipelines
for dependency in dependencies:
if dependency not in pipeline_name_to_id:
raise Exception('Pipeline not found: %s.' % dependency)
# Map from dependency id to name
dependencies = {pipeline_name_to_id[dep]: dep for dep in dependencies}
print 'Start checking for dependencies'
start_time = datetime.now()
failures = []
dependencies_ready = False
# Loop until all dependent pipelines have finished or failed
while not dependencies_ready:
print 'checking'
dependencies_ready, new_failures = check_dependencies_ready(
dependencies, args.start_date, dependencies_to_ignore)
failures.extend(new_failures)
if not dependencies_ready:
time.sleep(float(args.refresh_rate))
# Send message through SNS if there are failures
if failures:
if args.sns_topic_arn:
message = 'Failed dependencies: %s.' % ', '.join(set(failures))
subject = 'Dependency error for pipeline: %s.' % args.pipeline_name
SNSConnection().publish(args.sns_topic_arn, message, subject)
else:
raise Exception('ARN for SNS topic not specified in ETL config')
print 'Finished checking for dependencies. Total time spent: ',
print(datetime.now() - start_time).total_seconds(), ' seconds'
elif opt in ('-v'):
vault_name = arg;
if vault_name == '':
usage();
log('Start!');
log('Configure Property.');
aws_account_id = boto.config.get('Credentials', 'aws_account_id');
aws_access_key_id = boto.config.get('Credentials', 'aws_access_key_id');
aws_secret_access_key = boto.config.get('Credentials', 'aws_secret_access_key');
aws_region_name = boto.config.get('Environments', 'aws_region_name');
job_output_file = boto.config.get('Environments', 'job_output_file');
log('Set up SNS and SQS.');
sns_region = RegionInfo(name = aws_region_name, endpoint = 'sns.' + aws_region_name + '.amazonaws.com');
sns_con = SNSConnection(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, region = sns_region);
sns_topic = sns_con.create_topic('empty_vault-topic');
# SQS endpoint URL (sqs.xxx.amazonaws.com) doesn't work in boto 2.8.0 due to certification error.
# sqs_region = RegionInfo(name = aws_region_name, endpoint = 'sqs.' + aws_region_name + '.amazonaws.com')
# sqs_con = SQSConnection(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, region = sqs_region);
sqs_con = SQSConnection(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key);
sqs_queue = sqs_con.create_queue('empty_vault-queue');
sns_topic_arn = sns_topic['CreateTopicResponse']['CreateTopicResult']['TopicArn'];
sns_con.subscribe_sqs_queue(sns_topic_arn, sqs_queue);
log('Point to Glacier vault.');
glacier_layer1 = Layer1(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, region_name = aws_region_name);
glacier_layer1.describe_vault(vault_name);
evaluationId = environ['EVALUATION_ID']
numerateBucketAccessEmailAddress= environ['NUMERATE_BUCKET_ACCESS_EMAIL_ADDRESS']
synapseUserId = environ['SYNAPSE_USER_ID']
synapseUserPw = environ['SYNAPSE_USER_PW']
snsTopic = environ['SNS_TOPIC']
# we allow multiple AWS accounts. this gets us past the 100 bucket per account limit
synapseAccessKeyProjectId=environ["SYNAPSE_ACCESS_KEY_PROJECT_ID"]
s3Connections = []
iamConnections = []
i=0
while True:
aws_access_key_id = environ.get("AWS_ACCESS_KEY_ID_"+str(i+1))
aws_secret_access_key = environ.get("AWS_SECRET_ACCESS_KEY_"+str(i+1))
if (i==0):
snsConnection = SNSConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
if ((aws_access_key_id is None) or (aws_secret_access_key is None)):
break
else:
s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key))
iamConnections.append(IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key))
i=i+1
if (len(s3Connections)==0):
raise("No AWS crdentials provided")
MAXIMUM_USER_NAME_LENGTH = 63
## connect to Synapse
syn = Synapse()
def __init__(self, topic, subject, instance_id=None):
logging.Handler.__init__(self)
self.sns_conn = SNSConnection()
self.topic = topic
self.subject = subject
self.instance_id = instance_id
elif node_env == 'staging':
key_prefix = 'efins_dbbackups/staging'
else:
node_env = node_env if node_env else '-none-'
print("NODE_ENV '" + node_env + "' is not valid. Try setting it to one of {production, staging, development}.")
exit(-1)
# Get a SNS connection so we can report on our status
regions = boto.sns.regions()
myRegion = ''
for region in regions:
print region
if region.name == 'us-west-2':
myRegion = region
sns = SNSConnection('AKIAIBYUXG6UOLBFSRKA', 'FvFbsW2C9rS9ayA1AHvHmBqL07iU+oz5X803xdot', region=myRegion)
# Take the DB backup
db_suffix = "_" + node_env if node_env in ["development", "staging"] else ""
db_name = "efins" + db_suffix
print("Starting the backup of " + db_name)
now = datetime.datetime.now()
backup_name = "%s_%s_%s_%s_%s_%s_%s" % (now.month, now.day, now.year, now.hour, now.minute, now.second, os.getenv('USER'))
print backup_name
backup_path = "/tmp/%s" % backup_name
print("Dumping to %s" % backup_path)
orig_dir = os.getcwd()
os.chdir('/tmp')
evaluationId = environ["EVALUATION_ID"]
numerateBucketAccessEmailAddress = environ["NUMERATE_BUCKET_ACCESS_EMAIL_ADDRESS"]
synapseUserId = environ["SYNAPSE_USER_ID"]
synapseUserPw = environ["SYNAPSE_USER_PW"]
snsTopic = environ["SNS_TOPIC"]
# we allow multiple AWS accounts. this gets us past the 100 bucket per account limit
synapseAccessKeyProjectId = environ["SYNAPSE_ACCESS_KEY_PROJECT_ID"]
s3Connections = []
iamConnections = []
i = 0
while True:
aws_access_key_id = environ.get("AWS_ACCESS_KEY_ID_" + str(i + 1))
aws_secret_access_key = environ.get("AWS_SECRET_ACCESS_KEY_" + str(i + 1))
if i == 0:
snsConnection = SNSConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
if (aws_access_key_id is None) or (aws_secret_access_key is None):
break
else:
s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key))
iamConnections.append(
IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
)
i = i + 1
if len(s3Connections) == 0:
raise ("No AWS crdentials provided")
MAXIMUM_USER_NAME_LENGTH = 63
def create_topic(self, *args, **kw):
topic = Topic()
topic.creator_id = request.identity['user'].user_id
log.debug("Query string: %s"%kw)
if kw.has_key('title'):
topic.title = kw['title']
else:
#TODO: create topic named after all member
return dict(success=False)
if kw.has_key('logourl'):
topic.logourl = kw['logourl']
#creator is always a member of the topic
membertopic = MemberTopic(role='c', local_title=topic.title, member_id=topic.creator_id)
topic.members.append(membertopic)
#do not change the order of the following if clause
members = []
if kw.has_key('members'):
if isinstance(kw['members'], list):
members = kw['members']
elif kw['members']!='' and isinstance(kw['members'], unicode):
members = [kw['members']]
try:
# check if each member is in the User table
# if not add him to nonexists list
nonregistered_users = []
registered_users = DBSession.query(User.email_address, User.user_id)\
.filter(User.email_address.in_(members)).all()
registered_email_address = [user[0] for user in registered_users]
registered_user_ids = [user[1] for user in registered_users]
log.debug(registered_users)
if len(registered_users)!=len(members):
for user in members:
if user not in registered_email_address:
log.info("User %s is not in the database"%user)
nonregistered_users.append(user)
for member_id in registered_user_ids:
if member_id == topic.creator_id:
continue
member_topic = MemberTopic(role='c',
local_title=topic.title,
member_id=member_id)
topic.members.append(member_topic)
DBSession.add(topic)
DBSession.flush()
main_res = dict()
#if there is subject to be created, then create it, return error_code if failed
if kw.has_key('subject'):
if kw.has_key('message'):
res=self.create_subject(topic_id=topic.uid, title=kw['subject'], content=kw['message'])
else:
res=self.create_subject(topic_id=topic.uid, title=kw['subject'])
if res['success']==False:
main_res['error_code'] = ErrorCode.CREATSUBJECTFAILED
#TODO: check main_res before sending notification
#send notification to users
notif=dict(type='new_topic',topic=topic.title,registered_users=members,
user_name=request.identity['user'].user_name)
log.info('Sending SNS notification: %s'%notif)
sns=SNSConnection(config.get('AWS_ACCESS_KEY_ID'), config.get("AWS_SECRET_ACCESS_KEY"))
sns.publish(config.get('sqs_user_notification'),js.dumps(notif))
if len(nonregistered_users):
main_res.update(dict(success=True,
topic=dict(uid=topic.uid, time=topic.time),
nonregistered_users=nonregistered_users))
else:
main_res.update(dict(success=True,
topic=dict(uid=topic.uid, time=topic.time,
creator_id=topic.creator_id, update_time=topic.time,
title=topic.title, logourl=topic.logourl)))
return main_res
except Exception as e:
log.exception("Got exception: %s"%e)
return dict(success=False)
from boto.sns import SNSConnection
#create connection
conn = SNSConnection()
topic_name = 'test-topic'
#create new SNS topic
# conn.create_topic('test-topic')
# ARN Number of topic
arn_num = 'arn:aws:sns:us-east-1:926344641371:'+topic_name
# get attr for the ARN topic
# topic_attr = conn.get_topic_attributes(arn_num)
#subscribe to a topic
protocol = 'email'
endpoint = '*****@*****.**'
#conn.subscribe(arn_num,protocol,endpoint)
#publish to a topic
message = "This is a test message. <br> -Kiran"
subject = "Test SNS subscription"
message_structure = 'html'
conn.publish(message=message, subject=subject, target_arn=arn_num, message_structure=message_structure)
def create_subject(self, **kw):
try:
#check if this user can create a subject in this topic
creator = DBSession.query(MemberTopic).\
filter(MemberTopic.topic_id==kw['topic_id'],
MemberTopic.member_id==request.identity['user'].user_id,
'role = "c" or role = "s"').\
first()
if not creator:
#TODO: this should never happen, log this event and return only False
return dict(success=False, error_code=ErrorCode.UNAUTHORIZED)
#add new subject to the database
subject = Subject()
subject.topic_id = kw['topic_id']
subject.title = kw['title']
subject.creator_id = request.identity['user'].user_id
DBSession.add(subject)
DBSession.flush()
comment = None
if kw.has_key('content'):
comment = Comment()
comment.subject_id = subject.uid
comment.creator_id = request.identity['user'].user_id
comment.content = kw['content']
DBSession.add(comment)
DBSession.flush()
#add creator to member_subject
if comment:
member_subject = MemberSubject(member_id = creator.member_id,
subject_id = subject.uid,
last_read = comment.uid)
else:
member_subject = MemberSubject(member_id = creator.member_id,
subject_id = subject.uid,
last_read = 0)
subject.members.append(member_subject)
# registered_users = DBSession.query(User.email_address).filter(User.email_address.in_(members)).all()
members = DBSession.query(MemberTopic.member_id,User.email_address).\
filter(MemberTopic.topic_id==kw['topic_id'],
MemberTopic.member_id!=creator.member_id,
MemberTopic.deleted == False).\
join(User).\
all()
#add other members to member_subject table
member_ids = [member[0] for member in members]
for member_id in member_ids:
member_subject = MemberSubject(member_id = member_id,
subject_id = subject.uid,
last_read = 0)
subject.members.append(member_subject)
topic = DBSession.query(Topic).\
filter(Topic.uid==kw['topic_id']).first()
#TODO: need to rename/rethink this time vars
topic.update_time = subject.time
log.info("User %s creates subject %s"%(creator.member_id, subject))
member_emails = [member[1] for member in members]
#send notification to users
if len(member_emails)>0:
notif=dict(type='new_subject',topic=topic.title,subject=subject.title,
registered_users=member_emails,user_name=request.identity['user'].user_name)
log.info('Sending SNS notification: %s'%notif)
sns=SNSConnection(config.get('AWS_ACCESS_KEY_ID'), config.get("AWS_SECRET_ACCESS_KEY"))
sns.publish(config.get('sqs_user_notification'),js.dumps(notif))
return dict(success=True, subject=dict(uid=subject.uid, time=subject.time))
except Exception as e:
log.exception('Got exception %s'%e)
#traceback.print_exc(file=sys.stdout)
return dict(success=False)
