def publish_msg_to_sns_topic(region, aws_access_key, aws_secret_key, topic, message, subject): connect_to_region(region) conn = SNSConnection(aws_access_key, aws_secret_key) conn.publish(topic, message, subject)
def notify_usage( function ): try: from boto.sns import SNSConnection c = SNSConnection(*s3accounts.accounts.get(s3accounts.preferred_account)) message = "Using sc2s3 at {0}".format( datetime.now().isoformat() ) c.publish(s3accounts.arn_sns, message ) except: pass return
def create_comment(self, **kw): try: #check if this user can create a comment in this topic #for now every member of the topic can create a comment creator = DBSession.query(MemberTopic).\ filter(MemberTopic.topic_id==kw['topic_id'], MemberTopic.member_id==request.identity['user'].user_id).\ first() if not creator: #TODO: this should never happen, log this event and return only False return dict(success=False, error_code=ErrorCode.UNAUTHORIZED) comment = Comment() comment.subject_id = kw['subject_id'] comment.creator_id = request.identity['user'].user_id comment.content = kw['content'] DBSession.add(comment) DBSession.flush() #TODO: should not query then update, should only update topic = DBSession.query(Topic).\ filter(Topic.uid==kw['topic_id']).first() #TODO: need to rethink this time var, how about subject update_time topic.update_time = comment.time log.info("User %s creates comment %s"%(comment.creator_id, comment)) subject_title = DBSession.query(Subject.title).\ filter(Subject.uid==kw['subject_id']).first()[0] members = DBSession.query(User.email_address).\ filter(MemberTopic.topic_id==kw['topic_id'], MemberTopic.member_id!=creator.member_id, MemberTopic.deleted==False, MemberSubject.subject_id==kw['subject_id'], MemberSubject.muted==False).\ join(MemberTopic,MemberSubject).\ all() member_emails = [member[0] for member in members] #send notification to users notif=dict(type='new_comment',topic=topic.title,subject=subject_title,comment=comment.content, registered_users=member_emails,user_name=request.identity['user'].user_name) log.info('Sending SNS notification: %s'%notif) sns=SNSConnection(config.get('AWS_ACCESS_KEY_ID'), config.get("AWS_SECRET_ACCESS_KEY")) sns.publish(config.get('sqs_user_notification'),js.dumps(notif)) return dict(success=True, comment=dict(uid=comment.uid, time=comment.time)) except Exception as e: log.exception('Got exception %s'%e) #traceback.print_exc(file=sys.stdout) return dict(success=False)
def delete_sns_topic(): """ Deletes the SNS topic. Subscriptions must be deleted prior to that. """ con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY, region=RegionInfo(name=REGION, endpoint='sns.%s.amazonaws.com' % REGION)) topic_arn = get_topic_arn() print "Deleting topic : %s" % topic_arn con.delete_topic(topic_arn) con.close()
def create_sns_topic(): """ Creates the SNS topic and returns the topic arn string """ con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY, region=RegionInfo(name=REGION, endpoint='sns.%s.amazonaws.com' % REGION)) topic_arn = con.create_topic(SNS_TOPIC_NAME) topic_arn = topic_arn['CreateTopicResponse']['CreateTopicResult']['TopicArn'] print "Topic created, arn is : %s" % topic_arn con.close() return topic_arn
def create_email_topic_subscription(): """ Creates the email subscription for our topic. Topic must be created prior to that. returns nothing. """ con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY, region=RegionInfo(name=REGION, endpoint='sns.%s.amazonaws.com' % REGION)) topic_arn = get_topic_arn() subscription = con.subscribe(topic_arn, "email", SNS_EMAIL) print "Subscribed email : %s to SNS notifications" % SNS_EMAIL print subscription print "Please make sure to check your inbox and confirm your subscription" con.close()
class SNSLogHandler(logging.Handler): def __init__(self, topic, subject, instance_id=None): logging.Handler.__init__(self) self.sns_conn = SNSConnection() self.topic = topic self.subject = subject self.instance_id = instance_id def emit(self, record): if self.instance_id is None: msg = record.message else: msg = "[from: %s] %s" % (self.instance_id, record.message) self.sns_conn.publish(self.topic, msg, subject=self.subject)
def get_sns_alert_function(self): """Get a lamdda function for SNS alert publishing """ if self.sns_topic_arn is None: return None return lambda message, subject: \ SNSConnection().publish(self.sns_topic_arn, message, subject)
def updateActions(self, actions): '''Update the actions on this account based on the supplied actions. Actions should a dictionary of Amazon Simple Notification Service topic names, and their associated subscriptions.''' # First, we need a SNS Connection to make this changes conn = SNSConnection(**self.kwargs) # Now make sure each subscription is registered to the topic for name, subscriptions in actions.items(): logger.info('Creating topic %s' % name) # Try to make a topic try: arn = conn.create_topic(name)['CreateTopicResponse']['CreateTopicResult']['TopicArn'] self.actions[name] = arn except KeyError: raise EmitterException('Bad response creating topic %s' % name) if len(subscriptions) == 0: raise EmitterException('No subscriptions for action %s' % name) # Now try to arrange for subscriptions # Oddly enough, calling create_topic doesn't have any effect # if the topic already exists, but calling subscribe() for an # existing subscription causes a second subscription to be added # So, we have to get a list of current subscriptions, and then # make sure to only add the subscription if it's currently there logger.info('Getting a list of current subscriptions...') current = conn.get_all_subscriptions_by_topic(arn) current = current['ListSubscriptionsByTopicResponse'] current = current['ListSubscriptionsByTopicResult'] current = current['Subscriptions'] current = set(s['Endpoint'] for s in current) # For all desired subscriptions not present, subscribe for s in subscriptions: if s['endpoint'] not in current: logger.info('Adding %s to action %s' % (s['endpoint'], name)) conn.subscribe(arn, s.get('protocol', 'email'), s['endpoint']) else: logger.info('%s already subscribed to action' % s['endpoint']) # Check for subscriptions that are active, but not listed... activeUnlisted = set(current) - set([s['endpoint'] for s in subscriptions]) for s in activeUnlisted: logger.warn('Subscript "%s" active, but not listed in config' % s)
def connect_sns(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): """ :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.sns.SNSConnection` :return: A connection to Amazon's SNS """ from boto.sns import SNSConnection return SNSConnection(aws_access_key_id, aws_secret_access_key, **kwargs)
def updateActions(self, actions): '''Update the actions on this account based on the supplied actions. Actions should a dictionary of Amazon Simple Notification Service topic names, and their associated subscriptions.''' # First, we need a SNS Connection to make this changes conn = SNSConnection(**self.kwargs) # Now make sure each subscription is registered to the topic for name, subscriptions in actions.items(): logger.info('Creating topic %s' % name) # Try to make a topic try: arn = conn.create_topic(name)['CreateTopicResponse'][ 'CreateTopicResult']['TopicArn'] self.actions[name] = arn except KeyError: raise EmitterException('Bad response creating topic %s' % name) if len(subscriptions) == 0: raise EmitterException('No subscriptions for action %s' % name) # Now try to arrange for subscriptions # Oddly enough, calling create_topic doesn't have any effect # if the topic already exists, but calling subscribe() for an # existing subscription causes a second subscription to be added # So, we have to get a list of current subscriptions, and then # make sure to only add the subscription if it's currently there logger.info('Getting a list of current subscriptions...') current = conn.get_all_subscriptions_by_topic(arn) current = current['ListSubscriptionsByTopicResponse'] current = current['ListSubscriptionsByTopicResult'] current = current['Subscriptions'] current = set(s['Endpoint'] for s in current) # For all desired subscriptions not present, subscribe for s in subscriptions: if s['endpoint'] not in current: logger.info('Adding %s to action %s' % (s['endpoint'], name)) conn.subscribe(arn, s.get('protocol', 'email'), s['endpoint']) else: logger.info('%s already subscribed to action' % s['endpoint']) # Check for subscriptions that are active, but not listed... activeUnlisted = set(current) - set( [s['endpoint'] for s in subscriptions]) for s in activeUnlisted: logger.warn('Subscript "%s" active, but not listed in config' % s)
def get_topic_arn(topicname = SNS_TOPIC_NAME): """ Returns the topic arn, using our default SNS_TOPIC_NAME or a given parameter """ con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY, region=RegionInfo(name=REGION, endpoint='sns.%s.amazonaws.com' % REGION)) for t in con.get_all_topics()['ListTopicsResponse']['ListTopicsResult']['Topics']: topicarn = t['TopicArn'] if SNS_TOPIC_NAME in topicarn: con.close() return topicarn con.close() return None
def delete_sns_subscription(): """ Deletes the SNS email subscription """ con = SNSConnection(aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY, region=RegionInfo(name=REGION, endpoint='sns.%s.amazonaws.com' % REGION)) topic_arn = get_topic_arn() subscriptions = con.get_all_subscriptions_by_topic(topic_arn)['ListSubscriptionsByTopicResponse']\ ['ListSubscriptionsByTopicResult']\ ['Subscriptions'] for s in subscriptions: try: print "Unsubscribing %s" % s con.unsubscribe(s['SubscriptionArn']) except: print "Could not unsubscribe %s" % s con.close()
def dependency_check(): parser = argparse.ArgumentParser() parser.add_argument('--dependencies', type=str, nargs='+', default=[]) parser.add_argument('--dependencies_ok_to_fail', type=str, nargs='+', default=[]) parser.add_argument('--pipeline_name', dest='pipeline_name') parser.add_argument('--refresh_rate', dest='refresh_rate', default='900') parser.add_argument('--start_date', dest='start_date') parser.add_argument('--sns_topic_arn', dest="sns_topic_arn") args = parser.parse_args() # Exit if there are no dependencies if not args.dependencies and not args.dependencies_ok_to_fail: sys.exit() # Create mapping from pipeline name to id pipeline_name_to_id = dict( (pipeline['name'], pipeline['id']) for pipeline in list_pipelines()) # Remove whitespace from dependency lists dependencies = map(str.strip, args.dependencies) dependencies_to_ignore = map(str.strip, args.dependencies_ok_to_fail) # Add the dependencies which can fail to the list of dependencies dependencies.extend(dependencies_to_ignore) # Check if all dependencies are valid pipelines for dependency in dependencies: if dependency not in pipeline_name_to_id: raise Exception('Pipeline not found: %s.' % dependency) # Map from dependency id to name dependencies = {pipeline_name_to_id[dep]: dep for dep in dependencies} print 'Start checking for dependencies' start_time = datetime.now() failures = [] dependencies_ready = False # Loop until all dependent pipelines have finished or failed while not dependencies_ready: print 'checking' dependencies_ready, new_failures = check_dependencies_ready( dependencies, args.start_date, dependencies_to_ignore) failures.extend(new_failures) if not dependencies_ready: time.sleep(float(args.refresh_rate)) # Send message through SNS if there are failures if failures: if args.sns_topic_arn: message = 'Failed dependencies: %s.' % ', '.join(set(failures)) subject = 'Dependency error for pipeline: %s.' % args.pipeline_name SNSConnection().publish(args.sns_topic_arn, message, subject) else: raise Exception('ARN for SNS topic not specified in ETL config') print 'Finished checking for dependencies. Total time spent: ', print(datetime.now() - start_time).total_seconds(), ' seconds'
elif opt in ('-v'): vault_name = arg; if vault_name == '': usage(); log('Start!'); log('Configure Property.'); aws_account_id = boto.config.get('Credentials', 'aws_account_id'); aws_access_key_id = boto.config.get('Credentials', 'aws_access_key_id'); aws_secret_access_key = boto.config.get('Credentials', 'aws_secret_access_key'); aws_region_name = boto.config.get('Environments', 'aws_region_name'); job_output_file = boto.config.get('Environments', 'job_output_file'); log('Set up SNS and SQS.'); sns_region = RegionInfo(name = aws_region_name, endpoint = 'sns.' + aws_region_name + '.amazonaws.com'); sns_con = SNSConnection(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, region = sns_region); sns_topic = sns_con.create_topic('empty_vault-topic'); # SQS endpoint URL (sqs.xxx.amazonaws.com) doesn't work in boto 2.8.0 due to certification error. # sqs_region = RegionInfo(name = aws_region_name, endpoint = 'sqs.' + aws_region_name + '.amazonaws.com') # sqs_con = SQSConnection(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, region = sqs_region); sqs_con = SQSConnection(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key); sqs_queue = sqs_con.create_queue('empty_vault-queue'); sns_topic_arn = sns_topic['CreateTopicResponse']['CreateTopicResult']['TopicArn']; sns_con.subscribe_sqs_queue(sns_topic_arn, sqs_queue); log('Point to Glacier vault.'); glacier_layer1 = Layer1(aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, region_name = aws_region_name); glacier_layer1.describe_vault(vault_name);
evaluationId = environ['EVALUATION_ID'] numerateBucketAccessEmailAddress= environ['NUMERATE_BUCKET_ACCESS_EMAIL_ADDRESS'] synapseUserId = environ['SYNAPSE_USER_ID'] synapseUserPw = environ['SYNAPSE_USER_PW'] snsTopic = environ['SNS_TOPIC'] # we allow multiple AWS accounts. this gets us past the 100 bucket per account limit synapseAccessKeyProjectId=environ["SYNAPSE_ACCESS_KEY_PROJECT_ID"] s3Connections = [] iamConnections = [] i=0 while True: aws_access_key_id = environ.get("AWS_ACCESS_KEY_ID_"+str(i+1)) aws_secret_access_key = environ.get("AWS_SECRET_ACCESS_KEY_"+str(i+1)) if (i==0): snsConnection = SNSConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) if ((aws_access_key_id is None) or (aws_secret_access_key is None)): break else: s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key)) iamConnections.append(IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)) i=i+1 if (len(s3Connections)==0): raise("No AWS crdentials provided") MAXIMUM_USER_NAME_LENGTH = 63 ## connect to Synapse syn = Synapse()
def __init__(self, topic, subject, instance_id=None): logging.Handler.__init__(self) self.sns_conn = SNSConnection() self.topic = topic self.subject = subject self.instance_id = instance_id
elif node_env == 'staging': key_prefix = 'efins_dbbackups/staging' else: node_env = node_env if node_env else '-none-' print("NODE_ENV '" + node_env + "' is not valid. Try setting it to one of {production, staging, development}.") exit(-1) # Get a SNS connection so we can report on our status regions = boto.sns.regions() myRegion = '' for region in regions: print region if region.name == 'us-west-2': myRegion = region sns = SNSConnection('AKIAIBYUXG6UOLBFSRKA', 'FvFbsW2C9rS9ayA1AHvHmBqL07iU+oz5X803xdot', region=myRegion) # Take the DB backup db_suffix = "_" + node_env if node_env in ["development", "staging"] else "" db_name = "efins" + db_suffix print("Starting the backup of " + db_name) now = datetime.datetime.now() backup_name = "%s_%s_%s_%s_%s_%s_%s" % (now.month, now.day, now.year, now.hour, now.minute, now.second, os.getenv('USER')) print backup_name backup_path = "/tmp/%s" % backup_name print("Dumping to %s" % backup_path) orig_dir = os.getcwd() os.chdir('/tmp')
evaluationId = environ["EVALUATION_ID"] numerateBucketAccessEmailAddress = environ["NUMERATE_BUCKET_ACCESS_EMAIL_ADDRESS"] synapseUserId = environ["SYNAPSE_USER_ID"] synapseUserPw = environ["SYNAPSE_USER_PW"] snsTopic = environ["SNS_TOPIC"] # we allow multiple AWS accounts. this gets us past the 100 bucket per account limit synapseAccessKeyProjectId = environ["SYNAPSE_ACCESS_KEY_PROJECT_ID"] s3Connections = [] iamConnections = [] i = 0 while True: aws_access_key_id = environ.get("AWS_ACCESS_KEY_ID_" + str(i + 1)) aws_secret_access_key = environ.get("AWS_SECRET_ACCESS_KEY_" + str(i + 1)) if i == 0: snsConnection = SNSConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) if (aws_access_key_id is None) or (aws_secret_access_key is None): break else: s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key)) iamConnections.append( IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) ) i = i + 1 if len(s3Connections) == 0: raise ("No AWS crdentials provided") MAXIMUM_USER_NAME_LENGTH = 63
def create_topic(self, *args, **kw): topic = Topic() topic.creator_id = request.identity['user'].user_id log.debug("Query string: %s"%kw) if kw.has_key('title'): topic.title = kw['title'] else: #TODO: create topic named after all member return dict(success=False) if kw.has_key('logourl'): topic.logourl = kw['logourl'] #creator is always a member of the topic membertopic = MemberTopic(role='c', local_title=topic.title, member_id=topic.creator_id) topic.members.append(membertopic) #do not change the order of the following if clause members = [] if kw.has_key('members'): if isinstance(kw['members'], list): members = kw['members'] elif kw['members']!='' and isinstance(kw['members'], unicode): members = [kw['members']] try: # check if each member is in the User table # if not add him to nonexists list nonregistered_users = [] registered_users = DBSession.query(User.email_address, User.user_id)\ .filter(User.email_address.in_(members)).all() registered_email_address = [user[0] for user in registered_users] registered_user_ids = [user[1] for user in registered_users] log.debug(registered_users) if len(registered_users)!=len(members): for user in members: if user not in registered_email_address: log.info("User %s is not in the database"%user) nonregistered_users.append(user) for member_id in registered_user_ids: if member_id == topic.creator_id: continue member_topic = MemberTopic(role='c', local_title=topic.title, member_id=member_id) topic.members.append(member_topic) DBSession.add(topic) DBSession.flush() main_res = dict() #if there is subject to be created, then create it, return error_code if failed if kw.has_key('subject'): if kw.has_key('message'): res=self.create_subject(topic_id=topic.uid, title=kw['subject'], content=kw['message']) else: res=self.create_subject(topic_id=topic.uid, title=kw['subject']) if res['success']==False: main_res['error_code'] = ErrorCode.CREATSUBJECTFAILED #TODO: check main_res before sending notification #send notification to users notif=dict(type='new_topic',topic=topic.title,registered_users=members, user_name=request.identity['user'].user_name) log.info('Sending SNS notification: %s'%notif) sns=SNSConnection(config.get('AWS_ACCESS_KEY_ID'), config.get("AWS_SECRET_ACCESS_KEY")) sns.publish(config.get('sqs_user_notification'),js.dumps(notif)) if len(nonregistered_users): main_res.update(dict(success=True, topic=dict(uid=topic.uid, time=topic.time), nonregistered_users=nonregistered_users)) else: main_res.update(dict(success=True, topic=dict(uid=topic.uid, time=topic.time, creator_id=topic.creator_id, update_time=topic.time, title=topic.title, logourl=topic.logourl))) return main_res except Exception as e: log.exception("Got exception: %s"%e) return dict(success=False)
from boto.sns import SNSConnection #create connection conn = SNSConnection() topic_name = 'test-topic' #create new SNS topic # conn.create_topic('test-topic') # ARN Number of topic arn_num = 'arn:aws:sns:us-east-1:926344641371:'+topic_name # get attr for the ARN topic # topic_attr = conn.get_topic_attributes(arn_num) #subscribe to a topic protocol = 'email' endpoint = '*****@*****.**' #conn.subscribe(arn_num,protocol,endpoint) #publish to a topic message = "This is a test message. <br> -Kiran" subject = "Test SNS subscription" message_structure = 'html' conn.publish(message=message, subject=subject, target_arn=arn_num, message_structure=message_structure)
def create_subject(self, **kw): try: #check if this user can create a subject in this topic creator = DBSession.query(MemberTopic).\ filter(MemberTopic.topic_id==kw['topic_id'], MemberTopic.member_id==request.identity['user'].user_id, 'role = "c" or role = "s"').\ first() if not creator: #TODO: this should never happen, log this event and return only False return dict(success=False, error_code=ErrorCode.UNAUTHORIZED) #add new subject to the database subject = Subject() subject.topic_id = kw['topic_id'] subject.title = kw['title'] subject.creator_id = request.identity['user'].user_id DBSession.add(subject) DBSession.flush() comment = None if kw.has_key('content'): comment = Comment() comment.subject_id = subject.uid comment.creator_id = request.identity['user'].user_id comment.content = kw['content'] DBSession.add(comment) DBSession.flush() #add creator to member_subject if comment: member_subject = MemberSubject(member_id = creator.member_id, subject_id = subject.uid, last_read = comment.uid) else: member_subject = MemberSubject(member_id = creator.member_id, subject_id = subject.uid, last_read = 0) subject.members.append(member_subject) # registered_users = DBSession.query(User.email_address).filter(User.email_address.in_(members)).all() members = DBSession.query(MemberTopic.member_id,User.email_address).\ filter(MemberTopic.topic_id==kw['topic_id'], MemberTopic.member_id!=creator.member_id, MemberTopic.deleted == False).\ join(User).\ all() #add other members to member_subject table member_ids = [member[0] for member in members] for member_id in member_ids: member_subject = MemberSubject(member_id = member_id, subject_id = subject.uid, last_read = 0) subject.members.append(member_subject) topic = DBSession.query(Topic).\ filter(Topic.uid==kw['topic_id']).first() #TODO: need to rename/rethink this time vars topic.update_time = subject.time log.info("User %s creates subject %s"%(creator.member_id, subject)) member_emails = [member[1] for member in members] #send notification to users if len(member_emails)>0: notif=dict(type='new_subject',topic=topic.title,subject=subject.title, registered_users=member_emails,user_name=request.identity['user'].user_name) log.info('Sending SNS notification: %s'%notif) sns=SNSConnection(config.get('AWS_ACCESS_KEY_ID'), config.get("AWS_SECRET_ACCESS_KEY")) sns.publish(config.get('sqs_user_notification'),js.dumps(notif)) return dict(success=True, subject=dict(uid=subject.uid, time=subject.time)) except Exception as e: log.exception('Got exception %s'%e) #traceback.print_exc(file=sys.stdout) return dict(success=False)