Exemple #1
0
 def test_s3_filter(self):
     t = Template()
     t.add_resource(
         Function(
             "ProcessorFunction",
             Handler='process_file.handler',
             CodeUri='.',
             Runtime='python3.6',
             Policies='AmazonS3FullAccess',
             Events={
                 'FileUpload': S3Event(
                     'FileUpload',
                     Bucket="bucket",
                     Events=['s3:ObjectCreated:*'],
                     Filter=Filter(S3Key=S3Key(
                         Rules=[
                             Rules(Name="prefix", Value="upload/"),
                             Rules(Name="suffix", Value=".txt"),
                         ],
                     ))
                 )
             }
         )
     )
     t.to_json()
    def add_submit_lambda(self):
        ## We will make event triggers for all affiliates.
        all_affiliates = self.config["UXData"]["Affiliates"]
        ## Make Rule sets for each affiliate:
        all_events = {}
        for affiliate in all_affiliates:
            ## Get necessary properties:
            affiliatename = affiliate["AffiliateName"]
            ## If user input, reads directly from input directory. If other function output, reads from output directory.
            assert type(affiliate["UserInput"]
                        ) == bool, "must provide a json boolean for UserInput"
            if affiliate["UserInput"] == True:
                readdir = self.config['Lambda']['LambdaConfig']['SUBMITDIR']
            elif affiliate["UserInput"] == False:
                readdir = self.config['Lambda']['LambdaConfig']['OUTDIR']

            aff_filter = Filter(
                'Filter' + affiliatename,
                S3Key=S3Key('S3Key' + affiliatename,
                            Rules=[
                                Rules('PrefixRule' + affiliatename,
                                      Name='prefix',
                                      Value=affiliatename + '/' + readdir),
                                Rules('SuffixRule' + affiliatename,
                                      Name='suffix',
                                      Value='submit.json')
                            ]))
            event_name = 'BucketEvent' + affiliatename
            all_events[event_name] = {
                'Type': 'S3',
                'Properties': {
                    'Bucket': Ref('PipelineMainBucket'),
                    'Events': ['s3:ObjectCreated:*'],
                    'Filter': aff_filter
                }
            }
        ## We're going to add in all of the lambda configuration items to the runtime environment.
        lambdaconfig = self.config['Lambda']['LambdaConfig']
        #lambdaconfig ={}
        ### Most of the config can be done through the config file, but we will pass certain elements from the template.
        lambdaconfig['figlambid'] = Ref(self.figurelamb)
        lambdaconfig['figlambarn'] = GetAtt(self.figurelamb, 'Arn')
        lambdaconfig['cwrolearn'] = GetAtt(self.cwrole, 'Arn')
        ## Now add to a lambda function:
        function = Function(
            'MainLambda',
            CodeUri='../../protocols',
            Runtime='python3.6',
            Handler='submit_start.handler',
            Description='Main Lambda Function for Serverless',
            MemorySize=128,
            Timeout=self.config["Lambda"]['LambdaConfig']["EXECUTION_TIMEOUT"],
            Role=
            'arn:aws:iam::739988523141:role/lambda_dataflow',  ## TODO: Create this in template
            Events=all_events,
            #Environment = Environment(Variables={'figlambid':Ref(self.figurelamb),'figlambarn':GetAtt(self.figurelamb,'Arn'),'cwrolearn':GetAtt(self.cwrole,'Arn')})
            Environment=Environment(Variables=lambdaconfig))
        self.template.add_resource(function)
Exemple #3
0
 def test_s3_filter(self):
     t = Template()
     t.add_resource(
         Function(
             "ProcessorFunction",
             Handler="process_file.handler",
             CodeUri=".",
             Runtime="python3.6",
             Policies="AmazonS3FullAccess",
             Events={
                 "FileUpload":
                 S3Event(
                     "FileUpload",
                     Bucket="bucket",
                     Events=["s3:ObjectCreated:*"],
                     Filter=Filter(S3Key=S3Key(Rules=[
                         Rules(Name="prefix", Value="upload/"),
                         Rules(Name="suffix", Value=".txt"),
                     ], )),
                 )
             },
         ))
     t.to_json()
            )
        ]),
        LifecycleConfiguration=LifecycleConfiguration(Rules=[
            LifecycleRule(
                Id='DeleteUploadsAfterOneDay',
                Status='Enabled',
                ExpirationInDays=1,
                Prefix='upload/',
            )
        ], ),
        NotificationConfiguration=NotificationConfiguration(
            TopicConfigurations=[
                TopicConfigurations(Event='s3:ObjectCreated:*',
                                    Filter=Filter(S3Key=S3Key(Rules=[
                                        Rules(
                                            Name='prefix',
                                            Value='upload/',
                                        )
                                    ], ), ),
                                    Topic=Ref(upload_topic))
            ], ),
    ))

video_bucket = template.add_resource(
    Bucket(
        'VideoBucket',
        CorsConfiguration=CorsConfiguration(CorsRules=[
            CorsRules(
                AllowedOrigins=['*'],
                AllowedMethods=['GET', 'HEAD'],
                AllowedHeaders=['*'],
            )
              Type="String",
              Default="gziptosnappy"))
"""
Create S3 Bucket With a Notifications Configuration.
The bucket notifies the lambda function in case there is a PUT event.
"""

S3Bucket = t.add_resource(
    Bucket("S3Bucket",
           BucketName=Ref(inputBucketName),
           NotificationConfiguration=NotificationConfiguration(
               LambdaConfigurations=[
                   LambdaConfigurations(Event="s3:ObjectCreated:*",
                                        Filter=Filter(S3Key=S3Key(Rules=[
                                            Rules(Name="prefix",
                                                  Value=Ref(inputKeyPrefix)),
                                            Rules(Name="suffix", Value=".gz")
                                        ])),
                                        Function=Ref(functionName))
               ])))

BucketPermission = t.add_resource(
    Permission("BucketPermission",
               Action="lambda:InvokeFunction",
               FunctionName=Ref(functionName),
               Principal="s3.amazonaws.com",
               SourceAccount=Ref("AWS::AccountId"),
               SourceArn=Join("",
                              ["aws:arn:s3:::", Ref(S3Bucket)])))
"""
Create Lambda Function