Exemplo n.º 1
0
def processLambda(config_fname,
                  profile,
                  noWrap=False,
                  update=False,
                  deleteThem=False,
                  noBotocore=False,
                  spotTableRegion='us-west-2',
                  spotTableName='gammaRays',
                  saveBucket=False,
                  tracing=False,
                  useGammaRay=False,
                  useFleece=False):
    # Config
    try:
        config = json.loads(open(config_fname, 'r').read())
    except:
        print(
            'Error loading json file, please fix formatting problems (commas?), and retry...({})'
            .format(config_fname))
        sys.exit(1)
    region = config['region']
    fns = config['functions']

    if profile:
        boto3.setup_default_session(profile_name=profile)
    config = botocore.client.Config(connect_timeout=50, read_timeout=100)
    lambda_client = boto3.client('lambda', config=config, region_name=region)
    s3 = None
    s3_client = None
    if not noBotocore:
        s3 = boto3.resource('s3')
        config = botocore.client.Config(connect_timeout=50, read_timeout=200)
    s3_client = boto3.client('s3', config=config,
                             region_name=region)  #used to set trigger
    gammawraptemplate = 'GammaRay.py.template'
    gammawrapfile = 'GammaRay.py'
    spotwraptemplate = 'SpotWrap.py.template'
    spotwrapfile = 'SpotWrap.py'

    #Create the lambda functions
    for fn in fns:
        name = fn['name']
        if deleteThem:  #don't do anything more if we are deleting
            try:
                l_fn = lambdautils.LambdaManager(lambda_client, region, None,
                                                 name, None)
                l_fn.delete_function()
                print('setupApps: {} deleted'.format(name))
            except Exception as e:
                stre = str(e)
                if 'ResourceNotFoundException' not in stre:
                    print('EXCEPTION in lambda delete: {}'.format(e))
            try:
                if not saveBucket and 'permission' in fn:
                    bkt = fn['permission']
                    print('setupApps: bucket contents for {} deleted'.format(
                        bkt))
                    lambdautils.LambdaManager.deleteBucketContents(s3, bkt)
            except:
                pass
            try:
                lambdautils.LambdaManager.cleanup_logs(name)
            except:
                pass
            continue  #get the next function to delete

        #else process the rest and create the Lambdas
        lambda_memory = fn['lambdaMemory']
        ziplist = fn['files_and_dirs']
        zipfile = fn['zip']
        handler = fn['handler']  #must be of the form filename.handlername
        periods = handler.count('.')
        peridx = handler.find('.')
        if periods != 1 or peridx == 0:
            print(
                'Error, the handler entry for {} must be of the form filename.handlername.  Please fix and rerun.'
                .format(name))
            sys.exit(1)

        tmp_dir = None
        if useGammaRay:  #inject GammaWrap support (produce functions with and without GammaWrap support
            if not os.path.isfile(gammawraptemplate):
                print(
                    'Error, {} Not Found!  To inject GammaWrap support, rerun this program in the same directory as GammaWrap.py. Not injecting GammaWrap support...'
                    .format(gammawraptemplate))
                sys.exit(1)

            else:  #inject GammaWrap support (produce functions with GammaWrap support)
                tmp_dir = tempfile.mkdtemp()
                target = '{}/{}'.format(tmp_dir, gammawrapfile)
                print('target: {}'.format(target))
                shutil.copyfile(gammawraptemplate,
                                target)  #copy the template into temp dir
                ziplist.append(
                    target
                )  #add temp dir and file to the zip list so we include it

                #next update the template to call the user's handler
                orig_file = handler[:
                                    peridx]  #filename containing original handler
                orig_handler = handler
                handler = 'GammaRay.handleRequest'
                print('orig: {}:{} -- {}'.format(orig_file, orig_handler,
                                                 handler))

                # Read in the file, replace the string, and write it out
                with open(target, 'r') as f:
                    filedata = f.read()
                filedata = filedata.replace('import GammaRayTemplate',
                                            'import {}'.format(orig_file))
                filedata = filedata.replace('GammaRayTemplate.handler',
                                            orig_handler)
                filedata = filedata.replace('ZZZZ', spotTableRegion)
                filedata = filedata.replace('QQQQ', spotTableName)
                # Write the file out
                with open(target, 'w') as f:
                    f.write(filedata)

                print('setupApps: GammaRay support inserted')

        if useFleece:  #inject Fleece
            tmp_dir = tempfile.mkdtemp()
            fname2find = '{}.py'.format(
                handler[:peridx])  #filename containing original handler
            found = False
            for fnameInOriginalLoc in ziplist:
                if fnameInOriginalLoc.endswith(fname2find):
                    found = True
                    break
            if not found:
                print('Error, {} not found in ziplist {}'.format(
                    fname2find, ziplist))
                sys.exit(1)
            ziplist.remove(fnameInOriginalLoc)  #remove from ziplist
            target = '{}/{}'.format(tmp_dir,
                                    fname2find)  #same file only in tmp_dir
            ziplist.append(target)  #add file in new location to ziplist
            print('target: {}'.format(target))

            #next update the code in target to include fleece
            with open(fnameInOriginalLoc, 'r') as f:
                filedata = f.read()
            # Write the file out
            with open(target, 'w') as f:
                f.write(
                    'from fleece.xray import (monkey_patch_botocore_for_xray, monkey_patch_requests_for_xray, trace_xray_subsegment)\n'
                )
                f.write('monkey_patch_botocore_for_xray()\n')
                f.write('monkey_patch_requests_for_xray()\n')
                f.write(filedata)

            print('setupApps: Fleece support inserted')

        elif not noWrap:  #do not useGammaRay, no not use Fleece, so use spotwrap (not noWrap) or nothing:
            ''' Process the patched botocore file (place in S3 for SpotWrap.py to download)'''
            botozipdir = None
            zipbase = 'botocore_patched.zip'
            zipname = '/tmp/{}'.format(
                zipbase)  #this much match same in SpotWrap.py.template

            if 'patched_botocore_dir' not in fn or fn[
                    'patched_botocore_dir'] == '':  #no patch dir specified
                print(
                    'patched_botocore_dir not set in configuration file. To inject SpotWrap support, set this value and rerun this program. Not injecting SpotWrap support...'
                )
                noWrap = True
            else:
                if 'patched_botocore_dir' in fn:
                    botozipdir = fn['patched_botocore_dir']
                    if not os.path.isdir(botozipdir):
                        noWrap = True

            s3bkt = 'spotbucket'
            if 's3bucket' not in fn or fn[
                    's3bucket'] == '':  #double check that this exists
                print(
                    'No s3 bucket name (s3bucket) found in configuration file. Not injecting SpotWrap support...'
                )
                noWrap = True
            else:  #s3bucket is set, check that it is a valid S3 bucket
                s3bkt = fn['s3bucket']
                if not noBotocore:
                    noBotocore = True
                    if lambdautils.LambdaManager.S3BktExists(
                            s3, s3bkt, region):
                        #zip up the patched botocore directory and place in S3
                        if not botozipdir:  #sanity check
                            print(
                                'setupApps Error:  botozipdir is None unexpectedly here'
                            )
                            sys.exit(1)
                        here = os.path.abspath(os.path.dirname(__file__))
                        dirname = os.path.dirname(botozipdir)
                        fname = os.path.basename(botozipdir)
                        if dirname != '':
                            os.chdir(
                                dirname)  #go to dir to place file at top level
                        if os.path.exists(zipname):  #remove the zip file first
                            os.remove(zipname)
                        subprocess.call([
                            'zip', '-9', '-u', '-r', '--exclude=*.pyc',
                            zipname, fname
                        ])
                        try:
                            lambdautils.LambdaManager.copyToS3(
                                s3, s3bkt, zipname)
                        except Exception as e:
                            print(
                                'setupApps: S3 error on zip file storage:\n{}'.
                                format(e))
                            sys.exit(1)
                        os.chdir(here)
            ''' Inject SpotWrap Support '''
            #first check that SpotWrap.py is in the current working directory
            if not os.path.isfile(spotwraptemplate):
                print(
                    'Error, {} Not Found!  To inject SpotWrap support, rerun this program in the same directory as SpotWrap.py. Not injecting SpotWrap support...'
                    .format(spotwraptemplate))
            else:  #inject SpotWrap support
                #first remove SpotWrap.py so we don't confuse things, if added by mistake
                todel = []
                for fname in ziplist:
                    #if fname.endswith(spotwrapfile) or fname.endswith('botocore'):
                    if fname.find('boto') != -1:
                        print(
                            'Found boto in the list, AWS Lambda installs boto for you, please remove to reduce the size of your zip, and rerun: {}'
                            .format(fname))
                        sys.exit(1)
                    if fname.endswith(spotwrapfile):
                        todel.append(fname)  #can't remove while iterating...
                for fname in todel:
                    ziplist.remove(fname)
                tmp_dir = tempfile.mkdtemp()
                target = '{}/{}'.format(tmp_dir, spotwrapfile)
                shutil.copyfile(spotwraptemplate,
                                target)  #copy the template into temp dir
                ziplist.append(
                    target
                )  #add temp dir and file to the zip list so we include it

                #next update the template to call the user's handler
                orig_file = handler[:
                                    peridx]  #filename containing original handler
                orig_handler = handler
                handler = 'SpotWrap.handleRequest'
                # Read in the file, replace the string, and write it out
                with open(target, 'r') as f:
                    filedata = f.read()
                filedata = filedata.replace('import SpotTemplate',
                                            'import {}'.format(orig_file))
                filedata = filedata.replace('SpotTemplate.handler',
                                            orig_handler)
                filedata = filedata.replace('XXXX', s3bkt)
                filedata = filedata.replace('YYYY', zipbase)
                filedata = filedata.replace('ZZZZ', spotTableRegion)
                filedata = filedata.replace('QQQQ', spotTableName)
                # Write the file out
                with open(target, 'w') as f:
                    f.write(filedata)

                print('setupApps: SpotWrap support inserted')

        else:  #no GammaWrap and not noSpotWrap and no Fleece (clean deploy)
            print(
                'setupApps: no GammaWrap and no Fleece and no SpotWrap support inserted'
            )

        l_zip = zipLambda(zipfile, ziplist, update)
        if tmp_dir:
            shutil.rmtree(tmp_dir)

        print('handler: {}'.format(handler))
        #create lambdas
        l_fn = lambdautils.LambdaManager(lambda_client, region, l_zip, name,
                                         handler, tracing, lambda_memory)
        l_fn.update_code_or_create_on_noexist()
        if 'permission' in fn:
            job_bucket = fn['permission']
            if 'reducerCoordinator' in name:  #for map-reduce only
                job_id = '{}/{}'.format(fn['job_id'], 'task')
            else:
                job_id = fn['job_id']
            print(
                'adding permission and notification to {} for JOBID {}'.format(
                    job_bucket, job_id))
            l_fn.add_lambda_permission(random.randint(1, 1000), job_bucket)
            l_fn.create_s3_eventsource_notification(s3_client, job_bucket,
                                                    job_id)
        if 'bucket_listener' in fn:
            job_bucket = fn['bucket_listener']
            job_id = '{}/{}'.format(fn['job_id'], 'result')
            l_fn.add_lambda_permission(random.randint(1, 1000), job_bucket)
            l_fn.create_s3_eventsource_notification(s3_client, job_bucket,
                                                    job_id)
Exemplo n.º 2
0
# write job config
write_job_config(job_id, job_bucket, n_mappers, reducer_lambda_name,
                 config["reducer"]["handler"])

zipLambda(config["mapper"]["name"], config["mapper"]["zip"])
zipLambda(config["reducer"]["name"], config["reducer"]["zip"])
zipLambda(config["reducerCoordinator"]["name"],
          config["reducerCoordinator"]["zip"])
xray_recorder.end_subsegment()  #Prepare Lambda functions

# mapper
xray_recorder.begin_subsegment('Create mapper Lambda function')
l_mapper = lambdautils.LambdaManager(lambda_client, s3_client, region,
                                     config["mapper"]["zip"], job_id,
                                     mapper_lambda_name,
                                     config["mapper"]["handler"],
                                     config["mapper"]["layers"])
l_mapper.update_code_or_create_on_noexist()
xray_recorder.end_subsegment()  #Create mapper Lambda function

# Reducer func
xray_recorder.begin_subsegment('Create reducer Lambda function')
l_reducer = lambdautils.LambdaManager(lambda_client, s3_client, region,
                                      config["reducer"]["zip"], job_id,
                                      reducer_lambda_name,
                                      config["reducer"]["handler"],
                                      config["reducer"]["layers"])
l_reducer.update_code_or_create_on_noexist()
xray_recorder.end_subsegment()  #Create reducer Lambda function
Exemplo n.º 3
0
rc_lambda_name = L_PREFIX + "-rc-" + job_id

# write job config
write_job_config(job_id, job_bucket, n_mappers, reducer_lambda_name,
                 reducer_config["handler"], lambda_memory, concurrent_lambdas)

zipLambda(mapper_config["name"], mapper_config["zip"])
zipLambda(reducer_config["name"], reducer_config["zip"])
zipLambda(reducerCoordinator_config["name"], reducerCoordinator_config["zip"])
xray_recorder.end_subsegment()  #Prepare Lambda functions

# mapper
xray_recorder.begin_subsegment('Create mapper Lambda function')
l_mapper = lambdautils.LambdaManager(lambda_client, s3_client, region,
                                     mapper_config["zip"], job_id,
                                     mapper_lambda_name,
                                     mapper_config["handler"],
                                     lambda_execution_role, lambda_memory)
l_mapper.update_code_or_create_on_noexist()
xray_recorder.end_subsegment()  #Create mapper Lambda function

# Reducer func
xray_recorder.begin_subsegment('Create reducer Lambda function')
l_reducer = lambdautils.LambdaManager(lambda_client, s3_client, region,
                                      reducer_config["zip"], job_id,
                                      reducer_lambda_name,
                                      reducer_config["handler"],
                                      lambda_execution_role, lambda_memory)
l_reducer.update_code_or_create_on_noexist()
xray_recorder.end_subsegment()  #Create reducer Lambda function