def _first_init(self, **kwargs): # If the coordinates were supplied explicitly, they may need to be deserialized. if isinstance(kwargs.get("coordinates"), OrderedDict): kwargs["coordinates"] = Coordinates.from_definition( kwargs["coordinates"]) elif isinstance(kwargs.get("coordinates"), string_types): kwargs["coordinates"] = Coordinates.from_json( kwargs["coordinates"]) return kwargs
def _first_init(self, **kwargs): if 'reprojected_coordinates' in kwargs: if isinstance(kwargs['reprojected_coordinates'], list): kwargs[ 'reprojected_coordinates'] = Coordinates.from_definition( kwargs['reprojected_coordinates']) elif isinstance(kwargs['reprojected_coordinates'], str): kwargs['reprojected_coordinates'] = Coordinates.from_json( kwargs['reprojected_coordinates']) return kwargs
def _first_init(self, **kwargs): if "reprojected_coordinates" in kwargs: if isinstance(kwargs["reprojected_coordinates"], dict): kwargs[ "reprojected_coordinates"] = Coordinates.from_definition( kwargs["reprojected_coordinates"]) elif isinstance(kwargs["reprojected_coordinates"], string_types): kwargs["reprojected_coordinates"] = Coordinates.from_json( kwargs["reprojected_coordinates"]) return super(ReprojectedSource, self)._first_init(**kwargs)
def _f(definition, coords, q, outputkw): try: n = Node.from_json(definition) c = Coordinates.from_json(coords) o = n.eval(c) o._pp_serialize() _log.debug("o.shape: {}, output_format: {}".format(o.shape, outputkw)) if outputkw: _log.debug( "Saving output results to output format {}".format(outputkw)) o = o.to_format(outputkw["format"], **outputkw.get("format_kwargs")) q.put(o) except Exception as e: q.put(str(e))
def _first_init(self, **kwargs): warnings.warn( "ReprojectedSource has been replaced by the Reproject algorithm node " "and will be removed in a future version of podpac.", DeprecationWarning, ) if "reprojected_coordinates" in kwargs: if isinstance(kwargs["reprojected_coordinates"], dict): kwargs[ "reprojected_coordinates"] = Coordinates.from_definition( kwargs["reprojected_coordinates"]) elif isinstance(kwargs["reprojected_coordinates"], string_types): kwargs["reprojected_coordinates"] = Coordinates.from_json( kwargs["reprojected_coordinates"]) return super(ReprojectedSource, self)._first_init(**kwargs)
def handler(event, context, get_deps=True, ret_pipeline=False): bucket_name = event['Records'][0]['s3']['bucket']['name'] if get_deps: s3.download_file(bucket_name, 'podpac/updated/' + deps, '/tmp/' + deps) subprocess.call(['unzip', '/tmp/' + deps, '-d', '/tmp']) sys.path.append('/tmp/') subprocess.call(['rm', '/tmp/' + deps]) file_key = urllib.unquote_plus( event['Records'][0]['s3']['object']['key']) _json = '' # get the object obj = s3.get_object(Bucket=bucket_name, Key=file_key) # get lines lines = obj['Body'].read().split(b'\n') for r in lines: if len(_json) > 0: _json += '\n' _json += r.decode() _json = json.loads( _json, object_pairs_hook=OrderedDict) pipeline_json = _json['pipeline'] # Need to set matplotlib backend to 'Agg' before importing it elsewhere import matplotlib matplotlib.use('agg') from podpac import settings from podpac.core.pipeline import Pipeline from podpac.core.coordinates import Coordinates from podpac.core.utils import JSONEncoder pipeline = Pipeline(definition=pipeline_json, do_write_output=False) coords = Coordinates.from_json( json.dumps(_json['coordinates'], indent=4, cls=JSONEncoder)) pipeline.eval(coords) if ret_pipeline: return pipeline filename = file_key.replace('.json', '.' + pipeline.output.format) filename = filename.replace(settings['S3_JSON_FOLDER'], settings['S3_OUTPUT_FOLDER']) body = cPickle.dumps(pipeline._output) s3.put_object(Bucket=bucket_name, Key=filename, Body=body) return
def handler(event, context): """Lambda function handler Parameters ---------- event : dict Description context : TYPE Description get_deps : bool, optional Description ret_pipeline : bool, optional Description """ print (event) # Add /tmp/ path to handle python path for dependencies sys.path.append("/tmp/") # handle triggers trigger = get_trigger(event) # parse event pipeline = parse_event(trigger, event) # bail if we can't parse if pipeline is None: return # ----- # TODO: remove when layers is configured # get configured bucket to download dependencies # If specified in the environmental variables, we cannot overwrite it. Otherwise it HAS to be # specified in the settings. bucket = os.environ.get("S3_BUCKET_NAME", pipeline["settings"].get("S3_BUCKET_NAME")) # get dependencies path if "FUNCTION_DEPENDENCIES_KEY" in pipeline["settings"] or "FUNCTION_DEPENDENCIES_KEY" in os.environ: dependencies = os.environ.get( "FUNCTION_DEPENDENCIES_KEY", pipeline["settings"].get("FUNCTION_DEPENDENCIES_KEY") ) else: dependencies = "podpac_deps_{}.zip".format( os.environ.get("PODPAC_VERSION", pipeline["settings"].get("PODPAC_VERSION")) ) if 'None' in dependencies: dependencies = 'podpac_deps.zip' # Development version of podpac # this should be equivalent to version.semver() # Check to see if this function is "hot", in which case the dependencies have already been downloaded and are # available for use right away. if os.path.exists("/tmp/scipy"): print ( "Scipy has been detected in the /tmp/ directory. Assuming this function is hot, dependencies will" " not be downloaded." ) else: # Download dependencies from specific bucket/object print ("Downloading and extracting dependencies from {} {}".format(bucket, dependencies)) s3 = boto3.client("s3") s3.download_file(bucket, dependencies, "/tmp/" + dependencies) subprocess.call(["unzip", "/tmp/" + dependencies, "-d", "/tmp"]) sys.path.append("/tmp/") subprocess.call(["rm", "/tmp/" + dependencies]) # ----- # Load PODPAC # Need to set matplotlib backend to 'Agg' before importing it elsewhere import matplotlib matplotlib.use("agg") from podpac import settings from podpac.core.node import Node from podpac.core.coordinates import Coordinates from podpac.core.utils import JSONEncoder, _get_query_params_from_url import podpac.datalib # update podpac settings with inputs from the trigger settings.update(json.loads(os.environ.get("SETTINGS", "{}"))) settings.update(pipeline["settings"]) # build the Node and Coordinates if trigger in ("eval", "S3"): node = Node.from_definition(pipeline["pipeline"]) coords = Coordinates.from_json(json.dumps(pipeline["coordinates"], indent=4, cls=JSONEncoder)) # TODO: handle API Gateway better - is this always going to be WCS? elif trigger == "APIGateway": node = Node.from_url(pipeline["url"]) coords = Coordinates.from_url(pipeline["url"]) # make sure pipeline is allowed to be run if "PODPAC_RESTRICT_PIPELINES" in os.environ: whitelist = json.loads(os.environ["PODPAC_RESTRICT_PIPELINES"]) if node.hash not in whitelist: raise ValueError("Node hash is not in the whitelist for this function") # run analysis output = node.eval(coords) # convert to output format body = output.to_format(pipeline["output"]["format"], **pipeline["output"]["format_kwargs"]) # Response if trigger == "eval": return body elif trigger == "S3": s3.put_object(Bucket=settings["S3_BUCKET_NAME"], Key=pipeline["output"]["filename"], Body=body) elif trigger == "APIGateway": # TODO: can we handle the deserialization better? try: json.dumps(body) except Exception as e: print ("Output body is not serializable, attempting to decode.") body = body.decode() return { "statusCode": 200, "headers": {"Content-Type": pipeline["output"]["format"]}, "isBase64Encoded": pipeline["output"]["format_kwargs"]["return_base64"], "body": body, }