def _first_init(self, **kwargs):
# If the coordinates were supplied explicitly, they may need to be deserialized.
if isinstance(kwargs.get("coordinates"), OrderedDict):
kwargs["coordinates"] = Coordinates.from_definition(
kwargs["coordinates"])
elif isinstance(kwargs.get("coordinates"), string_types):
kwargs["coordinates"] = Coordinates.from_json(
kwargs["coordinates"])
return kwargs
def _first_init(self, **kwargs):
if 'reprojected_coordinates' in kwargs:
if isinstance(kwargs['reprojected_coordinates'], list):
kwargs[
'reprojected_coordinates'] = Coordinates.from_definition(
kwargs['reprojected_coordinates'])
elif isinstance(kwargs['reprojected_coordinates'], str):
kwargs['reprojected_coordinates'] = Coordinates.from_json(
kwargs['reprojected_coordinates'])
return kwargs
def _first_init(self, **kwargs):
if "reprojected_coordinates" in kwargs:
if isinstance(kwargs["reprojected_coordinates"], dict):
kwargs[
"reprojected_coordinates"] = Coordinates.from_definition(
kwargs["reprojected_coordinates"])
elif isinstance(kwargs["reprojected_coordinates"], string_types):
kwargs["reprojected_coordinates"] = Coordinates.from_json(
kwargs["reprojected_coordinates"])
return super(ReprojectedSource, self)._first_init(**kwargs)
def _f(definition, coords, q, outputkw):
try:
n = Node.from_json(definition)
c = Coordinates.from_json(coords)
o = n.eval(c)
o._pp_serialize()
_log.debug("o.shape: {}, output_format: {}".format(o.shape, outputkw))
if outputkw:
_log.debug(
"Saving output results to output format {}".format(outputkw))
o = o.to_format(outputkw["format"],
**outputkw.get("format_kwargs"))
q.put(o)
except Exception as e:
q.put(str(e))
def _first_init(self, **kwargs):
warnings.warn(
"ReprojectedSource has been replaced by the Reproject algorithm node "
"and will be removed in a future version of podpac.",
DeprecationWarning,
)
if "reprojected_coordinates" in kwargs:
if isinstance(kwargs["reprojected_coordinates"], dict):
kwargs[
"reprojected_coordinates"] = Coordinates.from_definition(
kwargs["reprojected_coordinates"])
elif isinstance(kwargs["reprojected_coordinates"], string_types):
kwargs["reprojected_coordinates"] = Coordinates.from_json(
kwargs["reprojected_coordinates"])
return super(ReprojectedSource, self)._first_init(**kwargs)
def handler(event, context, get_deps=True, ret_pipeline=False):
bucket_name = event['Records'][0]['s3']['bucket']['name']
if get_deps:
s3.download_file(bucket_name, 'podpac/updated/' + deps, '/tmp/' + deps)
subprocess.call(['unzip', '/tmp/' + deps, '-d', '/tmp'])
sys.path.append('/tmp/')
subprocess.call(['rm', '/tmp/' + deps])
file_key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'])
_json = ''
# get the object
obj = s3.get_object(Bucket=bucket_name, Key=file_key)
# get lines
lines = obj['Body'].read().split(b'\n')
for r in lines:
if len(_json) > 0:
_json += '\n'
_json += r.decode()
_json = json.loads(
_json, object_pairs_hook=OrderedDict)
pipeline_json = _json['pipeline']
# Need to set matplotlib backend to 'Agg' before importing it elsewhere
import matplotlib
matplotlib.use('agg')
from podpac import settings
from podpac.core.pipeline import Pipeline
from podpac.core.coordinates import Coordinates
from podpac.core.utils import JSONEncoder
pipeline = Pipeline(definition=pipeline_json, do_write_output=False)
coords = Coordinates.from_json(
json.dumps(_json['coordinates'], indent=4, cls=JSONEncoder))
pipeline.eval(coords)
if ret_pipeline:
return pipeline
filename = file_key.replace('.json', '.' + pipeline.output.format)
filename = filename.replace(settings['S3_JSON_FOLDER'], settings['S3_OUTPUT_FOLDER'])
body = cPickle.dumps(pipeline._output)
s3.put_object(Bucket=bucket_name,
Key=filename, Body=body)
return
def handler(event, context):
"""Lambda function handler
Parameters
----------
event : dict
Description
context : TYPE
Description
get_deps : bool, optional
Description
ret_pipeline : bool, optional
Description
"""
print (event)
# Add /tmp/ path to handle python path for dependencies
sys.path.append("/tmp/")
# handle triggers
trigger = get_trigger(event)
# parse event
pipeline = parse_event(trigger, event)
# bail if we can't parse
if pipeline is None:
return
# -----
# TODO: remove when layers is configured
# get configured bucket to download dependencies
# If specified in the environmental variables, we cannot overwrite it. Otherwise it HAS to be
# specified in the settings.
bucket = os.environ.get("S3_BUCKET_NAME", pipeline["settings"].get("S3_BUCKET_NAME"))
# get dependencies path
if "FUNCTION_DEPENDENCIES_KEY" in pipeline["settings"] or "FUNCTION_DEPENDENCIES_KEY" in os.environ:
dependencies = os.environ.get(
"FUNCTION_DEPENDENCIES_KEY", pipeline["settings"].get("FUNCTION_DEPENDENCIES_KEY")
)
else:
dependencies = "podpac_deps_{}.zip".format(
os.environ.get("PODPAC_VERSION", pipeline["settings"].get("PODPAC_VERSION"))
)
if 'None' in dependencies:
dependencies = 'podpac_deps.zip' # Development version of podpac
# this should be equivalent to version.semver()
# Check to see if this function is "hot", in which case the dependencies have already been downloaded and are
# available for use right away.
if os.path.exists("/tmp/scipy"):
print (
"Scipy has been detected in the /tmp/ directory. Assuming this function is hot, dependencies will"
" not be downloaded."
)
else:
# Download dependencies from specific bucket/object
print ("Downloading and extracting dependencies from {} {}".format(bucket, dependencies))
s3 = boto3.client("s3")
s3.download_file(bucket, dependencies, "/tmp/" + dependencies)
subprocess.call(["unzip", "/tmp/" + dependencies, "-d", "/tmp"])
sys.path.append("/tmp/")
subprocess.call(["rm", "/tmp/" + dependencies])
# -----
# Load PODPAC
# Need to set matplotlib backend to 'Agg' before importing it elsewhere
import matplotlib
matplotlib.use("agg")
from podpac import settings
from podpac.core.node import Node
from podpac.core.coordinates import Coordinates
from podpac.core.utils import JSONEncoder, _get_query_params_from_url
import podpac.datalib
# update podpac settings with inputs from the trigger
settings.update(json.loads(os.environ.get("SETTINGS", "{}")))
settings.update(pipeline["settings"])
# build the Node and Coordinates
if trigger in ("eval", "S3"):
node = Node.from_definition(pipeline["pipeline"])
coords = Coordinates.from_json(json.dumps(pipeline["coordinates"], indent=4, cls=JSONEncoder))
# TODO: handle API Gateway better - is this always going to be WCS?
elif trigger == "APIGateway":
node = Node.from_url(pipeline["url"])
coords = Coordinates.from_url(pipeline["url"])
# make sure pipeline is allowed to be run
if "PODPAC_RESTRICT_PIPELINES" in os.environ:
whitelist = json.loads(os.environ["PODPAC_RESTRICT_PIPELINES"])
if node.hash not in whitelist:
raise ValueError("Node hash is not in the whitelist for this function")
# run analysis
output = node.eval(coords)
# convert to output format
body = output.to_format(pipeline["output"]["format"], **pipeline["output"]["format_kwargs"])
# Response
if trigger == "eval":
return body
elif trigger == "S3":
s3.put_object(Bucket=settings["S3_BUCKET_NAME"], Key=pipeline["output"]["filename"], Body=body)
elif trigger == "APIGateway":
# TODO: can we handle the deserialization better?
try:
json.dumps(body)
except Exception as e:
print ("Output body is not serializable, attempting to decode.")
body = body.decode()
return {
"statusCode": 200,
"headers": {"Content-Type": pipeline["output"]["format"]},
"isBase64Encoded": pipeline["output"]["format_kwargs"]["return_base64"],
"body": body,
}
