def eval_source(self, coordinates, coordinates_index, out, i, source=None): if source is None: source = self.source # Make a copy to prevent any possibility of memory corruption source = Node.from_definition(source.definition) success = False o = None while not success: if self.check_worker_available(): try: o = source.eval(coordinates, output=out) success = True except self.async_exception: # This exception is fine and constitutes a success o = None success = True except self.no_worker_exception as e: response = e.response if not (response and response.get("Error", {}).get("Code") == "TooManyRequestsException"): raise e # Raise error again, not the right error _log.debug("Worker {} exception {}".format(i, e)) success = False time.sleep(self.sleep_time) else: _log.debug("Worker unavailable for {}".format(i, e)) time.sleep(self.sleep_time) _log.info("Submitting source {}".format(i)) return (o, coordinates_index)
def eval_source(self, coordinates, coordinates_index, out, i, source=None): if source is None: source = self.source # Make a copy to prevent any possibility of memory corruption source = Node.from_definition(source.definition) _log.info("Submitting source {}".format(i)) return (source.eval(coordinates, output=out), coordinates_index)
def test_definition_inputs_dict(self): global MyNodeWithDictInput class MyNodeWithDictInput(Node): my_dict = tl.Dict().tag(attr=True) node1 = MyNodeWithDictInput(my_dict={"a": podpac.algorithm.Arange()}) node2 = Node.from_definition(node1.definition) assert node2 is not node1 and node2 == node1
def test_definition_inputs_array(self): global MyNodeWithArrayInput class MyNodeWithArrayInput(Node): my_array = ArrayTrait().tag(attr=True) node1 = MyNodeWithArrayInput(my_array=[podpac.algorithm.Arange()]) node2 = Node.from_definition(node1.definition) assert node2 is not node1 and node2 == node1
def eval_source(self, coordinates, coordinates_index, out, i, source=None): if source is None: source = self.source if self.skip_existing: # This section allows previously computed chunks to be skipped dk = self.zarr_data_key if isinstance(dk, list): dk = dk[0] try: exists = self.zarr_node.chunk_exists(coordinates_index, data_key=dk, list_dir=self._list_dir, chunks=self._chunks) except ValueError as e: # This was needed in cases where a poor internet connection caused read errors exists = False if exists: _log.info("Skipping {} (already exists)".format(i)) return out, coordinates_index # Make a copy to prevent any possibility of memory corruption source = Node.from_definition(source.definition) _log.debug("Creating output format.") output = dict( format="zarr_part", format_kwargs=dict( part=[[s.start, min(s.stop, self._shape[i]), s.step] for i, s in enumerate(coordinates_index)], source=self.zarr_file, mode="a", ), ) _log.debug("Finished creating output format.") if source.has_trait("output_format"): source.set_trait("output_format", output) _log.debug("output: {}, coordinates.shape: {}".format( output, coordinates.shape)) _log.debug("Evaluating node.") o, slc = super(ZarrOutputMixin, self).eval_source(coordinates, coordinates_index, out, i, source) if not source.has_trait("output_format"): o.to_format(output["format"], **output["format_kwargs"]) return o, slc
def test_definition(self): # definition d = self.node.definition assert isinstance(d, OrderedDict) assert len(d) == 5 # from_definition with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Insecure evaluation.*") node = Node.from_definition(d) assert node is not self.node assert node == self.node assert isinstance(node, podpac.algorithm.Arithmetic) assert isinstance(node.inputs["A"], podpac.algorithm.Arange) assert isinstance(node.inputs["B"], podpac.data.Array) assert isinstance(node.inputs["C"], podpac.compositor.OrderedCompositor)
def test_style(self): node = podpac.data.Array( source=[10, 20, 30], coordinates=podpac.Coordinates([[0, 1, 2]], dims=["lat"]), style=Style(name="test", units="m"), ) d = node.definition assert "style" in d[node.base_ref] node2 = Node.from_definition(d) assert node2 is not node assert isinstance(node2, podpac.data.Array) assert node2.style is not node.style assert node2.style == node.style assert node2.style.name == "test" assert node2.style.units == "m" # default style node = podpac.data.Array(source=[10, 20, 30], coordinates=podpac.Coordinates([[0, 1, 2]], dims=["lat"])) d = node.definition assert "style" not in d[node.base_ref]
def handler(event, context): """Lambda function handler Parameters ---------- event : dict Description context : TYPE Description get_deps : bool, optional Description ret_pipeline : bool, optional Description """ print (event) # Add /tmp/ path to handle python path for dependencies sys.path.append("/tmp/") # handle triggers trigger = get_trigger(event) # parse event pipeline = parse_event(trigger, event) # bail if we can't parse if pipeline is None: return # ----- # TODO: remove when layers is configured # get configured bucket to download dependencies # If specified in the environmental variables, we cannot overwrite it. Otherwise it HAS to be # specified in the settings. bucket = os.environ.get("S3_BUCKET_NAME", pipeline["settings"].get("S3_BUCKET_NAME")) # get dependencies path if "FUNCTION_DEPENDENCIES_KEY" in pipeline["settings"] or "FUNCTION_DEPENDENCIES_KEY" in os.environ: dependencies = os.environ.get( "FUNCTION_DEPENDENCIES_KEY", pipeline["settings"].get("FUNCTION_DEPENDENCIES_KEY") ) else: dependencies = "podpac_deps_{}.zip".format( os.environ.get("PODPAC_VERSION", pipeline["settings"].get("PODPAC_VERSION")) ) if 'None' in dependencies: dependencies = 'podpac_deps.zip' # Development version of podpac # this should be equivalent to version.semver() # Check to see if this function is "hot", in which case the dependencies have already been downloaded and are # available for use right away. if os.path.exists("/tmp/scipy"): print ( "Scipy has been detected in the /tmp/ directory. Assuming this function is hot, dependencies will" " not be downloaded." ) else: # Download dependencies from specific bucket/object print ("Downloading and extracting dependencies from {} {}".format(bucket, dependencies)) s3 = boto3.client("s3") s3.download_file(bucket, dependencies, "/tmp/" + dependencies) subprocess.call(["unzip", "/tmp/" + dependencies, "-d", "/tmp"]) sys.path.append("/tmp/") subprocess.call(["rm", "/tmp/" + dependencies]) # ----- # Load PODPAC # Need to set matplotlib backend to 'Agg' before importing it elsewhere import matplotlib matplotlib.use("agg") from podpac import settings from podpac.core.node import Node from podpac.core.coordinates import Coordinates from podpac.core.utils import JSONEncoder, _get_query_params_from_url import podpac.datalib # update podpac settings with inputs from the trigger settings.update(json.loads(os.environ.get("SETTINGS", "{}"))) settings.update(pipeline["settings"]) # build the Node and Coordinates if trigger in ("eval", "S3"): node = Node.from_definition(pipeline["pipeline"]) coords = Coordinates.from_json(json.dumps(pipeline["coordinates"], indent=4, cls=JSONEncoder)) # TODO: handle API Gateway better - is this always going to be WCS? elif trigger == "APIGateway": node = Node.from_url(pipeline["url"]) coords = Coordinates.from_url(pipeline["url"]) # make sure pipeline is allowed to be run if "PODPAC_RESTRICT_PIPELINES" in os.environ: whitelist = json.loads(os.environ["PODPAC_RESTRICT_PIPELINES"]) if node.hash not in whitelist: raise ValueError("Node hash is not in the whitelist for this function") # run analysis output = node.eval(coords) # convert to output format body = output.to_format(pipeline["output"]["format"], **pipeline["output"]["format_kwargs"]) # Response if trigger == "eval": return body elif trigger == "S3": s3.put_object(Bucket=settings["S3_BUCKET_NAME"], Key=pipeline["output"]["filename"], Body=body) elif trigger == "APIGateway": # TODO: can we handle the deserialization better? try: json.dumps(body) except Exception as e: print ("Output body is not serializable, attempting to decode.") body = body.decode() return { "statusCode": 200, "headers": {"Content-Type": pipeline["output"]["format"]}, "isBase64Encoded": pipeline["output"]["format_kwargs"]["return_base64"], "body": body, }