def latest_run(model, valid_time): with tracer.span(name="download_content"): download_content() pattern = re.compile("./%s" \ "/grib" \ "\\/(\d{2})" \ "/t" \ "/icon-eu_europe_regular-lat-lon_model-level_(\d{10})_(\d{3})_1_T.grib2.bz2" % model) max_t: int = 0 result = None with tracer.span(name="parse_content"): for i, line in enumerate(open('tmp/content.log')): for match in re.finditer(pattern, line): matches = match.groups() match_valid_at = datetime.datetime.strptime(matches[1], "%Y%m%d%H") match_valid_at = pytz.timezone('UTC').localize(match_valid_at) match_valid_at = match_valid_at + datetime.timedelta(hours=int(matches[2])) delta_t = abs((match_valid_at - valid_time).total_seconds()) if delta_t <= 30 * 60 and int(matches[1]) > max_t: result = matches max_t = int(matches[1]) return result
def hello(): if request.method == 'POST': n = int(request.get_json().get("n")) else: n = 10 tracer = initialize_tracer(PROJECT_ID) with tracer.span(name="gae_gapic") as span: latencies = [] temp = time.time() response = client.publish(topic_path, [{'data': b'first'}]) assert response.message_ids logger.log_text(f"{int((time.time()- temp)*1000)}") for i in range(n): messages = [{'data': str(i).encode('utf-8')}] temp = time.time() response = client.publish(topic_path, messages) assert response.message_ids latencies.append(time.time() - temp) logger.log_text(f"{','.join([str(int(i*1000)) for i in latencies])}") return ("Published messages.\n")
def hello_world(request): request_json = request.get_json(silent=True) if 'n' in request_json: n = int(request_json['n']) else: n = 10 tracer = initialize_tracer(PROJECT_ID) with tracer.span(name="cf_gapic") as span: latencies = [] temp = time.time() response = client.publish(topic_path, [{'data': b'first'}]) assert response.message_ids logger.log_text(f"{int((time.time()- temp)*1000)}") for i in range(n): messages = [{'data': str(i).encode('utf-8')}] temp = time.time() response = client.publish(topic_path, messages) assert response.message_ids latencies.append(time.time() - temp) logger.log_text(f"{','.join([str(int(i*1000)) for i in latencies])}") return ("Published messages.")
def hello(): if request.method == 'POST': n = int(request.get_json().get("n")) else: n = 10 tracer = initialize_tracer(PROJECT_ID) with tracer.span(name="gae_apiary") as span: latencies = [] message = base64.b64encode(b'first') body = {'messages': [{'data': message.decode('utf-8')}]} temp = time.time() response = service.projects().topics().publish( topic=topic_path, body=body, ).execute() assert response['messageIds'] logger.log_text(f"{int((time.time()- temp)*1000)}") for i in range(n): message = base64.b64encode(str(i).encode('utf-8')) body = {'messages': [{'data': message.decode('utf-8')}]} temp = time.time() response = service.projects().topics().publish( topic=topic_path, body=body, ).execute() assert response['messageIds'] latencies.append(time.time() - temp) logger.log_text(f"{','.join([str(int(i*1000)) for i in latencies])}") return("Published messages.\n")
def hello(): if request.method == 'POST': n = int(request.get_json().get("n")) else: n = 10 tracer = initialize_tracer(PROJECT_ID) def get_callback(f, data): def callback(f): try: f.result() logger.log_text(f"{int((time.time()- futures[data])*1000)}") except Exception: raise Exception return callback with tracer.span(name="gae_cl_mixed") as span: futures = dict() latencies = [] futures.update({b"first": time.time()}) future = client.publish(topic_path, data=b"first") future.add_done_callback(get_callback(future, b"first")) for i in range(n): data = str(i) temp = time.time() future = client.publish(topic_path, data=data.encode('utf-8')) future.result() latencies.append(time.time() - temp) logger.log_text(f"{','.join([str(int(i*1000)) for i in latencies])}") return("Published messages.\n")
def skewt(latitude, longitude, valid_at): with tracer.span(name="skewt") as span: span.add_attribute("latitude", str(latitude)) span.add_attribute("longitude", str(longitude)) span.add_attribute("valid_at", str(valid_at)) valid_at_parsed = datetime.datetime.strptime(valid_at, "%Y%m%d%H") valid_at_parsed = pytz.timezone('UTC').localize(valid_at_parsed) with tracer.span(name="sounding"): sounding = load_weather_model_sounding(latitude, longitude, valid_at_parsed) with tracer.span(name="plotting"): model_time = str(np.datetime_as_string(sounding.metadata.model_time)) valid_time = str(np.datetime_as_string(sounding.metadata.valid_time)) model_time_for_file_name = str(np.datetime_as_string(sounding.metadata.model_time, unit='m')).replace(":", "_") valid_time_for_file_name = str(np.datetime_as_string(sounding.metadata.valid_time, unit='m')).replace(":", "_") full_plot = plot_skewt_icon(sounding=sounding, parcel="surface-based") full_plot_filename = f"plot_{sounding.latitude_pretty}_{sounding.longitude_pretty}_" \ f"{model_time_for_file_name}_{valid_time_for_file_name}_full.png" full_plot.savefig(full_plot_filename) detail_plot = plot_skewt_icon(sounding=sounding, parcel="surface-based", base=1000, top=500, skew=15) detail_plot_filename = f"plot_{sounding.latitude_pretty}_{sounding.longitude_pretty}_" \ f"{model_time_for_file_name}_{valid_time_for_file_name}_detail.png" detail_plot.savefig(detail_plot_filename) with tracer.span(name="cloud_upload"): # Google Cloud Upload storage_client = storage.Client() bucket = storage_client.bucket(config.bucket_name) blob_full = bucket.blob(full_plot_filename) blob_detail = bucket.blob(detail_plot_filename) with ThreadPoolExecutor(max_workers=2) as executor: executor.submit(blob_full.upload_from_filename, full_plot_filename) executor.submit(blob_detail.upload_from_filename, detail_plot_filename) result = json.dumps(SkewTResult(model_time, valid_time, config.bucket_public_url + full_plot_filename, config.bucket_public_url + detail_plot_filename).__dict__) response = make_response(result) response.mimetype = 'application/json' return response
def parameter_all_levels(model, latitude, longitude, run_hour, run_datetime, timestep, parameter, level_type="model_level", base_level=60, top_level=1): with tracer.span(name="download"): levels = list(range(base_level, top_level - 1, -1)) paths = [level_path(model, run_hour, run_datetime, timestep, parameter, level, level_type) for level in levels] session = requests.sessions.Session() with concurrent.futures.ThreadPoolExecutor(max_workers=config.http_download_pool) as executor: futures = list(executor.submit(download_file(path, session)) for path in paths) concurrent.futures.wait(futures, timeout=None, return_when=ALL_COMPLETED) with tracer.span(name="parsing"): data_set = xr.open_mfdataset(paths, engine="cfgrib", concat_dim="generalVerticalLayer", combine='nested', parallel=config.cfgrib_parallel) interpolated = data_set.to_array()[0].interp(latitude=latitude, longitude=longitude) data = AllLevelDataResult(interpolated.values, interpolated.time.values, interpolated.valid_time.values) data_set.close() return data
def parameter_all_levels(model, run, parameter, latitude, longitude, level_type="model_level", session=requests.Session()): with tracer.span(name="load_parameters") as span: run_hour = run[0] run_datetime = run[1] timestep = int(run[2]) logging.info(f"Loading sounding for latitude={latitude} longitude={longitude} with " f"run_hour={run_hour} run_datetime={run_datetime} timestep={timestep} " f"level_type={level_type} and parameter={parameter}") span.add_attribute("parameter", parameter) span.add_attribute("run_hour", str(run_hour)) span.add_attribute("run_datetime", str(run_datetime)) span.add_attribute("timestep", str(timestep)) levels = np.floor(np.linspace(60, 0, config.level_workers)).astype(int).tolist() urls = list() for i in range(0, len(levels) - 1): base = levels[i] top = levels[i + 1] + 1 # example URL: # https://nwp-sounding-mw5zsrftba-ew.a.run.app/48.21/16.37/06/2020030406/4/p url = f"{config.sounding_api}" \ f"/{latitude}" \ f"/{longitude}" \ f"/{run_hour}" \ f"/{run_datetime}" \ f"/{timestep}" \ f"/{parameter}" \ f"?level_type={level_type}" \ f"&base={base}" \ f"&top={top}" urls.append(url) result = AllLevelData(data=np.empty(0), model_time=None, valid_time=None) with ThreadPoolExecutor(max_workers=config.level_workers) as executor: responses = list(executor.map(session.get, urls)) for response in responses: response.raise_for_status() json_result = json.loads(response.content) result.data = np.append(result.data, np.array(json_result["data"])) json_first = json.loads(responses[0].content) result.model_time = np.datetime64(json_result["model_time"]) result.valid_time = np.datetime64(json_result["valid_time"]) return result
def handleHttp(request: 'flask.Request') -> str: tracer = initialize_tracer(request) req = WebhookRequest() res = WebhookResponse() try: json_format.Parse(request.data, req, ignore_unknown_fields=True) if req.query_result.action == "roll": with tracer.span(name='roll'): handleRoll(req, res) except UnfulfillableRequestError as e: logging.exception(e) if STACKDRIVER_ERROR_REPORTING: try: client = error_reporting.Client() client.report_exception( http_context=error_reporting.build_flask_context(request)) except Exception: logging.exception("Failed to send error report to Google") add_fulfillment_messages(res, str(e)) return json_format.MessageToJson(res)
def sounding(latitude, longitude, run_hour, run_datetime, timestep, parameter): with tracer.span(name="sounding") as span: span.add_attribute("latitude", str(latitude)) span.add_attribute("longitude", str(longitude)) span.add_attribute("run_hour", str(run_hour)) span.add_attribute("run_datetime", str(run_datetime)) span.add_attribute("timestep", str(timestep)) span.add_attribute("parameter", str(parameter)) level_type = request.args.get("level_type", "model_level") base_level = int(request.args.get("base", "60")) top_level = int(request.args.get("top", "1")) sounding = parameter_all_levels(config.model, latitude, longitude, run_hour, run_datetime, int(timestep), parameter, level_type, base_level, top_level) response = make_response(json.dumps(sounding.__dict__)) response.mimetype = 'application/json' return response
def trace_requests(request): """Responds to any HTTP request. Args: request (flask.Request): HTTP request object. Returns: The response text or any set of values that can be turned into a Response object using `make_response <http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>`. """ tracer = initTracer() base_name = os.environ.get("FUNCTION_NAME") + "-trace-" with tracer.span(name=base_name + "metadata-call"): import requests r = requests.get( "http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id", headers={'Metadata-Flavor': 'Google'}) return r.content
def load_weather_model_sounding(latitude, longitude, valid_time): with tracer.span(name="latest_run"): model = "icon-eu" run = latest_run(model, valid_time) http_session = session() with ThreadPoolExecutor(max_workers=config.parameter_all_levels_workers) as executor: p_future = executor.submit(parameter_all_levels, model, run, "p", latitude, longitude, session=http_session) T_future = executor.submit(parameter_all_levels, model, run, "T", latitude, longitude, session=http_session) QV_future = executor.submit(parameter_all_levels, model, run, "QV", latitude, longitude, session=http_session) U_future = executor.submit(parameter_all_levels, model, run, "U", latitude, longitude, session=http_session) V_future = executor.submit(parameter_all_levels, model, run, "V", latitude, longitude, session=http_session) HHL_future = executor.submit(parameter_all_levels, model, run, "HHL", latitude, longitude, "time_invariant", session=http_session) # Pressure Pa p_raw = p_future.result() p = p_raw.data # Temperature K T = T_future.result().data # Specific Humidty kg/kg QV = QV_future.result().data # Dewpoint K Td = mpcalc.dewpoint_from_specific_humidity(QV * units("kg/kg"), T * units.K, p * units.Pa) # Wind m/s U = U_future.result().data V = V_future.result().data # Height above MSL for model level HHL = HHL_future.result().data meta_data = WeatherModelSoundingMetaData(p_raw.model_time, p_raw.valid_time) return WeatherModelSounding(latitude, longitude, p, T, QV, Td, U, V, HHL, meta_data)