def register_model_to_server(server_port, dataset_port, dataset_name): """ Send notification to the server with the training name and port to register the microservice It retries until a connection with the server is established """ while not dependency.shutdown: try: headers = { 'api_key': API_KEY } r = requests.post('http://host.docker.internal:' + str(server_port) + '/training/register', headers=headers, json={"name": dataset_name, "socket": 'http://host.docker.internal:'+dataset_port}) r.raise_for_status() dependency.connected = True except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError): dependency.connected = False logger.debug('Registering to server fails. Retry in ' + str(dependency.WAIT_TIME) + ' seconds') # Delay for WAIT_TIME between server registration pings for increment in range(dependency.WAIT_TIME): if not dependency.shutdown: # Check between increments to stop hanging on shutdown time.sleep(1) logger.debug("[Healthcheck] Server Registration Thread Halted.")
async def get_stream_apps( start: Optional[int] = 0, # pylint: disable=unsubscriptable-object size: Optional[int] = 10, # pylint: disable=unsubscriptable-object ): """ Get all streaming applications. start: Start index of the applications size: Number of sessions to fetch """ conf = get_config() livyConf = conf["livy"] url = livyConf["url"] + "/batches" parameters = { "from": start, "size": size, } headers = createLivyCommonHeaders() livyResp = requests.get(url, params=parameters, headers=headers, timeout=get_global_request_timeout(), ) livyRespJson = livyResp.json() if livyResp.status_code != status.HTTP_200_OK: raise HTTPException( status_code=livyResp.status_code, detail=livyRespJson ) livyRespJson["start"] = livyRespJson["from"] resp = LivyGetBatchResp.parse_obj(livyRespJson) fastapi_logger.debug(resp) return resp
async def login(user_name: str, password: str) -> str: """Obtain a token form RW API using given user name and password.""" headers = {"Content-Type": "application/json"} payload = {"email": user_name, "password": password} logger.debug( f"Requesting Bearer token from GFW production API for user {user_name}" ) url = f"{RW_API_URL}/auth/login" try: async with AsyncClient() as client: response: HTTPXResponse = await client.post( url, json=payload, headers=headers ) except ReadTimeout: raise HTTPException( status_code=500, detail="Call to authorization server timed-out. Please try again.", ) if response.status_code != 200: logger.warning( f"Authentication for user {user_name} failed. API responded with status code {response.status_code} and message {response.text}" ) raise UnauthorizedError("Authentication failed") return response.json()["data"]["token"]
async def _worker(self, _id: int, init_wait: float, db: SQLiteDatasource): wait_time = init_wait while True: wait_time = self.calc_next_wait(wait_time) logger.debug("Worker id: {} - Waiting for {}".format( _id, wait_time)) await sleep(wait_time) # Locking reduces transaction deadlocks in sqlite to zero # Pulling one task a time from the queue is acceptable behavior to me async with self._lock: logger.debug("Worker id: {} - Getting task".format(_id)) task: Optional[DownloadTask] = await db.get() if task is not None: logger.info("Worker id: {} - Downloading {}".format( _id, task.submitted_task.url)) process: Union[TaskProcessorException, Process] = await self.task_processor.process( task) if isinstance(process, Process): self._running_tasks[_id] = (task, process) status = await self.wait_for_process_status(_id, process) del (self._running_tasks[_id]) else: status = await self.wait_for_process_status(_id, process) await db.set_status(task.submitted_task.uuid, status) wait_time = init_wait
def predict(self, qa: QA, model): logger.debug('Using slow Predictor') contexts = [self.get_context(r) for r in qa.results] return [ biobert_predict.predict(qa.question, context, model) for context in contexts ]
def create_submission(execution: schemas.Execution): try: logger.debug( f"Start code submission request with payload: {execution}") result = celery_client.send_task( 'worker.execute_code', args=(execution.language, execution.code, execution.timeout)).wait(timeout=None, interval=0.1) result['submission_id'] = str(uuid.uuid4()) if not result['has_error']: result['results'] = python_result_extractor(result['raw_output']) else: result['results'] = [] logger.debug(f"Code submission succeeded with result: {result}") return result except ValueError as vle: logger.error(f"Invalid raw output test format, error: {str(vle)}") raise HTTPException(status_code=400, detail="Invalid test code format") except Exception as exc: logger.error( f"Worker could not process submission, error {exc}, payload: {execution}" ) raise HTTPException( status_code=500, detail="The server could not process the code execution")
async def add(self, elem: str): utc = time.time() logger.debug( f'TempBag - {self.rds_name} - {self.ttl} - {utc} - {elem}') await RdsOpr.raw().zadd(self.rds_name, utc, elem) await RdsOpr.raw().zremrangebyscore(self.rds_name, max=utc - self.ttl) await RdsOpr.raw().expire(self.rds_name, self.ttl)
async def _serialized_asset(asset_orm: ORMAsset) -> Asset: data: Asset = Asset.from_orm(asset_orm) data.metadata = asset_metadata_factory(asset_orm.asset_type, asset_orm.metadata) logger.debug(f"Metadata: {data.metadata.dict(by_alias=True)}") return data
def init(): """ This method will be run once on startup. You should check if the supporting files your model needs have been created, and if not then you should create/fetch them. """ # Placeholder init code. Replace the sleep with check for model files required etc... logger.debug('Successful init of ImageShapeMicroservice')
async def fallback(path) -> HTMLResponse: """ Fallback URI. This gives both flexibility and responsibility of Routing to Vue.JS application side """ logger.debug(f"Sinking path: {path}") # Will bring user to SPA. You may want to show 404 later # for the paths irrelevant return await index()
def predict(self, qa: QA, model): logger.debug('Using FastPredictor') qas = [(qa.question, self.get_context(r)) for r in qa.results] results = biobert_fast_predict.fast_predict( qas, model, batch_size=self.config.batch_size, use_cuda=self.config.use_cuda) return [(answer, float(score)) for answer, score in results]
def update_ecs_service(cluster: str, service: str) -> Dict[str, Any]: ecs_client = get_ecs_client() logger.debug("Reload tile cache service") response = ecs_client.update_service( cluster=cluster, service=service, forceNewDeployment=True ) return response
async def get_engine(method: str) -> GinoEngine: """Select the database connection depending on request method.""" if method.upper() == "WRITE": logger.debug("Use write engine") engine: GinoEngine = WRITE_ENGINE else: logger.debug("Use read engine") engine = READ_ENGINE return engine
def __call__(self, user: User_Pydantic = Depends(get_current_active_user), role: str = Depends(get_current_user_role)): if role not in self.allowed_roles: logger.debug(f"User with role {role} not in {self.allowed_roles}") # print(f"User with role {role} not in {self.allowed_roles}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="You are not authorised to use this resource")
async def add_word_count(wordCountDict: AddWordCountReq, redis: Redis = Depends(depends_redis)): pipe = redis.pipeline() for key, count in wordCountDict.wordcount.items(): pipe.hincrby(WORD_COUNT_KEY, key, count) result = await pipe.execute() resp = {"result": result} fastapi_logger.debug(resp) return resp
async def create_wm_tile_set_job( dataset: str, version: str, creation_options: RasterTileSetSourceCreationOptions, job_name: str, parents: Optional[List[Job]] = None, use_resampler: bool = False, ) -> Tuple[Job, str]: asset_uri = get_asset_uri( dataset, version, AssetType.raster_tile_set, creation_options.dict(by_alias=True), "epsg:3857", ) # Create an asset record asset_options = AssetCreateIn( asset_type=AssetType.raster_tile_set, asset_uri=asset_uri, is_managed=True, creation_options=creation_options, metadata=RasterTileSetMetadata(), ).dict(by_alias=True) wm_asset_record = await create_asset(dataset, version, **asset_options) logger.debug(f"Created asset for {asset_uri}") # TODO: Consider removing the use_resampler argument and changing this # to "if creation_options.calc is None:" # Make sure to test different scenarios when done! if use_resampler: job = await create_resample_job( dataset, version, creation_options, int(creation_options.grid.strip("zoom_")), job_name, callback_constructor(wm_asset_record.asset_id), parents=parents, ) else: job = await create_pixetl_job( dataset, version, creation_options, job_name, callback_constructor(wm_asset_record.asset_id), parents=parents, ) zoom_level = int(creation_options.grid.strip("zoom_")) job = scale_batch_job(job, zoom_level) return job, asset_uri
def _update_lifecycle_rule(bucket, rule) -> Dict[str, Any]: """Add new lifecycle rule to bucket.""" client = get_s3_client() rules = _get_lifecycle_rules(bucket) rules.append(rule) logger.debug(f"Add lifecylce configuration rule {rules} to bucket {bucket}") response = client.put_bucket_lifecycle_configuration( Bucket=bucket, LifecycleConfiguration={"Rules": rules} ) return response
def sample(): logger.critical('CRITICAL message') logger.error('ERROR message') logger.warning('WARNING message') logger.info('INFO message') logger.debug('DEBUG message') module() return 'hello, world'
async def get_info_link(code: str): try: redirect, src = await Linker.get_link_or_none(code) logger.debug(f'Linker return - {code} - {redirect} - {src}') if redirect is None: return schm.ReportLink(ok=False) return schm.ReportLink(ok=True, link=redirect['link'], is_on=redirect['is_on']) except Exception as e: logger.warning(f"get_link - {code} - Exception [{type(e)}]: {e}") return schm.ReportLink(ok=False)
def bind(self): try: e = CURRENT_ENGINE.get() bind = e.result() logger.debug(f"Set bind to {bind.repr(color=True)}") return bind except LookupError: # not in a request logger.debug("Not in a request, using default bind") return self._bind
async def create_tile_cache( dataset: str, version: str, source_uri: str, zoom_level: int, implementation: str, callback: Callback, parents: List[Job], bit_depth: int, ): """Create batch job to generate raster tile cache for given zoom level.""" asset_prefix = os.path.dirname(source_uri) logger.debug( f"CREATING TILE CACHE JOB FOR ZOOM LEVEL {zoom_level} WITH PREFIX {asset_prefix}" ) cmd: List[str] = [ "raster_tile_cache.sh", "-d", dataset, "-v", version, "-I", implementation, "--target_bucket", TILE_CACHE_BUCKET, "--zoom_level", str(zoom_level), "--bit_depth", str(bit_depth), ] # TODO: GTC-1090, GTC 1091 # this should be the default. However there seems to be an issue # with some of the symbology function (discrete, date-conf-intensity) # which generate empty tiles at least during tests. # if zoom_level > 9: cmd += ["--skip"] cmd += [asset_prefix] tile_cache_job = GDAL2TilesJob( dataset=dataset, job_name=f"generate_tile_cache_zoom_{zoom_level}", command=cmd, environment=JOB_ENV, callback=callback, parents=[parent.job_name for parent in parents], ) tile_cache_job = scale_batch_job(tile_cache_job, zoom_level) return tile_cache_job
async def _serialized_asset(asset_orm: ORMAsset) -> Asset: version_orm: ORMVersion = await versions.get_version( asset_orm.dataset, asset_orm.version) data: Asset = Asset.from_orm(asset_orm) data.metadata = asset_metadata_factory(asset_orm.asset_type, asset_orm.metadata) data.creation_options = asset_creation_option_factory( version_orm.source_type, asset_orm.asset_type, asset_orm.creation_options) logger.debug(f"Metadata: {data.metadata.dict(by_alias=True)}") return data
async def create_user_area(geometry: Geometry) -> Geostore: # Sanitize the JSON by doing a round-trip with Postgres. We want the sort # order, whitespace, etc. to match what would be saved via other means # (in particular, via batch/scripts/add_gfw_fields.sh) geometry_str = geometry.json() sql = db.text("SELECT ST_AsGeoJSON(ST_GeomFromGeoJSON(:geo)::geometry);") bind_vals = {"geo": geometry_str} sql = sql.bindparams(**bind_vals) logger.debug(sql) sanitized_json = await db.scalar(sql) bbox: List[float] = await db.scalar( f""" SELECT ARRAY[ ST_XMin(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)), ST_YMin(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)), ST_XMax(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)), ST_YMax(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)) ]::NUMERIC[]; """ ) area: float = await db.scalar( f""" SELECT ST_Area( ST_GeomFromGeoJSON( '{sanitized_json}' )::geography ) """ ) area = area / 10000 # We could easily do this in Python but we want PostgreSQL's behavior # (if different) to be the source of truth. # geo_id = UUID(str(hashlib.md5(feature_json.encode("UTF-8")).hexdigest())) geo_id: UUID = await db.scalar(f"SELECT MD5('{sanitized_json}')::uuid;") try: user_area = await ORMUserArea.create( gfw_geostore_id=geo_id, gfw_geojson=sanitized_json, gfw_area__ha=area, gfw_bbox=bbox, ) geostore: Geostore = Geostore.from_orm(user_area) except UniqueViolationError: geostore = await get_geostore_from_anywhere(geo_id) return geostore
def produce_upload_message(json_payload): if not KAFKA_PRODUCER: raise Exception("Kafka not available") logger.debug("to producer.send()") future = KAFKA_PRODUCER.send(KAFKA_TOPIC, json_payload) try: record_metadata = future.get(timeout=10) logger.info("send future completed") return record_metadata except KafkaError: logger.exception('Failed to send to kafka') raise
async def actor_deactivation(actor_type_name: str, actor_id: str): try: await ActorRuntime.deactivate(actor_type_name, actor_id) except DaprInternalError as ex: return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR, ex.as_dict()) except Exception as ex: return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR, repr(ex), ERROR_CODE_UNKNOWN) msg = f'deactivated actor: {actor_type_name}.{actor_id}' logger.debug(msg) return _wrap_response(status.HTTP_200_OK, msg)
async def get_fields(dv: Tuple[str, str] = Depends(dataset_version_dependency)): dataset, version = dv asset = await assets.get_default_asset(dataset, version) logger.debug(f"Processing default asset type {asset.asset_type}") fields: Union[List[FieldMetadata], List[RasterFieldMetadata]] = [] if asset.asset_type == AssetType.raster_tile_set: fields = await _get_raster_fields(asset) else: fields = [FieldMetadata(**field) for field in asset.fields] return FieldMetadataResponse(data=fields)
async def parser_incm(request: Request) -> schemas.Incm: body = await request.json() out = await get_incm_cmd(body) if not out: out = await get_incm_txt(body) if not out: out = await get_callback_json(body) if not out: out = await get_callback(body) if not out: out = await get_incm_location(body) if not out: logger.warning(f"unknown income: {json.dumps(body)}") logger.debug(type(out)) return out
def notify_upload(url, account_id, tenant_id, bundle_id): logger.debug("notify_upload") bundle_file = get_bundle_path(bundle_id) bundle_size = os.stat(bundle_file).st_size payload = { 'account': account_id, 'b64_identity': '__=', 'category': 'analytics', 'metadata': {}, 'principal': tenant_id, 'request_id': bundle_id, 'service': 'tower', 'size': bundle_size, 'timestamp': datetime.datetime.now().astimezone().isoformat(), 'url': '{}/bundles/{}?done=True'.format(url, bundle_id) } return produce_upload_message(payload)
async def actor_timer(actor_type_name: str, actor_id: str, timer_name: str, request: Request): try: # Read raw bytes from request stream req_body = await request.body() await ActorRuntime.fire_timer(actor_type_name, actor_id, timer_name, req_body) except DaprInternalError as ex: return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR, ex.as_dict()) except Exception as ex: return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR, repr(ex), ERROR_CODE_UNKNOWN) msg = f'called timer. actor: {actor_type_name}.{actor_id}, timer: {timer_name}' logger.debug(msg) return _wrap_response(status.HTTP_200_OK, msg)
async def create_stream_app(data:CreateStreamAppReq): """ Create a streaming application through apache livy. """ if not data.verifySparkAppName(): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail={ "msg": "sparkAppName is invalid" }) conf = get_config() livyConf = conf["livy"] url = livyConf["url"] + "/batches" headers = createLivyCommonHeaders() wordCountUpdateUrl = livyConf['wordCountUpdateUrl'] json = { "file": livyConf["jarPath"], "className": livyConf["className"], "queue": livyConf["queue"], "numExecutors": livyConf["numExecutors"], "executorMemory": livyConf["executorMemory"], "executorCores": livyConf["executorCores"], "driverMemory": livyConf["driverMemory"], "args": [ "--spark.app.name", data.sparkAppName, "--kafka.brokers", livyConf["kafka"]["brokers"], "--kafka.topics", livyConf["kafka"]["topics"], "--kafka.groupId", livyConf["kafka"]["groupId"], "--batch.duration.seconds", livyConf["duration"]["batch"], "--wordcount.update.url", wordCountUpdateUrl, ], "name": data.sparkAppName } livyResp = requests.post(url, json=json, headers=headers, timeout=get_global_request_timeout(), ) livyRespJson = livyResp.json() if livyResp.status_code != status.HTTP_201_CREATED: raise HTTPException( status_code=livyResp.status_code, detail=livyRespJson ) resp = LivyBatch.parse_obj(livyRespJson) fastapi_logger.debug(resp) return resp