def register_model_to_server(server_port, dataset_port, dataset_name):
"""
Send notification to the server with the training name and port to register the microservice
It retries until a connection with the server is established
"""
while not dependency.shutdown:
try:
headers = {
'api_key': API_KEY
}
r = requests.post('http://host.docker.internal:' + str(server_port) + '/training/register',
headers=headers,
json={"name": dataset_name, "socket": 'http://host.docker.internal:'+dataset_port})
r.raise_for_status()
dependency.connected = True
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError):
dependency.connected = False
logger.debug('Registering to server fails. Retry in ' + str(dependency.WAIT_TIME) + ' seconds')
# Delay for WAIT_TIME between server registration pings
for increment in range(dependency.WAIT_TIME):
if not dependency.shutdown: # Check between increments to stop hanging on shutdown
time.sleep(1)
logger.debug("[Healthcheck] Server Registration Thread Halted.")
async def get_stream_apps(
start: Optional[int] = 0, # pylint: disable=unsubscriptable-object
size: Optional[int] = 10, # pylint: disable=unsubscriptable-object
):
"""
Get all streaming applications.
start: Start index of the applications
size: Number of sessions to fetch
"""
conf = get_config()
livyConf = conf["livy"]
url = livyConf["url"] + "/batches"
parameters = {
"from": start,
"size": size,
}
headers = createLivyCommonHeaders()
livyResp = requests.get(url,
params=parameters,
headers=headers,
timeout=get_global_request_timeout(),
)
livyRespJson = livyResp.json()
if livyResp.status_code != status.HTTP_200_OK:
raise HTTPException(
status_code=livyResp.status_code,
detail=livyRespJson
)
livyRespJson["start"] = livyRespJson["from"]
resp = LivyGetBatchResp.parse_obj(livyRespJson)
fastapi_logger.debug(resp)
return resp
async def login(user_name: str, password: str) -> str:
"""Obtain a token form RW API using given user name and password."""
headers = {"Content-Type": "application/json"}
payload = {"email": user_name, "password": password}
logger.debug(
f"Requesting Bearer token from GFW production API for user {user_name}"
)
url = f"{RW_API_URL}/auth/login"
try:
async with AsyncClient() as client:
response: HTTPXResponse = await client.post(
url, json=payload, headers=headers
)
except ReadTimeout:
raise HTTPException(
status_code=500,
detail="Call to authorization server timed-out. Please try again.",
)
if response.status_code != 200:
logger.warning(
f"Authentication for user {user_name} failed. API responded with status code {response.status_code} and message {response.text}"
)
raise UnauthorizedError("Authentication failed")
return response.json()["data"]["token"]
async def _worker(self, _id: int, init_wait: float, db: SQLiteDatasource):
wait_time = init_wait
while True:
wait_time = self.calc_next_wait(wait_time)
logger.debug("Worker id: {} - Waiting for {}".format(
_id, wait_time))
await sleep(wait_time)
# Locking reduces transaction deadlocks in sqlite to zero
# Pulling one task a time from the queue is acceptable behavior to me
async with self._lock:
logger.debug("Worker id: {} - Getting task".format(_id))
task: Optional[DownloadTask] = await db.get()
if task is not None:
logger.info("Worker id: {} - Downloading {}".format(
_id, task.submitted_task.url))
process: Union[TaskProcessorException,
Process] = await self.task_processor.process(
task)
if isinstance(process, Process):
self._running_tasks[_id] = (task, process)
status = await self.wait_for_process_status(_id, process)
del (self._running_tasks[_id])
else:
status = await self.wait_for_process_status(_id, process)
await db.set_status(task.submitted_task.uuid, status)
wait_time = init_wait
def predict(self, qa: QA, model):
logger.debug('Using slow Predictor')
contexts = [self.get_context(r) for r in qa.results]
return [
biobert_predict.predict(qa.question, context, model)
for context in contexts
]
def create_submission(execution: schemas.Execution):
try:
logger.debug(
f"Start code submission request with payload: {execution}")
result = celery_client.send_task(
'worker.execute_code',
args=(execution.language, execution.code,
execution.timeout)).wait(timeout=None, interval=0.1)
result['submission_id'] = str(uuid.uuid4())
if not result['has_error']:
result['results'] = python_result_extractor(result['raw_output'])
else:
result['results'] = []
logger.debug(f"Code submission succeeded with result: {result}")
return result
except ValueError as vle:
logger.error(f"Invalid raw output test format, error: {str(vle)}")
raise HTTPException(status_code=400, detail="Invalid test code format")
except Exception as exc:
logger.error(
f"Worker could not process submission, error {exc}, payload: {execution}"
)
raise HTTPException(
status_code=500,
detail="The server could not process the code execution")
async def add(self, elem: str):
utc = time.time()
logger.debug(
f'TempBag - {self.rds_name} - {self.ttl} - {utc} - {elem}')
await RdsOpr.raw().zadd(self.rds_name, utc, elem)
await RdsOpr.raw().zremrangebyscore(self.rds_name, max=utc - self.ttl)
await RdsOpr.raw().expire(self.rds_name, self.ttl)
async def _serialized_asset(asset_orm: ORMAsset) -> Asset:
data: Asset = Asset.from_orm(asset_orm)
data.metadata = asset_metadata_factory(asset_orm.asset_type,
asset_orm.metadata)
logger.debug(f"Metadata: {data.metadata.dict(by_alias=True)}")
return data
def init():
"""
This method will be run once on startup. You should check if the supporting files your
model needs have been created, and if not then you should create/fetch them.
"""
# Placeholder init code. Replace the sleep with check for model files required etc...
logger.debug('Successful init of ImageShapeMicroservice')
async def fallback(path) -> HTMLResponse:
"""
Fallback URI. This gives both flexibility and responsibility
of Routing to Vue.JS application side
"""
logger.debug(f"Sinking path: {path}")
# Will bring user to SPA. You may want to show 404 later
# for the paths irrelevant
return await index()
def predict(self, qa: QA, model):
logger.debug('Using FastPredictor')
qas = [(qa.question, self.get_context(r)) for r in qa.results]
results = biobert_fast_predict.fast_predict(
qas,
model,
batch_size=self.config.batch_size,
use_cuda=self.config.use_cuda)
return [(answer, float(score)) for answer, score in results]
def update_ecs_service(cluster: str, service: str) -> Dict[str, Any]:
ecs_client = get_ecs_client()
logger.debug("Reload tile cache service")
response = ecs_client.update_service(
cluster=cluster, service=service, forceNewDeployment=True
)
return response
async def get_engine(method: str) -> GinoEngine:
"""Select the database connection depending on request method."""
if method.upper() == "WRITE":
logger.debug("Use write engine")
engine: GinoEngine = WRITE_ENGINE
else:
logger.debug("Use read engine")
engine = READ_ENGINE
return engine
def __call__(self,
user: User_Pydantic = Depends(get_current_active_user),
role: str = Depends(get_current_user_role)):
if role not in self.allowed_roles:
logger.debug(f"User with role {role} not in {self.allowed_roles}")
# print(f"User with role {role} not in {self.allowed_roles}")
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="You are not authorised to use this resource")
async def add_word_count(wordCountDict: AddWordCountReq,
redis: Redis = Depends(depends_redis)):
pipe = redis.pipeline()
for key, count in wordCountDict.wordcount.items():
pipe.hincrby(WORD_COUNT_KEY, key, count)
result = await pipe.execute()
resp = {"result": result}
fastapi_logger.debug(resp)
return resp
async def create_wm_tile_set_job(
dataset: str,
version: str,
creation_options: RasterTileSetSourceCreationOptions,
job_name: str,
parents: Optional[List[Job]] = None,
use_resampler: bool = False,
) -> Tuple[Job, str]:
asset_uri = get_asset_uri(
dataset,
version,
AssetType.raster_tile_set,
creation_options.dict(by_alias=True),
"epsg:3857",
)
# Create an asset record
asset_options = AssetCreateIn(
asset_type=AssetType.raster_tile_set,
asset_uri=asset_uri,
is_managed=True,
creation_options=creation_options,
metadata=RasterTileSetMetadata(),
).dict(by_alias=True)
wm_asset_record = await create_asset(dataset, version, **asset_options)
logger.debug(f"Created asset for {asset_uri}")
# TODO: Consider removing the use_resampler argument and changing this
# to "if creation_options.calc is None:"
# Make sure to test different scenarios when done!
if use_resampler:
job = await create_resample_job(
dataset,
version,
creation_options,
int(creation_options.grid.strip("zoom_")),
job_name,
callback_constructor(wm_asset_record.asset_id),
parents=parents,
)
else:
job = await create_pixetl_job(
dataset,
version,
creation_options,
job_name,
callback_constructor(wm_asset_record.asset_id),
parents=parents,
)
zoom_level = int(creation_options.grid.strip("zoom_"))
job = scale_batch_job(job, zoom_level)
return job, asset_uri
def _update_lifecycle_rule(bucket, rule) -> Dict[str, Any]:
"""Add new lifecycle rule to bucket."""
client = get_s3_client()
rules = _get_lifecycle_rules(bucket)
rules.append(rule)
logger.debug(f"Add lifecylce configuration rule {rules} to bucket {bucket}")
response = client.put_bucket_lifecycle_configuration(
Bucket=bucket, LifecycleConfiguration={"Rules": rules}
)
return response
def sample():
logger.critical('CRITICAL message')
logger.error('ERROR message')
logger.warning('WARNING message')
logger.info('INFO message')
logger.debug('DEBUG message')
module()
return 'hello, world'
async def get_info_link(code: str):
try:
redirect, src = await Linker.get_link_or_none(code)
logger.debug(f'Linker return - {code} - {redirect} - {src}')
if redirect is None:
return schm.ReportLink(ok=False)
return schm.ReportLink(ok=True, link=redirect['link'], is_on=redirect['is_on'])
except Exception as e:
logger.warning(f"get_link - {code} - Exception [{type(e)}]: {e}")
return schm.ReportLink(ok=False)
def bind(self):
try:
e = CURRENT_ENGINE.get()
bind = e.result()
logger.debug(f"Set bind to {bind.repr(color=True)}")
return bind
except LookupError:
# not in a request
logger.debug("Not in a request, using default bind")
return self._bind
async def create_tile_cache(
dataset: str,
version: str,
source_uri: str,
zoom_level: int,
implementation: str,
callback: Callback,
parents: List[Job],
bit_depth: int,
):
"""Create batch job to generate raster tile cache for given zoom level."""
asset_prefix = os.path.dirname(source_uri)
logger.debug(
f"CREATING TILE CACHE JOB FOR ZOOM LEVEL {zoom_level} WITH PREFIX {asset_prefix}"
)
cmd: List[str] = [
"raster_tile_cache.sh",
"-d",
dataset,
"-v",
version,
"-I",
implementation,
"--target_bucket",
TILE_CACHE_BUCKET,
"--zoom_level",
str(zoom_level),
"--bit_depth",
str(bit_depth),
]
# TODO: GTC-1090, GTC 1091
# this should be the default. However there seems to be an issue
# with some of the symbology function (discrete, date-conf-intensity)
# which generate empty tiles at least during tests.
#
if zoom_level > 9:
cmd += ["--skip"]
cmd += [asset_prefix]
tile_cache_job = GDAL2TilesJob(
dataset=dataset,
job_name=f"generate_tile_cache_zoom_{zoom_level}",
command=cmd,
environment=JOB_ENV,
callback=callback,
parents=[parent.job_name for parent in parents],
)
tile_cache_job = scale_batch_job(tile_cache_job, zoom_level)
return tile_cache_job
async def _serialized_asset(asset_orm: ORMAsset) -> Asset:
version_orm: ORMVersion = await versions.get_version(
asset_orm.dataset, asset_orm.version)
data: Asset = Asset.from_orm(asset_orm)
data.metadata = asset_metadata_factory(asset_orm.asset_type,
asset_orm.metadata)
data.creation_options = asset_creation_option_factory(
version_orm.source_type, asset_orm.asset_type,
asset_orm.creation_options)
logger.debug(f"Metadata: {data.metadata.dict(by_alias=True)}")
return data
async def create_user_area(geometry: Geometry) -> Geostore:
# Sanitize the JSON by doing a round-trip with Postgres. We want the sort
# order, whitespace, etc. to match what would be saved via other means
# (in particular, via batch/scripts/add_gfw_fields.sh)
geometry_str = geometry.json()
sql = db.text("SELECT ST_AsGeoJSON(ST_GeomFromGeoJSON(:geo)::geometry);")
bind_vals = {"geo": geometry_str}
sql = sql.bindparams(**bind_vals)
logger.debug(sql)
sanitized_json = await db.scalar(sql)
bbox: List[float] = await db.scalar(
f"""
SELECT ARRAY[
ST_XMin(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)),
ST_YMin(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)),
ST_XMax(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry)),
ST_YMax(ST_Envelope(ST_GeomFromGeoJSON('{sanitized_json}')::geometry))
]::NUMERIC[];
"""
)
area: float = await db.scalar(
f"""
SELECT ST_Area(
ST_GeomFromGeoJSON(
'{sanitized_json}'
)::geography
)
"""
)
area = area / 10000
# We could easily do this in Python but we want PostgreSQL's behavior
# (if different) to be the source of truth.
# geo_id = UUID(str(hashlib.md5(feature_json.encode("UTF-8")).hexdigest()))
geo_id: UUID = await db.scalar(f"SELECT MD5('{sanitized_json}')::uuid;")
try:
user_area = await ORMUserArea.create(
gfw_geostore_id=geo_id,
gfw_geojson=sanitized_json,
gfw_area__ha=area,
gfw_bbox=bbox,
)
geostore: Geostore = Geostore.from_orm(user_area)
except UniqueViolationError:
geostore = await get_geostore_from_anywhere(geo_id)
return geostore
def produce_upload_message(json_payload):
if not KAFKA_PRODUCER:
raise Exception("Kafka not available")
logger.debug("to producer.send()")
future = KAFKA_PRODUCER.send(KAFKA_TOPIC, json_payload)
try:
record_metadata = future.get(timeout=10)
logger.info("send future completed")
return record_metadata
except KafkaError:
logger.exception('Failed to send to kafka')
raise
async def actor_deactivation(actor_type_name: str, actor_id: str):
try:
await ActorRuntime.deactivate(actor_type_name, actor_id)
except DaprInternalError as ex:
return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR,
ex.as_dict())
except Exception as ex:
return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR,
repr(ex), ERROR_CODE_UNKNOWN)
msg = f'deactivated actor: {actor_type_name}.{actor_id}'
logger.debug(msg)
return _wrap_response(status.HTTP_200_OK, msg)
async def get_fields(dv: Tuple[str,
str] = Depends(dataset_version_dependency)):
dataset, version = dv
asset = await assets.get_default_asset(dataset, version)
logger.debug(f"Processing default asset type {asset.asset_type}")
fields: Union[List[FieldMetadata], List[RasterFieldMetadata]] = []
if asset.asset_type == AssetType.raster_tile_set:
fields = await _get_raster_fields(asset)
else:
fields = [FieldMetadata(**field) for field in asset.fields]
return FieldMetadataResponse(data=fields)
async def parser_incm(request: Request) -> schemas.Incm:
body = await request.json()
out = await get_incm_cmd(body)
if not out:
out = await get_incm_txt(body)
if not out:
out = await get_callback_json(body)
if not out:
out = await get_callback(body)
if not out:
out = await get_incm_location(body)
if not out:
logger.warning(f"unknown income: {json.dumps(body)}")
logger.debug(type(out))
return out
def notify_upload(url, account_id, tenant_id, bundle_id):
logger.debug("notify_upload")
bundle_file = get_bundle_path(bundle_id)
bundle_size = os.stat(bundle_file).st_size
payload = {
'account': account_id,
'b64_identity': '__=',
'category': 'analytics',
'metadata': {},
'principal': tenant_id,
'request_id': bundle_id,
'service': 'tower',
'size': bundle_size,
'timestamp': datetime.datetime.now().astimezone().isoformat(),
'url': '{}/bundles/{}?done=True'.format(url, bundle_id)
}
return produce_upload_message(payload)
async def actor_timer(actor_type_name: str, actor_id: str,
timer_name: str, request: Request):
try:
# Read raw bytes from request stream
req_body = await request.body()
await ActorRuntime.fire_timer(actor_type_name, actor_id,
timer_name, req_body)
except DaprInternalError as ex:
return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR,
ex.as_dict())
except Exception as ex:
return _wrap_response(status.HTTP_500_INTERNAL_SERVER_ERROR,
repr(ex), ERROR_CODE_UNKNOWN)
msg = f'called timer. actor: {actor_type_name}.{actor_id}, timer: {timer_name}'
logger.debug(msg)
return _wrap_response(status.HTTP_200_OK, msg)
async def create_stream_app(data:CreateStreamAppReq):
"""
Create a streaming application through apache livy.
"""
if not data.verifySparkAppName():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail={
"msg": "sparkAppName is invalid"
})
conf = get_config()
livyConf = conf["livy"]
url = livyConf["url"] + "/batches"
headers = createLivyCommonHeaders()
wordCountUpdateUrl = livyConf['wordCountUpdateUrl']
json = {
"file": livyConf["jarPath"],
"className": livyConf["className"],
"queue": livyConf["queue"],
"numExecutors": livyConf["numExecutors"],
"executorMemory": livyConf["executorMemory"],
"executorCores": livyConf["executorCores"],
"driverMemory": livyConf["driverMemory"],
"args": [
"--spark.app.name", data.sparkAppName,
"--kafka.brokers", livyConf["kafka"]["brokers"],
"--kafka.topics", livyConf["kafka"]["topics"],
"--kafka.groupId", livyConf["kafka"]["groupId"],
"--batch.duration.seconds", livyConf["duration"]["batch"],
"--wordcount.update.url", wordCountUpdateUrl,
],
"name": data.sparkAppName
}
livyResp = requests.post(url,
json=json,
headers=headers,
timeout=get_global_request_timeout(),
)
livyRespJson = livyResp.json()
if livyResp.status_code != status.HTTP_201_CREATED:
raise HTTPException(
status_code=livyResp.status_code,
detail=livyRespJson
)
resp = LivyBatch.parse_obj(livyRespJson)
fastapi_logger.debug(resp)
return resp
