def create_table():
try:
if not db.dialect.has_table(db, "predictions"):
prediction_table.create()
logger.info("Prediction table created")
else:
logger.info("Prediction table already exists")
pass
except Exception as e:
logger.error(f"Error when trying to create/connect to the db {e}")
def create_if_not_exist(session, name: str = None):
test_slug = slugify(name)
result = session.query(Category).filter(
Category.slug == test_slug).one_or_none()
if result:
logger.info("Category exists, associating recipe")
return result
else:
logger.info("Category doesn't exists, creating tag")
return Category(name=name)
def bind(self):
try:
e = CURRENT_ENGINE.get()
bind = e.result()
logger.info(f"Set bind to {bind.repr(color=True)}")
return bind
except LookupError:
# not in a request
logger.info("Not in a request, using default bind")
return self._bind
async def initial_deal():
try:
query = sa.select([deals]).order_by(deals.c.created_at.desc()).limit(1)
rlt = await database.execute(query)
if not rlt:
query = deals.insert().values(uuid=uuid.uuid4(),
balance=0.0).returning(deals.c.uuid)
rlt = await database.execute(query)
logger.info(f"Initial deal created {rlt}")
except Exception as e:
logger.warning(f"initial_deal: Exception: {e}")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
while True:
data = await websocket.receive_text()
chat_message = base_models.ChatMessage(uid_src=1, uid_dest=2, message=data)
is_profanity, msg_dict = await api.chat_api.send_chat(chat_message)
if not is_profanity:
await websocket.send_text(f"Message text was: {data}")
else:
logger.info('### Profanity detected')
print('### Profanity detected')
def print_response(response):
if not CONFIG.print_log:
return
if response.history:
logger.info(' Request was redirected')
for resp in response.history:
logger.info(f' status_code = {resp.status_code}, url= {resp.url}')
logger.info(' Final destination:')
else:
logger.info(' Request was not redirected')
logger.info(f' status_code = {response.status_code}, url= {response.url}')
def default_user_init(session: Session):
default_user = {
"full_name": "Change Me",
"email": "*****@*****.**",
"password": get_password_hash(settings.DEFAULT_PASSWORD),
"group": settings.DEFAULT_GROUP,
"admin": True,
}
logger.info("Generating Default User")
db.users.create(session, default_user)
def produce_upload_message(json_payload):
if not KAFKA_PRODUCER:
raise Exception("Kafka not available")
logger.debug("to producer.send()")
future = KAFKA_PRODUCER.send(KAFKA_TOPIC, json_payload)
try:
record_metadata = future.get(timeout=10)
logger.info("send future completed")
return record_metadata
except KafkaError:
logger.exception('Failed to send to kafka')
raise
def load_model():
model_path = PATH_TO_MODEL
desc_path = PATH_TO_DESCRIPTION
global model
model = None
logger.info('Start to load model')
time.sleep(20)
with open(model_path, 'rb') as file:
model = pickle.load(file)
global desc
logger.info('Model has been loaded!')
desc = classes.read_attempt_params(desc_path)
async def get_items(q: str = Query(...,
min_length=5,
max_length=50,
regex="^fixedquery$")):
try:
results = [{"item_id": "Foo"}, {"item_id": "Bar"}]
if q:
logger.info(type(q))
results.append({"q": q})
return results
except AttributeError as e:
logger.warning(msg=e)
async def translate(text: str = Form(...),
source_language: str = Form(...),
target_language: str = Form(...)):
data = {"source": source_language, "target": target_language, "q": text}
req = requests.get("http://taln.upf.edu/mmt-es_en/translate", params=data)
json_response = req.json()
text = json_response['data']['translation']
response = {"text": text}
logger.info(response)
return response
async def callback(request: Request):
# get X-Line-Signature header value
signature: str = request.headers["X-Line-Signature"]
# get request body as text
body: bytes = await request.body()
decode_body: str = body.decode()
logger.info("Request body: {}".format(decode_body))
# handle webhook body
try:
handler.handle(decode_body, signature)
except InvalidSignatureError:
HTTPException(status_code=400, detail="Invalid Signature")
return "OK"
async def _bulk_start_all(self):
await self.db.ready.wait()
await self.app_ready.wait()
webhooks = await self.db.pool.fetch(
f"SELECT {self.target}, webhook_url FROM {self.table}")
for webhook in webhooks:
self._loop.create_task(
self._start_webhook(target=webhook[self.target],
webhook_url=webhook["webhook_url"]))
logger.info(f"started {len(webhooks)} {self.provider} hooks")
def create_bundle(config: BundleConfig, process: bool = True):
"""Create a bundle and return an ID for later reference."""
config.bundle_uuid = str(uuid.uuid4()).replace('-', '')
TestDataGenerator().generate_bundle(config)
if process:
notify_upload(
HOST_URL,
config.account_id,
config.tenant_id,
config.bundle_uuid)
else:
logger.info("Process=False, not sending message")
return config
def convert_float_to_int(
stats: Optional[Dict[str, Any]],
source_asset_co: RasterTileSetSourceCreationOptions,
) -> Tuple[RasterTileSetSourceCreationOptions, str]:
stats = generate_stats(stats)
logger.info("In convert_float_to_int()")
assert len(stats.bands) == 1
stats_min = stats.bands[0].min
stats_max = stats.bands[0].max
value_range = math.fabs(stats_max - stats_min)
logger.info(
f"stats_min: {stats_min} stats_max: {stats_max} value_range: {value_range}"
)
# Shift by 1 (and add 1 later) so any values of zero don't get counted as no_data
uint16_max = np.iinfo(np.uint16).max - 1
# Expand or squeeze to fit into a uint16
mult_factor = (uint16_max / value_range) if value_range else 1
logger.info(f"Multiplicative factor: {mult_factor}")
if isinstance(source_asset_co.no_data, list):
raise RuntimeError("Cannot apply colormap on multi band image")
elif source_asset_co.no_data is None:
old_no_data: str = "None"
elif source_asset_co.no_data == str(np.nan):
old_no_data = "np.nan"
else:
old_no_data = str(source_asset_co.no_data)
calc_str = (f"(A != {old_no_data}).astype(bool) * "
f"(1 + (A - {stats_min}) * {mult_factor}).astype(np.uint16)")
logger.info(f"Resulting calc string: {calc_str}")
source_asset_co.data_type = DataType.uint16
source_asset_co.no_data = 0
if source_asset_co.symbology and source_asset_co.symbology.colormap is not None:
source_asset_co.symbology.colormap = {
(1 + (float(k) - stats_min) * mult_factor): v
for k, v in source_asset_co.symbology.colormap.items()
}
logger.info(
f"Resulting colormap: {source_asset_co.symbology.colormap}")
return source_asset_co, calc_str
def health(background_task: BackgroundTasks, response: Response) -> bool:
global counter
if model is None:
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
return False
elif counter < MAX_RESPONSE:
counter += 1
msg = f'You just use {counter} of {MAX_RESPONSE} requests'
logger.info(msg)
return True
else:
logger.error('Service unavailable')
response.status_code = status.HTTP_404_NOT_FOUND
return False
async def process_input(self, input):
loop = asyncio.get_running_loop()
our_task = {"done_event": asyncio.Event(loop=loop),
"input": input,
"time": loop.time()}
async with self.queue_lock:
if len(self.queue) >= self.max_queue_size:
raise HandlingError("I'm too busy", code=503)
self.queue.append(our_task)
logger.info("enqueued task. new queue size {}".format(len(self.queue)))
self.schedule_processing_if_needed(loop)
await our_task["done_event"].wait()
return our_task["output"]
def upload_blob(img, ext: str, mime: str):
"""Given an img array and extension, uploads it to GStorage."""
if "." in ext:
ext = ext[1:]
filename = str(uuid.uuid4()) + "." + ext
fastapi_logger.info(f"Uploading to Storage: {filename}")
blob = bucket.blob(filename)
with tempfile.NamedTemporaryFile(suffix=ext) as temp:
temp_filename = temp.name + "." + ext
cv2.imwrite(temp_filename, img)
blob.upload_from_filename(temp_filename, content_type=mime)
blob.make_public()
return blob.public_url
async def send_mail(message: Union[Dict, EmailSchema, EmailAttachFileSchema]):
message = message if isinstance(message, Dict) else message.dict()
email = generate_email(**message)
succeed = False
try:
sg = get_sg_client()
response = sg.send(email)
logger.info('send mail status_code: ' + str(response.status_code))
succeed = response.status_code >= 200
# logger.info('response body: ' + str(response.body))
# logger.info('response header' + str(response.headers))
except Exception as e:
logger.warning(f'Error: {str(e)}')
return succeed
async def model_runner(self):
loop = asyncio.get_running_loop()
self.queue_lock = asyncio.Lock(loop=loop)
self.needs_processing = asyncio.Event(loop=loop)
logger.info("started model runner")
while True:
logger.info('Waiting for needs_processing')
await self.needs_processing.wait()
self.needs_processing.clear()
if self.needs_processing_timer is not None:
self.needs_processing_timer.cancel()
self.needs_processing_timer = None
logger.info('Locking queue_lock')
async with self.queue_lock:
if self.queue:
longest_wait = loop.time() - self.queue[0]["time"]
else: # oops
longest_wait = None
logger.info(
"launching processing. queue size: {}. longest wait: {}".format(len(self.queue), longest_wait))
to_process = self.queue[:self.max_batch_size]
del self.queue[:len(to_process)]
self.schedule_processing_if_needed(loop)
# so here we copy, it would be neater to avoid this
batch = [t["input"] for t in to_process]
result = await loop.run_in_executor(
None, functools.partial(self.run_model, batch)
)
for t, r in zip(to_process, result):
t["output"] = r
t["done_event"].set()
del to_process
async def is_service_account(token: str = Depends(oauth2_scheme)) -> bool:
"""Calls GFW API to authorize user.
User must be service account with email [email protected]
"""
response = who_am_i(token)
if response.status_code == 401 or not (
response.json()["email"] == "*****@*****.**"
and "gfw" in response.json()["extraUserData"]["apps"]):
logger.info("Unauthorized user")
raise HTTPException(status_code=401, detail="Unauthorized")
else:
return True
async def is_admin(token: str = Depends(oauth2_scheme)) -> bool:
"""Calls GFW API to authorize user.
User must be ADMIN for gfw app
"""
response = who_am_i(token)
if response.status_code == 401 or not (
response.json()["role"] == "ADMIN"
and "gfw" in response.json()["extraUserData"]["apps"]):
logger.info("Unauthorized user")
raise HTTPException(status_code=401, detail="Unauthorized")
else:
return True
async def get_user(token: str = Depends(oauth2_scheme)) -> Tuple[str, str]:
"""Calls GFW API to authorize user.
This functions check is user of any level is associated with the GFW
app and returns the user ID
"""
response: Response = await who_am_i(token)
if response.status_code == 401 or not (
"gfw" in response.json()["extraUserData"]["apps"]
):
logger.info("Unauthorized user")
raise HTTPException(status_code=401, detail="Unauthorized")
else:
return response.json()["id"], response.json()["role"]
async def _query_raster_lambda(
geometry: Geometry,
sql: str,
grid: Grid = Grid.ten_by_forty_thousand,
format: QueryFormat = QueryFormat.json,
delimiter: Delimiters = Delimiters.comma,
) -> Dict[str, Any]:
data_environment = await _get_data_environment(grid)
payload = {
"geometry": jsonable_encoder(geometry),
"query": sql,
"environment": data_environment.dict()["layers"],
"format": format,
}
logger.info(
f"Submitting raster analysis lambda request with payload: {payload}")
try:
response = await invoke_lambda(RASTER_ANALYSIS_LAMBDA_NAME, payload)
except httpx.TimeoutException:
raise HTTPException(500, "Query took too long to process.")
# invalid response codes are reserved by Lambda specific issues (e.g. too many requests)
if response.status_code >= 300:
raise HTTPException(
500,
f"Raster analysis geoprocessor returned invalid response code {response.status_code}",
)
# response must be in JSEND format or something unexpected happened
response_body = response.json()
if "status" not in response_body or ("data" not in response_body
and "message" not in response_body):
raise HTTPException(
500,
f"Raster analysis lambda received an unexpected response: {response.text}",
)
if response_body["status"] == "failed":
# validation error
raise HTTPException(422, response_body["message"])
elif response_body["status"] == "error":
# geoprocessing error
raise HTTPException(500, response_body["message"])
return response_body
async def update(self):
""" Fetch and cache latest data from Galaxy
Returns:
str of json data from Galaxy
"""
fastapi_logger.info('Fetching %s "%s" metadata',
self.__class__.__name__, self.name)
# Ensure no two lookups occur at the same time
async with asyncio.Lock():
text = await fetch_from_url(self.url(), self.__class__.__name__,
self.name)
if text is None:
return None
jdata = json.loads(text)
self.last_update = datetime.now()
return jdata
async def api_logger(request: Request, response=None, error=None):
time_format = "%Y/%m/%d %H:%M:%S"
t = time() - request.state.start
status_code = error.status_code if error else response.status_code
error_log = None
user = request.state.user
if error:
if request.state.inspect:
frame = request.state.inspect
error_file = frame.f_code.co_filename
error_func = frame.f_code.co_name
error_line = frame.f_lineno
else:
error_func = error_file = error_line = "UNKNOWN"
error_log = dict(
errorFunc=error_func,
location="{} line in {}".format(str(error_line), error_file),
raised=str(error.__class__.__name__),
msg=str(error.ex),
)
email = user.email.split("@") if user and user.email else None
user_log = dict(
client=request.state.ip,
user=user.id if user and user.id else None,
email="**" + email[0][2:-1] + "*@" +
email[1] if user and user.email else None,
)
log_dict = dict(
url=request.url.hostname + request.url.path,
method=str(request.method),
statusCode=status_code,
errorDetail=error_log,
client=user_log,
processedTime=str(round(t * 1000, 5)) + "ms",
datetimeUTC=datetime.utcnow().strftime(time_format),
datetimeKST=(datetime.utcnow() +
timedelta(hours=9)).strftime(time_format),
)
if error and error.status_code >= 500:
logger.error(json.dumps(log_dict))
else:
logger.info(json.dumps(log_dict))
def log_stats(request, data=None, user=None):
"""Log detailed data in JSON for incoming/outgoing API request."""
client_host = request.client.host
msg = {}
# TODO: Log user once authentication is connected. msg["user"] = str(user)
# request.state.time_started = time.time()
msg["time_started"] = str(time.time())
msg["method"] = str(request.method)
msg["url"] = str(request.url)
msg["host"] = str(client_host)
msg["query_params"] = str(request.query_params)
msg["path_params"] = str(request.path_params)
msg["headers"] = dict(request.headers)
msg["data"] = str(data)
fastapi_logger.info(json.dumps(msg))
es.index(index=idx_log, body=json.dumps(msg))
return
async def InitLoop(self, loop: asyncio.AbstractEventLoop):
while True:
fastapi_logger.debug("dsadas")
fastapi_logger.info("dsadas")
try:
self.Connection = await aio_pika.robust_connection.connect_robust(
"amqp://*****:*****@" + RABBITMQ_HOST,
# loop=loop
)
logging.info("Connected to rmq")
break
except Exception as err:
logging.info("Failed to establish connection with RMQ")
logging.info("Retring in 2 sec")
self.Connection = None
await asyncio.sleep(2)
def create_blog_site(*, new_site: CreateBlogSite, current_user: User = Depends(get_current_user)):
title = new_site.title
site_name = new_site.site_name
theme = new_site.theme
blog_site = BlogSite.select().where(BlogSite.user_id == current_user.uuid)
if blog_site:
return fail_response('您的博客站点已存在,不能重复创建')
try:
BlogSite.create(title=title,
site_name=site_name,
theme=theme,
user_id=current_user.uuid)
logger.info(f'创建博客站点name={site_name}站点成功')
return success_response('添加成功')
except Exception as e:
db.rollback()
logger.info(f'创建博客站点name={site_name}站点成功，失败原因：{e}')
return fail_response('创建博客站点失败')
def init_db(db: Session) -> None:
from app import crud
try:
Base.metadata.create_all(bind=db.bind)
if crud.user.get_by_nickname(db, nickname=settings.ADMIN_NAME) is None:
logger.info("Creating new admin user, "
f"as no existing with {settings.ADMIN_NAME} was found")
crud.user.create(
db,
obj_in=schemas.UserCreate(
nickname=settings.ADMIN_NAME,
password=settings.ADMIN_PASSWORD,
),
)
except SQLAlchemyError:
db.rollback()
raise
