def main(argv):
#
# Create an admin client. This is the client we will use to create the database.
#
scheme = "http"
domain = "127.0.0.1"
port = "8443"
secret = "secret"
adminClient = FaunaClient(secret=secret,
domain=domain,
scheme=scheme,
port=port)
#
# If you are using the the FaunaDB-Cloud use these lines to create the connection.
# Change the secret to your value and comment out the lines above
#
# secret = "Your Secret Goes here"
# adminClient = FaunaClient(secret=secret)
dbName = "TestDB"
#
# Call to Create the database
#
res = adminClient.query(q.create_database({"name": dbName}))
print('DB {0} created: {1}'.format(dbName, res))
#
# Call to check to see if database exists and to delete it id it does.
#
res = adminClient.query(
q.if_(q.exists(q.database(dbName)), q.delete(q.database(dbName)),
True))
print('DB {0} deleted: {1}'.format(dbName, res))
def create_database(scheme, domain, port, secret, db_name):
#
# Create an admin client. This is the client we will use to create the database.
#
# If you are using the the FaunaDB-Cloud you will need to replace the value of the
# 'secret' in the command below with your "secret".
#
adminClient = FaunaClient(secret=secret, domain=domain, scheme=scheme, port=port)
print("Connected to FaunaDB as admin!")
#
# The code below creates the Database that will be used for this example. Please note that
# the existence of the database is evaluated, deleted if it exists and recreated with a single
# call to the Fauna DB.
#
res = adminClient.query(
q.if_(
q.exists(q.database(db_name)),
[q.delete(q.database(db_name)), q.create_database({"name": db_name})],
q.create_database({"name": db_name}))
)
print('DB {0} created:'.format(db_name))
pprint.pprint(res)
#
# Create a key specific to the database we just created. We will use this to
# create a new client we will use in the remainder of the examples.
#
res = adminClient.query(q.select(["secret"], q.create_key({"database": q.database(db_name), "role": "server"})))
print('DB {0} secret: {1}'.format(db_name, res))
return res
def __init__(
self,
secret: str,
username: Optional[str] = None,
password: Optional[str] = None,
host: str = "test.mosquitto.org",
port: int = 8883,
client_id: Optional[str] = None,
):
"""
Parameters
----------
secret : str
Auth token for the FaunaDB server.
username : Optional[str], optional
The username to authenticate to the MQTT broker with.
Set to None if not using username/password for broker authentication.
password : Optional[str], optional
The password to authenticate to the MQTT broker with.
Set to None if not using password-based authentication.
host : str, optional
Host name or IP address of the remote broker, by default "test.mosquitto.org".
port : int, optional
Network port of the MQTT broker to connect to, by default 8883.
client_id : Optional[str], optional
The unique client id string used when connecting to the broker.
If it has zero length or is None, then one will be randomly generated.
"""
self.client = mqtt.Client(client_id=client_id)
self.client.on_connect = self._on_connect
self.client.on_message = self._on_message
self.client.tls_set(tls_version=mqtt.ssl.PROTOCOL_TLS)
self.client.username_pw_set(username, password=password)
self.client.connect(host, port=port)
self.db = FaunaClient(secret=secret)
def test_error_on_closed_client(self):
client = FaunaClient(secret="secret")
client.__del__()
self.assertRaisesRegexCompat(
UnexpectedError,
"Cannnot create a session client from a closed session",
lambda: client.new_session_client(secret="new_secret"))
def delete(collection, id):
try:
serverClient = FaunaClient(secret=os.environ.get("FAUNA_SERVER_SECRET"))
serverClient.query(q.delete(q.ref(q.collection(collection), id)))
return True
except Exception as ex:
raise ex
def get_multiple(index, data=None):
"""
Get multiple records by ID
"""
try:
serverClient = FaunaClient(
secret=os.environ.get("FAUNA_SERVER_SECRET"))
res_arr = []
if data is None:
res = serverClient.query(
q.map_(q.lambda_("data", q.get(q.var("data"))),
q.paginate(q.match(q.index(index)))))
res_arr.extend(res["data"])
elif isinstance(data, list):
for x in data:
res = serverClient.query(
q.map_(q.lambda_("data", q.get(q.var("data"))),
q.paginate(q.match(q.index(index), q.casefold(x)))))
res_arr.extend(res["data"])
else:
res = serverClient.query(
q.map_(q.lambda_("data", q.get(q.var("data"))),
q.paginate(q.match(q.index(index), q.casefold(data)))))
res_arr.extend(res["data"])
arr = []
for x in res_arr:
x["data"]["ref_id"] = x["ref"].id()
arr.append(x["data"])
return arr
except Exception as ex:
raise ex
def create_server_client():
"""
create server client, collections and indexes
:return: server client
"""
client = FaunaClient(secret=os.environ.get('FAUNA_SERVER_SECRET'))
client.query(q.create_collection({"name": "users"}))
client.query(q.create_index(
{
"name": "users_by_username",
"source": q.collection("users"),
"permissions": {"read": "public"},
"terms": [{"field": ["data", "username"]}],
"unique": True
}
))
client.query(q.create_collection({"name": "contacts"}))
client.query(q.create_index(
{
"name": "contacts_by_username",
"source": q.collection("contacts"),
"terms": [{"field": ["data", "username"]}]
}
))
return client
def update(collection, id, data):
try:
serverClient = FaunaClient(secret=os.environ.get("FAUNA_SERVER_SECRET"))
res = serverClient.query(q.update(q.ref(q.collection(collection), id), {"data": data}))
res["data"]["id"] = res["ref"].id()
return res["data"]
except Exception as ex:
raise ex
def get(index, data):
try:
serverClient = FaunaClient(secret=os.environ.get("FAUNA_SERVER_SECRET"))
res = serverClient.query(q.get(q.match(q.index(index), data)))
res["data"]["id"] = res["ref"].id()
return res["data"]
except Exception as ex:
raise ex
def logout(user, logout_type):
if logout_type == "all":
all_tokens = True
else:
all_tokens = False
user_client = FaunaClient(secret=session["user_secret"])
result = user_client.query(q.logout(all_tokens))
session.clear()
return redirect(url_for("index"))
def decorated(*args, **kwargs):
if "user_secret" in session:
try:
user_client = FaunaClient(secret=session["user_secret"])
result = user_client.query(q.current_identity())
except Unauthorized as e:
session.clear()
return redirect(url_for("login"))
else:
return redirect(url_for("login"))
return f(result, *args, **kwargs)
def get_by_ref_id(collection, id):
"""
Get record by ID
"""
try:
serverClient = FaunaClient(
secret=os.environ.get("FAUNA_SERVER_SECRET"))
res = serverClient.query(q.get(q.ref(q.collection(collection), id)))
res["data"]["ref_id"] = res["ref"].id()
return res["data"]
except Exception as ex:
raise ex
def create(collection, data):
"""
Create new Record
"""
try:
serverClient = FaunaClient(
secret=os.environ.get("FAUNA_SERVER_SECRET"))
res = serverClient.query(
q.create(q.collection(collection), {"data": data}))
res["data"]["ref_id"] = res["ref"].id()
return res["data"]
except Exception as ex:
raise ex
def login():
body = request.json
client = FaunaClient(secret=FAUNA_SECRET)
try:
result = client.query(
q.login(q.match(q.index("Users_by_username"), body["username"]),
{"password": body["password"]}))
return {"secret": result['secret']}
except faunadb.errors.BadRequest as exception:
error = exception.errors[0]
return {"code": error.code, "description": error.description}, 401
def create_db_client(scheme, domain, port, secret):
#
# Create the DB specific DB client using the DB specific key just created.
#
client = FaunaClient(secret=secret, domain="127.0.0.1", scheme="http", port=8443)
return client
def test_runtime_env_headers(self):
client = FaunaClient(secret="secret")
self.assertEqual(
client.session.headers['X-Driver-Env'],
"driver=python-{0}; runtime=python-{1} env={2}; os={3}".format(
pkg_version, "{0}.{1}.{2}-{3}".format(*sys.version_info),
"Unknown", "{0}-{1}".format(platform.system(),
platform.release())).lower())
class FaunaDB:
def __init__(self, database="Balthus", collection="domains"):
self.database = database
self.collection = collection
FAUNA_SECRET = os.getenv("FAUNA_SECRET")
self.client = FaunaClient(secret=FAUNA_SECRET)
try:
self.client.query(q.create_database({"name": database}))
except:
print("Database: {} already exist".format(database))
ref = self.client.query(
q.create_key({
"database": q.database(database),
"role": "server"
}))
self.client = FaunaClient(secret=ref['secret'])
try:
self.client.query(q.create_collection({"name": collection}))
except:
print("Collection: {} already exist".format(collection))
def fill_collection(self, records):
self.client.query(
q.map_(
lambda data: q.create(q.collection(self.collection),
{"data": data}), records))
def _get_client(cls):
args = {
"domain": _FAUNA_DOMAIN,
"scheme": _FAUNA_SCHEME,
"port": _FAUNA_PORT
}
# If None, use default instead
non_null_args = {k: v for k, v in args.items() if v is not None}
return FaunaClient(secret=_FAUNA_ROOT_KEY, **non_null_args)
class FaunaWrapper:
def __init__(self):
url = os.getenv("FAUNA_KEY")
self.client = FaunaClient(secret=url)
# initialize faunadb client
pass
def get_documents_in_index(self, index="unique_news_items", size=100000):
"""Assume index name exists
Validation not needed personal script
unique_halts, unique_news, unique_short_news
"""
documents = self.client.query(
q.paginate(q.match(q.index(index)), size=100000))
return documents
# Have a faunadb class with a refer to the client
def create_document_in_collection(self, collection, collection_data):
"""Assumes that collection name exists
collections are halts and news and short_news
Validation not needed, personal project <3
"""
try:
result = self.client.query(
q.create(q.collection(collection), {"data": collection_data}))
print(result)
return True
except BadRequest as error:
# get properties of bad request
# print(dir(bad))
if hasattr(error, "_get_description"):
if error._get_description() == "document is not unique.":
ticker = collection_data.get("ticker")
print(f"skipping {ticker} since doc is not unique")
return False
# unknown error, stop everything
except Exception as error:
print(collection_data)
# raise Exception(error)
pass
def session():
"""
Helper function to get faunadb session, if there is one already it just returns
"""
from .config import config
from app.models.environments import Environments
global _session
if _session is None and config.ENVIRONMENT == Environments.DEV:
# We only need the Fauna Host if we are running from other server that's not the faunadb.com official server
_session = FaunaClient(
secret=config.FAUNA_SERVER_KEY.get_secret_value(),
domain=config.FAUNA_DB_URL.host,
port=config.FAUNA_DB_URL.port,
scheme=config.FAUNA_DB_URL.scheme,
)
elif _session is None:
_session = FaunaClient(
secret=config.FAUNA_SERVER_KEY.get_secret_value())
return _session
async def get_pypi_data(package_name: str) -> dict:
console.log(f"Requesting PyPi data for {package_name}.")
request_url = f"https://pypi.org/pypi/{package_name}/json"
response = requests.get(request_url)
if response.status_code == 404:
raise PyPiPackageNotFound
assert response.status_code == 200
package_data = response.json()
package_data["normalized_name"] = normalize(package_data["info"]["name"])
console.log(f"Logging PyPi data for {package_name} to FaunaDB.")
client = FaunaClient(secret=settings.FAUNADB_KEY.get_secret_value())
try:
client.query(
q.create(
q.collection("packages"),
{"data": package_data},
))
except BadRequest:
package = await get_package_by_name(package_name)
ref = package["ref"]
client.query(q.update(ref, {"data": package_data}))
return package_data
def test_recognized_runtime_env_headers(self):
originalPath = os.environ["PATH"]
os.environ["PATH"] = originalPath + ".heroku"
client = FaunaClient(secret="secret")
self.assertEqual(
client.session.headers['X-Driver-Env'],
"driver=python-{0}; runtime=python-{1} env={2}; os={3}".format(
pkg_version, "{0}.{1}.{2}-{3}".format(*sys.version_info),
"Heroku", "{0}-{1}".format(platform.system(),
platform.release())).lower())
os.environ["PATH"] = originalPath
def get_fauna_connection(self) -> FaunaClient:
"""Get Fauna Connection.
input: client_token from class
input_type: str
output: fauna database connection
output_type: FaunaClient
"""
try:
client = FaunaClient(secret=self.client_token)
return client
except Exception as _errorinfo: # pragma: no cover
raise ValueError("error connecting") from _errorinfo
def things():
userSecret = request.headers.get('fauna-user-secret')
client = FaunaClient(secret=userSecret)
try:
result = client.query(
q.map_(q.lambda_("ref", q.get(q.var("ref"))),
q.paginate(q.documents(q.collection("Things")))))
things = map(
lambda doc: {
"id": doc["ref"].id(),
"name": doc["data"]["name"],
"color": doc["data"]["color"]
}, result["data"])
return {"things": list(things)}
except faunadb.errors.Unauthorized as exception:
error = exception.errors[0]
return {"code": error.code, "description": error.description}, 401
def signup():
body = request.json
client = FaunaClient(secret=FAUNA_SECRET)
try:
result = client.query(
q.create(
q.collection("Users"), {
"data": {
"username": body["username"]
},
"credentials": {
"password": body["password"]
}
}))
return {"userId": result['ref'].id()}
except faunadb.errors.BadRequest as exception:
error = exception.errors[0]
return {"code": error.code, "description": error.description}, 409
def do_GET(self):
query = self.path.split('?', 1)
params = parse_qs(query[1])
theme = params.get('theme', '')[0]
m = pymorphy2.MorphAnalyzer()
client = FaunaClient(secret=os.environ.get('DBSECRET'))
futr_news = []
# futr_news = self.getNews('world', client, m) + self.getNews('nation', client, m) + self.getNews('scitech', client, m)
futr_news = self.getNews(theme, client, m)
if len(futr_news) > 0:
client.query(
q.map_(
lambda post: q.create(q.collection('NewsPast'),
{'data': post}), futr_news))
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(json.dumps(futr_news).encode())
return
def __init__(self, collection_name: str, model):
"""
Each CRUD requires that:
- Fauna client is connected to db
- collection_name is created / exists
- Collection is created / exists
"""
self.collection_name = collection_name
self.model = model
self.client = FaunaClient(secret=settings.FAUNADB_SECRET)
if not self.client.query(
query.exists(query.database(
db_name=settings.FAUNADB_DBNAME, ))):
self.database = self.client.query(
query.create_database(db_params={
'name': settings.FAUNADB_DBNAME,
}), )
if not self.client.query(
query.exists(
query.collection(collection_name=collection_name, ))):
self.collection = self.client.query(
query.create_collection(collection_params={
'name': collection_name,
}), )
def create_server_client():
client = FaunaClient(secret=os.environ.get('FAUNA_SECRET'))
client.query(q.create_collection({"name": "users"}))
client.query(
q.create_index({
"name": "users_by_username",
"source": q.collection("users"),
"permissions": {
"read": "public"
},
"terms": [{
"field": ["data", "username"]
}],
"unique": True
}))
client.create_collection()
def get_client(self):
print('grabbing secret')
client = secretmanager.SecretManagerServiceClient()
secret_name = "fauna_deepcite_db"
project_id = "deepcite-306405"
request = {
"name":
f"projects/{project_id}/secrets/{secret_name}/versions/latest"
}
response = client.access_secret_version(request)
secret_string = response.payload.data.decode("UTF-8")
print('connecting to fauna client')
print()
return FaunaClient(secret=secret_string, domain='db.us.fauna.com')
from fastapi import FastAPI, Response, status
from pydantic import BaseModel
import requests
from bs4 import BeautifulSoup
from faunadb import query as q
from faunadb.objects import Ref
from faunadb.client import FaunaClient
class ReadingReference(BaseModel):
url: str
NHK_PREFIX = 'https://www3.nhk.or.jp/news/easy/'
faunadb_secret = os.getenv('FAUNADB_SECRET')
client = FaunaClient(secret=faunadb_secret)
app = FastAPI(openapi_url="/api/openapi.json",
docs_url='/api/docs',
redoc_url='/api/redoc')
@app.post("/api/scrape", status_code=200)
async def scrape_article(article: ReadingReference, response: Response):
if NHK_PREFIX in article.url:
try:
# see if an article exists
client.query(q.get(q.match(q.index('reading_by_url'),
article.url)))
return {"success": True, "status": "DocumentExists"}
except:
