def upload_db_model(
db_model: Model,
credentials_file: str,
transaction: Optional[Transaction] = None,
batch: Optional[WriteBatch] = None,
) -> Model:
"""
Upload or update an existing database model.
Parameters
----------
db_model: Model
The database model to upload.
credentials_file: str
Path to Google Service Account Credentials JSON file.
transaction: Optional[Transaction]
The transaction to write this model during.
batch: Optional[WriteBatch]
The write batch to use during uploading this model.
Returns
-------
db_model: Model
The uploaded, or updated, database model.
"""
# Init transaction and auth
fireo.connection(from_file=credentials_file)
# Generate id and upsert
db_model = generate_and_attach_doc_hash_as_id(db_model)
db_model = db_model.upsert(transaction=transaction, batch=batch)
return db_model
def upload_db_model(
db_model: Model,
ingestion_model: Optional[IngestionModel],
creds_file: str,
) -> Model:
"""
Upload or update an existing database model.
Parameters
----------
db_model: Model
The database model to upload.
ingestion_model: Optional[IngestionModel]
The accompanying ingestion model in the case the model already exists and needs
to be updated rather than inserted.
creds_file: str
Path to Google Service Account Credentials JSON file.
Returns
-------
db_model: Model
The uploaded, or updated, database model.
Raises
------
exceptions.UniquenessError
More than one (1) conflicting model was found in the database. This should
never occur and indicates that something is wrong with the database.
"""
# Initialize fireo connection
fireo.connection(from_file=creds_file)
uniqueness_validation = get_model_uniqueness(db_model)
if uniqueness_validation.is_unique:
db_model.save()
log.info(
f"Saved new {db_model.__class__.__name__} with document id={db_model.id}."
)
elif (len(uniqueness_validation.conflicting_models) == 1
and ingestion_model is not None):
updated_db_model = update_db_model(
uniqueness_validation.conflicting_models[0],
ingestion_model,
)
return updated_db_model
else:
raise exceptions.UniquenessError(
model=db_model,
conflicting_results=uniqueness_validation.conflicting_models)
return db_model
def run(self):
# init Firestore connection
firebase_auth = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
fireo.connection(from_file=firebase_auth)
try:
with open('tests/ghost_names.csv', mode='r') as self.csv_file:
self.csv_reader = csv.reader(self.csv_file, delimiter=',')
header = next(self.csv_reader, None) # skip header
for name, description in self.csv_reader:
print(name, "-->", description)
self.ghost_record.id = name
self.ghost_record.name = name
self.ghost_record.description = description
self.ghost_record.save()
except Exception as e:
print(e)
return False
def _clean_cdp_database(google_creds_path: Path,
clean_index: bool = False) -> None:
# Connect to database
fireo.connection(from_file=google_creds_path)
# Remove indexed event gram if we don't want to clean the index
if not clean_index:
for index_model in [
db_models.IndexedEventGram,
]:
DATABASE_MODELS.remove(index_model)
# Iter through database models and delete the whole collection
for model in DATABASE_MODELS:
log.info(f"Cleaning collection: {model.collection_name}")
model.collection.delete()
log.info("Database cleaning complete")
def main() -> None:
try:
args = Args()
# Connect to the database
fireo.connection(client=Client(
project=args.instance,
credentials=AnonymousCredentials(),
))
run_remote_search(args.query, args.sort_by, args.first)
run_local_search(args.query, args.local_index_glob, args.sort_by,
args.first)
except Exception as e:
log.error("=============================================")
log.error("\n\n" + traceback.format_exc())
log.error("=============================================")
log.error("\n\n" + str(e) + "\n")
log.error("=============================================")
sys.exit(1)
def get_all_of_collection(
db_model: Model, credentials_file: str, batch_size: int = 1000
) -> List[Model]:
"""
Get all documents in a collection as a single list but request in batches.
Parameters
----------
db_model: Model
The CDP database model to get all documents for.
credentials_file: str
Path to Google Service Account Credentials JSON file.
batch_size: int
How many documents to request at a single time.
Default: 1000
Returns
-------
documents: List[Model]
All documents in the model's collection.
"""
fireo.connection(from_file=credentials_file)
# Construct all documents list and fill as batches return
all_documents: List[Model] = []
paginator = db_model.collection.fetch(batch_size)
all_documents_gathered = False
while not all_documents_gathered:
batch = list(paginator)
all_documents += batch
if len(batch) == 0:
all_documents_gathered = True
else:
paginator.next_fetch()
return all_documents
import firebase_admin
from fireo.models import Model
from fireo import fields, connection
from fireo.fields import errors
from firebase_admin import firestore as _firestore
from firebase_admin import credentials
from google.cloud import firestore
from settings import FIREBASE_OPTIONS, DEV, FIREBASE_SERVICE_ACCOUNT
if not DEV:
cred = credentials.Certificate(FIREBASE_SERVICE_ACCOUNT)
firebase_admin.initialize_app(cred, options=FIREBASE_OPTIONS)
connection(from_file=FIREBASE_SERVICE_ACCOUNT)
else:
firebase_admin.initialize_app()
db = _firestore.client()
FAILED = 'failed'
SUCCESS = 'success'
RUNNING = 'running'
STASIS = 'stasis'
POSSIBLE_STATUS = {
FAILED,
RUNNING,
STASIS,
SUCCESS,
}
from typing import List, Any, Tuple, Iterable, TYPE_CHECKING, TypeVar, Generic
from flask import Flask
from flask_cors import CORS
from flask_graphql import GraphQLView
import graphene
from graphene import ObjectType # type: ignore
import random
from server import settings
import fireo
fireo.connection(from_file=settings.FIREBASE_ORM_CERTIFICATE)
from . import models
app = Flask(__name__)
CORS(app)
if TYPE_CHECKING:
class MyObjectType(ObjectType[Any]):
pass
else:
class MyObjectType(ObjectType):
pass
class Cell(MyObjectType):
x = graphene.Int(required=True)
y = graphene.Int(required=True)
import logging
import fireo
from flask import request, Flask
from flask_cors import CORS
from flask_restful import Resource, Api
from views.heroes import HeroesHandler, HeroHandler, TopHeroesHandler, \
DocumentationHandler
app = Flask(__name__)
CORS(app)
API = Api(app)
fireo.connection(
from_file='./api-heroes-51e3b-firebase-adminsdk-ika07-67ab2b50f9.json')
@app.before_request
def start_request():
"""Start api request"""
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.DEBUG)
if request.method == 'OPTIONS':
return '', 200
if not request.endpoint:
return 'Sorry, Nothing at this URL.', 404
class Index(Resource):
""" class return API index """
import streamlit as st
import numpy as np
import pandas as pd
import fireo
from fireo.models import Model, TextField, NumberField
fireo.connection(from_file="keyfile.json")
class Post(Model):
content = TextField()
urgency = NumberField()
category = TextField()
data = [[x.urgency, x.content, x.category] for x in Post.collection.fetch()]
df = pd.DataFrame(data, columns=(['Urgencia', 'Contenido', 'Categoria']))
st.title(':pill: COVIBOT ')
st.write('COVIBOT está buscando ayuda relacionada a COVID en twitter')
st.write('{} tweets hasta ahora'.format(len(data)))
cols = ["Urgencia", "Contenido", "Categoria"]
st_ms = st.multiselect("Columns", df.columns.tolist(), default=cols)
st.dataframe(df)
# if dist_key == 'Normal':
# nums = np.random.randn(1000)
# elif dist_key == 'Uniform':
# nums = np.array([np.random.randint(100) for i in range(1000)])
"""Start module, its only for dev purposes."""
from api.server import app
import os
from api.models.User import User
import fireo
from firebase_admin import credentials, initialize_app
os.environ["DEBUG"] = "true"
cred = None
if os.getenv("DEBUG") == 'true':
cred = credentials.Certificate(
"/auth/desafio-conecta-d4fbb-firebase-adminsdk-n45a5-99ce66d235.json")
else:
cred = credentials.Certificate()
initialize_app(cred, {'storageBucket': 'desafio-conecta-d4fbb.appspot.com'})
fireo.connection(
from_file="./desafio-conecta-d4fbb-firebase-adminsdk-n45a5-99ce66d235.json"
)
app.run(debug=False, host="0.0.0.0")
def connect():
fireo.connection(from_file=main_config.gckey)