def get_gcp_blob_content(bucket_name, source_file_name, ext="model"):
"""Downloads a model from bucket and returns its name
Args:
bucket_name: gcp bucket name
source_file_name: name of a model
ext: extension type
Returns:
file: file name
"""
file = f"/tmp/{source_file_name}"
if os.path.exists(file):
return file
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(source_file_name)
if ext in ["txt", "html", "xml", "htm", "xht"]:
file = blob.download_as_string().decode("utf-8")
else:
with open(file, "wb") as ff:
blob.download_to_file(ff)
logger.info(f"Downloaded model into {file}")
return file
def nullify_session(self, errmsg):
# Sets Variables to ensure error handling
logger.info(errmsg)
self.content = []
self.remaining = 0
self.limit = 0
self.error = errmsg
logger.error('Nullifying Session...')
async def startup_event():
setup_routers()
logger.info("Starting application")
try:
db = Session()
db.execute("SELECT 1")
except:
logger.fatal("Failed to connect to the DB")
os._exit(1)
def _get_link(self):
config_web = self.config_web
logger.info("STEP: DOWNLOAD - START")
link_dict = dict()
link_dict['server_url'] = config_web.server_url
link_dict['base_url'] = config_web.base_url
link_dict['file_name'] = config_web.file_name
url_prov = UrlProvider(**link_dict)
url = url_prov.get_download_link()
return url
def create_person(self, data: Person):
person = Register.create_person()
person.id = str(uuid4())
person.name = data.name
person.age = data.age
person.phone = data.phone
person.address = data.address
person.save()
logger.info(f'Person {data.name} has been registered.')
return self.serialize(person)
def get_converter(self):
logger.info("Starting convert part")
config = self.config
self.down_dict['dest_dir'] = config.temp_dir
self.down_dict['work_dir'] = config.work_dir
self.down_dict['output_dir'] = config.output_dir
try:
converter = DataConverter(**self.down_dict)
file_conv = converter.converter()
except Exception as err:
logger.warning('FAILED TO PROCESS STEP; {}'.format(err))
return False
logger.info("STEP: CONVERT - DONE")
return True
async def main():
"""Creates first admin user if on fresh db."""
first_admin_id = os.getenv('FIRST_ADMIN_ID')
first_admin_email = os.getenv('FIRST_ADMIN_EMAIL')
first_admin_password = os.getenv('FIRST_ADMIN_PASSWORD')
async with db:
admin = await User.objects.get_or_none(id=first_admin_id)
if not admin:
logger.info('Creating first admin user.')
await User.objects.create(
**{
'id': first_admin_id,
'email': first_admin_email,
'hashed_password': get_password_hash(first_admin_password),
'scopes': 'admin'
})
def download_file(self):
logger.info("Starting Download")
config = self.config
self.down_dict['dest_dir'] = config.temp_dir
self.down_dict['own_name'] = config.own_name
self.down_dict['url'] = self._get_link()
try:
downloader = HttpDownloader(**self.down_dict)
result = downloader.download()
except Exception as err:
logger.warning('FAILED TO PROCESS STEP; {}'.format(err))
return False
logger.info("STEP: DOWNLOAD ARCHIVE - DONE")
return result
def get_repo_data():
#TODO: Clean this shit up
folders = [f for f in repo.content if f.path.endswith('-code')]
repo_json = {'folders': {}}
errors = []
for folder in folders:
message_data = None
files_by_group = defaultdict(dict)
folder_name = folder.name
for file_data in folder.raw_data:
if file_data['name'].endswith('.name'):
folder_name = file_data['name'].replace('.name', '')
continue
if file_data['name'] == 'message.html':
message_data = file_data
continue
chunks = file_data['name'].split('_')
if len(chunks) != 2:
logger.error('Skipping Gist. Could not parse: ' + str(chunks))
errors.append('Could not Parse: {}'.format(str(chunks)))
continue
file_group, file_name = chunks
files_by_group[file_group][file_name] = {'name': file_name.split('.')[0],
'id': file_name.replace('.','-'),
'data': file_data
}
folder_json = {'name': folder_name, 'contents': files_by_group }
repo_json['folders'][folder.name] = folder_json
if message_data:
repo_json['folders'][folder.name]['message'] = message_data
logger.info('Gettings Repo. Rate: {}/{}'.format(repo.remaining,
repo.limit))
if hasattr(repo, 'error'):
errors.append(repo.error.data['message'])
if not repo_json['folders']:
errors.append('repository did not return valid data')
if errors:
repo_json['error'] = errors
return repo_json
def get_download_link(self):
"""
Finds web-link for downloading file
:return:
new_url: str
a formatted string that describes a web-link that will use for downloading file
Methods
-------
__check_url
Validates web-links
__check_driver
Setups configurations to Selenium
Raises
------
AttributeError
If self.latest_pattern cannot be find in web-link content (page_content)
ConnectionError
If parsing_web_result is empty
If self.__partial_url is empty
"""
self.__check_url()
self.__set_up_driver().get(self._base_url)
time.sleep(3)
parsing_web_result = self.driver.execute_script("return document.body.innerHTML")
if parsing_web_result is None:
raise ConnectionError("No access for {}".format(self._base_url))
soup = BeautifulSoup(parsing_web_result, 'lxml')
page_content = soup.find('div', {'class': 's-downloadable-resources__results'})
logger.info("Searching link for {}".format(self._file_name[:-2]))
try:
for tag in page_content.find_all(["a"]):
if self.latest_pattern.search(tag.text):
self.__partial_url = tag.get('href')
break
except AttributeError:
raise ConnectionError("We weren't able to get web-link for downloading file"
" due to {} was not found in current web-link".format(self._file_name)) from None
new_url = self._server_url + str(self.__partial_url)
logger.info("Searching web link is {}".format(new_url))
return new_url
def create_ceps(self):
if CEP.objects.count() == 0:
logger.info('Inserindo dados no DB')
dir_path = os.path.dirname(os.path.realpath(__file__))
filename = os.path.join(dir_path, 'ceps.txt')
with open(filename, newline='') as ceps:
ceps_reader = csv.reader(ceps, delimiter='\t')
for cep in ceps_reader:
c = CEP()
c.cep = cep[0]
c.cidade = cep[1]
c.bairro = cep[2]
c.endereco = cep[3]
c.save()
else:
logger.info('Os CEPs já foram inseridos na base de dados.')
def __init__(self):
logger.info('Initializing Gist Session')
self._session = Github(GithubRepoWrapper.OAUTH)
try:
self._session.get_user().id
except Exception as errmsg:
logger.error('GitWrapper Error:')
self.nullify_session(errmsg)
else:
self.update_rates()
self._user = self._session.get_user()
self.update_content()
self.limit = self._session.get_rate_limit().rate.limit
logger.info('Rate Limit: {}/{}'.format(self.remaining,
self.limit))
if self.limit < 1:
self.nullify_session('Rate Limit Exceeded')
def __check_url(self):
"""
Validates web-links
:return:
self._url: str
a formatted string that describes a web-link that will use for downloading file
Raises
------
AttributeError
If web-link are not as a web-link
"""
if UrlRegex(self._url).check_url() is None:
logger.info('Param for download web-link is fine')
return self._url
else:
raise AttributeError('Params for download web-links are not fine')
def config_connection_tortoise():
settings = Settings()
if settings.ENVIRONMENT == "PROD":
logger.info("Using POSTGRES PROD database")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return {
"connections": {
"default": {
"engine": "tortoise.backends.asyncpg",
"credentials": {
"database": settings.DATABASE_NAME,
"host": settings.DATABASE_HOST,
"password": settings.DATABASE_PASS,
"port": settings.DATABASE_PORT,
"user": settings.DATABASE_USER,
"ssl": ctx
}
}
},
"apps": {
"models": {
"models": settings.MODELS_TORTOISE,
"default_connection": "default",
}
}
}
else:
logger.info("Using SQLITE DEV database")
return {
"connections": {
"default": settings.DATABASE_URL
},
"apps": {
"models": {
"models": settings.MODELS_TORTOISE,
"default_connection": "default",
}
}
}
def download(self):
# todo: exception
"""
Downloads file
Methods
-------
__check_url
Validates web-links
__check_driver
Setups configurations to Selenium
:return:
filename
The path of saved file
"""
self.__check_url()
filename = os.path.join(self._dest_dir, self._own_name)
r = requests.get(self._url, stream=True)
logger.info("Starting download from url {} to {}".format(self._url, filename))
cnk_size = 1024 * 8 * 8
cnt = 1
with open(filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=cnk_size):
if chunk:
f.write(chunk)
f.flush()
logger.info('Downloaded {} KB'.format(cnt * cnk_size))
cnt += 1
logger.info('Finished file download: {}'.format(filename))
return filename
def __check_url(self):
"""
Validates web-links
:return:
self._server_url: str
a formatted string that describes a web-link that will be use as begin of query for downloading file
self._base_url: str
a formatted string that describes a web-link from where link to download file could be find
Raises
------
AttributeError
If web-link are not as a web-link
"""
if UrlRegex(self._server_url).check_url() is None:
logger.info('Param for server web-link is fine')
if UrlRegex(self._base_url).check_url() is None:
logger.info('Param for base web-link is fine')
return self._server_url, self._base_url
else:
raise AttributeError('Params for web-links are not fine')
def __set_up_driver(self):
"""
Setups configurations to Selenium
:return:
self.driver
Selenium WebDriver with needful configurations
Raises
------
SystemError
If will be problem with Chrome browser
"""
chrome_options = Options()
chrome_options.add_argument("--headless")
try:
self.driver = webdriver.Chrome(options=chrome_options, executable_path=ChromeDriverManager().install())
logger.info("Successfully set up driver for selenium")
return self.driver
except SystemError:
logger.warning("Need to update Chrome")
def __init__(
self,
uri: str,
dbname: str,
connection_timeout: int,
min_pool_size: int,
max_pool_size: int,
):
super().__init__()
self._database = dbname
self._db = motor_asyncio.AsyncIOMotorClient(
uri,
serverSelectionTimeoutMS=connection_timeout,
minPoolSize=min_pool_size,
maxPoolSize=max_pool_size,
).get_database(
dbname,
codec_options=bson.codec_options.CodecOptions(
uuid_representation=bson.binary.UUID_SUBTYPE),
)
logger.info(f"Database connection to: {dbname}")
from flask_caching import Cache
from flask_s3 import FlaskS3
from flask_debugtoolbar import DebugToolbarExtension
from app.assets import css_assets, js_assets, js_api_assets
from app.utils.logger import logger
# Config
app = Flask(__name__)
compress = Compress()
flasks3 = FlaskS3()
cache = Cache()
toolbar = DebugToolbarExtension()
flask_config = os.environ['FLASK_CONFIG']
logger.info('** FLASK_CONFIG: {}'.format(flask_config))
app.config.from_object('app.config.{}'.format(flask_config))
compress.init_app(app)
flasks3.init_app(app)
cache.init_app(app)
toolbar.init_app(app)
if bool(int(os.getenv('CACHE_CLEAR', 0))):
cache.clear()
logger.info('** Cached cleared [CLEAR_CACHE] True')
# ASSETS
assets = Environment(app)
assets.register('css_assets', css_assets)
assets.register('js_assets', js_assets)
import json
from collections import OrderedDict
import os
from app import app
from app.utils.logger import logger
DB_INDEX_FILENAME = 'db_index_min.json'
template_dir = app.config['TEMPLATEDIR']
json_dir = os.path.join(template_dir, 'json')
db_path = os.path.join(json_dir, DB_INDEX_FILENAME)
with open(db_path) as fp:
db_json = json.load(fp)
# TODO: Stor Pre-sorted DB?
logger.info('DB_INDEX.JSON loaded.')
available_apis = app.config['AVAILABLE_APIS']
namespace_jsons = {}
# REMOVED: SAVED ON S3 requested directly through AJAX
# for year in available_apis:
# filename = 'namespace_{year}_min.json'.format(year=year)
# fullpath = os.path.join(json_dir, filename)
# with open(fullpath) as fp:
# jdata = json.load(fp, object_hook=OrderedDict)
# namespace_jsons[year] = jdata
# logger.info('Namespace Tree loaded.')
from redis import Redis
from app.config import settings
from app.utils.logger import logger
logger.info('Connecting to redis...')
redisclient = Redis(host=settings.cache.host, port=settings.cache.port)
def __init__(self, model=None, lang=True, max_length=None):
self.tokenizer = tokenizers[lang]
self.model = SequenceTagger.load(model)
self.max_length = int(max_length) if max_length else None
logger.info(f"NER model was loaded. {model}")
# //Init DB session
import uuid
from cassandra.cqlengine.models import Model
from cassandra.cqlengine.management import sync_table
from cassandra.cqlengine import (columns, connection)
from app.utils.logger import logger
from app.config import settings
logger.info('Connecting to database...')
logger.debug('Contact points:')
logger.debug(settings.database.hosts)
"""USER MODELS"""
class User_by_id_Model(Model):
__table_name__ = "user_by_id"
user_id = columns.UUID(primary_key=True, default=uuid.uuid4, required=True)
username = columns.Text(required=True)
srp_salt = columns.Text(required=True)
srp_verifier = columns.Text(required=True)
kek_salt = columns.Text(required=True)
cek = columns.Text(required=True)
class User_by_username_Model(Model):
__table_name__ = "user_by_username"
username = columns.Text(primary_key=True, required=True)
user_id = columns.UUID(required=True)
def __connect(self):
logger.info('Conneting to DB')
mongoengine.connect(host=MongoConfigs.MONGO_URI)
logger.info('Connected to DB')
def run(self):
uvicorn.run(app)
logger.info("Busca de Cep finalizada...")
def __connect(self):
logger.info("Connecting on mongo")
mongoengine.connect(host=MongoConfigs.MONGO_URI)
logger.info("Mongo connected")
def run(self):
uvicorn.run(app)
logger.info("People register stoping...")
