def get_gcp_blob_content(bucket_name, source_file_name, ext="model"): """Downloads a model from bucket and returns its name Args: bucket_name: gcp bucket name source_file_name: name of a model ext: extension type Returns: file: file name """ file = f"/tmp/{source_file_name}" if os.path.exists(file): return file storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) blob = bucket.blob(source_file_name) if ext in ["txt", "html", "xml", "htm", "xht"]: file = blob.download_as_string().decode("utf-8") else: with open(file, "wb") as ff: blob.download_to_file(ff) logger.info(f"Downloaded model into {file}") return file
def nullify_session(self, errmsg): # Sets Variables to ensure error handling logger.info(errmsg) self.content = [] self.remaining = 0 self.limit = 0 self.error = errmsg logger.error('Nullifying Session...')
async def startup_event(): setup_routers() logger.info("Starting application") try: db = Session() db.execute("SELECT 1") except: logger.fatal("Failed to connect to the DB") os._exit(1)
def _get_link(self): config_web = self.config_web logger.info("STEP: DOWNLOAD - START") link_dict = dict() link_dict['server_url'] = config_web.server_url link_dict['base_url'] = config_web.base_url link_dict['file_name'] = config_web.file_name url_prov = UrlProvider(**link_dict) url = url_prov.get_download_link() return url
def create_person(self, data: Person): person = Register.create_person() person.id = str(uuid4()) person.name = data.name person.age = data.age person.phone = data.phone person.address = data.address person.save() logger.info(f'Person {data.name} has been registered.') return self.serialize(person)
def get_converter(self): logger.info("Starting convert part") config = self.config self.down_dict['dest_dir'] = config.temp_dir self.down_dict['work_dir'] = config.work_dir self.down_dict['output_dir'] = config.output_dir try: converter = DataConverter(**self.down_dict) file_conv = converter.converter() except Exception as err: logger.warning('FAILED TO PROCESS STEP; {}'.format(err)) return False logger.info("STEP: CONVERT - DONE") return True
async def main(): """Creates first admin user if on fresh db.""" first_admin_id = os.getenv('FIRST_ADMIN_ID') first_admin_email = os.getenv('FIRST_ADMIN_EMAIL') first_admin_password = os.getenv('FIRST_ADMIN_PASSWORD') async with db: admin = await User.objects.get_or_none(id=first_admin_id) if not admin: logger.info('Creating first admin user.') await User.objects.create( **{ 'id': first_admin_id, 'email': first_admin_email, 'hashed_password': get_password_hash(first_admin_password), 'scopes': 'admin' })
def download_file(self): logger.info("Starting Download") config = self.config self.down_dict['dest_dir'] = config.temp_dir self.down_dict['own_name'] = config.own_name self.down_dict['url'] = self._get_link() try: downloader = HttpDownloader(**self.down_dict) result = downloader.download() except Exception as err: logger.warning('FAILED TO PROCESS STEP; {}'.format(err)) return False logger.info("STEP: DOWNLOAD ARCHIVE - DONE") return result
def get_repo_data(): #TODO: Clean this shit up folders = [f for f in repo.content if f.path.endswith('-code')] repo_json = {'folders': {}} errors = [] for folder in folders: message_data = None files_by_group = defaultdict(dict) folder_name = folder.name for file_data in folder.raw_data: if file_data['name'].endswith('.name'): folder_name = file_data['name'].replace('.name', '') continue if file_data['name'] == 'message.html': message_data = file_data continue chunks = file_data['name'].split('_') if len(chunks) != 2: logger.error('Skipping Gist. Could not parse: ' + str(chunks)) errors.append('Could not Parse: {}'.format(str(chunks))) continue file_group, file_name = chunks files_by_group[file_group][file_name] = {'name': file_name.split('.')[0], 'id': file_name.replace('.','-'), 'data': file_data } folder_json = {'name': folder_name, 'contents': files_by_group } repo_json['folders'][folder.name] = folder_json if message_data: repo_json['folders'][folder.name]['message'] = message_data logger.info('Gettings Repo. Rate: {}/{}'.format(repo.remaining, repo.limit)) if hasattr(repo, 'error'): errors.append(repo.error.data['message']) if not repo_json['folders']: errors.append('repository did not return valid data') if errors: repo_json['error'] = errors return repo_json
def get_download_link(self): """ Finds web-link for downloading file :return: new_url: str a formatted string that describes a web-link that will use for downloading file Methods ------- __check_url Validates web-links __check_driver Setups configurations to Selenium Raises ------ AttributeError If self.latest_pattern cannot be find in web-link content (page_content) ConnectionError If parsing_web_result is empty If self.__partial_url is empty """ self.__check_url() self.__set_up_driver().get(self._base_url) time.sleep(3) parsing_web_result = self.driver.execute_script("return document.body.innerHTML") if parsing_web_result is None: raise ConnectionError("No access for {}".format(self._base_url)) soup = BeautifulSoup(parsing_web_result, 'lxml') page_content = soup.find('div', {'class': 's-downloadable-resources__results'}) logger.info("Searching link for {}".format(self._file_name[:-2])) try: for tag in page_content.find_all(["a"]): if self.latest_pattern.search(tag.text): self.__partial_url = tag.get('href') break except AttributeError: raise ConnectionError("We weren't able to get web-link for downloading file" " due to {} was not found in current web-link".format(self._file_name)) from None new_url = self._server_url + str(self.__partial_url) logger.info("Searching web link is {}".format(new_url)) return new_url
def create_ceps(self): if CEP.objects.count() == 0: logger.info('Inserindo dados no DB') dir_path = os.path.dirname(os.path.realpath(__file__)) filename = os.path.join(dir_path, 'ceps.txt') with open(filename, newline='') as ceps: ceps_reader = csv.reader(ceps, delimiter='\t') for cep in ceps_reader: c = CEP() c.cep = cep[0] c.cidade = cep[1] c.bairro = cep[2] c.endereco = cep[3] c.save() else: logger.info('Os CEPs já foram inseridos na base de dados.')
def __init__(self): logger.info('Initializing Gist Session') self._session = Github(GithubRepoWrapper.OAUTH) try: self._session.get_user().id except Exception as errmsg: logger.error('GitWrapper Error:') self.nullify_session(errmsg) else: self.update_rates() self._user = self._session.get_user() self.update_content() self.limit = self._session.get_rate_limit().rate.limit logger.info('Rate Limit: {}/{}'.format(self.remaining, self.limit)) if self.limit < 1: self.nullify_session('Rate Limit Exceeded')
def __check_url(self): """ Validates web-links :return: self._url: str a formatted string that describes a web-link that will use for downloading file Raises ------ AttributeError If web-link are not as a web-link """ if UrlRegex(self._url).check_url() is None: logger.info('Param for download web-link is fine') return self._url else: raise AttributeError('Params for download web-links are not fine')
def config_connection_tortoise(): settings = Settings() if settings.ENVIRONMENT == "PROD": logger.info("Using POSTGRES PROD database") ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE return { "connections": { "default": { "engine": "tortoise.backends.asyncpg", "credentials": { "database": settings.DATABASE_NAME, "host": settings.DATABASE_HOST, "password": settings.DATABASE_PASS, "port": settings.DATABASE_PORT, "user": settings.DATABASE_USER, "ssl": ctx } } }, "apps": { "models": { "models": settings.MODELS_TORTOISE, "default_connection": "default", } } } else: logger.info("Using SQLITE DEV database") return { "connections": { "default": settings.DATABASE_URL }, "apps": { "models": { "models": settings.MODELS_TORTOISE, "default_connection": "default", } } }
def download(self): # todo: exception """ Downloads file Methods ------- __check_url Validates web-links __check_driver Setups configurations to Selenium :return: filename The path of saved file """ self.__check_url() filename = os.path.join(self._dest_dir, self._own_name) r = requests.get(self._url, stream=True) logger.info("Starting download from url {} to {}".format(self._url, filename)) cnk_size = 1024 * 8 * 8 cnt = 1 with open(filename, 'wb') as f: for chunk in r.iter_content(chunk_size=cnk_size): if chunk: f.write(chunk) f.flush() logger.info('Downloaded {} KB'.format(cnt * cnk_size)) cnt += 1 logger.info('Finished file download: {}'.format(filename)) return filename
def __check_url(self): """ Validates web-links :return: self._server_url: str a formatted string that describes a web-link that will be use as begin of query for downloading file self._base_url: str a formatted string that describes a web-link from where link to download file could be find Raises ------ AttributeError If web-link are not as a web-link """ if UrlRegex(self._server_url).check_url() is None: logger.info('Param for server web-link is fine') if UrlRegex(self._base_url).check_url() is None: logger.info('Param for base web-link is fine') return self._server_url, self._base_url else: raise AttributeError('Params for web-links are not fine')
def __set_up_driver(self): """ Setups configurations to Selenium :return: self.driver Selenium WebDriver with needful configurations Raises ------ SystemError If will be problem with Chrome browser """ chrome_options = Options() chrome_options.add_argument("--headless") try: self.driver = webdriver.Chrome(options=chrome_options, executable_path=ChromeDriverManager().install()) logger.info("Successfully set up driver for selenium") return self.driver except SystemError: logger.warning("Need to update Chrome")
def __init__( self, uri: str, dbname: str, connection_timeout: int, min_pool_size: int, max_pool_size: int, ): super().__init__() self._database = dbname self._db = motor_asyncio.AsyncIOMotorClient( uri, serverSelectionTimeoutMS=connection_timeout, minPoolSize=min_pool_size, maxPoolSize=max_pool_size, ).get_database( dbname, codec_options=bson.codec_options.CodecOptions( uuid_representation=bson.binary.UUID_SUBTYPE), ) logger.info(f"Database connection to: {dbname}")
from flask_caching import Cache from flask_s3 import FlaskS3 from flask_debugtoolbar import DebugToolbarExtension from app.assets import css_assets, js_assets, js_api_assets from app.utils.logger import logger # Config app = Flask(__name__) compress = Compress() flasks3 = FlaskS3() cache = Cache() toolbar = DebugToolbarExtension() flask_config = os.environ['FLASK_CONFIG'] logger.info('** FLASK_CONFIG: {}'.format(flask_config)) app.config.from_object('app.config.{}'.format(flask_config)) compress.init_app(app) flasks3.init_app(app) cache.init_app(app) toolbar.init_app(app) if bool(int(os.getenv('CACHE_CLEAR', 0))): cache.clear() logger.info('** Cached cleared [CLEAR_CACHE] True') # ASSETS assets = Environment(app) assets.register('css_assets', css_assets) assets.register('js_assets', js_assets)
import json from collections import OrderedDict import os from app import app from app.utils.logger import logger DB_INDEX_FILENAME = 'db_index_min.json' template_dir = app.config['TEMPLATEDIR'] json_dir = os.path.join(template_dir, 'json') db_path = os.path.join(json_dir, DB_INDEX_FILENAME) with open(db_path) as fp: db_json = json.load(fp) # TODO: Stor Pre-sorted DB? logger.info('DB_INDEX.JSON loaded.') available_apis = app.config['AVAILABLE_APIS'] namespace_jsons = {} # REMOVED: SAVED ON S3 requested directly through AJAX # for year in available_apis: # filename = 'namespace_{year}_min.json'.format(year=year) # fullpath = os.path.join(json_dir, filename) # with open(fullpath) as fp: # jdata = json.load(fp, object_hook=OrderedDict) # namespace_jsons[year] = jdata # logger.info('Namespace Tree loaded.')
from redis import Redis from app.config import settings from app.utils.logger import logger logger.info('Connecting to redis...') redisclient = Redis(host=settings.cache.host, port=settings.cache.port)
def __init__(self, model=None, lang=True, max_length=None): self.tokenizer = tokenizers[lang] self.model = SequenceTagger.load(model) self.max_length = int(max_length) if max_length else None logger.info(f"NER model was loaded. {model}")
# //Init DB session import uuid from cassandra.cqlengine.models import Model from cassandra.cqlengine.management import sync_table from cassandra.cqlengine import (columns, connection) from app.utils.logger import logger from app.config import settings logger.info('Connecting to database...') logger.debug('Contact points:') logger.debug(settings.database.hosts) """USER MODELS""" class User_by_id_Model(Model): __table_name__ = "user_by_id" user_id = columns.UUID(primary_key=True, default=uuid.uuid4, required=True) username = columns.Text(required=True) srp_salt = columns.Text(required=True) srp_verifier = columns.Text(required=True) kek_salt = columns.Text(required=True) cek = columns.Text(required=True) class User_by_username_Model(Model): __table_name__ = "user_by_username" username = columns.Text(primary_key=True, required=True) user_id = columns.UUID(required=True)
def __connect(self): logger.info('Conneting to DB') mongoengine.connect(host=MongoConfigs.MONGO_URI) logger.info('Connected to DB')
def run(self): uvicorn.run(app) logger.info("Busca de Cep finalizada...")
def __connect(self): logger.info("Connecting on mongo") mongoengine.connect(host=MongoConfigs.MONGO_URI) logger.info("Mongo connected")
def run(self): uvicorn.run(app) logger.info("People register stoping...")