def handle( # type: ignore self, verbosity: int, start_election: Optional[int], start_precinct: int, ballot_limit: Optional[int], **_kwargs, ): log.reset() log.silence("datafiles") log.init(verbosity=verbosity if "-v" in sys.argv[-1] else 2) try: scrape_ballots( starting_election_id=start_election, starting_precinct_id=start_precinct, ballot_limit=ballot_limit, ) except Exception as e: if "HEROKU_APP_NAME" in os.environ: log.error("Unable to finish scraping data", exc_info=e) bugsnag.notify(e) sys.exit(1) else: raise e from None
def handle(self, verbosity: int, **_kwargs): # type: ignore log.reset() log.silence("datafiles") log.init(verbosity=verbosity if "-v" in sys.argv[-1] else 2) self.get_or_create_superuser() self.add_elections() self.fetch_districts()
def etl( input_file: str, date: datetime, export_table: Tuple[str], output_file: Optional[str], to_anvil_table: Optional[str], anvil_token: Optional[str], to_mongo: Optional[str], mongo_dbname: str, date_column: str, append: Optional[bool], verbosity: int, ) -> None: """Get info from a SEAP/RJ bulletin saved as a local .XLSX file""" log.reset() log.init(verbosity=verbosity) # TODO: deal with files in SharePoint # TODO: deal with files in e-mail # TODO: deal with multiple input files or files in directory bulletin = bussola_etl_seap.SEAPBulletin( input_file=input_file, date=date, ) # process 'all' is present if export_table == 'all': export_table = bulletin.tables.keys() else: # TODO: replace with real handler for multiple files export_table = [export_table] # export to local files # TODO: refactor to accept a directory path and export all inside it if output_file is not None: for table in export_table: bulletin.to_file( output_file=output_file, tablename=table, date_col=date_column, ) # export to Anvil if (to_anvil_table is not None) or (anvil_token is not None): for table in export_table: bulletin.to_anvil( tablename=table, output_table=to_anvil_table, token=anvil_token, date_col=date_column, ) # export to MongoDB if to_mongo is not None: for table in export_table: bulletin.to_mongo( tablename=table, connection_string=to_mongo, database=mongo_dbname, date_col=date_column, )
def handle(self, verbosity: int, election: Optional[int], **_kwargs): # type: ignore log.reset() log.silence("datafiles") log.init(reset=True, verbosity=verbosity if "-v" in sys.argv[-1] else 2) try: parse_ballots(election_id=election) except Exception as e: if "HEROKU_APP_NAME" in os.environ: log.error("Unable to finish parsing data", exc_info=e) bugsnag.notify(e) sys.exit(1) else: raise e from None
def handle(self, verbosity: int, **_kwargs): # type: ignore log.reset() log.silence("datafiles") log.init(verbosity=verbosity if "-v" in sys.argv[-1] else 2) defaults.initialize_parties() defaults.initialize_districts() self.update_elections() self.update_jurisdictions() self.update_candidates() self.import_descriptions() self.export_descriptions()
def configure_logging(count=0): """Configure logging using the provided verbosity count.""" if count == -1: level = settings.QUIET_LOGGING_LEVEL default_format = settings.DEFAULT_LOGGING_FORMAT verbose_format = settings.LEVELED_LOGGING_FORMAT elif count == 0: level = settings.DEFAULT_LOGGING_LEVEL default_format = settings.DEFAULT_LOGGING_FORMAT verbose_format = settings.LEVELED_LOGGING_FORMAT elif count == 1: level = settings.VERBOSE_LOGGING_LEVEL default_format = settings.VERBOSE_LOGGING_FORMAT verbose_format = settings.VERBOSE_LOGGING_FORMAT elif count == 2: level = settings.VERBOSE2_LOGGING_LEVEL default_format = settings.VERBOSE_LOGGING_FORMAT verbose_format = settings.VERBOSE_LOGGING_FORMAT elif count == 3: level = settings.VERBOSE2_LOGGING_LEVEL default_format = settings.VERBOSE2_LOGGING_FORMAT verbose_format = settings.VERBOSE2_LOGGING_FORMAT else: level = settings.VERBOSE2_LOGGING_LEVEL - 1 default_format = settings.VERBOSE2_LOGGING_FORMAT verbose_format = settings.VERBOSE2_LOGGING_FORMAT # Set a custom formatter log.reset() log.init(level=level) log.silence('datafiles', allow_warning=True) logging.captureWarnings(True) formatter = WarningFormatter( default_format, verbose_format, datefmt=settings.LOGGING_DATEFMT ) logging.root.handlers[0].setFormatter(formatter) # Warn about excessive verbosity if count > _Config.MAX_VERBOSITY: msg = "Maximum verbosity level is {}".format(_Config.MAX_VERBOSITY) log.warning(msg) _Config.verbosity = _Config.MAX_VERBOSITY else: _Config.verbosity = count
def input(self, event=None): log.reset() self.treeview.delete(*self.treeview.get_children()) self.explanation_text.delete(1.0, END) self.translation_text.delete(1.0, END) text = self.entry.get() translated = amem.main(text) if log.error != "": self.explanation_text.insert(END, log.error) log.clear_error() else: self.set_treeview(self) self.translation_text.insert(END, translated)
def setUp(self): self.category = self.level = self.message = None self.tester = LogTester() # we want to remove the default handler so it doesn't show up stuff log.reset()
def setUp(self): log.reset()
def setUp(self): self.category = self.level = self.message = None self.tester = LogFunctionTester() log.reset()
""" import json import os import log # type: ignore import hyp2rem.utils from hyp2rem import hypothesis as hyp from hyp2rem import remnote as rem HYP_KEY = os.environ["HYP_KEY"] REM_KEY = os.environ["REM_KEY"] REM_USER = os.environ["REM_USER"] log.reset() log.init(verbosity=3) def test_document_for_source(): """Test creating/getting RemNote documents for Hypothes.is source URIs.""" group_id = hyp.get_group_by_name(HYP_KEY, "RemNote")["id"] annotations = hyp.get_annotations(HYP_KEY, group=group_id) for annotation in annotations: document_rem = hyp2rem.utils.document_for_source( REM_KEY, REM_USER, annotation=annotation, ) log.debug("Retrived document:" + json.dumps(document_rem.source, indent=4))
def main( # type: ignore group: Optional[str] = Option( # TODO: Accept multiple groups None, help="Name of the Hypothes.is group where annotations are stored", show_default=False, ), sort: SortOption = Option( SortOption.UPDATED, help="Metric to sort results by", case_sensitive=False, ), after: Optional[datetime] = Option( None, help="Search for annotations created ou updated after the given date", ), uri: Optional[str] = Option( None, help="A web page address (URL) or a URN representing another kind " + "of resource whose annotations should be synced.", show_default=False, ), hyp_key: str = Option( ..., envvar="HYP_KEY", prompt=True, help="API key for Hypothes.is account", show_default=False, ), rem_user: str = Option( ..., envvar="REM_USER", prompt=True, help="User ID for RemNote account", show_default=False, ), rem_key: str = Option( ..., envvar="REM_KEY", prompt=True, help="API key for RemNote account", show_default=False, ), quiet: bool = Option( False, help="Silences the output printed to the terminal.", ), verbose: bool = Option( False, help="Increases the output printed to the terminal.", ), debug: bool = Option( False, help="Print every step to the terminal, for debugging purposes.", ), ): """ Sync Hypothes.is annotations with Rem's in a RemNote account. """ # pylint: disable=too-many-locals # set verbosity levels and initialize logs verbosity: int = 1 # Errors and warnings (WARN) - Default if quiet: verbosity = 0 # Only errors (ERROR) elif verbose: verbosity = 2 # Errors, warnings and information (INFO) elif debug: verbosity = 3 # All possible output (DEBUG) log.reset() log.init(verbosity=verbosity) # set up Hypothes.is and RemNote connections hyp_client: HypothesisV1Client = HypothesisV1Client( group_name=group, sort=sort, order="asc", search_after=after, uri=uri, key=hyp_key, ) rem_client: RemNoteV0Client = RemNoteV0Client( key=rem_key, user_id=rem_user, ) # set up Bridge object between fetched annotations and RemNote bridge: Bridge = Bridge(hyp_client, rem_client) bridge.sync_all() if verbosity > 1: echo(bridge.stats)