def handle( # type: ignore
self,
verbosity: int,
start_election: Optional[int],
start_precinct: int,
ballot_limit: Optional[int],
**_kwargs,
):
log.reset()
log.silence("datafiles")
log.init(verbosity=verbosity if "-v" in sys.argv[-1] else 2)
try:
scrape_ballots(
starting_election_id=start_election,
starting_precinct_id=start_precinct,
ballot_limit=ballot_limit,
)
except Exception as e:
if "HEROKU_APP_NAME" in os.environ:
log.error("Unable to finish scraping data", exc_info=e)
bugsnag.notify(e)
sys.exit(1)
else:
raise e from None
def handle(self, verbosity: int, **_kwargs): # type: ignore
log.reset()
log.silence("datafiles")
log.init(verbosity=verbosity if "-v" in sys.argv[-1] else 2)
self.get_or_create_superuser()
self.add_elections()
self.fetch_districts()
def etl(
input_file: str,
date: datetime,
export_table: Tuple[str],
output_file: Optional[str],
to_anvil_table: Optional[str],
anvil_token: Optional[str],
to_mongo: Optional[str],
mongo_dbname: str,
date_column: str,
append: Optional[bool],
verbosity: int,
) -> None:
"""Get info from a SEAP/RJ bulletin saved as a local .XLSX file"""
log.reset()
log.init(verbosity=verbosity)
# TODO: deal with files in SharePoint
# TODO: deal with files in e-mail
# TODO: deal with multiple input files or files in directory
bulletin = bussola_etl_seap.SEAPBulletin(
input_file=input_file,
date=date,
)
# process 'all' is present
if export_table == 'all':
export_table = bulletin.tables.keys()
else:
# TODO: replace with real handler for multiple files
export_table = [export_table]
# export to local files
# TODO: refactor to accept a directory path and export all inside it
if output_file is not None:
for table in export_table:
bulletin.to_file(
output_file=output_file,
tablename=table,
date_col=date_column,
)
# export to Anvil
if (to_anvil_table is not None) or (anvil_token is not None):
for table in export_table:
bulletin.to_anvil(
tablename=table,
output_table=to_anvil_table,
token=anvil_token,
date_col=date_column,
)
# export to MongoDB
if to_mongo is not None:
for table in export_table:
bulletin.to_mongo(
tablename=table,
connection_string=to_mongo,
database=mongo_dbname,
date_col=date_column,
)
def handle(self, verbosity: int, election: Optional[int], **_kwargs): # type: ignore
log.reset()
log.silence("datafiles")
log.init(reset=True, verbosity=verbosity if "-v" in sys.argv[-1] else 2)
try:
parse_ballots(election_id=election)
except Exception as e:
if "HEROKU_APP_NAME" in os.environ:
log.error("Unable to finish parsing data", exc_info=e)
bugsnag.notify(e)
sys.exit(1)
else:
raise e from None
def handle(self, verbosity: int, **_kwargs): # type: ignore
log.reset()
log.silence("datafiles")
log.init(verbosity=verbosity if "-v" in sys.argv[-1] else 2)
defaults.initialize_parties()
defaults.initialize_districts()
self.update_elections()
self.update_jurisdictions()
self.update_candidates()
self.import_descriptions()
self.export_descriptions()
def configure_logging(count=0):
"""Configure logging using the provided verbosity count."""
if count == -1:
level = settings.QUIET_LOGGING_LEVEL
default_format = settings.DEFAULT_LOGGING_FORMAT
verbose_format = settings.LEVELED_LOGGING_FORMAT
elif count == 0:
level = settings.DEFAULT_LOGGING_LEVEL
default_format = settings.DEFAULT_LOGGING_FORMAT
verbose_format = settings.LEVELED_LOGGING_FORMAT
elif count == 1:
level = settings.VERBOSE_LOGGING_LEVEL
default_format = settings.VERBOSE_LOGGING_FORMAT
verbose_format = settings.VERBOSE_LOGGING_FORMAT
elif count == 2:
level = settings.VERBOSE2_LOGGING_LEVEL
default_format = settings.VERBOSE_LOGGING_FORMAT
verbose_format = settings.VERBOSE_LOGGING_FORMAT
elif count == 3:
level = settings.VERBOSE2_LOGGING_LEVEL
default_format = settings.VERBOSE2_LOGGING_FORMAT
verbose_format = settings.VERBOSE2_LOGGING_FORMAT
else:
level = settings.VERBOSE2_LOGGING_LEVEL - 1
default_format = settings.VERBOSE2_LOGGING_FORMAT
verbose_format = settings.VERBOSE2_LOGGING_FORMAT
# Set a custom formatter
log.reset()
log.init(level=level)
log.silence('datafiles', allow_warning=True)
logging.captureWarnings(True)
formatter = WarningFormatter(
default_format, verbose_format, datefmt=settings.LOGGING_DATEFMT
)
logging.root.handlers[0].setFormatter(formatter)
# Warn about excessive verbosity
if count > _Config.MAX_VERBOSITY:
msg = "Maximum verbosity level is {}".format(_Config.MAX_VERBOSITY)
log.warning(msg)
_Config.verbosity = _Config.MAX_VERBOSITY
else:
_Config.verbosity = count
def input(self, event=None):
log.reset()
self.treeview.delete(*self.treeview.get_children())
self.explanation_text.delete(1.0, END)
self.translation_text.delete(1.0, END)
text = self.entry.get()
translated = amem.main(text)
if log.error != "":
self.explanation_text.insert(END, log.error)
log.clear_error()
else:
self.set_treeview(self)
self.translation_text.insert(END, translated)
def setUp(self):
self.category = self.level = self.message = None
self.tester = LogTester()
# we want to remove the default handler so it doesn't show up stuff
log.reset()
def setUp(self):
log.reset()
def setUp(self):
self.category = self.level = self.message = None
self.tester = LogFunctionTester()
log.reset()
"""
import json
import os
import log # type: ignore
import hyp2rem.utils
from hyp2rem import hypothesis as hyp
from hyp2rem import remnote as rem
HYP_KEY = os.environ["HYP_KEY"]
REM_KEY = os.environ["REM_KEY"]
REM_USER = os.environ["REM_USER"]
log.reset()
log.init(verbosity=3)
def test_document_for_source():
"""Test creating/getting RemNote documents for Hypothes.is source URIs."""
group_id = hyp.get_group_by_name(HYP_KEY, "RemNote")["id"]
annotations = hyp.get_annotations(HYP_KEY, group=group_id)
for annotation in annotations:
document_rem = hyp2rem.utils.document_for_source(
REM_KEY,
REM_USER,
annotation=annotation,
)
log.debug("Retrived document:" +
json.dumps(document_rem.source, indent=4))
def setUp(self):
self.category = self.level = self.message = None
self.tester = LogTester()
# we want to remove the default handler so it doesn't show up stuff
log.reset()
def setUp(self):
log.reset()
def setUp(self):
self.category = self.level = self.message = None
self.tester = LogFunctionTester()
log.reset()
def main( # type: ignore
group: Optional[str] = Option( # TODO: Accept multiple groups
None,
help="Name of the Hypothes.is group where annotations are stored",
show_default=False,
),
sort: SortOption = Option(
SortOption.UPDATED,
help="Metric to sort results by",
case_sensitive=False,
),
after: Optional[datetime] = Option(
None,
help="Search for annotations created ou updated after the given date",
),
uri: Optional[str] = Option(
None,
help="A web page address (URL) or a URN representing another kind " +
"of resource whose annotations should be synced.",
show_default=False,
),
hyp_key: str = Option(
...,
envvar="HYP_KEY",
prompt=True,
help="API key for Hypothes.is account",
show_default=False,
),
rem_user: str = Option(
...,
envvar="REM_USER",
prompt=True,
help="User ID for RemNote account",
show_default=False,
),
rem_key: str = Option(
...,
envvar="REM_KEY",
prompt=True,
help="API key for RemNote account",
show_default=False,
),
quiet: bool = Option(
False,
help="Silences the output printed to the terminal.",
),
verbose: bool = Option(
False,
help="Increases the output printed to the terminal.",
),
debug: bool = Option(
False,
help="Print every step to the terminal, for debugging purposes.",
),
):
"""
Sync Hypothes.is annotations with Rem's in a RemNote account.
"""
# pylint: disable=too-many-locals
# set verbosity levels and initialize logs
verbosity: int = 1 # Errors and warnings (WARN) - Default
if quiet:
verbosity = 0 # Only errors (ERROR)
elif verbose:
verbosity = 2 # Errors, warnings and information (INFO)
elif debug:
verbosity = 3 # All possible output (DEBUG)
log.reset()
log.init(verbosity=verbosity)
# set up Hypothes.is and RemNote connections
hyp_client: HypothesisV1Client = HypothesisV1Client(
group_name=group,
sort=sort,
order="asc",
search_after=after,
uri=uri,
key=hyp_key,
)
rem_client: RemNoteV0Client = RemNoteV0Client(
key=rem_key,
user_id=rem_user,
)
# set up Bridge object between fetched annotations and RemNote
bridge: Bridge = Bridge(hyp_client, rem_client)
bridge.sync_all()
if verbosity > 1:
echo(bridge.stats)
