def convert(input_encoding, output_encoding, input_locale, output_locale,
source, destination):
with rows.locale_context(input_locale):
table = import_from_uri(source)
with rows.locale_context(output_locale):
export_to_uri(destination, table)
def sort(input_encoding, output_encoding, input_locale, output_locale, key,
source, destination):
key = key.replace('^', '-')
with rows.locale_context(input_locale):
table = import_from_uri(source)
table.order_by(key)
with rows.locale_context(output_locale):
export_to_uri(destination, table)
def join(input_encoding, output_encoding, input_locale, output_locale, keys,
sources, destination):
keys = [key.strip() for key in keys.split(',')]
with rows.locale_context(input_locale):
tables = [import_from_uri(source) for source in sources]
result = rows.join(keys, tables)
with rows.locale_context(output_locale):
export_to_uri(destination, result)
def _import_table(source, encoding, verify_ssl=True, *args, **kwargs):
try:
table = import_from_uri(source,
default_encoding=DEFAULT_INPUT_ENCODING,
verify_ssl=verify_ssl,
encoding=encoding, *args, **kwargs)
except requests.exceptions.SSLError:
click.echo('ERROR: SSL verification failed! '
'Use `--verify-ssl=no` if you want to ignore.', err=True)
sys.exit(2)
else:
return table
def sum(input_encoding, output_encoding, input_locale, output_locale, sources,
destination):
with rows.locale_context(input_locale):
tables = [import_from_uri(source) for source in sources]
result = tables[0]
for table in tables[1:]:
result = result + table
with rows.locale_context(output_locale):
export_to_uri(destination, result)
def _import_table(source, encoding, verify_ssl=True, *args, **kwargs):
try:
table = import_from_uri(source,
default_encoding=DEFAULT_INPUT_ENCODING,
verify_ssl=verify_ssl,
encoding=encoding,
*args,
**kwargs)
except requests.exceptions.SSLError:
click.echo(
'ERROR: SSL verification failed! '
'Use `--verify-ssl=no` if you want to ignore.',
err=True)
sys.exit(2)
else:
return table
def _import_table(source, encoding, verify_ssl=True, *args, **kwargs):
# TODO: add `--quiet|-q` or `--progress|-p` to set `progress` properly
try:
table = import_from_uri(
source,
default_encoding=DEFAULT_INPUT_ENCODING,
verify_ssl=verify_ssl,
encoding=encoding,
progress=True,
*args,
**kwargs,
)
except requests.exceptions.SSLError:
click.echo(
'ERROR: SSL verification failed! '
'Use `--verify-ssl=no` if you want to ignore.',
err=True)
sys.exit(2)
else:
return table
def download_photos(year):
year = str(year)
url = f"http://agencia.tse.jus.br/estatistica/sead/eleicoes/eleicoes{year}/fotos/"
table = import_from_uri(url)
for row in table:
if row.name == "Parent Directory":
continue
filename = download_path / year / row.name
print(f"Downloading {filename.name}", end="")
if filename.exists():
print(" - downloaded already, skipping.")
else:
if not filename.parent.exists():
filename.parent.mkdir()
print()
download_file(urljoin(url, row.name),
progress=True,
filename=filename)
print(f" saved: {filename}")
photo_path = output_path / year
if not photo_path.exists():
photo_path.mkdir()
print(f" Exporting to: {photo_path}")
zf = ZipFile(filename)
for file_info in tqdm(zf.filelist, desc="Exporting pictures"):
internal_name = file_info.filename
internal_path = Path(internal_name)
extension = internal_path.name.split(".")[-1].lower()
info = internal_path.name.split(".")[0].split("_")[0]
state, sequence_number = info[1:3], info[3:]
new_filename = photo_path / state / f"{sequence_number}.{extension}"
if not new_filename.parent.exists():
new_filename.parent.mkdir()
zfobj = zf.open(internal_name)
with open(new_filename, mode="wb") as fobj:
fobj.write(zfobj.read())
