def convert(input_encoding, output_encoding, input_locale, output_locale, source, destination): with rows.locale_context(input_locale): table = import_from_uri(source) with rows.locale_context(output_locale): export_to_uri(destination, table)
def sort(input_encoding, output_encoding, input_locale, output_locale, key, source, destination): key = key.replace('^', '-') with rows.locale_context(input_locale): table = import_from_uri(source) table.order_by(key) with rows.locale_context(output_locale): export_to_uri(destination, table)
def join(input_encoding, output_encoding, input_locale, output_locale, keys, sources, destination): keys = [key.strip() for key in keys.split(',')] with rows.locale_context(input_locale): tables = [import_from_uri(source) for source in sources] result = rows.join(keys, tables) with rows.locale_context(output_locale): export_to_uri(destination, result)
def _import_table(source, encoding, verify_ssl=True, *args, **kwargs): try: table = import_from_uri(source, default_encoding=DEFAULT_INPUT_ENCODING, verify_ssl=verify_ssl, encoding=encoding, *args, **kwargs) except requests.exceptions.SSLError: click.echo('ERROR: SSL verification failed! ' 'Use `--verify-ssl=no` if you want to ignore.', err=True) sys.exit(2) else: return table
def sum(input_encoding, output_encoding, input_locale, output_locale, sources, destination): with rows.locale_context(input_locale): tables = [import_from_uri(source) for source in sources] result = tables[0] for table in tables[1:]: result = result + table with rows.locale_context(output_locale): export_to_uri(destination, result)
def _import_table(source, encoding, verify_ssl=True, *args, **kwargs): try: table = import_from_uri(source, default_encoding=DEFAULT_INPUT_ENCODING, verify_ssl=verify_ssl, encoding=encoding, *args, **kwargs) except requests.exceptions.SSLError: click.echo( 'ERROR: SSL verification failed! ' 'Use `--verify-ssl=no` if you want to ignore.', err=True) sys.exit(2) else: return table
def _import_table(source, encoding, verify_ssl=True, *args, **kwargs): # TODO: add `--quiet|-q` or `--progress|-p` to set `progress` properly try: table = import_from_uri( source, default_encoding=DEFAULT_INPUT_ENCODING, verify_ssl=verify_ssl, encoding=encoding, progress=True, *args, **kwargs, ) except requests.exceptions.SSLError: click.echo( 'ERROR: SSL verification failed! ' 'Use `--verify-ssl=no` if you want to ignore.', err=True) sys.exit(2) else: return table
def download_photos(year): year = str(year) url = f"http://agencia.tse.jus.br/estatistica/sead/eleicoes/eleicoes{year}/fotos/" table = import_from_uri(url) for row in table: if row.name == "Parent Directory": continue filename = download_path / year / row.name print(f"Downloading {filename.name}", end="") if filename.exists(): print(" - downloaded already, skipping.") else: if not filename.parent.exists(): filename.parent.mkdir() print() download_file(urljoin(url, row.name), progress=True, filename=filename) print(f" saved: {filename}") photo_path = output_path / year if not photo_path.exists(): photo_path.mkdir() print(f" Exporting to: {photo_path}") zf = ZipFile(filename) for file_info in tqdm(zf.filelist, desc="Exporting pictures"): internal_name = file_info.filename internal_path = Path(internal_name) extension = internal_path.name.split(".")[-1].lower() info = internal_path.name.split(".")[0].split("_")[0] state, sequence_number = info[1:3], info[3:] new_filename = photo_path / state / f"{sequence_number}.{extension}" if not new_filename.parent.exists(): new_filename.parent.mkdir() zfobj = zf.open(internal_name) with open(new_filename, mode="wb") as fobj: fobj.write(zfobj.read())