import os import argparse import pandas as pd from datapunt_processing.helpers.connections import postgres_engine_pandas from datapunt_processing import logger # Setup basic logging logger = logger() def load_xls(datadir, config_path, db_config_name): """Load xlsx into postgres for multiple files""" files = os.listdir(datadir) files_xls = [f for f in files if f.split('.')[-1] in ('xlsx', 'xls')] logger.info(files_xls) for filename in files_xls: df = pd.read_excel(datadir + '/' + filename) if df.empty: logger.info('No data') continue logger.info("added " + filename) logger.info(df.columns) # load the data into pg engine = postgres_engine_pandas(config_path, db_config_name) # TODO: link to to_sql function table_name = filename.split('.')[0]
#!/usr/bin/env python3 import subprocess import argparse from urllib.parse import urlencode from datapunt_processing import logger from datapunt_processing.helpers.connections import psycopg_connection_string # Setup basic logging logger = logger() class NonZeroReturnCode(Exception): """Used for subprocess error messages.""" pass def scrub(line): """Hide the login credentials of Postgres in the console.""" out = [] for x in line: if x.strip().startswith('PG:'): out.append('PG: <CONNECTION STRING REDACTED>') else: out.append(x) return out def run_command_sync(cmd, allow_fail=False): """ Run a string in the command line.
# utf-8 import requests import os import random import string import argparse from urllib.parse import urlparse, parse_qsl from datapunt_processing import logger log = logger() class GetAccessToken(object): """ Get an header authentication item for access token for using the internal API's by logging in as with email and password credentials and authenticated scopes or as type 'employee' To see the available scopes and types, see this file: https://github.com/Amsterdam/authorization_levels/blob/master/authorization_levels.py Usage: from authentication.getaccesstoken import GetAccessToken accessToken = GetAccessToken().getAccessToken(usertype='employee_plus', scopes=BRK/RS,BRK/RSN,BRK/RO) requests.get(url, headers=accessToken) Args: - scopes: Add scopes as a comma separated list. - usertype: Add the usertype - email: Set and get environment variable: export DATAPUNT_EMAIL=***** - password: Set and get environment variable: export DATAPUNT_PASSWORD=***** Returns: accesstoken