import os
import argparse
import pandas as pd
from datapunt_processing.helpers.connections import postgres_engine_pandas
from datapunt_processing import logger
# Setup basic logging
logger = logger()
def load_xls(datadir, config_path, db_config_name):
"""Load xlsx into postgres for multiple files"""
files = os.listdir(datadir)
files_xls = [f for f in files if f.split('.')[-1] in ('xlsx', 'xls')]
logger.info(files_xls)
for filename in files_xls:
df = pd.read_excel(datadir + '/' + filename)
if df.empty:
logger.info('No data')
continue
logger.info("added " + filename)
logger.info(df.columns)
# load the data into pg
engine = postgres_engine_pandas(config_path, db_config_name)
# TODO: link to to_sql function
table_name = filename.split('.')[0]
#!/usr/bin/env python3
import subprocess
import argparse
from urllib.parse import urlencode
from datapunt_processing import logger
from datapunt_processing.helpers.connections import psycopg_connection_string
# Setup basic logging
logger = logger()
class NonZeroReturnCode(Exception):
"""Used for subprocess error messages."""
pass
def scrub(line):
"""Hide the login credentials of Postgres in the console."""
out = []
for x in line:
if x.strip().startswith('PG:'):
out.append('PG: <CONNECTION STRING REDACTED>')
else:
out.append(x)
return out
def run_command_sync(cmd, allow_fail=False):
"""
Run a string in the command line.
# utf-8
import requests
import os
import random
import string
import argparse
from urllib.parse import urlparse, parse_qsl
from datapunt_processing import logger
log = logger()
class GetAccessToken(object):
"""
Get an header authentication item for access token
for using the internal API's by logging in as with email and password credentials and authenticated scopes or as type 'employee'
To see the available scopes and types, see this file:
https://github.com/Amsterdam/authorization_levels/blob/master/authorization_levels.py
Usage:
from authentication.getaccesstoken import GetAccessToken
accessToken = GetAccessToken().getAccessToken(usertype='employee_plus', scopes=BRK/RS,BRK/RSN,BRK/RO)
requests.get(url, headers=accessToken)
Args:
- scopes: Add scopes as a comma separated list.
- usertype: Add the usertype
- email: Set and get environment variable: export DATAPUNT_EMAIL=*****
- password: Set and get environment variable: export DATAPUNT_PASSWORD=*****
Returns:
accesstoken
