def get_objectstore_file(location, dir1):
path = "/".join(location.split("/")[1:])
output_path = dir1 + '/' + path
if os.path.isfile(output_path):
log.warning(f"File {output_path} exists. Skip download")
return
else:
log.warning(f"Get file {output_path}")
connection = objectstore.get_connection(OBJECTSTORE)
container = location.split("/")[0]
new_data = objectstore.get_object(connection, {'name': path}, container)
with open(output_path, 'wb') as file:
file.write(new_data)
def get_objectstore_file(location, dir1, connection):
path = "/".join(location.split("/")[1:])
output_path = dir1 + "/" + path
if os.path.isfile(output_path):
log.warning(f"File {output_path} exists. Skip download")
return
else:
log.warning(f"Get file {output_path}")
if connection:
credentials = fetch_objectstore_credentials(connection)
else:
credentials = fetch_objectstore_credentials()
connection = objectstore.get_connection(credentials)
container = location.split("/")[0]
new_data = objectstore.get_object(connection, {"name": path}, container)
output_dir = "/".join(output_path.split("/")[:-1])
if not os.path.exists(output_dir):
os.mkdir(output_dir)
with open(output_path, "wb") as file:
file.write(new_data)
def get_source(self):
if self.source_type == 'objectstore':
container = self.source_location.split("/")[0]
path = "/".join(self.source_location.split("/")[1:])
connection = objectstore.get_connection(OBJECTSTORE)
file = objectstore.get_object(connection, {'name': path},
container)
csv_file = file.decode(self.source_encoding).splitlines()
csvreader = list(csv.reader(csv_file, delimiter=self.delimiter))
headers = csvreader[0]
rows = [dict(zip(headers, row)) for row in csvreader[1:]]
return rows
elif self.source_type == 'file':
with open(self.source_location) as file:
csvreader = list(csv.reader(file, delimiter=self.delimiter))
headers = csvreader[0]
rows = [dict(zip(headers, row)) for row in csvreader[1:]]
return rows
else:
raise NotImplementedError
OBJECTSTORE = dict(
VERSION='2.0',
AUTHURL='https://identity.stack.cloudvps.com/v2.0',
TENANT_NAME='BGE000081 CMS',
TENANT_ID='8cf50dff44f54b5a9d59d5b77fe672b3',
USER=os.getenv('OBJECTSTORE_USER', 'cms_objectstore'),
PASSWORD=os.getenv(OBJECTSTORE_PASSWORD_ENV_KEY),
REGION_NAME='NL',
)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("swiftclient").setLevel(logging.WARNING)
_conn = objectstore.get_connection(OBJECTSTORE)
def _files_in_source_dir(dir):
return list(
filter(lambda f: os.path.isfile(f),
glob.glob(f'{dir}/**', recursive=True)))
if __name__ == "__main__":
file_list = objectstore.get_full_container_list(_conn,
SCHEMAS_CONTAINER,
prefix=ENVIRONMENT + "/")
last_modified = max([item['last_modified'] for item in file_list])
source_dir = os.path.dirname(os.path.realpath(__file__)) + "/target"
"""
We use the objectstore to get the latest and greatest of the mks dump
"""
import os
# import datetime
# from dateutil import parser
from objectstore import get_connection
from objectstore import databasedumps
assert os.getenv('HANDELSREGISTER_OBJECTSTORE_PASSWORD')
ENV = os.getenv('ENVIRONMENT', 'ACCEPTANCE')
OBJECTSTORE = dict(
VERSION='2.0',
AUTHURL='https://identity.stack.cloudvps.com/v2.0',
TENANT_NAME='BGE000081_Handelsregister',
TENANT_ID='0efc828b88584759893253f563b35f9b',
USER=os.getenv('OBJECTSTORE_USER', 'handelsregister'),
PASSWORD=os.getenv('HANDELSREGISTER_OBJECTSTORE_PASSWORD'),
REGION_NAME='NL',
)
handelsregister = get_connection(OBJECTSTORE)
dump = open('/tmp/backups/database.dump', 'rb')
databasedumps.run(handelsregister)
USER=os.getenv('OBJECTSTORE_USER', 'grondexpl'),
PASSWORD=os.getenv('GRONDEXPLOITATIE_OBJECTSTORE_PASSWORD'),
REGION_NAME='NL',
)
EXPECTED_FILES_AND_MAX_AGE = [
(os.getenv('MAPPING_FILE'), 0),
(os.getenv('PLANEX_FILE'), 180),
('{}.tgz'.format(os.getenv('GREX_SHAPES_BASE_NAME')), 180),
]
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("swiftclient").setLevel(logging.WARNING)
grondexploitatie_conn = objectstore.get_connection(OBJECTSTORE)
def download_files(file_list):
"""Download the latest data. """
for _, source_data_file in file_list:
sql_gz_name = source_data_file['name'].split('/')[-1]
msg = 'Downloading: %s' % sql_gz_name
log.debug(msg)
new_data = objectstore.get_object(grondexploitatie_conn,
source_data_file, 'grondexploitatie')
# save output to file!
with open('{}{}'.format(DATA_DIR, sql_gz_name), 'wb') as outputzip:
outputzip.write(new_data)
def main(datadir):
conn = objectstore.get_connection(OBJECTSTORE)
download_containers(conn, DATASETS, datadir)
EXPECTED_FILES = [
'kvkadr.sql.gz',
'kvkbeshdn.sql.gz',
'kvkhdn.sql.gz',
'kvkmac.sql.gz',
'kvkprs.sql.gz',
'kvkprsash.sql.gz',
'kvkves.sql.gz',
'kvkveshis.sql.gz',
]
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("swiftclient").setLevel(logging.WARNING)
handelsregister_conn = objectstore.get_connection(OBJECTSTORE)
def download_files(file_list):
"""Download the latest data. """
for _, source_data_file in file_list:
sql_gz_name = source_data_file['name'].split('/')[-1]
msg = 'Downloading: %s' % (sql_gz_name)
log.debug(msg)
new_data = objectstore.get_object(handelsregister_conn,
source_data_file, 'handelsregister')
# save output to file!
with open('data/{}'.format(sql_gz_name), 'wb') as outputzip:
outputzip.write(new_data)