def set_engine_encoding(self):
"""Set MySQL database encoding to match data encoding
Please update the encoding lookup table if the required encoding is not present.
"""
encoding = ENCODING.lower()
if self.script.encoding:
encoding = self.script.encoding.lower()
encoding_lookup = {'iso-8859-1': 'latin1', 'latin-1': 'latin1', 'utf-8': 'utf8'}
db_encoding = encoding_lookup.get(encoding)
self.execute("SET NAMES '{0}';".format(db_encoding))
def set_engine_encoding(self):
"""Set MySQL database encoding to match data encoding
Please update the encoding lookup table if the required encoding is not present.
"""
encoding = ENCODING.lower()
if self.script.encoding:
encoding = self.script.encoding.lower()
encoding_lookup = {'iso-8859-1': 'latin1', 'latin-1': 'latin1', 'utf-8': 'utf8'}
db_encoding = encoding_lookup.get(encoding)
self.execute("SET NAMES '{0}';".format(db_encoding))
def lookup_encoding(self):
"""Convert well known encoding to MySQL syntax
MySQL has a unique way of representing the encoding.
For example, latin-1 becomes latin1 in MySQL.
Please update the encoding lookup table if the required
encoding is not present."""
encoding = ENCODING.lower()
if self.script.encoding:
encoding = self.script.encoding.lower()
encoding_lookup = {'iso-8859-1': 'latin1', 'latin-1': 'latin1', 'utf-8': 'utf8'}
db_encoding = encoding_lookup.get(encoding)
return db_encoding
def get_connection(self):
"""Gets the db connection.
Please update the encoding lookup table if the required encoding is not present.
"""
import psycopg2 as dbapi
self.get_input()
conn = dbapi.connect(host=self.opts["host"],
port=int(self.opts["port"]),
user=self.opts["user"],
password=self.opts["password"],
database=self.opts["database"])
encoding = ENCODING.lower()
if self.script.encoding:
encoding = self.script.encoding.lower()
encoding_lookup = {'iso-8859-1': 'Latin1', 'latin-1': 'Latin1', 'utf-8': 'UTF8'}
db_encoding = encoding_lookup.get(encoding)
conn.set_client_encoding(db_encoding)
return conn
def get_connection(self):
"""Gets the db connection.
Please update the encoding lookup table if the required encoding is not present.
"""
import psycopg2 as dbapi
self.get_input()
conn = dbapi.connect(host=self.opts["host"],
port=int(self.opts["port"]),
user=self.opts["user"],
password=self.opts["password"],
database=self.opts["database"])
encoding = ENCODING.lower()
if self.script.encoding:
encoding = self.script.encoding.lower()
encoding_lookup = {'iso-8859-1': 'Latin1',
'latin-1': 'Latin1',
'utf-8': 'UTF8'}
db_encoding = encoding_lookup.get(encoding)
conn.set_client_encoding(db_encoding)
return conn
# -*- coding: latin-1 -*-
# """Integrations tests for Data Retriever"""
from __future__ import print_function
import json
import os
import shutil
import sys
from imp import reload
from retriever.lib.defaults import ENCODING
encoding = ENCODING.lower()
reload(sys)
if hasattr(sys, 'setdefaultencoding'):
sys.setdefaultencoding(encoding)
import pytest
from retriever.lib.load_json import read_json
from retriever.lib.defaults import HOME_DIR
from retriever.engines import engine_list
from retriever.lib.engine_tools import file_2list
from retriever.lib.engine_tools import create_file
# Set postgres password, Appveyor service needs the password given
# The Travis service obtains the password from the config file.
if os.name == "nt":
os_password = "******"
else:
os_password = ""
import csv
import re
import requests
from collections import OrderedDict
from math import ceil
from tqdm import tqdm
from retriever.lib.tools import open_fr, open_fw, open_csvw, walk_relative_path
from setuptools import archive_util
from retriever.lib.defaults import DATA_SEARCH_PATHS, DATA_WRITE_PATH, ENCODING
from retriever.lib.cleanup import no_cleanup
from retriever.lib.warning import Warning
from urllib.request import urlretrieve
from requests.exceptions import InvalidSchema
from imp import reload
encoding = ENCODING.lower()
# sys removes the setdefaultencoding method at startup; reload to get it back
reload(sys)
if hasattr(sys, 'setdefaultencoding'):
sys.setdefaultencoding(encoding)
class Engine(object):
"""A generic database system. Specific database platforms will inherit
from this class."""
_connection = None
_cursor = None
datatypes = []
db = None
debug = False
def set_engine_encoding(self):
"""Set up the encoding to be used."""
self.encoding = ENCODING.lower()
if self.script and self.script.encoding:
self.encoding = self.script.encoding.lower()