def set_engine_encoding(self): """Set MySQL database encoding to match data encoding Please update the encoding lookup table if the required encoding is not present. """ encoding = ENCODING.lower() if self.script.encoding: encoding = self.script.encoding.lower() encoding_lookup = {'iso-8859-1': 'latin1', 'latin-1': 'latin1', 'utf-8': 'utf8'} db_encoding = encoding_lookup.get(encoding) self.execute("SET NAMES '{0}';".format(db_encoding))
def set_engine_encoding(self): """Set MySQL database encoding to match data encoding Please update the encoding lookup table if the required encoding is not present. """ encoding = ENCODING.lower() if self.script.encoding: encoding = self.script.encoding.lower() encoding_lookup = {'iso-8859-1': 'latin1', 'latin-1': 'latin1', 'utf-8': 'utf8'} db_encoding = encoding_lookup.get(encoding) self.execute("SET NAMES '{0}';".format(db_encoding))
def lookup_encoding(self): """Convert well known encoding to MySQL syntax MySQL has a unique way of representing the encoding. For example, latin-1 becomes latin1 in MySQL. Please update the encoding lookup table if the required encoding is not present.""" encoding = ENCODING.lower() if self.script.encoding: encoding = self.script.encoding.lower() encoding_lookup = {'iso-8859-1': 'latin1', 'latin-1': 'latin1', 'utf-8': 'utf8'} db_encoding = encoding_lookup.get(encoding) return db_encoding
def get_connection(self): """Gets the db connection. Please update the encoding lookup table if the required encoding is not present. """ import psycopg2 as dbapi self.get_input() conn = dbapi.connect(host=self.opts["host"], port=int(self.opts["port"]), user=self.opts["user"], password=self.opts["password"], database=self.opts["database"]) encoding = ENCODING.lower() if self.script.encoding: encoding = self.script.encoding.lower() encoding_lookup = {'iso-8859-1': 'Latin1', 'latin-1': 'Latin1', 'utf-8': 'UTF8'} db_encoding = encoding_lookup.get(encoding) conn.set_client_encoding(db_encoding) return conn
def get_connection(self): """Gets the db connection. Please update the encoding lookup table if the required encoding is not present. """ import psycopg2 as dbapi self.get_input() conn = dbapi.connect(host=self.opts["host"], port=int(self.opts["port"]), user=self.opts["user"], password=self.opts["password"], database=self.opts["database"]) encoding = ENCODING.lower() if self.script.encoding: encoding = self.script.encoding.lower() encoding_lookup = {'iso-8859-1': 'Latin1', 'latin-1': 'Latin1', 'utf-8': 'UTF8'} db_encoding = encoding_lookup.get(encoding) conn.set_client_encoding(db_encoding) return conn
# -*- coding: latin-1 -*- # """Integrations tests for Data Retriever""" from __future__ import print_function import json import os import shutil import sys from imp import reload from retriever.lib.defaults import ENCODING encoding = ENCODING.lower() reload(sys) if hasattr(sys, 'setdefaultencoding'): sys.setdefaultencoding(encoding) import pytest from retriever.lib.load_json import read_json from retriever.lib.defaults import HOME_DIR from retriever.engines import engine_list from retriever.lib.engine_tools import file_2list from retriever.lib.engine_tools import create_file # Set postgres password, Appveyor service needs the password given # The Travis service obtains the password from the config file. if os.name == "nt": os_password = "******" else: os_password = ""
import csv import re import requests from collections import OrderedDict from math import ceil from tqdm import tqdm from retriever.lib.tools import open_fr, open_fw, open_csvw, walk_relative_path from setuptools import archive_util from retriever.lib.defaults import DATA_SEARCH_PATHS, DATA_WRITE_PATH, ENCODING from retriever.lib.cleanup import no_cleanup from retriever.lib.warning import Warning from urllib.request import urlretrieve from requests.exceptions import InvalidSchema from imp import reload encoding = ENCODING.lower() # sys removes the setdefaultencoding method at startup; reload to get it back reload(sys) if hasattr(sys, 'setdefaultencoding'): sys.setdefaultencoding(encoding) class Engine(object): """A generic database system. Specific database platforms will inherit from this class.""" _connection = None _cursor = None datatypes = [] db = None debug = False
def set_engine_encoding(self): """Set up the encoding to be used.""" self.encoding = ENCODING.lower() if self.script and self.script.encoding: self.encoding = self.script.encoding.lower()