def conn(cls):
"""
return the postgres connection handle to the configured server
"""
config = cls.config
try:
# conn_string here
conn_string_parts = []
conn_string_params = []
for key, value in config.iteritems():
# Build connection string based on what is defined in the config
if value:
if key == 'host':
conn_string_parts.append('host=%s')
conn_string_params.append(value)
elif key == 'username':
conn_string_parts.append('user=%s')
conn_string_params.append(value)
elif key == 'password':
conn_string_parts.append('password=%s')
conn_string_params.append(value)
elif key == 'revision_db_name':
conn_string_parts.append('dbname=%s')
conn_string_params.append(value)
elif key == 'port':
conn_string_parts.append('port=%s')
conn_string_params.append(value)
conn_string = ' '.join(conn_string_parts) % tuple(
conn_string_params)
conn = psycopg2.connect(conn_string)
except Exception, e:
raise DbError(
"Cannot connect to Postgres Db: %s\n"
"Ensure that the server is running and you can connect normally"
% e.message)
def conn(cls):
"""
return the vertica connection handle to the configured server
"""
config = cls.config
try:
conn_driver_dict = {}
conf_to_driver_map = {
'host': 'host',
'username': '******',
'password': '******',
'revision_db_name': 'database',
'port': 'port'
}
for conf_key, conf_value in config.iteritems():
try:
driver_key = conf_to_driver_map[conf_key]
driver_value = conf_value
# NOTE: Vertica Python driver requires non-unicode strings
if isinstance(driver_value, unicode):
driver_value = str(driver_value)
conn_driver_dict[driver_key] = driver_value
except KeyError:
pass
conn = vertica_python.connect(conn_driver_dict)
except Exception, e:
raise DbError(
"Cannot connect to Vertica Db: %s\n"
"Ensure that the server is running and you can connect normally"
% e.message)
class VerticaDb(Db):
DEFAULT_PORT = 5433
@classmethod
def new(cls, config):
super(VerticaDb, cls).new(config)
if 'revision_schema_name' in cls.config:
cls.history_table_name = cls.config['history_table_name']
cls.full_table_name = '"%s"."%s"' % (
cls.config['revision_schema_name'],
cls.config['history_table_name'])
else:
raise DbError(
'No schema found in config file. Please add one with the key: '
'revision_schema_name')
return cls
@classmethod
def init_conn(cls):
try:
vertica_python
except NameError:
raise DbError(
'Vertica module not found/loaded. Please make sure all dependencies are installed\n'
)
cls.conn = cls.conn()
cls.cursor = cls.conn.cursor()
cls.conn_initialized = True
return cls
@classmethod
def execute(cls, query, data=None):
if not cls.conn_initialized:
cls.init_conn()
try:
cursor = cls.cursor
cursor.execute('SET search_path TO %s' % cls.config['schema_name'])
if data:
cursor.execute(query, data)
else:
cursor.execute(query)
results = []
if cursor.rowcount > 0:
try:
results = cursor.fetchall()
except vertica_python.ProgrammingError, e:
raise vertica_python.ProgrammingError(e.message)
cls.conn.commit()
return results
except Exception, e:
raise DbError(
'Vertica execution error: %s\n. Query: %s - Data: %s\n.' %
(e.message, query, str(data)))
def new(cls, config):
super(PostgresDb, cls).new(config)
if 'revision_schema_name' in cls.config:
cls.full_table_name = '"%s"."%s"' % (cls.config['revision_schema_name'],
cls.config['history_table_name'])
else:
raise DbError('No schema found in config file. Please add one with the key: '
'revision_schema_name')
return cls
def init_conn(cls):
try:
hive
except NameError:
raise DbError('Hive client module not found/loaded. Please make sure all dependencies are installed\n')
cls.conn = cls.conn()
cls.cursor = cls.conn.cursor()
cls.conn_initialized = True
return cls
def execute(cls, query, data=None):
if not cls.conn_initialized:
cls.init_conn()
try:
cursor = cls.cursor
if data is not None:
cursor.execute(query, data)
else:
cursor.execute(query)
except mysql.connector.Error, e:
raise DbError('Could not query DB. Exception:\n%s\n\nQuery:%s' % (e, query))
def new(cls, config):
super(HiveDb, cls).new(config)
if 'revision_db_name' in cls.config and 'history_table_name' in cls.config:
cls.db_name = '`%s`' % cls.config['revision_db_name']
cls.history_table_name = cls.config['history_table_name']
cls.full_table_name = '`%s`.`%s`' % (cls.config['revision_db_name'],
cls.config['history_table_name'])
else:
raise DbError('No history schema found in config file. Please add values for the '
'following keys: revision_db_name, history_table_name\n')
return cls
def init_conn(cls):
try:
psycopg2
except NameError:
raise DbError(
'Postgres module not found/loaded. Please make sure psycopg2 is installed\n'
)
cls.conn = cls.conn()
cls.cursor = cls.conn.cursor()
cls.conn_initialized = True
return cls
def conn(cls):
"""
return the hive connection handle to the configured server
"""
config = cls.config
try:
connection = hive.connect(host=config['host'], port=config.get('port', cls.DEFAULT_PORT),
authMechanism='NOSASL', user=config['username'],
password=config['password'])
except Exception, e:
raise DbError("Cannot connect to Hive Server: %s\n"
"Ensure that the server is running and you can connect normally"
% e.message)
def execute(cls, query, data=None):
if not cls.conn_initialized:
cls.init_conn()
cursor = cls.cursor
results = []
try:
if data:
cursor.execute(query % data)
else:
cursor.execute(query)
results = cursor.fetch()
return results
except Pyhs2Exception, e:
raise DbError('Could not query DB. Exception:\n%s\n\nQuery:%s' % (e, query))
def conn(cls):
"""
return the mysql connection handle to the configured server
"""
config = cls.config
try:
if cls.config.get('password'):
conn = mysql.connector.Connect(user=config['username'],
password=config['password'],
host=config['host'],
port=config['port'])
else:
conn = mysql.connector.Connect(user=config['username'],
host=config['host'],
port=config['port'])
except mysql.connector.InterfaceError, ex:
raise DbError("Cannot connect to MySQL Db (1): %s\n"
"Ensure that the server is running and you can connect normally" % ex)
class PostgresDb(Db):
DEFAULT_PORT = 5432
@classmethod
def new(cls, config):
super(PostgresDb, cls).new(config)
if 'revision_schema_name' in cls.config:
cls.history_table_name = cls.config['history_table_name']
cls.full_table_name = '"%s"."%s"' % (
cls.config['revision_schema_name'],
cls.config['history_table_name'])
else:
raise DbError(
'No schema found in config file. Please add one with the key: '
'revision_schema_name')
return cls
@classmethod
def init_conn(cls):
try:
psycopg2
except NameError:
raise DbError(
'Postgres module not found/loaded. Please make sure psycopg2 is installed\n'
)
cls.conn = cls.conn()
cls.cursor = cls.conn.cursor()
cls.conn_initialized = True
return cls
@classmethod
def execute(cls, query, data=None):
if not cls.conn_initialized:
cls.init_conn()
try:
cursor = cls.cursor
cursor.execute('SET search_path TO %s' % cls.config['schema_name'])
if data:
cursor.execute(query, data)
else:
cursor.execute(query)
results = []
# If rowcount == 0, just return None.
#
# Note from psycopg2 docs:
#
# The rowcount attribute specifies the number of rows that the
# last execute*() produced (for DQL statements like SELECT) or
# affected (for DML statements like UPDATE or INSERT).
#
# http://initd.org/psycopg/docs/cursor.html
#
# Thus, it is possible that fetchone/fetchall will fail despite
# rowcount being > 0. That error is caught below and None is
# returned.
if cursor.rowcount > 0:
try:
results = cursor.fetchall()
except psycopg2.ProgrammingError, e:
if str(e) != 'no results to fetch':
raise psycopg2.ProgrammingError(e.message)
cls.conn.commit()
return results
except Exception, e:
raise DbError(
'Psycopg2 execution error: %s\n. Query: %s - Data: %s\n.' %
(e.message, query, str(data)))
config = cls.config
try:
if cls.config.get('password'):
conn = mysql.connector.Connect(user=config['username'],
password=config['password'],
host=config['host'],
port=config['port'])
else:
conn = mysql.connector.Connect(user=config['username'],
host=config['host'],
port=config['port'])
except mysql.connector.InterfaceError, ex:
raise DbError("Cannot connect to MySQL Db (1): %s\n"
"Ensure that the server is running and you can connect normally" % ex)
except mysql.connector.ProgrammingError, ex:
raise DbError("Cannot connect to MySQL Db (2): %s" % ex)
except db_errors.DatabaseError, er:
base = 'Cannot not connect to MySQL Db (3): %s, %s\n\n' % (er.errno, er.msg)
if er.errno == -1 and re.compile('.*insecure.*').match(er.msg) is not None:
# print some instructions on connecting with new mode
additional = ("Your MySQL version may be running with old_password compatibility mode."
"\nPlease check your CNF files and if necessary change the setting, restart,"
"\nand create a new-user or update your existing users to use new auth.")
raise DbError(base, additional)
else:
raise DbError(base)
return conn
@classmethod
def run_file_cmd(cls):