def update_finished(self, env):
"""Should be called once the config object is finished being updated"""
# Hack to avoid passing arguments to all invocations.
if self.netstat_cmd:
utils.netstat_cmd = self.netstat_cmd
utils.dummy = self.dummy
# Backward compatibility. To be removed in the future.
if (not self.send_mail_on_restart and
hasattr(self, 'send_mail_on_crash')):
self.send_mail_on_restart = self.send_mail_on_crash
# Convert log level to int
try:
self.log_level = int(self.log_level)
except ValueError:
levels = ['trace', 'debug', 'info', 'warn', 'fatal', 'none']
level_name_to_int = dict(zip(levels, range(-1, len(levels))))
if not self.log_level in level_name_to_int:
raise Exception(
'Invalid log level value {0!r} allowed: {1!r}'.format(
self.log_level, levels))
self.log_level = level_name_to_int[self.log_level]
self.remote_project_path = self.remote_project_path.format(
project_name=self.project_name)
# import here to prevent import cycle errors
from synthesepy import db_backends
self.ineo_db = None
if self.ineo_conn_string:
self.ineo_db = db_backends.create_backend(env, self.ineo_conn_string)
def run_python_tests(self, suite_args):
# Save data on our instance, for access from the tests.
self.backends = []
for conn_string in self.env.c.test_conn_strings:
self.backends.append(db_backends.create_backend(
self.env, conn_string))
sys_argv = sys.argv[0:1]
test_names = suite_args
if test_names:
sys_argv.append('-v')
sys_argv.extend(test_names)
else:
thisdir = os.path.dirname(os.path.abspath(__file__))
start_dir = os.path.join(thisdir, os.pardir)
sys_argv.extend([
'discover',
'-p', '*_unittest.py',
'-s', start_dir,
'-v'
])
test_prog = unittest.main(argv=sys_argv, module=None, exit=False)
return test_prog.result.wasSuccessful()
def db_remote_dump(self):
"""Dump database from remote server"""
if not self.config.remote_server:
raise Exception('No remote server defined in configuration')
@contextlib.contextmanager
def remote_transaction_sqlite(conn_info):
remote_db_local_path = join(self.db_path, 'remote_config.db3')
log.info('Fetching db to %r', remote_db_local_path)
utils.rsync(self.config, '{remote_server}:' + self.config.remote_db_path,
utils.to_cygwin_path(remote_db_local_path))
remote_conn_info = self.db_backend.conn_info.copy()
remote_conn_info['path'] = remote_db_local_path
yield remote_conn_info
@contextlib.contextmanager
def remote_transaction_mysql(conn_info):
MYSQL_FORWARDED_PORT = 33000
utils.kill_listening_processes(MYSQL_FORWARDED_PORT)
if utils.can_connect(MYSQL_FORWARDED_PORT):
raise Exception('MySQL tunnel port (%s) is not '
'available' % MYSQL_FORWARDED_PORT)
ssh_command_line = utils.ssh_command_line(
self.config,
extra_opts='-N -L {forwarded_port}:localhost:3306'.format(
forwarded_port=MYSQL_FORWARDED_PORT))
log.debug('ssh command line for tunnel: %s', ssh_command_line)
p = subprocess.Popen(ssh_command_line, shell=True)
log.info('Waiting a bit for the tunnel to establish...')
time.sleep(3)
remote_conn_info = db_backends.ConnectionInfo(
self.config.remote_conn_string)
remote_conn_info.data['port'] = MYSQL_FORWARDED_PORT
remote_conn_info.data['host'] = '127.0.0.1'
yield remote_conn_info
p.kill()
if self.db_backend.name == 'sqlite':
remote_transaction = remote_transaction_sqlite
elif self.db_backend.name == 'mysql':
remote_transaction = remote_transaction_mysql
else:
raise Exception('Unsupported backend: %r', self.db_backend.name)
with remote_transaction(self.db_backend.conn_info.copy()) as remote_conn_info:
remote_conn_string = remote_conn_info.conn_string
log.info('Remote connection string: %r', remote_conn_string)
remote_backend = db_backends.create_backend(self.env, remote_conn_string)
return self.db_dump(remote_backend, self.config.remote_dump_prefix)
def test_create_backend(self):
backend = db_backends.create_backend(DummyEnv(), 'dummy://')
self.assertEqual(backend.name, 'dummy')
def test_create_backend(self):
backend = db_backends.create_backend(DummyEnv(), 'dummy://')
self.assertEqual(backend.name, 'dummy')
def db_backend(self):
if self._db_backend:
return self._db_backend
self._db_backend = db_backends.create_backend(
self.env, self.config.conn_string)
return self._db_backend
def convert(env, sourceconn, targetconn):
sqlite_backend = db_backends.create_backend(env, sourceconn)
assert sqlite_backend.name == 'sqlite'
mysql_backend = db_backends.create_backend(env, targetconn)
assert mysql_backend.name == 'mysql'
# TODO: use get_cursor() instead.
mysql_cursor = mysql_backend.get_connection().cursor()
sqlite_conn = sqlite_backend.get_connection(spatialite=True)
sqlite_cursor = sqlite_conn.cursor()
table_re = re.compile('INSERT INTO "([^"]+)" VALUES\(([^,]+),')
remove_geom_re = re.compile("X'[0-9A-F]+'")
IGNORED_TABLES = ['spatial_ref_sys', 'geometry_columns']
# Obsolete tables.
IGNORED_TABLES.append('t005_service_dates')
initialized_tables = set()
for line in sqlite_conn.iterdump():
if not line.startswith('INSERT INTO '):
continue
m = table_re.match(line)
assert m, 'Parse error on line %s' % repr(line)
table, rowid = m.groups(1)
# Ignore rtree index tables.
if table.startswith('idx_'):
continue
if table in IGNORED_TABLES:
continue
if table not in initialized_tables:
initialized_tables.add(table)
log.debug('Filling table %s', table)
mysql_cursor.execute('TRUNCATE TABLE `%s`' % table)
# TODO: this assumes that only geometry columns are encoded as blobs,
# they are called 'geometry' and that there is only one.
# This is all true at this time, but might change in the future.
#log.debug('Inserting %s', line)
line, n_subs = remove_geom_re.subn('NULL', line)
assert n_subs <= 1
mysql_cursor.execute(line)
if n_subs == 1:
res = sqlite_cursor.execute(
'SELECT AsText(geometry) FROM %s WHERE id=%s' % (table, rowid))
geom_text = res.fetchone()[0]
mysql_cursor.execute(
'UPDATE `%s` SET geometry=GeomFromText(%%s)'
' WHERE id=%%s' % table, (geom_text, rowid))
for table in initialized_tables:
log.info('Verifying row count of table %s', table)
sqlite_count = sqlite_cursor.execute('SELECT count(1) FROM %s' %
table).fetchone()[0]
mysql_cursor.execute('SELECT count(1) FROM %s' % table)
mysql_count = mysql_cursor.fetchone()[0]
if sqlite_count != mysql_count:
raise Exception(('After import of table: %s, row count differs: '
'sqlite: %s - mysql: %s') %
(table, sqlite_count, mysql_count))
def convert(env, sourceconn, targetconn):
sqlite_backend = db_backends.create_backend(env, sourceconn)
assert sqlite_backend.name == 'sqlite'
mysql_backend = db_backends.create_backend(env, targetconn)
assert mysql_backend.name == 'mysql'
# TODO: use get_cursor() instead.
mysql_cursor = mysql_backend.get_connection().cursor()
sqlite_conn = sqlite_backend.get_connection(spatialite=True)
sqlite_cursor = sqlite_conn.cursor()
table_re = re.compile('INSERT INTO "([^"]+)" VALUES\(([^,]+),')
remove_geom_re = re.compile("X'[0-9A-F]+'")
IGNORED_TABLES = ['spatial_ref_sys', 'geometry_columns']
# Obsolete tables.
IGNORED_TABLES.append('t005_service_dates')
initialized_tables = set()
for line in sqlite_conn.iterdump():
if not line.startswith('INSERT INTO '):
continue
m = table_re.match(line)
assert m, 'Parse error on line %s' % repr(line)
table, rowid = m.groups(1)
# Ignore rtree index tables.
if table.startswith('idx_'):
continue
if table in IGNORED_TABLES:
continue
if table not in initialized_tables:
initialized_tables.add(table)
log.debug('Filling table %s', table)
mysql_cursor.execute('TRUNCATE TABLE `%s`' % table)
# TODO: this assumes that only geometry columns are encoded as blobs,
# they are called 'geometry' and that there is only one.
# This is all true at this time, but might change in the future.
#log.debug('Inserting %s', line)
line, n_subs = remove_geom_re.subn('NULL', line)
assert n_subs <= 1
mysql_cursor.execute(line)
if n_subs == 1:
res = sqlite_cursor.execute(
'SELECT AsText(geometry) FROM %s WHERE id=%s' % (table, rowid))
geom_text = res.fetchone()[0]
mysql_cursor.execute(
'UPDATE `%s` SET geometry=GeomFromText(%%s)'
' WHERE id=%%s' % table,
(geom_text, rowid))
for table in initialized_tables:
log.info('Verifying row count of table %s', table)
sqlite_count = sqlite_cursor.execute('SELECT count(1) FROM %s' % table).fetchone()[0]
mysql_cursor.execute('SELECT count(1) FROM %s' % table)
mysql_count = mysql_cursor.fetchone()[0]
if sqlite_count != mysql_count:
raise Exception(
('After import of table: %s, row count differs: '
'sqlite: %s - mysql: %s') % (table, sqlite_count, mysql_count))