def save(self):
temp_csv = tempfile.NamedTemporaryFile(suffix='.csv', delete=False, mode='w')
writer = csv.writer(temp_csv, delimiter=',',
doublequote=True,
quoting=csv.QUOTE_ALL)
writer.writerows(self.rows)
temp_csv.close()
template_file = sysparam.get_string('LABEL_TEMPLATE_PATH')
if not os.path.exists(template_file):
raise ValueError(_('Template file for printing labels was not found.'))
args = ['-f', str(self.skip + 1), '-o', self.filename, '-i',
temp_csv.name, template_file]
# FIXME: This is just a quick workaround. There must be a better way to
# do this.
# glables3 changed the script name. If the default (glables2) is not
# available, try the one from glables3
try:
p = Process(['glabels-batch'] + args)
except OSError:
p = Process(['glabels-3-batch'] + args)
# FIXME: We should use while so the print dialog can be canceled (see
# threadutils)
p.wait()
def restore_database(self, dump, new_name=None, clean_first=True):
"""Restores the current database.
:param dump: a database dump file to be used to restore the database.
:param new_name: optional name for the new restored database.
:param clean_first: if a clean_database will be performed before restoring.
"""
log.info("Restoring database %s using %s" % (self.dbname, dump))
if self.rdbms == 'postgres':
# This will create a new database
if not new_name:
new_name = "%s__backup_%s" % (self.dbname,
time.strftime("%Y%m%d_%H%M"))
if clean_first:
self.clean_database(new_name)
args = ['pg_restore', '-d', new_name]
args.extend(self.get_tool_args())
args.append(dump)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args, stderr=PIPE)
proc.wait()
return new_name
else:
raise NotImplementedError(self.rdbms)
def dump_database(self, filename, schema_only=False,
gzip=False, format='custom'):
"""Dump the contents of the current database
:param filename: filename to write the database dump to
:param schema_only: If only the database schema will be dumped
:param gzip: if the dump should be compressed using gzip -9
:param format: database dump format, defaults to ``custom``
"""
log.info("Dumping database to %s" % filename)
if self.rdbms == 'postgres':
args = ['pg_dump',
'--format=%s' % (format, ),
'--encoding=UTF-8']
if gzip:
args.append('--compress=9')
if schema_only:
args.append('--schema-only')
if filename is not None:
args.extend(['-f', filename])
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args)
return proc.wait() == 0
else:
raise NotImplementedError(self.rdbms)
def start_htsql(port):
config = get_config()
if config.get('General', 'disable_htsql'):
logger.info("Not starting htsql as requested in config file.")
return
logger.info("Starting htsql server")
if db_settings.password:
password = '******' + urllib.parse.quote_plus(db_settings.password)
else:
password = ''
uri = 'pgsql://{}{}@{}:{}/{}'.format(
db_settings.username, password,
db_settings.address, db_settings.port, db_settings.dbname)
config = library.get_resource_filename('stoqserver', 'htsql', 'config.yml')
popen = Process(['htsql-ctl', 'server', '-C', config, uri,
'--host', '127.0.0.1', '--port', port])
def _sigterm_handler(_signal, _stack_frame):
popen.poll()
if popen.returncode is None:
popen.terminate()
os._exit(0)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, _sigterm_handler)
popen.wait()
def _run(cmd, *args):
script = library.get_resource_filename('stoqserver', 'scripts',
'duplicitybackup.py')
p = Process(['python2', script, cmd] + list(args), stdout=PIPE, stderr=PIPE)
threadit(_watch_fd, p.stdout)
threadit(_watch_fd, p.stderr)
p.wait()
return p.returncode == 0
def start_rtc():
if not api.sysparam.get_bool('ONLINE_SERVICES'):
logger.info("ONLINE_SERVICES not enabled. Not starting rtc...")
return
config = get_config()
if config.get('General', 'disable_rtc'):
logger.info("Not starting rtc as requested in config file.")
return
logger.info("Starting webRTC")
cwd = library.get_resource_filename('stoqserver', 'webrtc')
retry = True
extra_args = []
camera_urls = config.get('Camera', 'url') or None
if camera_urls:
extra_args.append('-c=' + ' '.join(set(camera_urls.split(' '))))
xmlrpc_host = config.get('General', 'serveraddress') or '127.0.0.1'
extra_args.append('-h={}'.format(xmlrpc_host))
xmlrpc_port = config.get('General', 'serverport') or SERVER_XMLRPC_PORT
extra_args.append('-p={}'.format(xmlrpc_port))
while retry:
retry = False
popen = Process(
['bash', 'start.sh'] + extra_args, cwd=cwd)
def _sigterm_handler(_signal, _stack_frame):
popen.poll()
if popen.returncode is None:
popen.terminate()
os._exit(0)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, _sigterm_handler)
popen.wait()
if popen.returncode == 11:
logger.warning("libstdc++ too old, not running webRTC client. "
"A system upgrade may be required!")
retry = False
elif popen.returncode == 10:
logger.warning("Something failed when trying to start webrtc. "
"Retrying again in 10 minutes...")
time.sleep(10 * 60)
retry = True
elif popen.returncode == 12:
logger.warning("webrtc installation corrupted. Restarting it...")
time.sleep(1)
retry = True
elif popen.returncode == 139:
logger.warning("Segmentation fault caught on wrtc. Restarting...")
time.sleep(1)
retry = True
def check_version(self, store):
"""Verify that the database version is recent enough to be supported
by stoq. Emits a warning if the version isn't recent enough, suitable
for usage by an installer.
:param store: a store
"""
if self.rdbms == 'postgres':
version = store.execute('SELECT VERSION();').get_one()[0]
server_version = version.split(' ', 2)[1]
assert server_version.count('.') == 2, version
parts = server_version.split(".")[:2]
try:
svs = map(int, parts)
except ValueError:
log.info("Error getting server version: %s" % (server_version, ))
return
# Client version
kwargs = {}
args = ['psql']
if _system == 'Windows':
# FIXME: figure out why this isn't working
return
else:
args.append('--version')
p = Process(args, stdout=PIPE, **kwargs)
stdout = p.communicate()[0]
line = stdout.split('\n', 1)[0]
if line.endswith('\r'):
line = line[:-1]
parts = line.split(' ')
# assert len(parts) == 3, parts
if len(parts) != 3:
log.info("Error getting psql version: %s" % (line, ))
return
client_version = parts[2]
# assert client_version.count('.') == 2, line
if client_version.count('.') != 2:
log.info("Error getting pg version: %s" % (client_version, ))
return
cvs = map(int, client_version.split('.'))[:2]
if svs != cvs:
warning(_(u"Problem with PostgreSQL version"),
_(u"The version of the PostgreSQL database server (%s) and the "
"postgres client tools (%s) differ. I will let you use "
"Stoq, but you will always see this warning when "
"starting Stoq until you resolve the version "
"incompatibilty by upgrading the server or the client "
"tools.") % (server_version, client_version))
else:
raise NotImplementedError(self.rdbms)
def test_stoqlib_domain(self):
args = ["pylint",
"--rcfile=%s/tools/pylint.rcfile" % (self.root,),
"--load-plugins", "tools/pylint_stoq",
"-E",
"stoqlib.domain"]
p = Process(args)
retval = p.wait()
if retval:
raise Exception("Pylint errors")
def pylint(self, modules, args=None):
if not args:
args = []
args = ["pylint",
"--dummy-variables=unused,_",
"--disable=%s" % (",".join(DISABLED)),
"--include-ids=y",
"--rcfile=%s/tools/pylint.rcfile" % (self.root,),
"--reports=n"] + args + modules
p = Process(args)
retval = p.wait()
if retval:
raise Exception("Pylint errors")
def save(self):
temp_csv = tempfile.NamedTemporaryFile(suffix='.csv', delete=False)
writer = csv.writer(temp_csv, delimiter=',',
doublequote=True,
quoting=csv.QUOTE_ALL)
writer.writerows(self.rows)
temp_csv.close()
template_file = sysparam.get_string('LABEL_TEMPLATE_PATH')
if not os.path.exists(template_file):
raise ValueError(_('Template file for printing labels was not found.'))
args = ['-f', str(self.skip + 1), '-o', self.filename, '-i',
temp_csv.name, template_file]
# FIXME: This is just a quick workaround. There must be a better way to
# do this.
# glables3 changed the script name. If the default (glables2) is not
# available, try the one from glables3
try:
p = Process(['glabels-batch'] + args)
except OSError:
p = Process(['glabels-3-batch'] + args)
# FIXME: We should use while so the print dialog can be canceled (see
# threadutils)
p.wait()
def _create_dbuser(self, username):
import os
from stoqlib.lib.process import Process, PIPE
for envname in ['PGUSER', 'PGHOST']:
if envname in os.environ:
del os.environ[envname]
# See if we can connect to the database
args = ['psql', 'postgres', username, '-c', 'SELECT 1;']
proc = Process(args, stdout=PIPE)
proc.communicate()
if proc.returncode == 0:
return 0
from stoqlib.lib.kiwilibrary import library
createdbuser = library.get_resource_filename(
'stoq', 'scripts', 'createdbuser.sh')
args = ['pkexec',
'-u', 'postgres',
createdbuser,
username]
proc = Process(args)
proc.communicate()
if proc.returncode != 0:
print("ERROR: Failed to run %r" % (args, ))
return 30
return 0
def start_shell(self, command=None, quiet=False):
"""Runs a database shell
:param command: tell psql to execute the command string
:param quiet: sets psql quiet option (``-q``)
"""
if self.rdbms == 'postgres':
args = ['psql']
if command:
args.extend(['-c', command])
if quiet:
args.append('-q')
args.extend(self.get_tool_args())
args.append(self.dbname)
print('Connecting to %s' % (
self.get_store_dsn(filter_password=True), ))
proc = Process(args)
proc.wait()
else:
raise NotImplementedError(self.rdbms)
def backup(backup_dir, full=False, retry=1):
# Tell Stoq Link Admin that you're starting a backup
user_hash = api.sysparam.get_string('USER_HASH')
start_url = urllib.parse.urljoin(WebService.API_SERVER, 'api/backup/start')
response = requests.get(start_url, params={'hash': user_hash})
# If the server rejects the backup, don't even attempt to proceed. Log
# which error caused the backup to fail
if response.status_code != 200:
raise Exception('ERROR: ' + response.content)
cmd = [
_duplicati_exe, 'backup', _webservice_url, backup_dir,
'--log-id=' + response.content
] + _get_extra_args()
p = Process(cmd)
threadit(_watch_fd, p.stdout)
threadit(_watch_fd, p.stderr)
p.wait()
if p.returncode == 100 and retry > 0:
# If the password has changed, duplicati will refuse to do the
# backup, even tough we support that on our backend. Force remove
# the cache so it will work
duplicati_config = os.path.join(os.getenv('APPDATA'), 'Duplicati')
shutil.rmtree(duplicati_config, ignore_errors=True)
return backup(backup_dir, full=full, retry=retry - 1)
if p.returncode != 0:
raise Exception("Failed to backup the database: {}".format(
p.returncode))
# Tell Stoq Link Admin that the backup has finished
end_url = urllib.parse.urljoin(WebService.API_SERVER, 'api/backup/end')
requests.get(end_url,
params={
'log_id': response.content,
'hash': user_hash
})
def start_shell(self, command=None, quiet=False):
"""Runs a database shell
:param command: tell psql to execute the command string
:param quiet: sets psql quiet option (``-q``)
"""
if self.rdbms == 'postgres':
args = ['psql']
if command:
args.extend(['-c', command])
if quiet:
args.append('-q')
args.extend(self.get_tool_args())
args.append(self.dbname)
print('Connecting to %s' % (
self.get_store_uri(filter_password=True), ))
proc = Process(args)
proc.wait()
else:
raise NotImplementedError(self.rdbms)
def test_connection(self):
"""Test for database connectivity using command line tools
:returns: `True` if the database connection succeeded.
"""
log.info("Testing database connectivity using command line tools")
if self.rdbms == 'postgres':
# -w avoids password prompts, which causes this to hang.
args = [
'psql', '-n', '-q', '-w', '--variable', 'ON_ERROR_STOP=', '-c',
'SELECT 1;'
]
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args, stdin=PIPE, stdout=PIPE)
retval = proc.wait()
return retval == 0
else:
raise NotImplementedError(self.rdbms)
def test_connection(self):
"""Test for database connectivity using command line tools
:returns: `True` if the database connection succeeded.
"""
log.info("Testing database connectivity using command line tools")
if self.rdbms == 'postgres':
# -w avoids password prompts, which causes this to hang.
args = ['psql', '-n', '-q', '-w',
'--variable', 'ON_ERROR_STOP=',
'-c', 'SELECT 1;']
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args,
stdin=PIPE,
stdout=PIPE)
retval = proc.wait()
return retval == 0
else:
raise NotImplementedError(self.rdbms)
def execute_command(self, args):
self.feed('Executing: %s\r\n' % (' '.join(args)))
kwargs = {}
# On Windows you have to passin stdout/stdin = PIPE or
# it will result in an invalid handle, see
# * CR2012071248
# * http://bugs.python.org/issue3905
if self.listen_stdout or platform.system() == 'Windows':
kwargs['stdout'] = PIPE
if self.listen_stderr or platform.system() == 'Windows':
kwargs['stderr'] = PIPE
self.proc = Process(args, **kwargs)
if self.listen_stdout:
threadit(self._watch_fd, self.proc.stdout)
if self.listen_stderr:
threadit(self._watch_fd, self.proc.stderr)
# We could probably listen to SIGCHLD here instead
GLib.timeout_add(CHILD_TIMEOUT, self._check_child_finished)
def _terminate(self, restart=False, app=None):
log.info("Terminating Stoq")
# This removes all temporary files created when calling
# get_resource_filename() that extract files to the file system
import pkg_resources
pkg_resources.cleanup_resources()
log.debug('Stopping deamon')
from stoqlib.lib.daemonutils import stop_daemon
stop_daemon()
# Finally, go out of the reactor and show possible crash reports
log.debug("Show some crash reports")
self._show_crash_reports()
log.debug("Stoq Gtk.main")
from gi.repository import Gtk
Gtk.main_quit()
# Make sure that no connection is left open (specially on Windows)
try:
from stoqlib.database.runtime import get_default_store
get_default_store().close()
except Exception:
pass
if restart:
from stoqlib.lib.process import Process
log.info('Restarting Stoq')
args = [sys.argv[0], '--no-splash-screen']
if app is not None:
args.append(app)
Process(args)
# os._exit() forces a quit without running atexit handlers
# and does not block on any running threads
# FIXME: This is the wrong solution, we should figure out why there
# are any running threads/processes at this point
log.debug("Terminating by calling os._exit()")
os._exit(0)
raise AssertionError("Should never happen")
def start(self):
try:
self._get_port()
except TryAgainError:
pass
else:
return defer.succeed(self)
stoq_daemon = find_program('stoq-daemon')
if not stoq_daemon:
raise AssertionError
args = [sys.executable,
stoq_daemon,
'--daemon-id', self._daemon_id]
args.extend(db_settings.get_command_line_arguments())
self._process = Process(args)
reactor.callLater(0.1, self._check_active)
self._defer = defer.Deferred()
return self._defer
def _create_dbuser(self, username):
import os
from stoqlib.lib.process import Process, PIPE
for envname in ['PGUSER', 'PGHOST']:
if envname in os.environ:
del os.environ[envname]
# See if we can connect to the database
args = ['psql', 'postgres', username, '-c', 'SELECT 1;']
proc = Process(args, stdout=PIPE)
proc.communicate()
if proc.returncode == 0:
return 0
args = ['pkexec',
'-u', 'postgres',
'stoqcreatedbuser',
username]
proc = Process(args)
proc.communicate()
if proc.returncode != 0:
print("ERROR: Failed to run %r" % (args, ))
return 30
return 0
def _create_dbuser(self, username):
import os
from stoqlib.lib.process import Process, PIPE
for envname in ["PGUSER", "PGHOST"]:
if envname in os.environ:
del os.environ[envname]
# See if we can connect to the database
args = ["psql", "postgres", username, "-c", "SELECT 1;"]
proc = Process(args, stdout=PIPE)
proc.communicate()
if proc.returncode == 0:
return 0
args = ["pkexec", "-u", "postgres", "stoqcreatedbuser", username]
proc = Process(args)
proc.communicate()
if proc.returncode != 0:
print("ERROR: Failed to run %r" % (args,))
return 30
return 0
def _create_dbuser(self, username):
import os
from stoqlib.lib.process import Process, PIPE
for envname in ['PGUSER', 'PGHOST']:
if envname in os.environ:
del os.environ[envname]
# See if we can connect to the database
args = ['psql', 'postgres', username, '-c', 'SELECT 1;']
proc = Process(args, stdout=PIPE)
proc.communicate()
if proc.returncode == 0:
return 0
args = ['pkexec',
'-u', 'postgres',
'stoqcreatedbuser',
username]
proc = Process(args)
proc.communicate()
if proc.returncode != 0:
print "ERROR: Failed to run %r" % (args, )
return 30
return 0
def start_backup_scheduler(doing_backup):
_setup_signal_termination()
if not api.sysparam.get_bool('ONLINE_SERVICES'):
logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...")
return
logger.info("Starting backup scheduler")
config = get_config()
backup_schedule = config.get('Backup', 'schedule')
if backup_schedule is None:
# By defualt, we will do 2 backups. One in a random time between
# 9-11 or 14-17 and another one 12 hours after that.
# We are using 2, 3 and 4 because they will be summed with 12 bellow
hour = random.choice([2, 3, 4, 9, 10])
minute = random.randint(0, 59)
backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute)
config.set('Backup', 'schedule', backup_schedule)
config.flush()
backup_hours = [list(map(int, i.strip().split(':')))
for i in backup_schedule.split(',')]
now = datetime.datetime.now()
backup_dates = collections.deque(sorted(
now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0)
for bh in backup_hours))
while True:
now = datetime.datetime.now()
next_date = datetime.datetime.min
while next_date < now:
next_date = backup_dates.popleft()
backup_dates.append(next_date + datetime.timedelta(1))
time.sleep(max(1, (next_date - now).total_seconds()))
for i in range(3):
# FIXME: This is SO UGLY, we should be calling backup_database
# task directly, but duplicity messes with multiprocessing in a
# way that it will not work
args = sys.argv[:]
for i, arg in enumerate(args[:]):
if arg == 'run':
args[i] = 'backup_database'
break
doing_backup.value = 1
try:
p = Process(args)
stdout, stderr = p.communicate()
finally:
doing_backup.value = 0
if p.returncode == 0:
break
else:
# When duplicity fails in unpredicted situations (e.g. the
# power is shut down suddenly) it can leave a lockfile behind,
# and that can make any future backup attempts fail to.
# Check if that was the reason of the failure and, if the
# lockfile is older than 3h remove it and try again.
# Note that this only happens for duplicity (linux) and
# not for duplicati (windows)
match = re.search('/.*lockfile.lock', stderr)
if match is not None:
lockfile = match.group(0)
now = datetime.datetime.now()
mdate = datetime.datetime.fromtimestamp(os.path.getmtime(lockfile))
if (now - mdate) > _lock_remove_threshold:
os.unlink(lockfile)
logger.warning(
"Failed to backup database:\nstdout: %s\nstderr: %s",
stdout, stderr)
# Retry again with a exponential backoff
time.sleep((60 * 2) ** (i + 1))
def execute_sql(self, filename, lock_database=False):
"""Inserts raw SQL commands into the database read from a file.
:param filename: filename with SQL commands
:param lock_database: If the existing tables in the database should be
locked
:returns: return code, ``0`` if succeeded, positive integer for failure
"""
log.info("Executing SQL script %s database locked=%s" % (filename,
lock_database))
if self.rdbms == 'postgres':
# Okay, this might look crazy, but it's actually the only way
# to execute many SQL statements in PostgreSQL and
# 1) Stop immediatelly when an error occur
# 2) Print the error message, the filename and the line number where
# the error occurred.
# 3) Do not print anything on the output unless it's an warning or a
# an error
args = ['psql']
# -U needs to go in first or psql on windows get confused
args.extend(self.get_tool_args())
args.extend(['-n', '-q'])
kwargs = {}
if _system == 'Windows':
# Hide the console window
# For some reason XP doesn't like interacting with
# proceses via pipes
read_from_pipe = False
else:
read_from_pipe = True
# We have two different execution modes,
# 1) open stdin (-) and write the data via a pipe,
# this allows us to also disable noticies and info messages,
# so that only warnings are printed, we also fail if a warning
# or error is printed
# 2) Pass in the file normally to psql, no error reporting included
if read_from_pipe:
args.extend(['-f', '-'])
args.extend(['--variable', 'ON_ERROR_STOP='])
else:
args.extend(['-f', filename])
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
**kwargs)
proc.stdin.write('BEGIN TRANSACTION;')
if lock_database:
store = self.create_store()
lock_query = store.get_lock_database_query()
proc.stdin.write(lock_query)
store.close()
if read_from_pipe:
# We don't want to see notices on the output, skip them,
# this will make all reported line numbers offset by 1
proc.stdin.write("SET SESSION client_min_messages TO 'warning';")
data = open(filename).read()
# Rename serial into bigserial, for 64-bit id columns
data = data.replace('id serial', 'id bigserial')
data += '\nCOMMIT;'
else:
data = None
stdout, stderr = proc.communicate(data)
if read_from_pipe and stderr:
raise SQLError(stderr[:-1])
return proc.returncode
else:
raise NotImplementedError(self.rdbms)
def start_backup_scheduler(doing_backup):
_setup_signal_termination()
if not api.sysparam.get_bool('ONLINE_SERVICES'):
logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...")
return
logger.info("Starting backup scheduler")
config = get_config()
backup_schedule = config.get('Backup', 'schedule')
if backup_schedule is None:
# By defualt, we will do 2 backups. One in a random time between
# 9-11 or 14-17 and another one 12 hours after that.
# We are using 2, 3 and 4 because they will be summed with 12 bellow
hour = random.choice([2, 3, 4, 9, 10])
minute = random.randint(0, 59)
backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute)
config.set('Backup', 'schedule', backup_schedule)
config.flush()
backup_hours = [
list(map(int,
i.strip().split(':'))) for i in backup_schedule.split(',')
]
now = datetime.datetime.now()
backup_dates = collections.deque(
sorted(
now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0)
for bh in backup_hours))
while True:
now = datetime.datetime.now()
next_date = datetime.datetime.min
while next_date < now:
next_date = backup_dates.popleft()
backup_dates.append(next_date + datetime.timedelta(1))
time.sleep(max(1, (next_date - now).total_seconds()))
for i in range(3):
# FIXME: This is SO UGLY, we should be calling backup_database
# task directly, but duplicity messes with multiprocessing in a
# way that it will not work
args = sys.argv[:]
for i, arg in enumerate(args[:]):
if arg == 'run':
args[i] = 'backup_database'
break
doing_backup.value = 1
try:
p = Process(args)
stdout, stderr = p.communicate()
finally:
doing_backup.value = 0
if p.returncode == 0:
break
else:
# When duplicity fails in unpredicted situations (e.g. the
# power is shut down suddenly) it can leave a lockfile behind,
# and that can make any future backup attempts fail to.
# Check if that was the reason of the failure and, if the
# lockfile is older than 3h remove it and try again.
# Note that this only happens for duplicity (linux) and
# not for duplicati (windows)
match = re.search('/.*lockfile.lock', stderr)
if match is not None:
lockfile = match.group(0)
now = datetime.datetime.now()
mdate = datetime.datetime.fromtimestamp(
os.path.getmtime(lockfile))
if (now - mdate) > _lock_remove_threshold:
os.unlink(lockfile)
logger.warning(
"Failed to backup database:\nstdout: %s\nstderr: %s",
stdout, stderr)
# Retry again with a exponential backoff
time.sleep((60 * 2)**(i + 1))
def execute_sql(self, filename, lock_database=False):
"""Inserts raw SQL commands into the database read from a file.
:param filename: filename with SQL commands
:param lock_database: If the existing tables in the database should be
locked
:returns: return code, ``0`` if succeeded, positive integer for failure
"""
log.info("Executing SQL script %s database locked=%s" % (filename,
lock_database))
if self.rdbms == 'postgres':
# Okay, this might look crazy, but it's actually the only way
# to execute many SQL statements in PostgreSQL and
# 1) Stop immediatelly when an error occur
# 2) Print the error message, the filename and the line number where
# the error occurred.
# 3) Do not print anything on the output unless it's an warning or a
# an error
args = ['psql']
# -U needs to go in first or psql on windows get confused
args.extend(self.get_tool_args())
args.extend(['-n', '-q'])
kwargs = {}
if _system == 'Windows':
# Hide the console window
# For some reason XP doesn't like interacting with
# proceses via pipes
read_from_pipe = False
else:
read_from_pipe = True
# We have two different execution modes,
# 1) open stdin (-) and write the data via a pipe,
# this allows us to also disable noticies and info messages,
# so that only warnings are printed, we also fail if a warning
# or error is printed
# 2) Pass in the file normally to psql, no error reporting included
if read_from_pipe:
args.extend(['-f', '-'])
args.extend(['--variable', 'ON_ERROR_STOP='])
else:
args.extend(['-f', filename])
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
**kwargs)
proc.stdin.write('BEGIN TRANSACTION;')
if lock_database:
store = self.create_store()
lock_query = store.get_lock_database_query()
proc.stdin.write(lock_query)
store.close()
if read_from_pipe:
# We don't want to see notices on the output, skip them,
# this will make all reported line numbers offset by 1
proc.stdin.write("SET SESSION client_min_messages TO 'warning';")
data = open(filename).read()
# Rename serial into bigserial, for 64-bit id columns
data = data.replace('id serial', 'id bigserial')
data += '\nCOMMIT;'
else:
data = None
stdout, stderr = proc.communicate(data)
if read_from_pipe and stderr:
raise SQLError(stderr[:-1])
return proc.returncode
else:
raise NotImplementedError(self.rdbms)
