def restore_database(self, dump, new_name=None, clean_first=True):
"""Restores the current database.
:param dump: a database dump file to be used to restore the database.
:param new_name: optional name for the new restored database.
:param clean_first: if a clean_database will be performed before restoring.
"""
log.info("Restoring database %s using %s" % (self.dbname, dump))
if self.rdbms == 'postgres':
# This will create a new database
if not new_name:
new_name = "%s__backup_%s" % (self.dbname,
time.strftime("%Y%m%d_%H%M"))
if clean_first:
self.clean_database(new_name)
args = ['pg_restore', '-d', new_name]
args.extend(self.get_tool_args())
args.append(dump)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args, stderr=PIPE)
proc.wait()
return new_name
else:
raise NotImplementedError(self.rdbms)
def start_htsql(port):
logger.info("Starting htsql server")
if db_settings.password:
password = '******' + urllib.parse.quote_plus(db_settings.password)
else:
password = ''
uri = 'pgsql://{}{}@{}:{}/{}'.format(db_settings.username, password,
db_settings.address, db_settings.port,
db_settings.dbname)
config = library.get_resource_filename('stoqserver', 'htsql', 'config.yml')
popen = Process([
'htsql-ctl', 'server', '-C', config, uri, '--host', '127.0.0.1',
'--port', port
])
def _sigterm_handler(_signal, _stack_frame):
popen.poll()
if popen.returncode is None:
popen.terminate()
os._exit(0)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, _sigterm_handler)
popen.wait()
def save(self):
temp_csv = tempfile.NamedTemporaryFile(suffix='.csv', delete=False, mode='w')
writer = csv.writer(temp_csv, delimiter=',',
doublequote=True,
quoting=csv.QUOTE_ALL)
writer.writerows(self.rows)
temp_csv.close()
template_file = sysparam.get_string('LABEL_TEMPLATE_PATH')
if not os.path.exists(template_file):
raise ValueError(_('Template file for printing labels was not found.'))
args = ['-f', str(self.skip + 1), '-o', self.filename, '-i',
temp_csv.name, template_file]
# FIXME: This is just a quick workaround. There must be a better way to
# do this.
# glables3 changed the script name. If the default (glables2) is not
# available, try the one from glables3
try:
p = Process(['glabels-batch'] + args)
except OSError:
p = Process(['glabels-3-batch'] + args)
# FIXME: We should use while so the print dialog can be canceled (see
# threadutils)
p.wait()
def start_htsql(port):
config = get_config()
if config.get('General', 'disable_htsql'):
logger.info("Not starting htsql as requested in config file.")
return
logger.info("Starting htsql server")
if db_settings.password:
password = '******' + urllib.parse.quote_plus(db_settings.password)
else:
password = ''
uri = 'pgsql://{}{}@{}:{}/{}'.format(
db_settings.username, password,
db_settings.address, db_settings.port, db_settings.dbname)
config = library.get_resource_filename('stoqserver', 'htsql', 'config.yml')
popen = Process(['htsql-ctl', 'server', '-C', config, uri,
'--host', '127.0.0.1', '--port', port])
def _sigterm_handler(_signal, _stack_frame):
popen.poll()
if popen.returncode is None:
popen.terminate()
os._exit(0)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, _sigterm_handler)
popen.wait()
def restore_database(self, dump, new_name=None, clean_first=True):
"""Restores the current database.
:param dump: a database dump file to be used to restore the database.
:param new_name: optional name for the new restored database.
:param clean_first: if a clean_database will be performed before restoring.
"""
log.info("Restoring database %s using %s" % (self.dbname, dump))
if self.rdbms == 'postgres':
# This will create a new database
if not new_name:
new_name = "%s__backup_%s" % (self.dbname,
time.strftime("%Y%m%d_%H%M"))
if clean_first:
self.clean_database(new_name)
args = ['pg_restore', '-d', new_name]
args.extend(self.get_tool_args())
args.append(dump)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args, stderr=PIPE)
proc.wait()
return new_name
else:
raise NotImplementedError(self.rdbms)
def save(self):
temp_csv = tempfile.NamedTemporaryFile(suffix='.csv', delete=False, mode='w')
writer = csv.writer(temp_csv, delimiter=',',
doublequote=True,
quoting=csv.QUOTE_ALL)
writer.writerows(self.rows)
temp_csv.close()
template_file = sysparam.get_string('LABEL_TEMPLATE_PATH')
if not os.path.exists(template_file):
raise ValueError(_('Template file for printing labels was not found.'))
args = ['-f', str(self.skip + 1), '-o', self.filename, '-i',
temp_csv.name, template_file]
# FIXME: This is just a quick workaround. There must be a better way to
# do this.
# glables3 changed the script name. If the default (glables2) is not
# available, try the one from glables3
try:
p = Process(['glabels-batch'] + args)
except OSError:
p = Process(['glabels-3-batch'] + args)
# FIXME: We should use while so the print dialog can be canceled (see
# threadutils)
p.wait()
def backup(backup_dir, full=False, retry=1):
# Tell Stoq Link Admin that you're starting a backup
user_hash = api.sysparam.get_string('USER_HASH')
start_url = urlparse.urljoin(WebService.API_SERVER, 'api/backup/start')
response = requests.get(start_url, params={'hash': user_hash})
# If the server rejects the backup, don't even attempt to proceed. Log
# which error caused the backup to fail
if response.status_code != 200:
raise Exception('ERROR: ' + response.content)
cmd = [_duplicati_exe, 'backup', _webservice_url, backup_dir,
'--log-id=' + response.content] + _get_extra_args()
p = Process(cmd)
threadit(_watch_fd, p.stdout)
threadit(_watch_fd, p.stderr)
p.wait()
if p.returncode == 100 and retry > 0:
# If the password has changed, duplicati will refuse to do the
# backup, even tough we support that on our backend. Force remove
# the cache so it will work
duplicati_config = os.path.join(os.getenv('APPDATA'), 'Duplicati')
shutil.rmtree(duplicati_config, ignore_errors=True)
return backup(backup_dir, full=full, retry=retry - 1)
if p.returncode != 0:
raise Exception("Failed to backup the database: {}".format(p.returncode))
# Tell Stoq Link Admin that the backup has finished
end_url = urlparse.urljoin(WebService.API_SERVER, 'api/backup/end')
requests.get(end_url,
params={'log_id': response.content, 'hash': user_hash})
def _run(cmd, *args):
script = library.get_resource_filename('stoqserver', 'scripts',
'duplicitybackup.py')
p = Process(['python2', script, cmd] + list(args), stdout=PIPE, stderr=PIPE)
threadit(_watch_fd, p.stdout)
threadit(_watch_fd, p.stderr)
p.wait()
return p.returncode == 0
def start_rtc():
if not api.sysparam.get_bool('ONLINE_SERVICES'):
logger.info("ONLINE_SERVICES not enabled. Not starting rtc...")
return
config = get_config()
if config.get('General', 'disable_rtc'):
logger.info("Not starting rtc as requested in config file.")
return
logger.info("Starting webRTC")
cwd = library.get_resource_filename('stoqserver', 'webrtc')
retry = True
extra_args = []
camera_urls = config.get('Camera', 'url') or None
if camera_urls:
extra_args.append('-c=' + ' '.join(set(camera_urls.split(' '))))
xmlrpc_host = config.get('General', 'serveraddress') or '127.0.0.1'
extra_args.append('-h={}'.format(xmlrpc_host))
xmlrpc_port = config.get('General', 'serverport') or SERVER_XMLRPC_PORT
extra_args.append('-p={}'.format(xmlrpc_port))
while retry:
retry = False
popen = Process(
['bash', 'start.sh'] + extra_args, cwd=cwd)
def _sigterm_handler(_signal, _stack_frame):
popen.poll()
if popen.returncode is None:
popen.terminate()
os._exit(0)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, _sigterm_handler)
popen.wait()
if popen.returncode == 11:
logger.warning("libstdc++ too old, not running webRTC client. "
"A system upgrade may be required!")
retry = False
elif popen.returncode == 10:
logger.warning("Something failed when trying to start webrtc. "
"Retrying again in 10 minutes...")
time.sleep(10 * 60)
retry = True
elif popen.returncode == 12:
logger.warning("webrtc installation corrupted. Restarting it...")
time.sleep(1)
retry = True
elif popen.returncode == 139:
logger.warning("Segmentation fault caught on wrtc. Restarting...")
time.sleep(1)
retry = True
def start_rtc():
if not api.sysparam.get_bool('ONLINE_SERVICES'):
logger.info("ONLINE_SERVICES not enabled. Not starting rtc...")
return
config = get_config()
if config.get('General', 'disable_rtc'):
logger.info("Not starting rtc as requested in config file.")
return
logger.info("Starting webRTC")
cwd = library.get_resource_filename('stoqserver', 'webrtc')
retry = True
extra_args = []
camera_urls = config.get('Camera', 'url') or None
if camera_urls:
extra_args.append('-c=' + ' '.join(set(camera_urls.split(' '))))
xmlrpc_host = config.get('General', 'serveraddress') or '127.0.0.1'
extra_args.append('-h={}'.format(xmlrpc_host))
xmlrpc_port = config.get('General', 'serverport') or SERVER_XMLRPC_PORT
extra_args.append('-p={}'.format(xmlrpc_port))
while retry:
retry = False
popen = Process(['bash', 'start.sh'] + extra_args, cwd=cwd)
def _sigterm_handler(_signal, _stack_frame):
popen.poll()
if popen.returncode is None:
popen.terminate()
os._exit(0)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, _sigterm_handler)
popen.wait()
if popen.returncode == 11:
logger.warning("libstdc++ too old, not running webRTC client. "
"A system upgrade may be required!")
retry = False
elif popen.returncode == 10:
logger.warning("Something failed when trying to start webrtc. "
"Retrying again in 10 minutes...")
time.sleep(10 * 60)
retry = True
elif popen.returncode == 12:
logger.warning("webrtc installation corrupted. Restarting it...")
time.sleep(1)
retry = True
elif popen.returncode == 139:
logger.warning("Segmentation fault caught on wrtc. Restarting...")
time.sleep(1)
retry = True
def _run(cmd, *args):
script = library.get_resource_filename('stoqserver', 'scripts',
'duplicitybackup.py')
p = Process(['python2', script, cmd] + list(args),
stdout=PIPE,
stderr=PIPE)
threadit(_watch_fd, p.stdout)
threadit(_watch_fd, p.stderr)
p.wait()
return p.returncode == 0
def restore(restore_dir, user_hash, time=None):
cmd = [_duplicati_exe, 'restore', _webservice_url, '*',
'--restore-path="{}"'.format(restore_dir),
'--log-id=-1'] + _get_extra_args(user_hash=user_hash)
p = Process(cmd)
threadit(_watch_fd, p.stdout)
threadit(_watch_fd, p.stderr)
p.wait()
if p.returncode != 0:
raise Exception("Failed to restore the database: {}".format(p.returncode))
def dump_database(self, filename, schema_only=False,
gzip=False, format='custom'):
"""Dump the contents of the current database
:param filename: filename to write the database dump to
:param schema_only: If only the database schema will be dumped
:param gzip: if the dump should be compressed using gzip -9
:param format: database dump format, defaults to ``custom``
"""
log.info("Dumping database to %s" % filename)
if self.rdbms == 'postgres':
args = ['pg_dump',
'--format=%s' % (format, ),
'--encoding=UTF-8']
if gzip:
args.append('--compress=9')
if schema_only:
args.append('--schema-only')
if filename is not None:
args.extend(['-f', filename])
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args)
return proc.wait() == 0
else:
raise NotImplementedError(self.rdbms)
def dump_database(self, filename, schema_only=False,
gzip=False, format='custom'):
"""Dump the contents of the current database
:param filename: filename to write the database dump to
:param schema_only: If only the database schema will be dumped
:param gzip: if the dump should be compressed using gzip -9
:param format: database dump format, defaults to ``custom``
"""
log.info("Dumping database to %s" % filename)
if self.rdbms == 'postgres':
args = ['pg_dump',
'--format=%s' % (format, ),
'--encoding=UTF-8']
if gzip:
args.append('--compress=9')
if schema_only:
args.append('--schema-only')
if filename is not None:
args.extend(['-f', filename])
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args)
return proc.wait() == 0
else:
raise NotImplementedError(self.rdbms)
def test_stoqlib_domain(self):
args = ["pylint",
"--rcfile=%s/tools/pylint.rcfile" % (self.root,),
"--load-plugins", "tools/pylint_stoq",
"-E",
"stoqlib.domain"]
p = Process(args)
retval = p.wait()
if retval:
raise Exception("Pylint errors")
def pylint(self, modules, args=None):
if not args:
args = []
args = [
"pylint", "--dummy-variables=unused,_",
"--disable=%s" % (",".join(DISABLED)), "--include-ids=y",
"--rcfile=%s/tools/pylint.rcfile" % (self.root, ), "--reports=n"
] + args + modules
p = Process(args)
retval = p.wait()
if retval:
raise Exception("Pylint errors")
def pylint(self, modules, args=None):
if not args:
args = []
args = ["pylint",
"--dummy-variables=unused,_",
"--disable=%s" % (",".join(DISABLED)),
"--include-ids=y",
"--rcfile=%s/tools/pylint.rcfile" % (self.root,),
"--reports=n"] + args + modules
p = Process(args)
retval = p.wait()
if retval:
raise Exception("Pylint errors")
def start_shell(self, command=None, quiet=False):
"""Runs a database shell
:param command: tell psql to execute the command string
:param quiet: sets psql quiet option (``-q``)
"""
if self.rdbms == 'postgres':
args = ['psql']
if command:
args.extend(['-c', command])
if quiet:
args.append('-q')
args.extend(self.get_tool_args())
args.append(self.dbname)
print('Connecting to %s' % (
self.get_store_dsn(filter_password=True), ))
proc = Process(args)
proc.wait()
else:
raise NotImplementedError(self.rdbms)
def start_shell(self, command=None, quiet=False):
"""Runs a database shell
:param command: tell psql to execute the command string
:param quiet: sets psql quiet option (``-q``)
"""
if self.rdbms == 'postgres':
args = ['psql']
if command:
args.extend(['-c', command])
if quiet:
args.append('-q')
args.extend(self.get_tool_args())
args.append(self.dbname)
print('Connecting to %s' % (
self.get_store_uri(filter_password=True), ))
proc = Process(args)
proc.wait()
else:
raise NotImplementedError(self.rdbms)
def test_connection(self):
"""Test for database connectivity using command line tools
:returns: `True` if the database connection succeeded.
"""
log.info("Testing database connectivity using command line tools")
if self.rdbms == 'postgres':
# -w avoids password prompts, which causes this to hang.
args = [
'psql', '-n', '-q', '-w', '--variable', 'ON_ERROR_STOP=', '-c',
'SELECT 1;'
]
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args, stdin=PIPE, stdout=PIPE)
retval = proc.wait()
return retval == 0
else:
raise NotImplementedError(self.rdbms)
def test_connection(self):
"""Test for database connectivity using command line tools
:returns: `True` if the database connection succeeded.
"""
log.info("Testing database connectivity using command line tools")
if self.rdbms == 'postgres':
# -w avoids password prompts, which causes this to hang.
args = ['psql', '-n', '-q', '-w',
'--variable', 'ON_ERROR_STOP=',
'-c', 'SELECT 1;']
args.extend(self.get_tool_args())
args.append(self.dbname)
log.debug('executing %s' % (' '.join(args), ))
proc = Process(args,
stdin=PIPE,
stdout=PIPE)
retval = proc.wait()
return retval == 0
else:
raise NotImplementedError(self.rdbms)
