def save(self): temp_csv = tempfile.NamedTemporaryFile(suffix='.csv', delete=False, mode='w') writer = csv.writer(temp_csv, delimiter=',', doublequote=True, quoting=csv.QUOTE_ALL) writer.writerows(self.rows) temp_csv.close() template_file = sysparam.get_string('LABEL_TEMPLATE_PATH') if not os.path.exists(template_file): raise ValueError(_('Template file for printing labels was not found.')) args = ['-f', str(self.skip + 1), '-o', self.filename, '-i', temp_csv.name, template_file] # FIXME: This is just a quick workaround. There must be a better way to # do this. # glables3 changed the script name. If the default (glables2) is not # available, try the one from glables3 try: p = Process(['glabels-batch'] + args) except OSError: p = Process(['glabels-3-batch'] + args) # FIXME: We should use while so the print dialog can be canceled (see # threadutils) p.wait()
def restore_database(self, dump, new_name=None, clean_first=True): """Restores the current database. :param dump: a database dump file to be used to restore the database. :param new_name: optional name for the new restored database. :param clean_first: if a clean_database will be performed before restoring. """ log.info("Restoring database %s using %s" % (self.dbname, dump)) if self.rdbms == 'postgres': # This will create a new database if not new_name: new_name = "%s__backup_%s" % (self.dbname, time.strftime("%Y%m%d_%H%M")) if clean_first: self.clean_database(new_name) args = ['pg_restore', '-d', new_name] args.extend(self.get_tool_args()) args.append(dump) log.debug('executing %s' % (' '.join(args), )) proc = Process(args, stderr=PIPE) proc.wait() return new_name else: raise NotImplementedError(self.rdbms)
def dump_database(self, filename, schema_only=False, gzip=False, format='custom'): """Dump the contents of the current database :param filename: filename to write the database dump to :param schema_only: If only the database schema will be dumped :param gzip: if the dump should be compressed using gzip -9 :param format: database dump format, defaults to ``custom`` """ log.info("Dumping database to %s" % filename) if self.rdbms == 'postgres': args = ['pg_dump', '--format=%s' % (format, ), '--encoding=UTF-8'] if gzip: args.append('--compress=9') if schema_only: args.append('--schema-only') if filename is not None: args.extend(['-f', filename]) args.extend(self.get_tool_args()) args.append(self.dbname) log.debug('executing %s' % (' '.join(args), )) proc = Process(args) return proc.wait() == 0 else: raise NotImplementedError(self.rdbms)
def start_htsql(port): config = get_config() if config.get('General', 'disable_htsql'): logger.info("Not starting htsql as requested in config file.") return logger.info("Starting htsql server") if db_settings.password: password = '******' + urllib.parse.quote_plus(db_settings.password) else: password = '' uri = 'pgsql://{}{}@{}:{}/{}'.format( db_settings.username, password, db_settings.address, db_settings.port, db_settings.dbname) config = library.get_resource_filename('stoqserver', 'htsql', 'config.yml') popen = Process(['htsql-ctl', 'server', '-C', config, uri, '--host', '127.0.0.1', '--port', port]) def _sigterm_handler(_signal, _stack_frame): popen.poll() if popen.returncode is None: popen.terminate() os._exit(0) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, _sigterm_handler) popen.wait()
def _run(cmd, *args): script = library.get_resource_filename('stoqserver', 'scripts', 'duplicitybackup.py') p = Process(['python2', script, cmd] + list(args), stdout=PIPE, stderr=PIPE) threadit(_watch_fd, p.stdout) threadit(_watch_fd, p.stderr) p.wait() return p.returncode == 0
def start_rtc(): if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not starting rtc...") return config = get_config() if config.get('General', 'disable_rtc'): logger.info("Not starting rtc as requested in config file.") return logger.info("Starting webRTC") cwd = library.get_resource_filename('stoqserver', 'webrtc') retry = True extra_args = [] camera_urls = config.get('Camera', 'url') or None if camera_urls: extra_args.append('-c=' + ' '.join(set(camera_urls.split(' ')))) xmlrpc_host = config.get('General', 'serveraddress') or '127.0.0.1' extra_args.append('-h={}'.format(xmlrpc_host)) xmlrpc_port = config.get('General', 'serverport') or SERVER_XMLRPC_PORT extra_args.append('-p={}'.format(xmlrpc_port)) while retry: retry = False popen = Process( ['bash', 'start.sh'] + extra_args, cwd=cwd) def _sigterm_handler(_signal, _stack_frame): popen.poll() if popen.returncode is None: popen.terminate() os._exit(0) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, _sigterm_handler) popen.wait() if popen.returncode == 11: logger.warning("libstdc++ too old, not running webRTC client. " "A system upgrade may be required!") retry = False elif popen.returncode == 10: logger.warning("Something failed when trying to start webrtc. " "Retrying again in 10 minutes...") time.sleep(10 * 60) retry = True elif popen.returncode == 12: logger.warning("webrtc installation corrupted. Restarting it...") time.sleep(1) retry = True elif popen.returncode == 139: logger.warning("Segmentation fault caught on wrtc. Restarting...") time.sleep(1) retry = True
def check_version(self, store): """Verify that the database version is recent enough to be supported by stoq. Emits a warning if the version isn't recent enough, suitable for usage by an installer. :param store: a store """ if self.rdbms == 'postgres': version = store.execute('SELECT VERSION();').get_one()[0] server_version = version.split(' ', 2)[1] assert server_version.count('.') == 2, version parts = server_version.split(".")[:2] try: svs = map(int, parts) except ValueError: log.info("Error getting server version: %s" % (server_version, )) return # Client version kwargs = {} args = ['psql'] if _system == 'Windows': # FIXME: figure out why this isn't working return else: args.append('--version') p = Process(args, stdout=PIPE, **kwargs) stdout = p.communicate()[0] line = stdout.split('\n', 1)[0] if line.endswith('\r'): line = line[:-1] parts = line.split(' ') # assert len(parts) == 3, parts if len(parts) != 3: log.info("Error getting psql version: %s" % (line, )) return client_version = parts[2] # assert client_version.count('.') == 2, line if client_version.count('.') != 2: log.info("Error getting pg version: %s" % (client_version, )) return cvs = map(int, client_version.split('.'))[:2] if svs != cvs: warning(_(u"Problem with PostgreSQL version"), _(u"The version of the PostgreSQL database server (%s) and the " "postgres client tools (%s) differ. I will let you use " "Stoq, but you will always see this warning when " "starting Stoq until you resolve the version " "incompatibilty by upgrading the server or the client " "tools.") % (server_version, client_version)) else: raise NotImplementedError(self.rdbms)
def test_stoqlib_domain(self): args = ["pylint", "--rcfile=%s/tools/pylint.rcfile" % (self.root,), "--load-plugins", "tools/pylint_stoq", "-E", "stoqlib.domain"] p = Process(args) retval = p.wait() if retval: raise Exception("Pylint errors")
def pylint(self, modules, args=None): if not args: args = [] args = ["pylint", "--dummy-variables=unused,_", "--disable=%s" % (",".join(DISABLED)), "--include-ids=y", "--rcfile=%s/tools/pylint.rcfile" % (self.root,), "--reports=n"] + args + modules p = Process(args) retval = p.wait() if retval: raise Exception("Pylint errors")
def save(self): temp_csv = tempfile.NamedTemporaryFile(suffix='.csv', delete=False) writer = csv.writer(temp_csv, delimiter=',', doublequote=True, quoting=csv.QUOTE_ALL) writer.writerows(self.rows) temp_csv.close() template_file = sysparam.get_string('LABEL_TEMPLATE_PATH') if not os.path.exists(template_file): raise ValueError(_('Template file for printing labels was not found.')) args = ['-f', str(self.skip + 1), '-o', self.filename, '-i', temp_csv.name, template_file] # FIXME: This is just a quick workaround. There must be a better way to # do this. # glables3 changed the script name. If the default (glables2) is not # available, try the one from glables3 try: p = Process(['glabels-batch'] + args) except OSError: p = Process(['glabels-3-batch'] + args) # FIXME: We should use while so the print dialog can be canceled (see # threadutils) p.wait()
def _create_dbuser(self, username): import os from stoqlib.lib.process import Process, PIPE for envname in ['PGUSER', 'PGHOST']: if envname in os.environ: del os.environ[envname] # See if we can connect to the database args = ['psql', 'postgres', username, '-c', 'SELECT 1;'] proc = Process(args, stdout=PIPE) proc.communicate() if proc.returncode == 0: return 0 from stoqlib.lib.kiwilibrary import library createdbuser = library.get_resource_filename( 'stoq', 'scripts', 'createdbuser.sh') args = ['pkexec', '-u', 'postgres', createdbuser, username] proc = Process(args) proc.communicate() if proc.returncode != 0: print("ERROR: Failed to run %r" % (args, )) return 30 return 0
def start_shell(self, command=None, quiet=False): """Runs a database shell :param command: tell psql to execute the command string :param quiet: sets psql quiet option (``-q``) """ if self.rdbms == 'postgres': args = ['psql'] if command: args.extend(['-c', command]) if quiet: args.append('-q') args.extend(self.get_tool_args()) args.append(self.dbname) print('Connecting to %s' % ( self.get_store_dsn(filter_password=True), )) proc = Process(args) proc.wait() else: raise NotImplementedError(self.rdbms)
def backup(backup_dir, full=False, retry=1): # Tell Stoq Link Admin that you're starting a backup user_hash = api.sysparam.get_string('USER_HASH') start_url = urllib.parse.urljoin(WebService.API_SERVER, 'api/backup/start') response = requests.get(start_url, params={'hash': user_hash}) # If the server rejects the backup, don't even attempt to proceed. Log # which error caused the backup to fail if response.status_code != 200: raise Exception('ERROR: ' + response.content) cmd = [ _duplicati_exe, 'backup', _webservice_url, backup_dir, '--log-id=' + response.content ] + _get_extra_args() p = Process(cmd) threadit(_watch_fd, p.stdout) threadit(_watch_fd, p.stderr) p.wait() if p.returncode == 100 and retry > 0: # If the password has changed, duplicati will refuse to do the # backup, even tough we support that on our backend. Force remove # the cache so it will work duplicati_config = os.path.join(os.getenv('APPDATA'), 'Duplicati') shutil.rmtree(duplicati_config, ignore_errors=True) return backup(backup_dir, full=full, retry=retry - 1) if p.returncode != 0: raise Exception("Failed to backup the database: {}".format( p.returncode)) # Tell Stoq Link Admin that the backup has finished end_url = urllib.parse.urljoin(WebService.API_SERVER, 'api/backup/end') requests.get(end_url, params={ 'log_id': response.content, 'hash': user_hash })
def start_shell(self, command=None, quiet=False): """Runs a database shell :param command: tell psql to execute the command string :param quiet: sets psql quiet option (``-q``) """ if self.rdbms == 'postgres': args = ['psql'] if command: args.extend(['-c', command]) if quiet: args.append('-q') args.extend(self.get_tool_args()) args.append(self.dbname) print('Connecting to %s' % ( self.get_store_uri(filter_password=True), )) proc = Process(args) proc.wait() else: raise NotImplementedError(self.rdbms)
def test_connection(self): """Test for database connectivity using command line tools :returns: `True` if the database connection succeeded. """ log.info("Testing database connectivity using command line tools") if self.rdbms == 'postgres': # -w avoids password prompts, which causes this to hang. args = [ 'psql', '-n', '-q', '-w', '--variable', 'ON_ERROR_STOP=', '-c', 'SELECT 1;' ] args.extend(self.get_tool_args()) args.append(self.dbname) log.debug('executing %s' % (' '.join(args), )) proc = Process(args, stdin=PIPE, stdout=PIPE) retval = proc.wait() return retval == 0 else: raise NotImplementedError(self.rdbms)
def test_connection(self): """Test for database connectivity using command line tools :returns: `True` if the database connection succeeded. """ log.info("Testing database connectivity using command line tools") if self.rdbms == 'postgres': # -w avoids password prompts, which causes this to hang. args = ['psql', '-n', '-q', '-w', '--variable', 'ON_ERROR_STOP=', '-c', 'SELECT 1;'] args.extend(self.get_tool_args()) args.append(self.dbname) log.debug('executing %s' % (' '.join(args), )) proc = Process(args, stdin=PIPE, stdout=PIPE) retval = proc.wait() return retval == 0 else: raise NotImplementedError(self.rdbms)
def execute_command(self, args): self.feed('Executing: %s\r\n' % (' '.join(args))) kwargs = {} # On Windows you have to passin stdout/stdin = PIPE or # it will result in an invalid handle, see # * CR2012071248 # * http://bugs.python.org/issue3905 if self.listen_stdout or platform.system() == 'Windows': kwargs['stdout'] = PIPE if self.listen_stderr or platform.system() == 'Windows': kwargs['stderr'] = PIPE self.proc = Process(args, **kwargs) if self.listen_stdout: threadit(self._watch_fd, self.proc.stdout) if self.listen_stderr: threadit(self._watch_fd, self.proc.stderr) # We could probably listen to SIGCHLD here instead GLib.timeout_add(CHILD_TIMEOUT, self._check_child_finished)
def _terminate(self, restart=False, app=None): log.info("Terminating Stoq") # This removes all temporary files created when calling # get_resource_filename() that extract files to the file system import pkg_resources pkg_resources.cleanup_resources() log.debug('Stopping deamon') from stoqlib.lib.daemonutils import stop_daemon stop_daemon() # Finally, go out of the reactor and show possible crash reports log.debug("Show some crash reports") self._show_crash_reports() log.debug("Stoq Gtk.main") from gi.repository import Gtk Gtk.main_quit() # Make sure that no connection is left open (specially on Windows) try: from stoqlib.database.runtime import get_default_store get_default_store().close() except Exception: pass if restart: from stoqlib.lib.process import Process log.info('Restarting Stoq') args = [sys.argv[0], '--no-splash-screen'] if app is not None: args.append(app) Process(args) # os._exit() forces a quit without running atexit handlers # and does not block on any running threads # FIXME: This is the wrong solution, we should figure out why there # are any running threads/processes at this point log.debug("Terminating by calling os._exit()") os._exit(0) raise AssertionError("Should never happen")
def start(self): try: self._get_port() except TryAgainError: pass else: return defer.succeed(self) stoq_daemon = find_program('stoq-daemon') if not stoq_daemon: raise AssertionError args = [sys.executable, stoq_daemon, '--daemon-id', self._daemon_id] args.extend(db_settings.get_command_line_arguments()) self._process = Process(args) reactor.callLater(0.1, self._check_active) self._defer = defer.Deferred() return self._defer
def _create_dbuser(self, username): import os from stoqlib.lib.process import Process, PIPE for envname in ['PGUSER', 'PGHOST']: if envname in os.environ: del os.environ[envname] # See if we can connect to the database args = ['psql', 'postgres', username, '-c', 'SELECT 1;'] proc = Process(args, stdout=PIPE) proc.communicate() if proc.returncode == 0: return 0 args = ['pkexec', '-u', 'postgres', 'stoqcreatedbuser', username] proc = Process(args) proc.communicate() if proc.returncode != 0: print("ERROR: Failed to run %r" % (args, )) return 30 return 0
def _create_dbuser(self, username): import os from stoqlib.lib.process import Process, PIPE for envname in ["PGUSER", "PGHOST"]: if envname in os.environ: del os.environ[envname] # See if we can connect to the database args = ["psql", "postgres", username, "-c", "SELECT 1;"] proc = Process(args, stdout=PIPE) proc.communicate() if proc.returncode == 0: return 0 args = ["pkexec", "-u", "postgres", "stoqcreatedbuser", username] proc = Process(args) proc.communicate() if proc.returncode != 0: print("ERROR: Failed to run %r" % (args,)) return 30 return 0
def _create_dbuser(self, username): import os from stoqlib.lib.process import Process, PIPE for envname in ['PGUSER', 'PGHOST']: if envname in os.environ: del os.environ[envname] # See if we can connect to the database args = ['psql', 'postgres', username, '-c', 'SELECT 1;'] proc = Process(args, stdout=PIPE) proc.communicate() if proc.returncode == 0: return 0 args = ['pkexec', '-u', 'postgres', 'stoqcreatedbuser', username] proc = Process(args) proc.communicate() if proc.returncode != 0: print "ERROR: Failed to run %r" % (args, ) return 30 return 0
def start_backup_scheduler(doing_backup): _setup_signal_termination() if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...") return logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [list(map(int, i.strip().split(':'))) for i in backup_schedule.split(',')] now = datetime.datetime.now() backup_dates = collections.deque(sorted( now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0) for bh in backup_hours)) while True: now = datetime.datetime.now() next_date = datetime.datetime.min while next_date < now: next_date = backup_dates.popleft() backup_dates.append(next_date + datetime.timedelta(1)) time.sleep(max(1, (next_date - now).total_seconds())) for i in range(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break doing_backup.value = 1 try: p = Process(args) stdout, stderr = p.communicate() finally: doing_backup.value = 0 if p.returncode == 0: break else: # When duplicity fails in unpredicted situations (e.g. the # power is shut down suddenly) it can leave a lockfile behind, # and that can make any future backup attempts fail to. # Check if that was the reason of the failure and, if the # lockfile is older than 3h remove it and try again. # Note that this only happens for duplicity (linux) and # not for duplicati (windows) match = re.search('/.*lockfile.lock', stderr) if match is not None: lockfile = match.group(0) now = datetime.datetime.now() mdate = datetime.datetime.fromtimestamp(os.path.getmtime(lockfile)) if (now - mdate) > _lock_remove_threshold: os.unlink(lockfile) logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2) ** (i + 1))
def execute_sql(self, filename, lock_database=False): """Inserts raw SQL commands into the database read from a file. :param filename: filename with SQL commands :param lock_database: If the existing tables in the database should be locked :returns: return code, ``0`` if succeeded, positive integer for failure """ log.info("Executing SQL script %s database locked=%s" % (filename, lock_database)) if self.rdbms == 'postgres': # Okay, this might look crazy, but it's actually the only way # to execute many SQL statements in PostgreSQL and # 1) Stop immediatelly when an error occur # 2) Print the error message, the filename and the line number where # the error occurred. # 3) Do not print anything on the output unless it's an warning or a # an error args = ['psql'] # -U needs to go in first or psql on windows get confused args.extend(self.get_tool_args()) args.extend(['-n', '-q']) kwargs = {} if _system == 'Windows': # Hide the console window # For some reason XP doesn't like interacting with # proceses via pipes read_from_pipe = False else: read_from_pipe = True # We have two different execution modes, # 1) open stdin (-) and write the data via a pipe, # this allows us to also disable noticies and info messages, # so that only warnings are printed, we also fail if a warning # or error is printed # 2) Pass in the file normally to psql, no error reporting included if read_from_pipe: args.extend(['-f', '-']) args.extend(['--variable', 'ON_ERROR_STOP=']) else: args.extend(['-f', filename]) args.append(self.dbname) log.debug('executing %s' % (' '.join(args), )) proc = Process(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs) proc.stdin.write('BEGIN TRANSACTION;') if lock_database: store = self.create_store() lock_query = store.get_lock_database_query() proc.stdin.write(lock_query) store.close() if read_from_pipe: # We don't want to see notices on the output, skip them, # this will make all reported line numbers offset by 1 proc.stdin.write("SET SESSION client_min_messages TO 'warning';") data = open(filename).read() # Rename serial into bigserial, for 64-bit id columns data = data.replace('id serial', 'id bigserial') data += '\nCOMMIT;' else: data = None stdout, stderr = proc.communicate(data) if read_from_pipe and stderr: raise SQLError(stderr[:-1]) return proc.returncode else: raise NotImplementedError(self.rdbms)
def start_backup_scheduler(doing_backup): _setup_signal_termination() if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...") return logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [ list(map(int, i.strip().split(':'))) for i in backup_schedule.split(',') ] now = datetime.datetime.now() backup_dates = collections.deque( sorted( now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0) for bh in backup_hours)) while True: now = datetime.datetime.now() next_date = datetime.datetime.min while next_date < now: next_date = backup_dates.popleft() backup_dates.append(next_date + datetime.timedelta(1)) time.sleep(max(1, (next_date - now).total_seconds())) for i in range(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break doing_backup.value = 1 try: p = Process(args) stdout, stderr = p.communicate() finally: doing_backup.value = 0 if p.returncode == 0: break else: # When duplicity fails in unpredicted situations (e.g. the # power is shut down suddenly) it can leave a lockfile behind, # and that can make any future backup attempts fail to. # Check if that was the reason of the failure and, if the # lockfile is older than 3h remove it and try again. # Note that this only happens for duplicity (linux) and # not for duplicati (windows) match = re.search('/.*lockfile.lock', stderr) if match is not None: lockfile = match.group(0) now = datetime.datetime.now() mdate = datetime.datetime.fromtimestamp( os.path.getmtime(lockfile)) if (now - mdate) > _lock_remove_threshold: os.unlink(lockfile) logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2)**(i + 1))