def _get_network_info(): # We do not have this infos on database, so until we have them we get from config config = get_config() return { 'id': config.get('B1Food', 'network_id') or '', 'name': config.get('B1Food', 'network_name') or '', }
def xmlrpc_dbadmin(self, cmd_args, stdin=None): if sys.argv[0].endswith('.egg'): args = [sys.executable, sys.argv[0]] else: args = ['stoq'] args.append('dbadmin') args.extend(cmd_args) config = get_config() for setting_opt, setting in [('-H', db_settings.address), ('-d', db_settings.dbname), ('-f', config.filename), ('-p', db_settings.port and str(db_settings.port)), ('-u', db_settings.username), ('-w', db_settings.password)]: if setting: args.extend([setting_opt, setting]) p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p.communicate(input=stdin)
def set_backup_key(self, key): config = get_config() config.set('Backup', 'key', key) config.flush() # Restart stoqserver so the backup key will take effect immediately self.restart() return "Backup key set successfully"
def restore_database(user_hash, time=None): assert user_hash tmp_path = tempfile.mkdtemp() try: restore_path = os.path.join(tmp_path, 'stoq') config = get_config() dbname = config.get('Database', 'dbname') backup.restore(restore_path, user_hash, time=time) # Drop the database subprocess.check_call( ['dropdb'] + _get_pg_args(config) + [dbname]) # Create the database subprocess.check_call( ['createdb'] + _get_pg_args(config) + [dbname]) # Restore the backup subprocess.check_call( ['psql', '-d', dbname] + _get_pg_args(config) + ['-f', os.path.join(restore_path, 'stoq.dump')]) logging.info("Backup restore finished sucessfully") finally: shutil.rmtree(tmp_path, ignore_errors=True)
def _get_proxy(self): if self._proxy is None: config = get_config() address = config.get('General', 'serveraddress') if not address: with api.new_store() as store: query = ("SELECT client_addr FROM pg_stat_activity " "WHERE application_name LIKE ? AND " " datname = ? " "LIMIT 1") params = [u'stoqserver%', unicode(db_settings.dbname)] res = store.execute(query, params=params).get_one() address = res and res[0] if not address: raise ServerError(_("Stoq server not found")) port = config.get('General', 'serverport') or 6970 url = 'http://%s:%s/XMLRPC' % (address, port) self._proxy = Proxy(url) try: yield self._check_proxy(self._proxy) except Exception: self._proxy = None raise api.asyncReturn(self._proxy)
def cmd_exec_action(self, options, *args): """Run an action on an already running server instance""" setup_logging() cmd = args[0] cmd_args = args[1:] config = get_config() port = (options.server_port or config.get('General', 'serverport') or SERVER_XMLRPC_PORT) address = (options.server_address or config.get('General', 'serveraddress') or '127.0.0.1') remote = xmlrpc.client.ServerProxy('http://%s:%s/XMLRPC' % (address, port), allow_none=True) # Backup commands can take a while to execute. Wait at least 10 minutes # before timing out so we can give a better feedback to the user if cmd.startswith('backup'): socket.setdefaulttimeout(60 * 10) print("Executing '%s' on server. This might take a while..." % (cmd, )) try: print(getattr(remote, cmd)(*cmd_args)) except socket.timeout: print("Connection timed out. The action may still be executing...") return 1 except xmlrpc.client.Fault as e: print("Server fault (%s): %s" % (e.faultCode, e.faultString)) return 1 except Exception as e: print("Could not send action to server: %s" % (str(e), )) return 1
def restore_database(user_hash, time=None): assert user_hash tmp_path = tempfile.mkdtemp() try: restore_path = os.path.join(tmp_path, 'stoq') config = get_config() dbname = config.get('Database', 'dbname') backup.restore(restore_path, user_hash, time=time) # Drop the database subprocess.check_call(['dropdb'] + _get_pg_args(config) + [dbname]) # Create the database subprocess.check_call(['createdb'] + _get_pg_args(config) + [dbname]) # Restore the backup subprocess.check_call( ['psql', '-d', dbname] + _get_pg_args(config) + ['-f', os.path.join(restore_path, 'stoq.dump')]) logging.info("Backup restore finished sucessfully") finally: shutil.rmtree(tmp_path, ignore_errors=True)
def start_htsql(port): config = get_config() if config.get('General', 'disable_htsql'): logger.info("Not starting htsql as requested in config file.") return logger.info("Starting htsql server") if db_settings.password: password = '******' + urllib.parse.quote_plus(db_settings.password) else: password = '' uri = 'pgsql://{}{}@{}:{}/{}'.format( db_settings.username, password, db_settings.address, db_settings.port, db_settings.dbname) config = library.get_resource_filename('stoqserver', 'htsql', 'config.yml') popen = Process(['htsql-ctl', 'server', '-C', config, uri, '--host', '127.0.0.1', '--port', port]) def _sigterm_handler(_signal, _stack_frame): popen.poll() if popen.returncode is None: popen.terminate() os._exit(0) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, _sigterm_handler) popen.wait()
def cmd_exec_action(self, options, *args): """Run an action on an already running server instance""" setup_logging() cmd = args[0] cmd_args = args[1:] config = get_config() port = (options.server_port or config.get('General', 'serverport') or SERVER_XMLRPC_PORT) address = (options.server_address or config.get('General', 'serveraddress') or '127.0.0.1') remote = xmlrpc.client.ServerProxy( 'http://%s:%s/XMLRPC' % (address, port), allow_none=True) # Backup commands can take a while to execute. Wait at least 10 minutes # before timing out so we can give a better feedback to the user if cmd.startswith('backup'): socket.setdefaulttimeout(60 * 10) print("Executing '%s' on server. This might take a while..." % (cmd, )) try: print(getattr(remote, cmd)(*cmd_args)) except socket.timeout: print("Connection timed out. The action may still be executing...") return 1 except xmlrpc.client.Fault as e: print("Server fault (%s): %s" % (e.faultCode, e.faultString)) return 1 except Exception as e: print("Could not send action to server: %s" % (str(e), )) return 1
def start_htsql(port): config = get_config() if config.get('General', 'disable_htsql'): logger.info("Not starting htsql as requested in config file.") return logger.info("Starting htsql server") if db_settings.password: password = '******' + urllib.parse.quote_plus(db_settings.password) else: password = '' uri = 'pgsql://{}{}@{}:{}/{}'.format(db_settings.username, password, db_settings.address, db_settings.port, db_settings.dbname) config = library.get_resource_filename('stoqserver', 'htsql', 'config.yml') popen = Process([ 'htsql-ctl', 'server', '-C', config, uri, '--host', '127.0.0.1', '--port', port ]) def _sigterm_handler(_signal, _stack_frame): popen.poll() if popen.returncode is None: popen.terminate() os._exit(0) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, _sigterm_handler) popen.wait()
def start_daemon_manager(): logging.info("Starting daemon manager") config = get_config() port = config.get('General', 'serverport') or SERVER_DAEMON_PORT dm = DaemonManager(port=port and int(port)) reactor.callWhenRunning(dm.start) reactor.addSystemEventTrigger('before', 'shutdown', dm.stop)
def start_xmlrpc_server(pipe_conn): _setup_signal_termination() logger.info("Starting the xmlrpc server") config = get_config() port = int(config.get('General', 'serverport') or SERVER_XMLRPC_PORT) run_xmlrpcserver(pipe_conn, port)
def start_rtc(): if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not starting rtc...") return config = get_config() if config.get('General', 'disable_rtc'): logger.info("Not starting rtc as requested in config file.") return logger.info("Starting webRTC") cwd = library.get_resource_filename('stoqserver', 'webrtc') retry = True extra_args = [] camera_urls = config.get('Camera', 'url') or None if camera_urls: extra_args.append('-c=' + ' '.join(set(camera_urls.split(' ')))) xmlrpc_host = config.get('General', 'serveraddress') or '127.0.0.1' extra_args.append('-h={}'.format(xmlrpc_host)) xmlrpc_port = config.get('General', 'serverport') or SERVER_XMLRPC_PORT extra_args.append('-p={}'.format(xmlrpc_port)) while retry: retry = False popen = Process( ['bash', 'start.sh'] + extra_args, cwd=cwd) def _sigterm_handler(_signal, _stack_frame): popen.poll() if popen.returncode is None: popen.terminate() os._exit(0) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, _sigterm_handler) popen.wait() if popen.returncode == 11: logger.warning("libstdc++ too old, not running webRTC client. " "A system upgrade may be required!") retry = False elif popen.returncode == 10: logger.warning("Something failed when trying to start webrtc. " "Retrying again in 10 minutes...") time.sleep(10 * 60) retry = True elif popen.returncode == 12: logger.warning("webrtc installation corrupted. Restarting it...") time.sleep(1) retry = True elif popen.returncode == 139: logger.warning("Segmentation fault caught on wrtc. Restarting...") time.sleep(1) retry = True
def start_rtc(): if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not starting rtc...") return config = get_config() if config.get('General', 'disable_rtc'): logger.info("Not starting rtc as requested in config file.") return logger.info("Starting webRTC") cwd = library.get_resource_filename('stoqserver', 'webrtc') retry = True extra_args = [] camera_urls = config.get('Camera', 'url') or None if camera_urls: extra_args.append('-c=' + ' '.join(set(camera_urls.split(' ')))) xmlrpc_host = config.get('General', 'serveraddress') or '127.0.0.1' extra_args.append('-h={}'.format(xmlrpc_host)) xmlrpc_port = config.get('General', 'serverport') or SERVER_XMLRPC_PORT extra_args.append('-p={}'.format(xmlrpc_port)) while retry: retry = False popen = Process(['bash', 'start.sh'] + extra_args, cwd=cwd) def _sigterm_handler(_signal, _stack_frame): popen.poll() if popen.returncode is None: popen.terminate() os._exit(0) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, _sigterm_handler) popen.wait() if popen.returncode == 11: logger.warning("libstdc++ too old, not running webRTC client. " "A system upgrade may be required!") retry = False elif popen.returncode == 10: logger.warning("Something failed when trying to start webrtc. " "Retrying again in 10 minutes...") time.sleep(10 * 60) retry = True elif popen.returncode == 12: logger.warning("webrtc installation corrupted. Restarting it...") time.sleep(1) retry = True elif popen.returncode == 139: logger.warning("Segmentation fault caught on wrtc. Restarting...") time.sleep(1) retry = True
def start_backup_scheduler(): logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [map(int, i.strip().split(':')) for i in backup_schedule.split(',')] def _backup_task(): for i in xrange(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break p = subprocess.Popen(args) stdout, stderr = p.communicate() if p.returncode == 0: break else: logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2) ** (i + 1)) for backup_hour in backup_hours: now = datetime.datetime.now() backup_date = now.replace(hour=backup_hour[0], minute=backup_hour[1], second=0, microsecond=0) if backup_date < now: backup_date = backup_date + datetime.timedelta(1) delta = backup_date - now backup_task = task.LoopingCall(_backup_task) # Schedule the task to start at the backup hour and repeat # every 24 hours reactor.callWhenRunning(reactor.callLater, delta.seconds, backup_task.start, 24 * 60 * 60) reactor.addSystemEventTrigger( 'before', 'shutdown', lambda: getattr(task, 'running', False) and task.stop())
def start_flask_server(): _setup_signal_termination() logger.info("Starting the flask server") config = get_config() # XXX: Is flaskport a good name for this? port = int(config.get('General', 'flaskport') or SERVER_FLASK_PORT) run_flaskserver(port)
def _setup_stoq(self): info = AppInfo() info.set('name', "stoqserver") info.set('version', stoqserver.version_str) info.set('ver', stoqserver.version_str) provide_utility(IAppInfo, info, replace=True) # FIXME: Maybe we should check_schema and load plugins here? setup(config=get_config(), options=None, register_station=False, check_schema=False, load_plugins=True)
def setup_stoq(): info = AppInfo() info.set('name', "stoqserver") info.set('version', stoqserver.version_str) info.set('ver', stoqserver.version_str) provide_utility(IAppInfo, info, replace=True) # FIXME: Maybe we should check_schema and load plugins here? setup(config=get_config(), options=None, register_station=False, check_schema=False, load_plugins=True)
def start_xmlrpc_server(pipe_conn): logger.info("Starting the xmlrpc server") config = get_config() port = int(config.get('General', 'serverport') or SERVER_XMLRPC_PORT) r = resource.Resource() r.putChild('XMLRPC', ServerXMLRPCResource(r, pipe_conn)) site = server.Site(r) reactor.callWhenRunning(reactor.listenTCP, port, site)
def wrapper(*args, **kwargs): # B1Food documentation says that it is args but we use the headers auth = request.headers.get('Authorization', '').split('Bearer ') if len(auth) != 2: auth = request.args.get('Authorization', '').split('Bearer ') config = get_config() access_token = config.get("B1Food", "access_token") or "" if len(auth) != 2 or auth[1] != access_token or access_token == "": abort(401) return f(*args, **kwargs)
def _get_proxy(self): if self._proxy is None: config = get_config() address = config.get('General', 'serveraddress') if not address: with api.new_store() as store: query = ("SELECT client_addr FROM pg_stat_activity " "WHERE application_name LIKE ? AND " " datname = ? " "LIMIT 1") params = [u'stoqserver%', unicode(db_settings.dbname)] res = store.execute(query, params=params).get_one() if res is not None and res[0] is not None: address = res[0] # For now we only support ipv4 if address == '::1': address = '127.0.0.1' # Is there a better way to detect ipv6? #if ':' in address: # address = '[{}]'.format(address) elif res is not None: # If the client_addr is NULL, then stoqserver is connected # using the unix socket, which means that he is in the same # ip as postgresql address = db_settings.address if not address: # We are also on unix socket, so use localhost address = 'localhost' else: address = None if not address: raise ServerError(_("Stoq server not found")) port = config.get('General', 'serverport') or 6970 url = 'http://%s:%s/XMLRPC' % (address, port) default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(self._timeout) self._proxy = xmlrpclib.ServerProxy(url, allow_none=True) socket.setdefaulttimeout(default_timeout) try: retval = self._proxy.ping() except (Exception, AttributeError): self._proxy = None raise if not retval: raise ServerError(_("Server not responding to pings")) return self._proxy
def start_flask_server(debug=False): # We need to delay importing from restfull so that the plugin infrastructure gets setup correcly from stoqserver.lib.restful import run_flaskserver _setup_signal_termination() logger.info("Starting the flask server") config = get_config() # XXX: Is flaskport a good name for this? port = int(config.get('General', 'flaskport') or SERVER_FLASK_PORT) run_flaskserver(port, debug)
def _get_proxy(self): if self._proxy is None: config = get_config() if not config: raise ServerError(_('Configuration not found')) address = config.get('General', 'serveraddress') if not address: query = ("SELECT client_addr FROM pg_stat_activity " "WHERE application_name LIKE ? AND " " datname = ? " "LIMIT 1") params = [u'stoqserver%', str(db_settings.dbname)] res = api.get_default_store().execute(query, params=params).get_one() if res: # When stoqserver is located in another machine if res[0] not in ['127.0.0.1', '::1', '', None]: address = res[0] else: # XXX: For now we only support ipv4 # XXX: If the client_addr is NULL, then stoqserver is # connected using the unix socket, which means that he # is in the same ip as the postgresql address = db_settings.address if not address: address = 'localhost' else: address = None if not address: raise ServerError(_("Stoq server not found")) port = config.get('General', 'serverport') or 6970 url = 'http://%s:%s/XMLRPC' % (address, port) default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(self._timeout) self._proxy = xmlrpc.client.ServerProxy(url, allow_none=True) socket.setdefaulttimeout(default_timeout) try: retval = self._proxy.ping() except (Exception, AttributeError): self._proxy = None raise if not retval: raise ServerError(_("Server not responding to pings")) return self._proxy
def start_flask_server(debug=False, multiclient=False): # We need to delay importing so that the plugin infrastructure gets setup correcly # XXX: is this still needed? from stoqserver.app import run_flaskserver _setup_signal_termination() logger.info("Starting the flask server") config = get_config() # XXX: Is flaskport a good name for this? port = int(config.get('General', 'flaskport') or SERVER_FLASK_PORT) run_flaskserver(port, debug, multiclient)
def _get_extra_args(user_hash=None): passphrase = get_config().get('Backup', 'key') if not passphrase: raise Exception("No backup key set on configuration file") if user_hash is None: user_hash = api.sysparam.get_string('USER_HASH') return [ '--db-hash=' + user_hash, '--pw-hash=' + hashlib.sha256(passphrase).hexdigest(), '--passphrase=' + passphrase, ]
def bootstrap_app(): app = Flask(__name__) # Indexing some session data by the USER_HASH will help to avoid maintaining # sessions between two different databases. This could lead to some errors in # the POS in which the user making the sale does not exist. app.config['SECRET_KEY'] = _get_user_hash() flask_api = Api(app) for cls in _BaseResource.__subclasses__(): flask_api.add_resource(cls, *cls.routes) if has_ntk: global ntk config = get_config() if config: config_dir = config.get_config_directory() tef_dir = os.path.join(config_dir, 'ntk') else: # Tests don't have a config set. Use the plugin path as tef_dir, since it also has the # library import stoqntk tef_dir = os.path.dirname(os.path.dirname(stoqntk.__file__)) ntk = Ntk() ntk.init(tef_dir) if has_nfe: NfeProgressEvent.connect(_nfe_progress_event) NfeWarning.connect(_nfe_warning_event) NfeSuccess.connect(_nfe_success_event) NfeYesNoQuestion.connect(_nfe_yes_no_question_event) @app.errorhandler(Exception) def unhandled_exception(e): traceback_info = "\n".join(traceback.format_tb(e.__traceback__)) traceback_hash = hashlib.sha1(traceback_info.encode('utf-8')).hexdigest()[:8] traceback_exception = traceback.format_exception_only(type(e), e)[-1] timestamp = localnow().strftime('%Y%m%d-%H%M%S') log.exception('Unhandled Exception: {timestamp} {error} {traceback_hash}'.format( timestamp=timestamp, error=e, traceback_hash=traceback_hash)) main.sentry_report(type(e), e, e.__traceback__, traceback_hash=traceback_hash) return Response(json.dumps({'error': _('bad request!'), 'timestamp': timestamp, 'exception': traceback_exception, 'traceback_hash': traceback_hash}), 500, mimetype='application/json') return app
def backup(backup_dir, full=False): global _user_hash _user_hash = api.sysparam.get_string('USER_HASH') with _mock_environ(): config = get_config() os.environ.setdefault('PASSPHRASE', config.get('Backup', 'key')) sys.argv.append(_duplicity_bin) if full: sys.argv.append('full') sys.argv.extend([backup_dir, _webservice_url]) _duplicity_main.main()
def backup_database(full=False): config = get_config() if not os.path.exists(APP_BACKUP_DIR): os.makedirs(APP_BACKUP_DIR) filename = os.path.join(APP_BACKUP_DIR, 'stoq.dump') subprocess.check_call(['pg_dump', '-Fp', '-f', filename] + _get_pg_args(config) + [config.get('Database', 'dbname')]) backup.backup(APP_BACKUP_DIR, full=full) logging.info("Database backup finished sucessfully")
def setup_stoq(register_station=False, name='stoqserver', version=stoqserver.version_str): info = AppInfo() info.set('name', name) info.set('version', version) info.set('ver', version) provide_utility(IAppInfo, info, replace=True) setup(config=get_config(), options=None, register_station=register_station, check_schema=True, load_plugins=True) # This is needed for api calls that requires the current branch set, # e.g. Sale.confirm main_company = api.sysparam.get_object( api.get_default_store(), 'MAIN_COMPANY') provide_utility(ICurrentBranch, main_company, replace=True)
def backup_database(full=False): config = get_config() if not os.path.exists(APP_BACKUP_DIR): os.makedirs(APP_BACKUP_DIR) filename = os.path.join(APP_BACKUP_DIR, 'stoq.dump') subprocess.check_call( ['pg_dump', '-Fp', '-f', filename] + _get_pg_args(config) + [config.get('Database', 'dbname')]) backup.backup(APP_BACKUP_DIR, full=full) logging.info("Database backup finished sucessfully")
def setup_stoq(register_station=False, name='stoqserver', version=stoqserver.version_str): info = AppInfo() info.set('name', name) info.set('version', version) info.set('ver', version) provide_utility(IAppInfo, info, replace=True) # FIXME: Maybe we should check_schema and load plugins here? setup(config=get_config(), options=None, register_station=register_station, check_schema=False, load_plugins=True) # This is needed for api calls that requires the current branch set, # e.g. Sale.confirm main_company = api.sysparam.get_object( api.get_default_store(), 'MAIN_COMPANY') provide_utility(ICurrentBranch, main_company, replace=True)
def _get_proxy(self): if self._proxy is None: config = get_config() address = config.get('General', 'serveraddress') if not address: with api.new_store() as store: query = ("SELECT client_addr FROM pg_stat_activity " "WHERE application_name LIKE ? AND " " datname = ? " "LIMIT 1") params = [u'stoqserver%', unicode(db_settings.dbname)] res = store.execute(query, params=params).get_one() if res is not None and res[0] is not None: address = res[0] elif res is not None: # If the client_addr is NULL, then stoqserver is connected # using the unix socket, which means that he is in the same # ip as postgresql address = db_settings.address if not address: # We are also on unix socket, so use localhost address = 'localhost' else: address = None if not address: raise ServerError(_("Stoq server not found")) port = config.get('General', 'serverport') or 6970 url = 'http://%s:%s/XMLRPC' % (address, port) default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(self._timeout) self._proxy = xmlrpclib.ServerProxy(url, allow_none=True) socket.setdefaulttimeout(default_timeout) try: retval = self._proxy.ping() except (Exception, AttributeError): self._proxy = None raise if not retval: raise ServerError(_("Server not responding to pings")) return self._proxy
def get(self): data = request.args if 'client_id' not in data: abort(400, 'Missing client_id') client_id = data['client_id'] config = get_config() config_client_id = config.get("B1Food", "client_id") or "" access_token = config.get("B1Food", "access_token") or "" if client_id != config_client_id and config_client_id != "": log.error('Login failed for client_id %s', client_id) abort(403, 'Login failed for client_id {}'.format(client_id)) return make_response( jsonify({ 'token_type': 'Bearer', 'expires_in': -1, 'access_token': access_token }), 200)
def restore(restore_dir, user_hash, time=None): global _user_hash _user_hash = user_hash with _mock_environ(): config = get_config() backup_key = config.get('Backup', 'key') if not backup_key: raise ValueError("No backup key set on configuration file") os.environ.setdefault('PASSPHRASE', backup_key) # Close the main store so the database can be dropped after this api.get_default_store().rollback(close=True) sys.argv.extend( [_duplicity_bin, 'restore', _webservice_url, restore_dir]) if time is not None: sys.argv.extend(['--time', time]) _duplicity_main.main()
def restore(restore_dir, user_hash, time=None): global _user_hash _user_hash = user_hash with _mock_environ(): config = get_config() backup_key = config.get('Backup', 'key') if not backup_key: raise ValueError("No backup key set on configuration file") os.environ.setdefault('PASSPHRASE', backup_key) # Close the main store so the database can be dropped after this api.get_default_store().rollback(close=True) sys.argv.extend([_duplicity_bin, 'restore', _webservice_url, restore_dir]) if time is not None: sys.argv.extend(['--time', time]) _duplicity_main.main()
def _get_proxy(self): if self._proxy is None: config = get_config() address = config.get('General', 'serveraddress') if not address: with api.new_store() as store: query = ("SELECT client_addr FROM pg_stat_activity " "WHERE application_name LIKE ? AND " " datname = ? " "LIMIT 1") params = [u'stoqserver%', unicode(db_settings.dbname)] res = store.execute(query, params=params).get_one() if res is not None and res[0] is not None: address = res[0] elif res is not None: # If the client_addr is NULL, then stoqserver is connected # using the unix socket, which means that he is in the same # ip as postgresql address = db_settings.address if not address: # We are also on unix socket, so use localhost address = 'localhost' else: address = None if not address: raise ServerError(_("Stoq server not found")) port = config.get('General', 'serverport') or 6970 url = 'http://%s:%s/XMLRPC' % (address, port) self._proxy = Proxy(url) try: yield self._check_proxy(self._proxy) except Exception: self._proxy = None raise api.asyncReturn(self._proxy)
def _mock_environ(): old_argv = sys.argv[:] while sys.argv: sys.argv.pop() old_environ = os.environ.copy() def _restore_environ(): while sys.argv: sys.argv.pop() sys.argv.extend(old_argv) os.environ.clear() os.environ.update(old_environ) backup_key = get_config().get('Backup', 'key') if not backup_key: _restore_environ() raise Exception("No backup key set on configuration file") os.environ['PASSPHRASE'] = backup_key yield _restore_environ()
def xmlrpc_dbadmin(self, cmd_args, stdin=None): if sys.argv[0].endswith('.egg'): args = [sys.executable, sys.argv[0]] else: args = ['stoq'] args.append('dbadmin') args.extend(cmd_args) config = get_config() for setting_opt, setting in [ ('-H', db_settings.address), ('-d', db_settings.dbname), ('-f', config.filename), ('-p', db_settings.port and str(db_settings.port)), ('-u', db_settings.username), ('-w', db_settings.password)]: if setting: args.extend([setting_opt, setting]) p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p.communicate(input=stdin)
def start_backup_scheduler(): global _doing_backup _setup_signal_termination() if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...") return logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [map(int, i.strip().split(':')) for i in backup_schedule.split(',')] now = datetime.datetime.now() backup_dates = collections.deque(sorted( now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0) for bh in backup_hours)) while True: now = datetime.datetime.now() next_date = datetime.datetime.min while next_date < now: next_date = backup_dates.popleft() backup_dates.append(next_date + datetime.timedelta(1)) time.sleep(max(1, (next_date - now).total_seconds())) for i in xrange(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break _doing_backup.value = 1 try: p = subprocess.Popen(args) stdout, stderr = p.communicate() finally: _doing_backup.value = 0 if p.returncode == 0: break else: logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2) ** (i + 1))
def start_backup_scheduler(doing_backup): _setup_signal_termination() if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...") return logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [list(map(int, i.strip().split(':'))) for i in backup_schedule.split(',')] now = datetime.datetime.now() backup_dates = collections.deque(sorted( now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0) for bh in backup_hours)) while True: now = datetime.datetime.now() next_date = datetime.datetime.min while next_date < now: next_date = backup_dates.popleft() backup_dates.append(next_date + datetime.timedelta(1)) time.sleep(max(1, (next_date - now).total_seconds())) for i in range(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break doing_backup.value = 1 try: p = Process(args) stdout, stderr = p.communicate() finally: doing_backup.value = 0 if p.returncode == 0: break else: # When duplicity fails in unpredicted situations (e.g. the # power is shut down suddenly) it can leave a lockfile behind, # and that can make any future backup attempts fail to. # Check if that was the reason of the failure and, if the # lockfile is older than 3h remove it and try again. # Note that this only happens for duplicity (linux) and # not for duplicati (windows) match = re.search('/.*lockfile.lock', stderr) if match is not None: lockfile = match.group(0) now = datetime.datetime.now() mdate = datetime.datetime.fromtimestamp(os.path.getmtime(lockfile)) if (now - mdate) > _lock_remove_threshold: os.unlink(lockfile) logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2) ** (i + 1))
def get_backup_key(self): config = get_config() return config.get('Backup', 'key')
def start_backup_scheduler(): global _doing_backup _setup_signal_termination() if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...") return logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [ map(int, i.strip().split(':')) for i in backup_schedule.split(',') ] now = datetime.datetime.now() backup_dates = collections.deque( sorted( now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0) for bh in backup_hours)) while True: now = datetime.datetime.now() next_date = datetime.datetime.min while next_date < now: next_date = backup_dates.popleft() backup_dates.append(next_date + datetime.timedelta(1)) time.sleep(max(1, (next_date - now).total_seconds())) for i in xrange(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break _doing_backup.value = 1 try: p = subprocess.Popen(args) stdout, stderr = p.communicate() finally: _doing_backup.value = 0 if p.returncode == 0: break else: logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2)**(i + 1))
def start_backup_scheduler(doing_backup): _setup_signal_termination() if not api.sysparam.get_bool('ONLINE_SERVICES'): logger.info("ONLINE_SERVICES not enabled. Not scheduling backups...") return logger.info("Starting backup scheduler") config = get_config() backup_schedule = config.get('Backup', 'schedule') if backup_schedule is None: # By defualt, we will do 2 backups. One in a random time between # 9-11 or 14-17 and another one 12 hours after that. # We are using 2, 3 and 4 because they will be summed with 12 bellow hour = random.choice([2, 3, 4, 9, 10]) minute = random.randint(0, 59) backup_schedule = '%d:%d,%s:%d' % (hour, minute, hour + 12, minute) config.set('Backup', 'schedule', backup_schedule) config.flush() backup_hours = [ list(map(int, i.strip().split(':'))) for i in backup_schedule.split(',') ] now = datetime.datetime.now() backup_dates = collections.deque( sorted( now.replace(hour=bh[0], minute=bh[1], second=0, microsecond=0) for bh in backup_hours)) while True: now = datetime.datetime.now() next_date = datetime.datetime.min while next_date < now: next_date = backup_dates.popleft() backup_dates.append(next_date + datetime.timedelta(1)) time.sleep(max(1, (next_date - now).total_seconds())) for i in range(3): # FIXME: This is SO UGLY, we should be calling backup_database # task directly, but duplicity messes with multiprocessing in a # way that it will not work args = sys.argv[:] for i, arg in enumerate(args[:]): if arg == 'run': args[i] = 'backup_database' break doing_backup.value = 1 try: p = Process(args) stdout, stderr = p.communicate() finally: doing_backup.value = 0 if p.returncode == 0: break else: # When duplicity fails in unpredicted situations (e.g. the # power is shut down suddenly) it can leave a lockfile behind, # and that can make any future backup attempts fail to. # Check if that was the reason of the failure and, if the # lockfile is older than 3h remove it and try again. # Note that this only happens for duplicity (linux) and # not for duplicati (windows) match = re.search('/.*lockfile.lock', stderr) if match is not None: lockfile = match.group(0) now = datetime.datetime.now() mdate = datetime.datetime.fromtimestamp( os.path.getmtime(lockfile)) if (now - mdate) > _lock_remove_threshold: os.unlink(lockfile) logger.warning( "Failed to backup database:\nstdout: %s\nstderr: %s", stdout, stderr) # Retry again with a exponential backoff time.sleep((60 * 2)**(i + 1))
def get_redis_server(): config = get_config() assert config url = config.get('General', 'redis_server') or 'redis://localhost' return redis.from_url(url)
def __init__(self): config = get_config() self._port = int( config.get('General', 'serveravahiport') or SERVER_AVAHI_PORT)
def __init__(self): config = get_config() self._port = int(config.get('General', 'serveravahiport') or SERVER_AVAHI_PORT)
def _setup_stoq(self): # FIXME: Maybe we should check_schema and load plugins here? setup(config=get_config(), options=None, register_station=False, check_schema=False, load_plugins=True)