def create_app(server_config): """Create Flask app with defined resource endpoints.""" app = Flask("api-server") CORS(app, resources={r"/api/*": {"origins": "*"}}) app.logger = get_pbench_logger(__name__, server_config) app.config["DEBUG"] = False app.config["TESTING"] = False api = Api(app) register_endpoints(api, app, server_config) try: Database.init_db(server_config=server_config, logger=app.logger) except Exception: app.logger.exception( "Exception while initializing sqlalchemy database") sys.exit(1) @app.teardown_appcontext def shutdown_session(exception=None): Database.db_session.remove() return app
def db_session(server_config): """ Construct a temporary DB session for the test case that will reset on completion. NOTE: the client fixture does something similar, but without the implicit cleanup, and with the addition of a Flask context that non-API tests don't require. Args: server_config: pbench-server.cfg fixture """ Database.init_db(server_config, None) yield Database.db_session.remove()
def run_migrations_offline(): """Run migrations in 'offline' mode. This configures the context with just a URL and not an Engine, though an Engine is acceptable here as well. By skipping the Engine creation we don't even need a DBAPI to be available. Calls to context.execute() here emit the given string to the script output. """ try: server_config = get_server_config() logger = get_pbench_logger(__name__, server_config) except Exception as e: print(e) sys.exit(1) url = Database.get_engine_uri(server_config, logger) if url is None: sys.exit(1) context.configure( url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, ) with context.begin_transaction(): context.run_migrations()
def main(options): if not options.cfg_name: print( f"{_NAME_}: ERROR: No config file specified; set" " _PBENCH_SERVER_CONFIG env variable", file=sys.stderr, ) return 1 try: config = PbenchServerConfig(options.cfg_name) except BadConfig as e: print(f"{_NAME_}: {e}", file=sys.stderr) return 2 logger = get_pbench_logger(_NAME_, config) # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(config, logger) # NOTE: the importer will ignore datasets that already exist in the DB; # we do the ACTION links first to get set up, and then sweep other links # to record the state of remaining datasets, especially to record those # which are quarantined. # # FIXME: This doesn't sweep the "<root>/quarantine" directory, which might # have additional datasets. Are they worth importing? actions = { "TO-UNPACK": States.UPLOADED, "TO-INDEX": States.UNPACKED, "INDEXED": States.INDEXED, "UNPACKED": States.UNPACKED, "WONT-UNPACK": States.QUARANTINED, "WONT-INDEX*": States.QUARANTINED, "BAD-MD5": States.QUARANTINED, } importer = Import(logger, options, config) return_value = 0 for link, state in actions.items(): status = importer.process(link, state) if status != 0: return_value = 1 return return_value
def main(): try: server_config = get_server_config() except (ConfigFileNotSpecified, BadConfig) as e: print(e) sys.exit(1) logger = get_pbench_logger(__name__, server_config) try: host = str(server_config.get("pbench-server", "bind_host")) port = str(server_config.get("pbench-server", "bind_port")) db = str(server_config.get("Postgres", "db_uri")) workers = str(server_config.get("pbench-server", "workers")) worker_timeout = str( server_config.get("pbench-server", "worker_timeout")) # Multiple gunicorn workers will attempt to connect to the DB; rather # than attempt to synchronize them, detect a missing DB (from the # postgres URI) and create it here. It's safer to do this here, # where we're single-threaded. if not database_exists(db): logger.info("Postgres DB {} doesn't exist", db) create_database(db) logger.info("Created DB {}", db) Database.init_db(server_config, logger) except (NoOptionError, NoSectionError): logger.exception(f"{__name__}: ERROR") sys.exit(1) subprocess.run([ "gunicorn", "--workers", workers, "--timeout", worker_timeout, "--pid", "/run/pbench-server/gunicorn.pid", "--bind", f"{host}:{port}", "pbench.cli.server.shell:app()", ])
def execute(self): config = PbenchServerConfig(self.context.config) logger = get_pbench_logger(_NAME_, config) # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(config, logger) user = User( username=self.context.username, password=self.context.password, first_name=self.context.first_name, last_name=self.context.last_name, email=self.context.email, role=self.context.role if self.context.role else "", ) user.add() if user.is_admin(): click.echo(f"Admin user {self.context.username} registered") else: click.echo(f"User {self.context.username} registered")
def run_migrations_online(): """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context. """ try: server_config = get_server_config() logger = get_pbench_logger(__name__, server_config) except Exception as e: print(e) sys.exit(1) connectable = create_engine(Database.get_engine_uri(server_config, logger)) with connectable.connect() as connection: context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): context.run_migrations()
def main(options): try: if not options.cfg_name: print( f"{_NAME_}: ERROR: No config file specified; set" " _PBENCH_SERVER_CONFIG env variable", file=sys.stderr, ) return 1 try: config = PbenchServerConfig(options.cfg_name) except BadConfig as e: print(f"{_NAME_}: {e}", file=sys.stderr) return 2 logger = get_pbench_logger(_NAME_, config) # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(config, logger) args = {} if options.create: user = options.create try: user = Auth.validate_user(user) except UnknownUser: # FIXME: I don't want to be creating the user here or # dealing with a non-existing user. The unittest setup # should create the test users we want ahead of time # using a pbench-user-manager command and we should # depend on having them here! The following is a hack # until that command exists. # # The desired behavior would be to remove this try and # except and allow UnknownUser to be handled below with # an error message and termination. User( username=user, first_name=user.capitalize(), last_name="Account", password=f"{user}_password", email=f"{user}@example.com", ).add() args["owner"] = user if options.controller: args["controller"] = options.controller if options.path: args["path"] = options.path if options.name: args["name"] = options.name if options.md5: args["md5"] = options.md5 if options.state: try: new_state = States[options.state.upper()] except KeyError: print( f"{_NAME_}: Specified string '{options.state}' is not a Pbench dataset state", file=sys.stderr, ) return 1 args["state"] = new_state if "path" not in args and ("controller" not in args or "name" not in args): print( f"{_NAME_}: Either --path or both --controller and --name must be specified", file=sys.stderr, ) return 1 # Either create a new dataset or attach to an existing dataset doit = Dataset.create if options.create else Dataset.attach # Find or create the specified dataset. doit(**args) except Exception as e: # Stringify any exception and report it; then fail logger.exception("Failed") print(f"{_NAME_}: {e}", file=sys.stderr) return 1 else: return 0
def main(cfg_name): if not cfg_name: print( f"{_NAME_}: ERROR: No config file specified; set" " _PBENCH_SERVER_CONFIG env variable or use --config <file> on the" " command line", file=sys.stderr, ) return 2 try: config = PbenchServerConfig(cfg_name) except BadConfig as e: print(f"{_NAME_}: {e} (config file {cfg_name})", file=sys.stderr) return 1 logger = get_pbench_logger(_NAME_, config) # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(config, logger) qdir, receive_dir = fetch_config_val(config, logger) if qdir is None and receive_dir is None: return 2 qdir_md5 = qdirs_check("quarantine", Path(qdir, "md5-002"), logger) duplicates = qdirs_check("duplicates", Path(qdir, "duplicates-002"), logger) # The following directory holds tarballs that are quarantined because # of operational errors on the server. They should be retried after # the problem is fixed: basically, move them back into the reception # area for 002 agents and wait. errors = qdirs_check("errors", Path(qdir, "errors-002"), logger) if qdir_md5 is None or duplicates is None or errors is None: return 1 counts = process_tb(config, logger, receive_dir, qdir_md5, duplicates, errors) result_string = (f"{config.TS}: Processed {counts.ntotal} entries," f" {counts.ntbs} tarballs successful," f" {counts.nquarantined} quarantined tarballs," f" {counts.ndups} duplicately-named tarballs," f" {counts.nerrs} errors.") logger.info(result_string) # prepare and send report with tempfile.NamedTemporaryFile(mode="w+t", dir=config.TMP) as reportfp: reportfp.write(f"{counts.nstatus}{result_string}\n") reportfp.seek(0) report = Report(config, _NAME_) report.init_report_template() try: report.post_status(config.timestamp(), "status", reportfp.name) except Exception as exc: logger.warning("Report post Unsuccesful: '{}'", exc) return 0
def main(cfg_name): if not cfg_name: print( f"{_NAME_}: ERROR: No config file specified; set" " _PBENCH_SERVER_CONFIG env variable or use --config <file> on the" " command line", file=sys.stderr, ) return 2 try: config = PbenchServerConfig(cfg_name) except BadConfig as e: print(f"{_NAME_}: {e}", file=sys.stderr) return 1 logger = get_pbench_logger(_NAME_, config) # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(config, logger) # Add a BACKUP and QDIR field to the config object config.BACKUP = config.conf.get("pbench-server", "pbench-backup-dir") config.QDIR = config.get("pbench-server", "pbench-quarantine-dir") # call the LocalBackupObject class lb_obj = LocalBackupObject(config) # call the S3Config class s3_obj = S3Config(config, logger) lb_obj, s3_obj = sanity_check(lb_obj, s3_obj, config, logger) if lb_obj is None and s3_obj is None: return 3 logger.info("start-{}", config.TS) # Initiate the backup counts = backup_data(lb_obj, s3_obj, config, logger) result_string = (f"Total processed: {counts.ntotal}," f" Local backup successes: {counts.nbackup_success}," f" Local backup failures: {counts.nbackup_fail}," f" S3 upload successes: {counts.ns3_success}," f" S3 upload failures: {counts.ns3_fail}," f" Quarantined: {counts.nquaran}") logger.info(result_string) prog = Path(sys.argv[0]).name # prepare and send report with tempfile.NamedTemporaryFile(mode="w+t", dir=config.TMP) as reportfp: reportfp.write( f"{prog}.{config.timestamp()}({config.PBENCH_ENV})\n{result_string}\n" ) reportfp.seek(0) report = Report(config, _NAME_) report.init_report_template() try: report.post_status(config.timestamp(), "status", reportfp.name) except Exception: pass logger.info("end-{}", config.TS) return 0
def main(options): if not options.cfg_name: print( f"{_NAME_}: ERROR: No config file specified; set" " _PBENCH_SERVER_CONFIG env variable", file=sys.stderr, ) return 1 try: config = PbenchServerConfig(options.cfg_name) except BadConfig as e: print(f"{_NAME_}: {e}", file=sys.stderr) return 2 logger = get_pbench_logger(_NAME_, config) # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(config, logger) archive_p = config.ARCHIVE try: incoming_p = config.INCOMING.resolve(strict=True) except FileNotFoundError: print( f"The configured INCOMING directory, {config.INCOMING}, does not exist", file=sys.stderr, ) return 5 else: if not incoming_p.is_dir(): print( f"The configured INCOMING directory, {config.INCOMING}, is not a valid directory", file=sys.stderr, ) return 6 _fmt = "%Y-%m-%d" try: oldest_dt = datetime.strptime(options.oldest, _fmt) newest_dt = datetime.strptime(options.newest, _fmt) except Exception as exc: print( f"Invalid time range, {options.oldest} to {options.newest}, " f"'{exc}', expected time range values in the form YYYY-MM-DD", file=sys.stderr, ) return 7 else: if newest_dt < oldest_dt: # For convenience, swap oldest and newest dates that are reversed. oldest_dt, newest_dt = newest_dt, oldest_dt print(f"Re-indexing tar balls in the range {oldest_dt} to {newest_dt}") actions = [] start = pbench.server._time() for _val in gen_reindex_list(archive_p, oldest_dt, newest_dt): controller_name, tb_name = _val act_set = reindex( controller_name, tb_name, archive_p, incoming_p, options.dry_run ) actions.append(act_set) end = pbench.server._time() for act_set in sorted(actions): print(f"{act_set!r}") print(f"Run-time: {start} {end} {end - start}") return 0
def main(options, name): """Main entry point to pbench-index. The caller is required to pass the "options" argument with the following expected attributes: cfg_name - Name of the configuration file to use dump_index_patterns - Don't do any indexing, but just emit the list of index patterns that would be used dump_templates - Dump the templates that would be used index_tool_data - Index tool data only re_index - Consider tar balls marked for re-indexing All exceptions are caught and logged to syslog with the stacktrace of the exception in a sub-object of the logged JSON document. Signal Handlers used to establish different patterns for the three behaviors: 1. Gracefully stop processing tar balls - SIGQUIT - The current tar ball is indexed until completion, but no other tar balls are processed. - Handler Behavior: - Sets a flag that causes the code flow to break out of the for loop. - Does not raise an exception. 2. Interrupt the current tar ball being indexed, and proceed to the next one, if any - SIGINT - Handler Behavior: - try/except/finally placed immediately around the es_index() call so that the signal handler will only be established for the duration of the call. - Raises an exception caught by above try/except/finally. - The finally clause would take down the signal handler. 3. Stop processing tar balls immediately and exit gracefully - SIGTERM - Handler Behavior: - Raises an exception caught be a new, outer-most, try/except block that does not have a finally clause (as you don't want any code execution in the finally block). 4. Re-evaluate the list of tar balls to index after indexing of the current tar ball has finished - SIGHUP - Report the current state of indexing - Current tar ball being processed - Count of Remaining tarballs - Count of Completed tarballs - No. of Errors encountered - Handler Behavior: - No exception raised """ _name_suf = "-tool-data" if options.index_tool_data else "" _name_re = "-re" if options.re_index else "" name = f"{name}{_name_re}{_name_suf}" error_code = Index.error_code if not options.cfg_name: print( f"{name}: ERROR: No config file specified; set" " _PBENCH_SERVER_CONFIG env variable or" " use --config <file> on the command line", file=sys.stderr, ) return error_code["CFG_ERROR"].value idxctx = None try: idxctx = IdxContext(options, name, _dbg=_DEBUG) except (ConfigFileError, ConfigParserError) as e: print(f"{name}: {e}", file=sys.stderr) return error_code["CFG_ERROR"].value except BadConfig as e: print(f"{name}: {e}", file=sys.stderr) return error_code["BAD_CFG"].value except JsonFileError as e: print(f"{name}: {e}", file=sys.stderr) return error_code["MAPPING_ERROR"].value if options.dump_index_patterns: idxctx.templates.dump_idx_patterns() return 0 if options.dump_templates: idxctx.templates.dump_templates() return 0 res = error_code["OK"] ARCHIVE_rp = idxctx.config.ARCHIVE INCOMING_rp = idxctx.config.INCOMING INCOMING_path = idxctx.config.get_valid_dir_option("INCOMING", INCOMING_rp, idxctx.logger) if not INCOMING_path: res = error_code["BAD_CFG"] qdir = idxctx.config.get_conf("QUARANTINE", "pbench-server", "pbench-quarantine-dir", idxctx.logger) if not qdir: res = error_code["BAD_CFG"] else: qdir_path = idxctx.config.get_valid_dir_option("QDIR", Path(qdir), idxctx.logger) if not qdir_path: res = error_code["BAD_CFG"] if not res.success: # Exit early if we encounter any errors. return res.value # We're going to need the Postgres DB to track dataset state, so setup # DB access. Database.init_db(idxctx.config, idxctx.logger) idxctx.logger.debug("{}.{}: starting", name, idxctx.TS) index_obj = Index(name, options, idxctx, INCOMING_rp, ARCHIVE_rp, qdir) status, tarballs = index_obj.collect_tb() if status == 0 and tarballs: status = index_obj.process_tb(tarballs) return status