def _get_or_create_run(build): """Gets a run for a build or creates it if it does not exist.""" release_name, release_number = _get_release_params() """ Spaces in the run name were replaced with underscores, put them back """ run_name = request.form.get('run_name', type=str).replace("_", " ") utils.jsonify_assert(run_name, 'run_name required') release = (models.Release.query.filter_by(build_id=build.id, name=release_name, number=release_number).first()) utils.jsonify_assert(release, 'release does not exist') run = (models.Run.query.filter_by(release_id=release.id, name=run_name).first()) if not run: # Ignore re-reports of the same run name for this release. logging.info( 'Created run: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r', build.id, release.name, release.number, run_name) run = models.Run(release_id=release.id, name=run_name, status=models.Run.DATA_PENDING) db.session.add(run) db.session.flush() return release, run
def create_release(): """Creates a new release candidate for a build.""" build = g.build release_name = request.form.get('release_name') utils.jsonify_assert(release_name, 'release_name required') url = request.form.get('url') utils.jsonify_assert(release_name, 'url required') release = models.Release( name=release_name, url=url, number=1, build_id=build.id) last_candidate = ( models.Release.query .filter_by(build_id=build.id, name=release_name) .order_by(models.Release.number.desc()) .first()) if last_candidate: release.number += last_candidate.number db.session.add(release) db.session.commit() logging.info('Created release: build_id=%r, release_name=%r, url=%r, ' 'release_number=%d', build.id, release.name, url, release.number) return flask.jsonify( success=True, build_id=build.id, release_name=release.name, release_number=release.number, url=url)
def _get_or_create_run(build): """Gets a run for a build or creates it if it does not exist.""" release_name, release_number = _get_release_params() run_name = request.form.get('run_name', type=str) utils.jsonify_assert(run_name, 'run_name required') release = ( models.Release.query .filter_by(build_id=build.id, name=release_name, number=release_number) .first()) utils.jsonify_assert(release, 'release does not exist') run = ( models.Run.query .filter_by(release_id=release.id, name=run_name) .first()) if not run: # Ignore re-reports of the same run name for this release. logging.info('Created run: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r', build.id, release.name, release.number, run_name) run = models.Run( release_id=release.id, name=run_name, status=models.Run.DATA_PENDING) db.session.add(run) db.session.flush() return release, run
def can_api_key_access_build(param_name): """Determines if the current API key can access the build in the request. Args: param_name: Parameter name to use for getting the build ID from the request. Will fetch from GET or POST requests. Returns: (api_key, build) The API Key and the Build it has access to. """ build_id = ( request.args.get(param_name, type=int) or request.form.get(param_name, type=int) or request.json[param_name]) utils.jsonify_assert(build_id, 'build_id required') if app.config.get('IGNORE_AUTH'): api_key = models.ApiKey( id='anonymous_superuser', secret='', superuser=True) build = models.Build.query.get(build_id) utils.jsonify_assert(build is not None, 'build must exist', 404) else: ops = _get_api_key_ops() api_key, build = ops.can_access_build(build_id) return api_key, build
def create_build(): passed_key = request.form.get('G5_DPXDT_API_KEY', default=None, type=str) if passed_key != os.environ['G5_DPXDT_API_KEY']: return flask.jsonify(error="invalid or missing API key") build_name = request.form.get('name', type=str) utils.jsonify_assert(build_name, 'supply a build name') #check for another build by this name build = db.session.query(models.Build).filter(models.Build.name == build_name).first() if build: return flask.jsonify( success=False, existing_build_id=build.id, message="A build by that name already exists" ) _create_build(build_name) return flask.jsonify( success=True, build_id=build.id, )
def runs_done(): """Marks a release candidate as having all runs reported.""" build = g.build release_name, release_number = _get_release_params() release = (models.Release.query.filter_by( build_id=build.id, name=release_name, number=release_number).with_lockmode('update').first()) utils.jsonify_assert(release, 'Release does not exist') release.status = models.Release.PROCESSING db.session.add(release) _check_release_done_processing(release) db.session.commit() signals.release_updated_via_api.send(app, build=build, release=release) logging.info( 'Runs done for release: build_id=%r, release_name=%r, ' 'release_number=%d', build.id, release.name, release.number) results_url = url_for('view_release', id=build.id, name=release.name, number=release.number, _external=True) return flask.jsonify(success=True, results_url=results_url)
def _get_release_params(): """Gets the release params from the current request.""" release_name = request.form.get('release_name') utils.jsonify_assert(release_name, 'release_name required') release_number = request.form.get('release_number', type=int) utils.jsonify_assert(release_number is not None, 'release_number required') return release_name, release_number
def runs_done(): """Marks a release candidate as having all runs reported.""" build = g.build release_name, release_number = _get_release_params() release = ( models.Release.query .filter_by(build_id=build.id, name=release_name, number=release_number) .first()) utils.jsonify_assert(release, 'Release does not exist') release.status = models.Release.PROCESSING db.session.add(release) _check_release_done_processing(release) db.session.commit() logging.info('Runs done for release: build_id=%r, release_name=%r, ' 'release_number=%d', build.id, release.name, release.number) results_url = url_for( 'view_release', id=build.id, name=release.name, number=release.number, _external=True) return flask.jsonify( success=True, results_url=results_url)
def _find_last_good_run(build): """Finds the last good release and run for a build.""" """ Spaces in the run name were replaced with underscores, put them back """ run_name = request.form.get('run_name', type=str).replace("_", " ") utils.jsonify_assert(run_name, 'run_name required') last_good_release = (models.Release.query.filter_by( build_id=build.id, status=models.Release.GOOD).order_by( models.Release.created.desc()).first()) last_good_run = None if last_good_release: logging.debug( 'Found last good release for: build_id=%r, ' 'release_name=%r, release_number=%d', build.id, last_good_release.name, last_good_release.number) last_good_run = (models.Run.query.filter_by( release_id=last_good_release.id, name=run_name).first()) if last_good_run: logging.debug( 'Found last good run for: build_id=%r, ' 'release_name=%r, release_number=%d, ' 'run_name=%r', build.id, last_good_release.name, last_good_release.number, last_good_run.name) return last_good_release, last_good_run
def can_api_key_access_build(param_name): """Determines if the current API key can access the build in the request. Args: param_name: Parameter name to use for getting the build ID from the request. Will fetch from GET or POST requests. Returns: (api_key, build) The API Key and the Build it has access to. """ build_id = (request.args.get(param_name, type=int) or request.form.get(param_name, type=int) or request.json[param_name]) utils.jsonify_assert(build_id, 'build_id required') if app.config.get('IGNORE_AUTH'): api_key = models.ApiKey(id='anonymous_superuser', secret='', superuser=True) build = models.Build.query.get(build_id) utils.jsonify_assert(build is not None, 'build must exist', 404) else: ops = _get_api_key_ops() api_key, build = ops.can_access_build(build_id) return api_key, build
def runs_done(): """Marks a release candidate as having all runs reported.""" build = g.build release_name, release_number = _get_release_params() release = ( models.Release.query.filter_by(build_id=build.id, name=release_name, number=release_number) .with_lockmode("update") .first() ) utils.jsonify_assert(release, "Release does not exist") release.status = models.Release.PROCESSING db.session.add(release) _check_release_done_processing(release) db.session.commit() signals.release_updated_via_api.send(app, build=build, release=release) logging.info( "Runs done for release: build_id=%r, release_name=%r, " "release_number=%d", build.id, release.name, release.number, ) results_url = url_for("view_release", id=build.id, name=release.name, number=release.number, _external=True) return flask.jsonify(success=True, results_url=results_url)
def create_release(): """Creates a new release candidate for a build.""" build = g.build release_name = request.form.get("release_name") utils.jsonify_assert(release_name, "release_name required") url = request.form.get("url") utils.jsonify_assert(release_name, "url required") release = models.Release(name=release_name, url=url, number=1, build_id=build.id) last_candidate = ( models.Release.query.filter_by(build_id=build.id, name=release_name) .order_by(models.Release.number.desc()) .first() ) if last_candidate: release.number += last_candidate.number db.session.add(release) db.session.commit() signals.release_updated_via_api.send(app, build=build, release=release) logging.info( "Created release: build_id=%r, release_name=%r, url=%r, " "release_number=%d", build.id, release.name, url, release.number, ) return flask.jsonify( success=True, build_id=build.id, release_name=release.name, release_number=release.number, url=url )
def _find_last_good_run(build): """Finds the last good release and run for a build.""" run_name = request.form.get('run_name', type=str) utils.jsonify_assert(run_name, 'run_name required') last_good_release = ( models.Release.query .filter_by( build_id=build.id, status=models.Release.GOOD) .order_by(models.Release.created.desc()) .first()) last_good_run = None if last_good_release: logging.debug('Found last good release for: build_id=%r, ' 'release_name=%r, release_number=%d', build.id, last_good_release.name, last_good_release.number) last_good_run = ( models.Run.query .filter_by(release_id=last_good_release.id, name=run_name) .first()) if last_good_run: logging.debug('Found last good run for: build_id=%r, ' 'release_name=%r, release_number=%d, ' 'run_name=%r', build.id, last_good_release.name, last_good_release.number, last_good_run.name) return last_good_release, last_good_run
def _get_release_params(): """Gets the release params from the current request.""" """ Spaces in the release name were replaced with underscores, put them back """ release_name = request.form.get('release_name').replace("_", " ") utils.jsonify_assert(release_name, 'release_name required') release_number = request.form.get('release_number', type=int) utils.jsonify_assert(release_number is not None, 'release_number required') return release_name, release_number
def create_release(): """Creates a new release candidate for a build.""" build = g.build release_name = request.form.get('release_name') utils.jsonify_assert(release_name, 'release_name required') url = request.form.get('url') utils.jsonify_assert(release_name, 'url required') release = models.Release(name=release_name, url=url, number=1, build_id=build.id) last_candidate = (models.Release.query.filter_by( build_id=build.id, name=release_name).order_by(models.Release.number.desc()).first()) if last_candidate: release.number += last_candidate.number if last_candidate.status == models.Release.PROCESSING: canceled_task_count = work_queue.cancel( release_id=last_candidate.id) logging.info( 'Canceling %d tasks for previous attempt ' 'build_id=%r, release_name=%r, release_number=%d', canceled_task_count, build.id, last_candidate.name, last_candidate.number) last_candidate.status = models.Release.BAD db.session.add(last_candidate) db.session.add(release) releases = models.Release.query.all() if len(releases) > 10: now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') for data in releases: createtime = datetime.datetime.strptime( str(data.created)[:-7], '%Y-%m-%d %H:%M:%S') nowtime = datetime.datetime.strptime(now, '%Y-%m-%d %H:%M:%S') if ( nowtime - createtime ).days + 1 > 5 and data.status != 'good' and data.build_id == build.id: db.session.delete(data) runs = models.Run.query.all() for runs_data in runs: if runs_data.release_id is None or runs_data.release_id == data.id: db.session.delete(runs_data) db.session.commit() signals.release_updated_via_api.send(app, build=build, release=release) logging.info( 'Created release: build_id=%r, release_name=%r, url=%r, ' 'release_number=%d', build.id, release.name, url, release.number) return flask.jsonify(success=True, build_id=build.id, release_name=release.name, release_number=release.number, url=url)
def wrapped(*args, **kwargs): api_key = current_api_key() g.api_key = api_key utils.jsonify_assert(api_key.superuser, 'API key=%r must be a super user' % api_key.id, 403) return f(*args, **kwargs)
def wrapped(*args, **kwargs): api_key = current_api_key() g.api_key = api_key utils.jsonify_assert( api_key.superuser, 'API key=%r must be a super user' % api_key.id, 403) return f(*args, **kwargs)
def can_access_build(self, build_id): api_key = self.get() build = models.Build.query.get(build_id) utils.jsonify_assert(build is not None, 'build must exist', 404) if not api_key.superuser: utils.jsonify_assert(api_key.build_id == build_id, 'API key must have access', 404) return api_key, build
def upload(): """Uploads an artifact referenced by a run.""" build = g.build utils.jsonify_assert(len(request.files) == 1, "Need exactly one uploaded file") file_storage = request.files.values()[0] data = file_storage.read() content_type, _ = mimetypes.guess_type(file_storage.filename) artifact = _save_artifact(build, data, content_type) db.session.add(artifact) db.session.commit() return flask.jsonify(success=True, build_id=build.id, sha1sum=artifact.id, content_type=content_type)
def create_release(): """Creates a new release candidate for a build.""" build = g.build release_name = request.form.get('release_name') utils.jsonify_assert(release_name, 'release_name required') url = request.form.get('url') utils.jsonify_assert(release_name, 'url required') release = models.Release( name=release_name, url=url, number=1, build_id=build.id) last_candidate = ( models.Release.query .filter_by(build_id=build.id, name=release_name) .order_by(models.Release.number.desc()) .first()) if last_candidate: release.number += last_candidate.number if last_candidate.status == models.Release.PROCESSING: canceled_task_count = work_queue.cancel( release_id=last_candidate.id) logging.info('Canceling %d tasks for previous attempt ' 'build_id=%r, release_name=%r, release_number=%d', canceled_task_count, build.id, last_candidate.name, last_candidate.number) last_candidate.status = models.Release.BAD db.session.add(last_candidate) db.session.add(release) db.session.commit() signals.release_updated_via_api.send(app, build=build, release=release) logging.info('Created release: build_id=%r, release_name=%r, url=%r, ' 'release_number=%d', build.id, release.name, url, release.number) return flask.jsonify( success=True, build_id=build.id, release_name=release.name, release_number=release.number, url=url)
def create_release(): """Creates a new release candidate for a build.""" build = g.build release_name = request.form.get('release_name').replace("_", " ") utils.jsonify_assert(release_name, 'release_name required') url = request.form.get('url') utils.jsonify_assert(release_name, 'url required') release = models.Release(name=release_name, url=url, number=1, build_id=build.id) last_candidate = (models.Release.query.filter_by( build_id=build.id, name=release_name).order_by(models.Release.number.desc()).first()) if last_candidate: release.number += last_candidate.number if last_candidate.status == models.Release.PROCESSING: canceled_task_count = work_queue.cancel( release_id=last_candidate.id) logging.info( 'Canceling %d tasks for previous attempt ' 'build_id=%r, release_name=%r, release_number=%d', canceled_task_count, build.id, last_candidate.name, last_candidate.number) last_candidate.status = models.Release.BAD db.session.add(last_candidate) db.session.add(release) db.session.commit() signals.release_updated_via_api.send(app, build=build, release=release) logging.info( 'Created release: build_id=%r, release_name=%r, url=%r, ' 'release_number=%d', build.id, release.name, url, release.number) return flask.jsonify(success=True, build_id=build.id, release_name=release.name, release_number=release.number, url=url)
def upload(): """Uploads an artifact referenced by a run.""" build = g.build utils.jsonify_assert( len(request.files) == 1, 'Need exactly one uploaded file') file_storage = request.files.values()[0] data = file_storage.read() content_type, _ = mimetypes.guess_type(file_storage.filename) artifact = _save_artifact(build, data, content_type) db.session.add(artifact) db.session.commit() return flask.jsonify(success=True, build_id=build.id, sha1sum=artifact.id, content_type=content_type)
def get(self): api_key = models.ApiKey.query.get(self.client_id) utils.jsonify_assert(api_key, 'API key must exist', 403) utils.jsonify_assert(api_key.active, 'API key must be active', 403) utils.jsonify_assert(api_key.secret == self.client_secret, 'Must have good credentials', 403) return api_key
def current_api_key(): """Determines the API key for the current request. Returns: The API key. """ if app.config.get('IGNORE_AUTH'): return models.ApiKey( id='anonymous_superuser', secret='', superuser=True) auth_header = request.authorization if not auth_header: logging.debug('API request lacks authorization header') abort(flask.Response( 'API key required', 401, {'WWW-Authenticate': 'Basic realm="API key required"'})) api_key = models.ApiKey.query.get(auth_header.username) utils.jsonify_assert(api_key, 'API key must exist', 403) utils.jsonify_assert(api_key.active, 'API key must be active', 403) utils.jsonify_assert(api_key.secret == auth_header.password, 'Must have good credentials', 403) logging.debug('Authenticated as API key=%r', api_key.id) return api_key
def current_api_key(): """Determines the API key for the current request. Returns: The API key. """ if app.config.get('IGNORE_AUTH'): return models.ApiKey(id='anonymous_superuser', secret='', superuser=True) auth_header = request.authorization if not auth_header: logging.debug('API request lacks authorization header') abort( flask.Response( 'API key required', 401, {'WWW-Authenticate': 'Basic realm="API key required"'})) api_key = models.ApiKey.query.get(auth_header.username) utils.jsonify_assert(api_key, 'API key must exist', 403) utils.jsonify_assert(api_key.active, 'API key must be active', 403) utils.jsonify_assert(api_key.secret == auth_header.password, 'Must have good credentials', 403) logging.debug('Authenticated as API key=%r', api_key.id) return api_key
def request_run(): """Requests a new run for a release candidate.""" build = g.build current_release, current_run = _get_or_create_run(build) current_url = request.form.get('url', type=str) config_data = request.form.get('config', default='{}', type=str) utils.jsonify_assert(current_url, 'url to capture required') utils.jsonify_assert(config_data, 'config document required') config_artifact = _enqueue_capture(build, current_release, current_run, current_url, config_data) ref_url = request.form.get('ref_url', type=str) ref_config_data = request.form.get('ref_config', type=str) utils.jsonify_assert( bool(ref_url) == bool(ref_config_data), 'ref_url and ref_config must both be specified or not specified') if ref_url and ref_config_data: ref_config_artifact = _enqueue_capture(build, current_release, current_run, ref_url, ref_config_data, baseline=True) else: _, last_good_run = _find_last_good_run(build) if last_good_run: current_run.ref_url = last_good_run.url current_run.ref_image = last_good_run.image current_run.ref_log = last_good_run.log current_run.ref_config = last_good_run.config db.session.add(current_run) db.session.commit() signals.run_updated_via_api.send(app, build=build, release=current_release, run=current_run) return flask.jsonify(success=True, build_id=build.id, release_name=current_release.name, release_number=current_release.number, run_name=current_run.name, url=current_run.url, config=current_run.config, ref_url=current_run.ref_url, ref_config=current_run.ref_config)
def request_run(): """Requests a new run for a release candidate.""" build = g.build current_release, current_run = _get_or_create_run(build) current_url = request.form.get('url', type=str) config_data = request.form.get('config', default='{}', type=str) utils.jsonify_assert(current_url, 'url to capture required') utils.jsonify_assert(config_data, 'config document required') config_artifact = _enqueue_capture( build, current_release, current_run, current_url, config_data) ref_url = request.form.get('ref_url', type=str) ref_config_data = request.form.get('ref_config', type=str) utils.jsonify_assert( bool(ref_url) == bool(ref_config_data), 'ref_url and ref_config must both be specified or not specified') if ref_url and ref_config_data: ref_config_artifact = _enqueue_capture( build, current_release, current_run, ref_url, ref_config_data, baseline=True) else: _, last_good_run = _find_last_good_run(build) if last_good_run: current_run.ref_url = last_good_run.url current_run.ref_image = last_good_run.image current_run.ref_log = last_good_run.log current_run.ref_config = last_good_run.config db.session.add(current_run) db.session.commit() signals.run_updated_via_api.send( app, build=build, release=current_release, run=current_run) return flask.jsonify( success=True, build_id=build.id, release_name=current_release.name, release_number=current_release.number, run_name=current_run.name, url=current_run.url, config=current_run.config, ref_url=current_run.ref_url, ref_config=current_run.ref_config)
def can_api_key_access_build(param_name): """Determines if the current API key can access the build in the request. Args: param_name: Parameter name to use for getting the build ID from the request. Will fetch from GET or POST requests. Returns: (api_key, build) The API Key and the Build it has access to. """ api_key = current_api_key() build_id = (request.args.get(param_name, type=int) or request.form.get(param_name, type=int)) utils.jsonify_assert(build_id, 'build_id required') build = models.Build.query.get(build_id) utils.jsonify_assert(build is not None, 'build must exist', 404) if not api_key.superuser: utils.jsonify_assert(api_key.build_id == build_id, 'API key must have access', 404) return api_key, build
def can_api_key_access_build(param_name): """Determines if the current API key can access the build in the request. Args: param_name: Parameter name to use for getting the build ID from the request. Will fetch from GET or POST requests. Returns: (api_key, build) The API Key and the Build it has access to. """ api_key = current_api_key() build_id = ( request.args.get(param_name, type=int) or request.form.get(param_name, type=int)) utils.jsonify_assert(build_id, 'build_id required') build = models.Build.query.get(build_id) utils.jsonify_assert(build is not None, 'build must exist', 404) if not api_key.superuser: utils.jsonify_assert(api_key.build_id == build_id, 'API key must have access', 404) return api_key, build
def release_and_run(): passed_key = request.form.get('G5_DPXDT_API_KEY', default=None, type=str) if passed_key != os.environ['G5_DPXDT_API_KEY']: return flask.jsonify(error="invalid or missing API key") build = request.form.get('build', default=None, type=int) url = request.form.get('url', default=None, type=str) name = request.form.get('name', default=None, type=str) depth = request.form.get('depth', default=1, type=int) #name supercedes build if name: msg = "Build determined via name: %s. " % name bd = models.Build.query.filter_by(name=name).first() if not bd: bd = _create_build(name) msg += " Build did not exist, created it. " # return flask.jsonify(error="build by that name does not exist.") build = bd.id else: msg = "Build id taken from passed arg: %s. " % build #however we determined the build, make sure we have a url if not url: rel = models.Release.query.filter_by(build_id=build)\ .order_by(models.Release.created.desc()).first() if not rel: return flask.jsonify(error="no url provided and no previous releases to extrapolate from.") url = rel.url msg += "url determined via last release in the build: %s. " % url else: msg += "url taken from passed arg: %s. " % url utils.jsonify_assert(build, 'must supply a build or a name') utils.jsonify_assert(url, 'must supply a url or a name') FLAGS.crawl_depth = depth pull_inject_code() coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() sd = SiteDiff( start_url=url, ignore_prefixes=None, upload_build_id=build, upload_release_name=url, heartbeat=workers.PrintWorkflow) sd.root = True coordinator.input_queue.put(sd) msg += "Job(s) started." return flask.jsonify( success=True, msg=msg, )