def _validate_input(values, user): topic_id = values.pop('topic_id') remoteci_id = values.get('remoteci_id') values.update({ 'id': utils.gen_uuid(), 'created_at': datetime.datetime.utcnow().isoformat(), 'updated_at': datetime.datetime.utcnow().isoformat(), 'etag': utils.gen_etag(), 'status': 'new' }) remoteci = v1_utils.verify_existence_and_get(remoteci_id, models.REMOTECIS) v1_utils.verify_existence_and_get(topic_id, models.TOPICS) # let's kill existing running jobs for the remoteci where_clause = sql.expression.and_( _TABLE.c.remoteci_id == remoteci_id, _TABLE.c.status.in_(('new', 'pre-run', 'running', 'post-run')) ) kill_query = _TABLE .update().where(where_clause).values(status='killed') flask.g.db_conn.execute(kill_query) if remoteci['state'] != 'active': message = 'RemoteCI "%s" is disabled.' % remoteci_id raise dci_exc.DCIException(message, status_code=412) # The user belongs to the topic then we can start the scheduling v1_utils.verify_team_in_topic(user, topic_id) return topic_id, remoteci
def _verify_team_in_topic(user, topic_id): topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True) # verify user's team in the topic v1_utils.verify_team_in_topic(user, topic_id) return topic_id
def get_all_tests(user, topic_id): args = schemas.args(flask.request.args.to_dict()) v1_utils.verify_team_in_topic(user, topic_id) v1_utils.verify_existence_and_get(topic_id, _TABLE) query = sql.select([models.TESTS]).\ select_from(models.JOIN_TOPICS_TESTS.join(models.TESTS)).\ where(models.JOIN_TOPICS_TESTS.c.topic_id == topic_id) T_COLUMNS = v1_utils.get_columns_name_with_objects(models.TESTS) sort_list = v1_utils.sort_query(args['sort'], T_COLUMNS) where_list = v1_utils.where_query(args['where'], models.TESTS, T_COLUMNS) query = v1_utils.add_sort_to_query(query, sort_list) query = v1_utils.add_where_to_query(query, where_list) if args.get('limit', None): query = query.limit(args.get('limit')) if args.get('offset', None): query = query.offset(args.get('offset')) rows = flask.g.db_conn.execute(query).fetchall() query_nb_rows = sql.select([func.count(models.TESTS.c.id)]). \ select_from(models.JOIN_TOPICS_TESTS.join(models.TESTS)). \ where(models.JOIN_TOPICS_TESTS.c.topic_id == topic_id) nb_rows = flask.g.db_conn.execute(query_nb_rows).scalar() res = flask.jsonify({'tests': rows, '_meta': {'count': nb_rows}}) res.status_code = 200 return res
def get_topic_by_id(user, topic_id): args = schemas.args(flask.request.args.to_dict()) topic = v1_utils.verify_existence_and_get(topic_id, _TABLE) v1_utils.verify_team_in_topic(user, topic_id) if not auth.is_admin(user): if 'teams' in args['embed']: raise dci_exc.DCIException('embed=teams not authorized.', status_code=401) return base.get_resource_by_id(user, topic, _TABLE, _EMBED_MANY)
def get_topic_by_id(user, topic_id): args = check_and_get_args(flask.request.args.to_dict()) topic = v1_utils.verify_existence_and_get(topic_id, _TABLE) if user.is_not_super_admin() and user.is_not_epm() and user.is_not_feeder( ): # noqa if not user.is_read_only_user(): v1_utils.verify_team_in_topic(user, topic_id) if 'teams' in args['embed']: raise dci_exc.Unauthorized() return base.get_resource_by_id(user, topic, _TABLE, _EMBED_MANY)
def download_component_file(user, c_id, f_id): swift = dci_config.get_store('components') component = v1_utils.verify_existence_and_get(c_id, _TABLE) v1_utils.verify_team_in_topic(user, component['topic_id']) v1_utils.verify_existence_and_get(f_id, models.COMPONENT_FILES) auth.check_export_control(user, component) file_path = swift.build_file_path(component['topic_id'], c_id, f_id) # Check if file exist on the storage engine swift.head(file_path) return flask.Response(swift.get_object(file_path))
def get_topic_by_id(user, topic_id): args = schemas.args(flask.request.args.to_dict()) topic = v1_utils.verify_existence_and_get(topic_id, _TABLE) if not user.is_super_admin() and not user.is_product_owner(): v1_utils.verify_team_in_topic(user, topic_id) if 'teams' in args['embed']: raise dci_exc.DCIException('embed=teams not authorized.', status_code=401) if not user.is_super_admin() and user.product_id != topic['product_id']: raise auth.UNAUTHORIZED return base.get_resource_by_id(user, topic, _TABLE, _EMBED_MANY)
def delete_test_from_topic(user, t_id, test_id): if not auth.is_admin(user): v1_utils.verify_team_in_topic(user, t_id) v1_utils.verify_existence_and_get(test_id, _TABLE) JDC = models.JOIN_REMOTECIS_TESTS where_clause = sql.and_(JDC.c.topic_id == t_id, JDC.c.test_id == test_id) query = JDC.delete().where(where_clause) result = flask.g.db_conn.execute(query) if not result.rowcount: raise dci_exc.DCIConflict('Test', test_id) return flask.Response(None, 204, content_type='application/json')
def list_component_file(user, c_id, f_id): component = v1_utils.verify_existence_and_get(c_id, _TABLE) auth.check_export_control(user, component) v1_utils.verify_team_in_topic(user, component['topic_id']) COMPONENT_FILES = models.COMPONENT_FILES where_clause = sql.and_(COMPONENT_FILES.c.id == f_id, COMPONENT_FILES.c.component_id == c_id) query = sql.select([COMPONENT_FILES]).where(where_clause) component_f = flask.g.db_conn.execute(query).fetchone() if component_f is None: raise dci_exc.DCINotFound('Component File', f_id) res = flask.jsonify({'component_file': component_f}) return res
def list_components_files(user, c_id): component = v1_utils.verify_existence_and_get(c_id, _TABLE) v1_utils.verify_team_in_topic(user, component['topic_id']) args = schemas.args(flask.request.args.to_dict()) query = v1_utils.QueryBuilder(models.COMPONENTFILES, args, _CF_COLUMNS) query.add_extra_condition(models.COMPONENTFILES.c.component_id == c_id) nb_rows = query.get_number_of_rows( models.COMPONENTFILES, models.COMPONENTFILES.c.component_id == c_id) # noqa rows = query.execute(fetchall=True) rows = v1_utils.format_result(rows, models.COMPONENTFILES.name, None, None) return flask.jsonify({ 'component_files': rows, '_meta': { 'count': nb_rows } })
def create_jobs(user): values = v1_utils.common_values_dict(user) values.update(schemas.job.post(flask.request.json)) components_ids = values.pop('components') values['team_id'] = values.get('team_id', user['team_id']) # Only super admin can create job for other teams if not user.is_super_admin() and not user.is_in_team(values['team_id']): raise auth.UNAUTHORIZED if values['topic_id'] is not None: v1_utils.verify_team_in_topic(user, values['topic_id']) values.update({ 'status': 'new', 'topic_id': values['topic_id'], 'rconfiguration_id': values['rconfiguration_id'], 'user_agent': flask.request.environ.get('HTTP_USER_AGENT'), 'client_version': flask.request.environ.get( 'HTTP_CLIENT_VERSION' ), }) # create the job and feed the jobs_components table with flask.g.db_conn.begin(): query = _TABLE.insert().values(**values) flask.g.db_conn.execute(query) jobs_components_to_insert = [] for cmpt_id in components_ids: v1_utils.verify_existence_and_get(cmpt_id, models.COMPONENTS) jobs_components_to_insert.append({'job_id': values['id'], 'component_id': cmpt_id}) if jobs_components_to_insert: flask.g.db_conn.execute(models.JOIN_JOBS_COMPONENTS.insert(), jobs_components_to_insert) return flask.Response(json.dumps({'job': values}), 201, headers={'ETag': values['etag']}, content_type='application/json')
def get_all_components(user, topic_id): """Get all components of a topic.""" args = schemas.args(flask.request.args.to_dict()) v1_utils.verify_team_in_topic(user, topic_id) query = v1_utils.QueryBuilder(_TABLE, args, _C_COLUMNS) query.add_extra_condition( sql.and_(_TABLE.c.topic_id == topic_id, _TABLE.c.state != 'archived')) nb_rows = query.get_number_of_rows() rows = query.execute(fetchall=True) rows = v1_utils.format_result(rows, _TABLE.name, args['embed'], _EMBED_MANY) # Return only the component which have the export_control flag set to true # if not (auth.is_admin(user)): rows = [row for row in rows if row['export_control']] return flask.jsonify({'components': rows, '_meta': {'count': nb_rows}})
def create_jobs(user): values = flask.request.json check_json_is_valid(create_job_schema, values) values.update(v1_utils.common_values_dict()) components_ids = values.pop('components') if user.is_not_remoteci(): raise dci_exc.DCIException('Only remoteci can create job') topic_id = values.get('topic_id') topic = v1_utils.verify_existence_and_get(topic_id, models.TOPICS) product_id = topic['product_id'] if topic_id: v1_utils.verify_team_in_topic(user, topic_id) previous_job_id = values.get('previous_job_id') if previous_job_id: v1_utils.verify_existence_and_get(previous_job_id, _TABLE) values.update({ 'status': 'new', 'remoteci_id': user.id, 'topic_id': topic_id, 'user_agent': flask.request.environ.get('HTTP_USER_AGENT'), 'client_version': flask.request.environ.get('HTTP_CLIENT_VERSION'), 'previous_job_id': previous_job_id, 'team_id': user.teams_ids[0], 'product_id': product_id, 'duration': 0 }) # create the job and feed the jobs_components table with flask.g.db_conn.begin(): query = _TABLE.insert().values(**values) flask.g.db_conn.execute(query) jobs_components_to_insert = [] for cmpt_id in components_ids: v1_utils.verify_existence_and_get(cmpt_id, models.COMPONENTS) jobs_components_to_insert.append({ 'job_id': values['id'], 'component_id': cmpt_id }) if jobs_components_to_insert: flask.g.db_conn.execute(models.JOIN_JOBS_COMPONENTS.insert(), jobs_components_to_insert) return flask.Response(json.dumps({'job': values}), 201, headers={'ETag': values['etag']}, content_type='application/json')
def schedule_jobs(user): """Dispatch jobs to remotecis. The remoteci can use this method to request a new job. Before a job is dispatched, the server will flag as 'killed' all the running jobs that were associated with the remoteci. This is because they will never be finished. """ values = flask.request.json check_json_is_valid(schedule_job_schema, values) values.update({ 'id': utils.gen_uuid(), 'created_at': get_utc_now().isoformat(), 'updated_at': get_utc_now().isoformat(), 'etag': utils.gen_etag(), 'status': 'new', 'remoteci_id': user.id, 'duration': 0, 'user_agent': flask.request.environ.get('HTTP_USER_AGENT'), 'client_version': flask.request.environ.get('HTTP_CLIENT_VERSION'), }) topic_id = values.pop('topic_id') dry_run = values.pop('dry_run') if dry_run: component_types = components.get_component_types_from_topic(topic_id) _components = components.get_last_components_by_type( component_types, topic_id) return flask.Response(json.dumps({ 'components': _components, 'job': None }), 201, content_type='application/json') # check remoteci remoteci = v1_utils.verify_existence_and_get(user.id, models.REMOTECIS) if remoteci['state'] != 'active': message = 'RemoteCI "%s" is disabled.' % remoteci['id'] raise dci_exc.DCIException(message, status_code=412) # check primary topic topic = v1_utils.verify_existence_and_get(topic_id, models.TOPICS) product_id = topic['product_id'] if topic['state'] != 'active': msg = 'Topic %s:%s not active.' % (topic_id, topic['name']) raise dci_exc.DCIException(msg, status_code=412) v1_utils.verify_team_in_topic(user, topic_id) # check secondary topic topic_id_secondary = values.pop('topic_id_secondary') if topic_id_secondary: topic_secondary = v1_utils.verify_existence_and_get( topic_id_secondary, models.TOPICS) if topic_secondary['state'] != 'active': msg = 'Topic %s:%s not active.' % (topic_id_secondary, topic['name']) raise dci_exc.DCIException(msg, status_code=412) v1_utils.verify_team_in_topic(user, topic_id_secondary) remotecis.kill_existing_jobs(remoteci['id']) components_ids = values.pop('components_ids') values = _build_job(product_id, topic_id, remoteci, components_ids, values, topic_id_secondary=topic_id_secondary) return flask.Response(json.dumps({'job': values}), 201, headers={'ETag': values['etag']}, content_type='application/json')
def get_jobs_status_from_components(user, topic_id, type_id): # List of job meaningfull job status for global overview # # ie. If current job status is running, we should retrieve status # from prior job. valid_status = ['failure', 'product-failure', 'deployment-failure', 'success'] topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True) v1_utils.verify_team_in_topic(user, topic_id) # Get list of all remotecis that are attached to a topic this type belongs # to fields = [models.REMOTECIS.c.id.label('remoteci_id'), models.REMOTECIS.c.name.label('remoteci_name'), models.TEAMS.c.id.label('team_id'), models.TEAMS.c.name.label('team_name'), models.TOPICS.c.name.label('topic_name'), models.COMPONENTS.c.id.label('component_id'), models.COMPONENTS.c.name.label('component_name'), models.COMPONENTS.c.type.label('component_type'), models.JOBS.c.id.label('job_id'), models.JOBS.c.status.label('job_status'), models.JOBS.c.created_at.label('job_created_at')] query = (sql.select(fields) .select_from( sql.join( models.REMOTECIS, models.JOBS, models.REMOTECIS.c.id == models.JOBS.c.remoteci_id, isouter=True) .join( models.JOIN_JOBS_COMPONENTS, models.JOIN_JOBS_COMPONENTS.c.job_id == models.JOBS.c.id) .join( models.COMPONENTS, models.COMPONENTS.c.id == models.JOIN_JOBS_COMPONENTS.c.component_id) # noqa .join( models.TOPICS, models.TOPICS.c.id == models.COMPONENTS.c.topic_id) .join( models.TEAMS, models.TEAMS.c.id == models.JOBS.c.team_id)) .where( sql.and_( models.REMOTECIS.c.state == 'active', models.TEAMS.c.external == True, # noqa models.JOBS.c.status.in_(valid_status), models.JOBS.c.state != 'archived', models.COMPONENTS.c.type == type_id, models.TOPICS.c.id == topic_id)) .order_by( models.REMOTECIS.c.id, models.JOBS.c.created_at.desc()) .distinct(models.REMOTECIS.c.id)) if not user.is_super_admin(): query.append_whereclause(models.TEAMS.c.id.in_(user.teams)) rcs = flask.g.db_conn.execute(query).fetchall() nb_row = len(rcs) return flask.jsonify({'jobs': rcs, '_meta': {'count': nb_row}})
def get_all_components(user, topic_id): topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True) v1_utils.verify_team_in_topic(user, topic_id) return components.get_all_components(user, topic_id=topic_id)
def get_all_jobdefinitions_by_topic(user, topic_id): topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True) v1_utils.verify_team_in_topic(user, topic_id) return jobdefinitions.list_jobdefinitions(user, [topic_id], by_topic=True)
def get_component_by_id(user, c_id): component = v1_utils.verify_existence_and_get(c_id, _TABLE) v1_utils.verify_team_in_topic(user, component['topic_id']) auth.check_export_control(user, component) return base.get_resource_by_id(user, component, _TABLE, _EMBED_MANY)