def _validate_input(values, user):
topic_id = values.pop('topic_id')
remoteci_id = values.get('remoteci_id')
values.update({
'id': utils.gen_uuid(),
'created_at': datetime.datetime.utcnow().isoformat(),
'updated_at': datetime.datetime.utcnow().isoformat(),
'etag': utils.gen_etag(),
'status': 'new'
})
remoteci = v1_utils.verify_existence_and_get(remoteci_id, models.REMOTECIS)
v1_utils.verify_existence_and_get(topic_id, models.TOPICS)
# let's kill existing running jobs for the remoteci
where_clause = sql.expression.and_(
_TABLE.c.remoteci_id == remoteci_id,
_TABLE.c.status.in_(('new', 'pre-run', 'running', 'post-run'))
)
kill_query = _TABLE .update().where(where_clause).values(status='killed')
flask.g.db_conn.execute(kill_query)
if remoteci['state'] != 'active':
message = 'RemoteCI "%s" is disabled.' % remoteci_id
raise dci_exc.DCIException(message, status_code=412)
# The user belongs to the topic then we can start the scheduling
v1_utils.verify_team_in_topic(user, topic_id)
return topic_id, remoteci
def _verify_team_in_topic(user, topic_id):
topic_id = v1_utils.verify_existence_and_get(topic_id,
_TABLE,
get_id=True)
# verify user's team in the topic
v1_utils.verify_team_in_topic(user, topic_id)
return topic_id
def get_all_tests(user, topic_id):
args = schemas.args(flask.request.args.to_dict())
v1_utils.verify_team_in_topic(user, topic_id)
v1_utils.verify_existence_and_get(topic_id, _TABLE)
query = sql.select([models.TESTS]).\
select_from(models.JOIN_TOPICS_TESTS.join(models.TESTS)).\
where(models.JOIN_TOPICS_TESTS.c.topic_id == topic_id)
T_COLUMNS = v1_utils.get_columns_name_with_objects(models.TESTS)
sort_list = v1_utils.sort_query(args['sort'], T_COLUMNS)
where_list = v1_utils.where_query(args['where'], models.TESTS, T_COLUMNS)
query = v1_utils.add_sort_to_query(query, sort_list)
query = v1_utils.add_where_to_query(query, where_list)
if args.get('limit', None):
query = query.limit(args.get('limit'))
if args.get('offset', None):
query = query.offset(args.get('offset'))
rows = flask.g.db_conn.execute(query).fetchall()
query_nb_rows = sql.select([func.count(models.TESTS.c.id)]). \
select_from(models.JOIN_TOPICS_TESTS.join(models.TESTS)). \
where(models.JOIN_TOPICS_TESTS.c.topic_id == topic_id)
nb_rows = flask.g.db_conn.execute(query_nb_rows).scalar()
res = flask.jsonify({'tests': rows,
'_meta': {'count': nb_rows}})
res.status_code = 200
return res
def get_topic_by_id(user, topic_id):
args = schemas.args(flask.request.args.to_dict())
topic = v1_utils.verify_existence_and_get(topic_id, _TABLE)
v1_utils.verify_team_in_topic(user, topic_id)
if not auth.is_admin(user):
if 'teams' in args['embed']:
raise dci_exc.DCIException('embed=teams not authorized.',
status_code=401)
return base.get_resource_by_id(user, topic, _TABLE, _EMBED_MANY)
def get_topic_by_id(user, topic_id):
args = check_and_get_args(flask.request.args.to_dict())
topic = v1_utils.verify_existence_and_get(topic_id, _TABLE)
if user.is_not_super_admin() and user.is_not_epm() and user.is_not_feeder(
): # noqa
if not user.is_read_only_user():
v1_utils.verify_team_in_topic(user, topic_id)
if 'teams' in args['embed']:
raise dci_exc.Unauthorized()
return base.get_resource_by_id(user, topic, _TABLE, _EMBED_MANY)
def download_component_file(user, c_id, f_id):
swift = dci_config.get_store('components')
component = v1_utils.verify_existence_and_get(c_id, _TABLE)
v1_utils.verify_team_in_topic(user, component['topic_id'])
v1_utils.verify_existence_and_get(f_id, models.COMPONENT_FILES)
auth.check_export_control(user, component)
file_path = swift.build_file_path(component['topic_id'], c_id, f_id)
# Check if file exist on the storage engine
swift.head(file_path)
return flask.Response(swift.get_object(file_path))
def get_topic_by_id(user, topic_id):
args = schemas.args(flask.request.args.to_dict())
topic = v1_utils.verify_existence_and_get(topic_id, _TABLE)
if not user.is_super_admin() and not user.is_product_owner():
v1_utils.verify_team_in_topic(user, topic_id)
if 'teams' in args['embed']:
raise dci_exc.DCIException('embed=teams not authorized.',
status_code=401)
if not user.is_super_admin() and user.product_id != topic['product_id']:
raise auth.UNAUTHORIZED
return base.get_resource_by_id(user, topic, _TABLE, _EMBED_MANY)
def delete_test_from_topic(user, t_id, test_id):
if not auth.is_admin(user):
v1_utils.verify_team_in_topic(user, t_id)
v1_utils.verify_existence_and_get(test_id, _TABLE)
JDC = models.JOIN_REMOTECIS_TESTS
where_clause = sql.and_(JDC.c.topic_id == t_id, JDC.c.test_id == test_id)
query = JDC.delete().where(where_clause)
result = flask.g.db_conn.execute(query)
if not result.rowcount:
raise dci_exc.DCIConflict('Test', test_id)
return flask.Response(None, 204, content_type='application/json')
def list_component_file(user, c_id, f_id):
component = v1_utils.verify_existence_and_get(c_id, _TABLE)
auth.check_export_control(user, component)
v1_utils.verify_team_in_topic(user, component['topic_id'])
COMPONENT_FILES = models.COMPONENT_FILES
where_clause = sql.and_(COMPONENT_FILES.c.id == f_id,
COMPONENT_FILES.c.component_id == c_id)
query = sql.select([COMPONENT_FILES]).where(where_clause)
component_f = flask.g.db_conn.execute(query).fetchone()
if component_f is None:
raise dci_exc.DCINotFound('Component File', f_id)
res = flask.jsonify({'component_file': component_f})
return res
def list_components_files(user, c_id):
component = v1_utils.verify_existence_and_get(c_id, _TABLE)
v1_utils.verify_team_in_topic(user, component['topic_id'])
args = schemas.args(flask.request.args.to_dict())
query = v1_utils.QueryBuilder(models.COMPONENTFILES, args, _CF_COLUMNS)
query.add_extra_condition(models.COMPONENTFILES.c.component_id == c_id)
nb_rows = query.get_number_of_rows(
models.COMPONENTFILES,
models.COMPONENTFILES.c.component_id == c_id) # noqa
rows = query.execute(fetchall=True)
rows = v1_utils.format_result(rows, models.COMPONENTFILES.name, None, None)
return flask.jsonify({
'component_files': rows,
'_meta': {
'count': nb_rows
}
})
def create_jobs(user):
values = v1_utils.common_values_dict(user)
values.update(schemas.job.post(flask.request.json))
components_ids = values.pop('components')
values['team_id'] = values.get('team_id', user['team_id'])
# Only super admin can create job for other teams
if not user.is_super_admin() and not user.is_in_team(values['team_id']):
raise auth.UNAUTHORIZED
if values['topic_id'] is not None:
v1_utils.verify_team_in_topic(user, values['topic_id'])
values.update({
'status': 'new',
'topic_id': values['topic_id'],
'rconfiguration_id': values['rconfiguration_id'],
'user_agent': flask.request.environ.get('HTTP_USER_AGENT'),
'client_version': flask.request.environ.get(
'HTTP_CLIENT_VERSION'
),
})
# create the job and feed the jobs_components table
with flask.g.db_conn.begin():
query = _TABLE.insert().values(**values)
flask.g.db_conn.execute(query)
jobs_components_to_insert = []
for cmpt_id in components_ids:
v1_utils.verify_existence_and_get(cmpt_id, models.COMPONENTS)
jobs_components_to_insert.append({'job_id': values['id'],
'component_id': cmpt_id})
if jobs_components_to_insert:
flask.g.db_conn.execute(models.JOIN_JOBS_COMPONENTS.insert(),
jobs_components_to_insert)
return flask.Response(json.dumps({'job': values}), 201,
headers={'ETag': values['etag']},
content_type='application/json')
def get_all_components(user, topic_id):
"""Get all components of a topic."""
args = schemas.args(flask.request.args.to_dict())
v1_utils.verify_team_in_topic(user, topic_id)
query = v1_utils.QueryBuilder(_TABLE, args, _C_COLUMNS)
query.add_extra_condition(
sql.and_(_TABLE.c.topic_id == topic_id, _TABLE.c.state != 'archived'))
nb_rows = query.get_number_of_rows()
rows = query.execute(fetchall=True)
rows = v1_utils.format_result(rows, _TABLE.name, args['embed'],
_EMBED_MANY)
# Return only the component which have the export_control flag set to true
#
if not (auth.is_admin(user)):
rows = [row for row in rows if row['export_control']]
return flask.jsonify({'components': rows, '_meta': {'count': nb_rows}})
def create_jobs(user):
values = flask.request.json
check_json_is_valid(create_job_schema, values)
values.update(v1_utils.common_values_dict())
components_ids = values.pop('components')
if user.is_not_remoteci():
raise dci_exc.DCIException('Only remoteci can create job')
topic_id = values.get('topic_id')
topic = v1_utils.verify_existence_and_get(topic_id, models.TOPICS)
product_id = topic['product_id']
if topic_id:
v1_utils.verify_team_in_topic(user, topic_id)
previous_job_id = values.get('previous_job_id')
if previous_job_id:
v1_utils.verify_existence_and_get(previous_job_id, _TABLE)
values.update({
'status':
'new',
'remoteci_id':
user.id,
'topic_id':
topic_id,
'user_agent':
flask.request.environ.get('HTTP_USER_AGENT'),
'client_version':
flask.request.environ.get('HTTP_CLIENT_VERSION'),
'previous_job_id':
previous_job_id,
'team_id':
user.teams_ids[0],
'product_id':
product_id,
'duration':
0
})
# create the job and feed the jobs_components table
with flask.g.db_conn.begin():
query = _TABLE.insert().values(**values)
flask.g.db_conn.execute(query)
jobs_components_to_insert = []
for cmpt_id in components_ids:
v1_utils.verify_existence_and_get(cmpt_id, models.COMPONENTS)
jobs_components_to_insert.append({
'job_id': values['id'],
'component_id': cmpt_id
})
if jobs_components_to_insert:
flask.g.db_conn.execute(models.JOIN_JOBS_COMPONENTS.insert(),
jobs_components_to_insert)
return flask.Response(json.dumps({'job': values}),
201,
headers={'ETag': values['etag']},
content_type='application/json')
def schedule_jobs(user):
"""Dispatch jobs to remotecis.
The remoteci can use this method to request a new job.
Before a job is dispatched, the server will flag as 'killed' all the
running jobs that were associated with the remoteci. This is because they
will never be finished.
"""
values = flask.request.json
check_json_is_valid(schedule_job_schema, values)
values.update({
'id':
utils.gen_uuid(),
'created_at':
get_utc_now().isoformat(),
'updated_at':
get_utc_now().isoformat(),
'etag':
utils.gen_etag(),
'status':
'new',
'remoteci_id':
user.id,
'duration':
0,
'user_agent':
flask.request.environ.get('HTTP_USER_AGENT'),
'client_version':
flask.request.environ.get('HTTP_CLIENT_VERSION'),
})
topic_id = values.pop('topic_id')
dry_run = values.pop('dry_run')
if dry_run:
component_types = components.get_component_types_from_topic(topic_id)
_components = components.get_last_components_by_type(
component_types, topic_id)
return flask.Response(json.dumps({
'components': _components,
'job': None
}),
201,
content_type='application/json')
# check remoteci
remoteci = v1_utils.verify_existence_and_get(user.id, models.REMOTECIS)
if remoteci['state'] != 'active':
message = 'RemoteCI "%s" is disabled.' % remoteci['id']
raise dci_exc.DCIException(message, status_code=412)
# check primary topic
topic = v1_utils.verify_existence_and_get(topic_id, models.TOPICS)
product_id = topic['product_id']
if topic['state'] != 'active':
msg = 'Topic %s:%s not active.' % (topic_id, topic['name'])
raise dci_exc.DCIException(msg, status_code=412)
v1_utils.verify_team_in_topic(user, topic_id)
# check secondary topic
topic_id_secondary = values.pop('topic_id_secondary')
if topic_id_secondary:
topic_secondary = v1_utils.verify_existence_and_get(
topic_id_secondary, models.TOPICS)
if topic_secondary['state'] != 'active':
msg = 'Topic %s:%s not active.' % (topic_id_secondary,
topic['name'])
raise dci_exc.DCIException(msg, status_code=412)
v1_utils.verify_team_in_topic(user, topic_id_secondary)
remotecis.kill_existing_jobs(remoteci['id'])
components_ids = values.pop('components_ids')
values = _build_job(product_id,
topic_id,
remoteci,
components_ids,
values,
topic_id_secondary=topic_id_secondary)
return flask.Response(json.dumps({'job': values}),
201,
headers={'ETag': values['etag']},
content_type='application/json')
def get_jobs_status_from_components(user, topic_id, type_id):
# List of job meaningfull job status for global overview
#
# ie. If current job status is running, we should retrieve status
# from prior job.
valid_status = ['failure', 'product-failure', 'deployment-failure',
'success']
topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True)
v1_utils.verify_team_in_topic(user, topic_id)
# Get list of all remotecis that are attached to a topic this type belongs
# to
fields = [models.REMOTECIS.c.id.label('remoteci_id'),
models.REMOTECIS.c.name.label('remoteci_name'),
models.TEAMS.c.id.label('team_id'),
models.TEAMS.c.name.label('team_name'),
models.TOPICS.c.name.label('topic_name'),
models.COMPONENTS.c.id.label('component_id'),
models.COMPONENTS.c.name.label('component_name'),
models.COMPONENTS.c.type.label('component_type'),
models.JOBS.c.id.label('job_id'),
models.JOBS.c.status.label('job_status'),
models.JOBS.c.created_at.label('job_created_at')]
query = (sql.select(fields)
.select_from(
sql.join(
models.REMOTECIS,
models.JOBS,
models.REMOTECIS.c.id == models.JOBS.c.remoteci_id,
isouter=True)
.join(
models.JOIN_JOBS_COMPONENTS,
models.JOIN_JOBS_COMPONENTS.c.job_id == models.JOBS.c.id)
.join(
models.COMPONENTS,
models.COMPONENTS.c.id == models.JOIN_JOBS_COMPONENTS.c.component_id) # noqa
.join(
models.TOPICS,
models.TOPICS.c.id == models.COMPONENTS.c.topic_id)
.join(
models.TEAMS,
models.TEAMS.c.id == models.JOBS.c.team_id))
.where(
sql.and_(
models.REMOTECIS.c.state == 'active',
models.TEAMS.c.external == True, # noqa
models.JOBS.c.status.in_(valid_status),
models.JOBS.c.state != 'archived',
models.COMPONENTS.c.type == type_id,
models.TOPICS.c.id == topic_id))
.order_by(
models.REMOTECIS.c.id,
models.JOBS.c.created_at.desc())
.distinct(models.REMOTECIS.c.id))
if not user.is_super_admin():
query.append_whereclause(models.TEAMS.c.id.in_(user.teams))
rcs = flask.g.db_conn.execute(query).fetchall()
nb_row = len(rcs)
return flask.jsonify({'jobs': rcs,
'_meta': {'count': nb_row}})
def get_all_components(user, topic_id):
topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True)
v1_utils.verify_team_in_topic(user, topic_id)
return components.get_all_components(user, topic_id=topic_id)
def get_all_jobdefinitions_by_topic(user, topic_id):
topic_id = v1_utils.verify_existence_and_get(topic_id, _TABLE, get_id=True)
v1_utils.verify_team_in_topic(user, topic_id)
return jobdefinitions.list_jobdefinitions(user, [topic_id], by_topic=True)
def get_component_by_id(user, c_id):
component = v1_utils.verify_existence_and_get(c_id, _TABLE)
v1_utils.verify_team_in_topic(user, component['topic_id'])
auth.check_export_control(user, component)
return base.get_resource_by_id(user, component, _TABLE, _EMBED_MANY)
