def setup(config: dict):
if not get_runtime_config_private_dict()['setup_needed']:
error_abort('Setup already performed', code=requests.codes.conflict)
unified_config = _DEFAULTS.copy()
unified_config.update({key: value for key, value in config.items() if key in _DEFAULTS})
unified_config['setup_needed'] = False
#known preferences
for key, value in unified_config.items():
db.session.execute('''
INSERT INTO app_config (key, value) VALUES (:key, :value)
ON CONFLICT(key) DO NOTHING
''', {'key': key, 'value': json.dumps(value)})
db.session.commit()
# admin user
admin_user_email = config.get('admin_user_email')
admin_user_password = config.get('admin_user_password')
if admin_user_email and admin_user_password and not db.session.query(models.roles_users).count():
try:
user = user_datastore.create_user(
active=True,
password=encrypt_password(admin_user_password),
email=admin_user_email,
)
user_datastore.add_role_to_user(user, 'admin')
db.session.commit()
except IntegrityError:
_logger.debug('User already exists. Skipping admin user creation')
def _paginate(self, query, metadata):
args = pagination_parser.parse_args()
page_size = metadata['page_size'] = args.page_size
max_page_size = current_app.config['MAX_QUERY_PAGE_SIZE']
if page_size > max_page_size:
error_abort(
'Query attempted to fetch more than the maximum number of results per page ({})'
.format(max_page_size))
returned = query.offset(
(args.page - 1) * args.page_size).limit(args.page_size + 1).all()
metadata['page'] = args.page
if len(returned) > args.page_size:
metadata['has_more'] = True
returned.pop(-1)
else:
metadata['has_more'] = False
if returned and not isinstance(returned[0], self.MODEL):
# Assume ID query
returned = self._sort(
self.MODEL.query.filter(self.MODEL.id.in_(returned)),
metadata).all()
return returned
def report_session_end(id: int, duration: (int, NoneType)=None, has_fatal_errors: bool=False):
try:
session = Session.query.filter(Session.id == id).one()
except NoResultFound:
error_abort('Session not found', code=requests.codes.not_found)
session.notify_subject_activity()
if session.status not in (statuses.RUNNING, statuses.INTERRUPTED):
error_abort('Session is not running', code=requests.codes.conflict)
if duration is None:
session.mark_ended()
else:
session.mark_ended_at(session.start_time + duration)
# TODO: handle interrupted sessions
if session.num_error_tests or session.num_errors:
session.status = statuses.ERROR
elif session.num_failed_tests or session.num_failures:
session.status = statuses.FAILURE
elif session.status != statuses.INTERRUPTED:
session.status = statuses.SUCCESS
session.has_fatal_errors = has_fatal_errors
session.in_pdb = False
if session.ttl_seconds is not None:
session.delete_at = flux.current_timeline.time() + session.ttl_seconds
db.session.add(session)
db.session.commit()
def _validate_session(session_id):
session = Session.query.get(session_id)
if session is None:
abort(requests.codes.not_found)
if session.end_time is not None:
error_abort('Session already ended', code=requests.codes.conflict)
return session
def _validate_session(session_id):
session = Session.query.get(session_id)
if session is None:
abort(requests.codes.not_found)
if session.end_time is not None:
error_abort('Session already ended', code=requests.codes.conflict)
return session
def add_warning(message:str, filename:str=None, lineno:int=None, test_id:int=None, session_id:int=None, timestamp:(int,float)=None):
# pylint: disable=superfluous-parens
if not ((test_id is not None) ^ (session_id is not None)):
error_abort('Either session_id or test_id required')
if session_id is not None:
obj = Session.query.get_or_404(session_id)
else:
obj = Test.query.get_or_404(test_id)
if timestamp is None:
timestamp = get_current_time()
warning = Warning.query.filter_by(session_id=session_id, test_id=test_id, lineno=lineno, filename=filename, message=message).first()
if warning is None:
if obj.num_warnings < current_app.config['MAX_WARNINGS_PER_ENTITY']:
warning = Warning(message=message, timestamp=timestamp, filename=filename, lineno=lineno, test_id=test_id, session_id=session_id)
db.session.add(warning)
else:
warning.num_warnings = Warning.num_warnings + 1
warning.timestamp = timestamp
obj.num_warnings = type(obj).num_warnings + 1
if session_id is None:
obj.session.num_test_warnings = Session.num_test_warnings + 1
db.session.add(obj.session)
db.session.commit()
def report_session_end(id: int,
duration: (int, NoneType) = None,
has_fatal_errors: bool = False):
try:
session = Session.query.filter(Session.id == id).one()
except NoResultFound:
error_abort('Session not found', code=requests.codes.not_found)
session.notify_subject_activity()
if session.status not in (statuses.RUNNING, statuses.INTERRUPTED):
error_abort('Session is not running', code=requests.codes.conflict)
if duration is None:
session.mark_ended()
else:
session.mark_ended_at(session.start_time + duration)
# TODO: handle interrupted sessions
if session.num_error_tests or session.num_errors:
session.status = statuses.ERROR
elif session.num_failed_tests or session.num_failures:
session.status = statuses.FAILURE
elif session.status != statuses.INTERRUPTED:
session.status = statuses.SUCCESS
session.has_fatal_errors = has_fatal_errors
session.in_pdb = False
if session.ttl_seconds is not None:
session.delete_at = flux.current_timeline.time() + session.ttl_seconds
db.session.add(session)
db.session.commit()
def _get_metadata_model(entity_type):
if entity_type == 'session':
return SessionMetadata
if entity_type == 'test':
return TestMetadata
error_abort('Unknown entity type')
def _get_metadata_model(entity_type):
if entity_type == 'session':
return SessionMetadata
if entity_type == 'test':
return TestMetadata
error_abort('Unknown entity type')
def _check_alowed_email_domain(user_info):
email = user_info['email']
domain = email.split('@', 1)
allowed = current_app.config.get('ALLOWED_EMAIL_DOMAINS')
if allowed is None:
return
if domain not in allowed:
error_abort('Disallowed email domain for {}'.format(email), code=requests.codes.unauthorized)
def _get_iterator(self):
args = session_test_query_parser.parse_args()
if not ((args.session_id is not None) ^ (args.test_id is not None)): # pylint: disable=superfluous-parens
error_abort(
'Either test_id or session_id must be passed to the query')
return models.Comment.query.filter_by(session_id=args.session_id,
test_id=args.test_id)
def report_session_interrupted(id: int):
s = Session.query.get_or_404(id)
if s.end_time is not None:
error_abort('Ended session cannot be marked as interrupted',
code=requests.codes.conflict)
s.status = statuses.INTERRUPTED
if s.parent:
s.parent.status = statuses.INTERRUPTED
db.session.commit()
def discard_sessions_search(search_string: str, grace_period_seconds: int=_DEFAULT_DELETE_GRACE_PERIOD_SECONDS):
if not search_string:
error_abort('Invadlid search string')
delete_at = flux.current_timeline.time() + grace_period_seconds
search_query = get_orm_query_from_search_string('session', search_string).filter(Session.delete_at == None)
Session.query.filter(Session.id.in_(db.session.query(search_query.subquery().c.id))).update({
'delete_at': delete_at
}, synchronize_session=False)
db.session.commit()
def _check_alowed_email_domain(user_info):
email = user_info['email']
domain = email.split('@', 1)
allowed = current_app.config.get('ALLOWED_EMAIL_DOMAINS')
if allowed is None:
return
if domain not in allowed:
error_abort('Disallowed email domain for {}'.format(email),
code=requests.codes.unauthorized)
def report_test_start(
session_id: int,
name: str,
file_name: (str, NoneType)=None,
class_name: (str, NoneType)=None,
test_logical_id: str=None,
scm: (str, NoneType)=None,
file_hash: (str, NoneType)=None,
scm_revision: (str, NoneType)=None,
scm_dirty: bool=False,
is_interactive: bool=False,
variation: (dict, NoneType)=None,
metadata: (dict, NoneType)=None,
test_index: (int, NoneType)=None,
parameters: (dict, NoneType)=None,
):
session = Session.query.get(session_id)
if session is None:
abort(requests.codes.not_found)
if session.end_time is not None:
error_abort('Session already ended', code=requests.codes.conflict)
test_info_id = get_or_create_test_information_id(
file_name=file_name, name=name, class_name=class_name)
if test_index is None:
test_index = Test.query.filter(Test.session_id == session_id).count() + 1
if is_interactive:
session.total_num_tests = Session.total_num_tests + 1
db.session.add(session)
returned = Test(
session_id=session.id,
logical_id=test_logical_id,
test_info_id=test_info_id,
status=statuses.RUNNING,
scm_dirty=scm_dirty,
scm_revision=scm_revision,
scm=scm,
is_interactive=is_interactive,
file_hash=file_hash,
test_index=test_index,
parameters=sanitize_json(parameters),
)
if variation is not None:
returned.test_variation = TestVariation(variation=sanitize_json(variation))
if metadata is not None:
for key, value in metadata.items():
returned.metadatas.append(TestMetadata(key=key, metadata_item=value))
try:
db.session.add(returned)
db.session.commit()
except DataError:
returned.test_variation.variation = sanitize_json(variation)
db.session.add(returned)
db.session.commit()
return returned
def new_func(*args, **kwargs):
if not current_user.is_authenticated:
if not allow_runtoken:
error_abort('Run token not alowed for API', code=requests.codes.unauthorized)
g.token_user = _get_user_from_run_token()
try:
return func(*args, **kwargs)
finally:
if hasattr(g, 'token_user'):
del g.token_user
def get_orm_query_from_search_string(object_type, query, abort_on_syntax_error=False):
with SearchContext.get_for_type(object_type) as ctx:
base_query = ctx.get_base_query()
try:
returned = transform_to_query(base_query, query)
except SearchSyntaxError as e:
if not abort_on_syntax_error:
raise
error_abort('Syntax Error', code=requests.codes.bad_request, extra={'errors': [{'syntax_error': e.reason}]})
return returned
def get_orm_query_from_search_string(object_type, query, abort_on_syntax_error=False):
with SearchContext.get_for_type(object_type) as ctx:
base_query = ctx.get_base_query()
try:
returned = transform_to_query(base_query, query)
except SearchSyntaxError as e:
if not abort_on_syntax_error:
raise
error_abort('Syntax Error', code=requests.codes.bad_request, extra={'errors': [{'detail': e.reason}]})
return returned
def put(self, object_id=None):
if object_id is None:
error_abort('Not implemented', code=requests.codes.not_implemented)
comment = models.Comment.query.get_or_404(object_id)
if comment.user_id != current_user.id:
error_abort('Not allowed to delete comment', code=requests.codes.forbidden)
comment.comment = request.get_json().get('comment', {}).get('comment')
comment.edited = True
models.db.session.add(comment)
models.db.session.commit()
return jsonify({'comment': self._render_single(comment, in_collection=False)})
def _login_with_google_oauth2(auth_code):
user_info = get_oauth2_identity(auth_code)
if not user_info:
error_abort('Could not complete OAuth2 exchange', code=requests.codes.unauthorized)
_check_alowed_email_domain(user_info)
user = get_or_create_user(user_info)
login_user(user)
return _make_success_login_response(user, user_info)
def put(self, object_id=None):
if object_id is None:
error_abort('Not implemented', code=requests.codes.not_implemented)
comment = models.Comment.query.get_or_404(object_id)
if comment.user_id != current_user.id:
error_abort('Not allowed to delete comment', code=requests.codes.forbidden)
comment.comment = request.get_json().get('comment', {}).get('comment')
comment.edited = True
models.db.session.add(comment)
models.db.session.commit()
return jsonify({'comment': self._render_single(comment, in_collection=False)})
def _login_with_google_oauth2(auth_code):
user_info = get_oauth2_identity(auth_code)
if not user_info:
error_abort('Could not complete OAuth2 exchange',
code=requests.codes.unauthorized)
_check_alowed_email_domain(user_info)
user = get_or_create_user(user_info)
login_user(user)
return _make_success_login_response(user, user_info)
def _get_iterator(self):
args = related_entity_parser.parse_args()
if not ((args.session_id is None) ^ (args.test_id is None)):
error_abort('Either test_id or session_id must be provided')
if args.session_id is not None:
return self._get_all_children_entities(args.session_id)
elif args.test_id is not None:
return models.Entity.query.join(models.test_entity).filter(models.test_entity.c.test_id == args.test_id)
else:
raise NotImplementedError() # pragma: no cover
def _get_iterator(self):
args = related_entity_parser.parse_args()
if not ((args.session_id is None) ^ (args.test_id is None)):
error_abort('Either test_id or session_id must be provided')
if args.session_id is not None:
return models.Entity.query.join(models.session_entity).filter(models.session_entity.c.session_id == args.session_id)
elif args.test_id is not None:
return models.Entity.query.join(models.test_entity).filter(models.test_entity.c.test_id == args.test_id)
else:
raise NotImplementedError() # pragma: no cover
def add_error(message: str,
exception_type: (str, NoneType) = None,
traceback: (list, NoneType) = None,
timestamp: (float, int) = None,
test_id: int = None,
session_id: int = None,
is_failure: bool = False): # pylint: disable=bad-whitespace
# pylint: disable=superfluous-parens
if not ((test_id is not None) ^ (session_id is not None)):
error_abort('Either test_id or session_id required')
if timestamp is None:
timestamp = get_current_time()
if test_id is not None:
cls = Test
object_id = test_id
else:
cls = Session
object_id = session_id
try:
obj = cls.query.filter(cls.id == object_id).one()
increment_field = cls.num_failures if is_failure else cls.num_errors
cls.query.filter(cls.id == object_id).update(
{increment_field: increment_field + 1})
err = Error(message=message,
exception_type=exception_type,
traceback_url=_normalize_traceback_get_url(traceback),
is_failure=is_failure,
timestamp=timestamp)
obj.errors.append(err)
if obj.end_time is not None:
if cls is Test:
if is_failure and obj.status not in (statuses.FAILURE,
statuses.ERROR):
obj.status = statuses.FAILURE
obj.session.num_failed_tests = Session.num_failed_tests + 1
elif not is_failure and obj.status != statuses.ERROR:
if obj.status == statuses.FAILURE:
obj.session.num_failed_tests = Session.num_failed_tests - 1
if obj.session.parent:
obj.session.parent.num_failed_tests = obj.session.parent.num_failed_tests + 1
obj.status = statuses.ERROR
obj.session.num_error_tests = Session.num_error_tests + 1
if obj.session.parent:
obj.session.parent.num_error_tests = obj.session.parent.num_error_tests + 1
db.session.add(obj)
except NoResultFound:
error_abort('Entity not found', code=requests.codes.not_found)
db.session.commit()
return err
def put(self, object_id=None):
if object_id is None:
error_abort('Not implemented', code=requests.codes.not_implemented)
if not has_role(current_user, 'admin'):
error_abort('Forbidden', code=requests.codes.forbidden)
replication = models.Replication.query.get_or_404(object_id)
request_json = request.get_json().get("replication", {})
for field_name in {'username', 'url', 'password'}:
value = request_json.get(field_name)
if value is not None:
setattr(replication, field_name, value)
models.db.session.commit()
return jsonify({'replication': self._render_single(replication, in_collection=False)})
def _update_running_test_status(test_id, status, ignore_conflict=False, additional_updates=None):
logbook.debug('marking test {} as {}', test_id, status)
updates = {'status': status}
if additional_updates:
updates.update(additional_updates)
if not Test.query.filter(Test.id == test_id, Test.status == statuses.RUNNING).update(updates):
if Test.query.filter(Test.id == test_id).count():
# we have a test, but it already ended
if not ignore_conflict:
error_abort('Test already ended', requests.codes.conflict)
else:
abort(requests.codes.not_found)
def put(self, object_id=None):
if object_id is None:
error_abort('Not implemented', code=requests.codes.not_implemented)
if not has_role(current_user, 'admin'):
error_abort('Forbidden', code=requests.codes.forbidden)
replication = models.Replication.query.get_or_404(object_id)
request_json = request.get_json().get("replication", {})
for field_name in {'username', 'url', 'password'}:
value = request_json.get(field_name)
if value is not None:
setattr(replication, field_name, value)
models.db.session.commit()
return jsonify({'replication': self._render_single(replication, in_collection=False)})
def delete(self, object_id=None):
if object_id is None:
error_abort('Not implemented', code=requests.codes.not_implemented)
comment = models.Comment.query.get_or_404(object_id)
if comment.session_id is not None:
obj = models.Session.query.get(comment.session_id)
else:
obj = models.Test.query.get(comment.test_id)
if comment.user_id != current_user.id:
error_abort('Not allowed to delete comment', code=requests.codes.forbidden)
obj.num_comments = type(obj).num_comments - 1
models.db.session.add(obj)
models.db.session.delete(comment)
models.db.session.commit()
def delete(self, object_id=None):
if object_id is None:
error_abort('Not implemented', code=requests.codes.not_implemented)
comment = models.Comment.query.get_or_404(object_id)
if comment.session_id is not None:
obj = models.Session.query.get(comment.session_id)
else:
obj = models.Test.query.get(comment.test_id)
if comment.user_id != current_user.id:
error_abort('Not allowed to delete comment', code=requests.codes.forbidden)
obj.num_comments = type(obj).num_comments - 1
models.db.session.add(obj)
models.db.session.delete(comment)
models.db.session.commit()
def login():
credentials = request.get_json(silent=True)
if not isinstance(credentials, dict):
error_abort('Credentials provided are not a JSON object')
if credentials.get('username'):
return _login_with_credentials(credentials)
auth_code = credentials.get('authorizationCode')
if auth_code:
return _login_with_google_oauth2(auth_code)
error_abort('No credentials were specified', code=requests.codes.unauthorized)
def _sort(self, iterator, metadata):
sort_fields_expr = request.args.get('sort', None)
if sort_fields_expr:
sort_fields = sort_fields_expr.split(',')
if not all((sort_field in self.SORTABLE_FIELDS)
for sort_field in sort_fields):
error_abort(
'Cannot sort according to given criteria - can only sort by {}'
.format(', '.join(self.SORTABLE_FIELDS)))
iterator = iterator.order_by(
*[self._build_sort_expr(self.MODEL, f) for f in sort_fields])
elif self.DEFAULT_SORT is not None:
iterator = iterator.order_by(*self.DEFAULT_SORT) # pylint: disable=not-an-iterable
return iterator
def _get_metadata_query(*, entity_type, entity_id):
model = _get_metadata_model(entity_type)
query = db.session.query(func.json_object_agg(model.key, model.metadata_item))
if entity_type == 'session':
related = Session
elif entity_type == 'test':
related = Test
else:
error_abort('Invalid entity type', requests.codes.bad_request)
query = query.join(related)
if isinstance(entity_id, int):
query = query.filter(related.id == entity_id)
else:
query = query.filter(related.logical_id == entity_id)
return query
def login():
credentials = request.get_json(silent=True)
if not isinstance(credentials, dict):
error_abort('Credentials provided are not a JSON object')
if credentials.get('username'):
return _login_with_credentials(credentials)
auth_code = credentials.get('authorizationCode')
if auth_code:
return _login_with_google_oauth2(auth_code)
error_abort('No credentials were specified',
code=requests.codes.unauthorized)
def _get_metadata_query(*, entity_type, entity_id):
model = _get_metadata_model(entity_type)
query = db.session.query(json_object_agg(model.key, model.metadata_item))
if entity_type == 'session':
related = Session
elif entity_type == 'test':
related = Test
else:
error_abort('Invalid entity type', requests.codes.bad_request)
query = query.join(related)
if isinstance(entity_id, int):
query = query.filter(related.id == entity_id)
else:
query = query.filter(related.logical_id == entity_id)
return query
def report_timing_start(name: str, session_id: int, test_id: (int, NoneType)=None): # pylint: disable=bad-whitespace
interval = -flux.current_timeline.time()
try:
db.session.execute(
Timing.__table__.insert().values(
session_id=session_id, test_id=test_id, name=name, total=interval))
db.session.commit()
except IntegrityError:
db.session.rollback()
res = db.session.execute(
Timing.__table__.update()
.values(total=Timing.total+interval)
.where(and_(Timing.session_id==session_id, Timing.test_id==test_id, Timing.name==name, Timing.total >= 0)))
if res.rowcount != 1:
error_abort('Attempted to start measurement on an already started metric', code=requests.codes.conflict) # pylint: disable=no-member, line-too-long
db.session.commit()
def get_activities():
args = session_test_user_query_parser.parse_args()
if not ((args.session_id is not None) ^
(args.test_id is not None) ^
(args.user_id is not None)):
error_abort('Either test_id, session_id or user_id must be passed to the query')
results = models.db.session.execute(
activity.get_activity_query(user_id=args.user_id, test_id=args.test_id, session_id=args.session_id))
return jsonify({
'activities': [
_fix_action_string(dict(obj.items())) for obj in results
]
})
def reauth():
token = (request.json or {}).get('auth_token')
if token is None:
error_abort('Missing reauth token')
try:
token_data = _get_token_serializer().loads(token,
max_age=_MAX_TOKEN_AGE)
except BadSignature:
error_abort('Reauth token invalid', code=requests.codes.unauthorized)
user = User.query.get_or_404(token_data['user_id'])
login_user(user)
return jsonify({
'auth_token': token,
'user_info': token_data['user_info'],
})
def set_preference(preference: str, value: (str, Number)):
user_id = current_user.id
if preference not in current_app.config['DEFAULT_PREFERENCES']:
error_abort('Unknown preference specified: {}'.format(preference))
db.session.execute('''
INSERT INTO user_preference (user_id, preference, value)
VALUES (:user_id, :pref, :value)
ON CONFLICT (user_id, preference) DO UPDATE SET value=EXCLUDED.value
''', params={
'user_id': user_id,
'pref': preference,
'value': json.dumps({'value': value})
})
db.session.commit()
return value
def add_related_entity(type: str, name: str, test_id: int=None, session_id: int=None):
#pylint: disable=superfluous-parens
if not ((test_id is not None) ^ (session_id is not None)):
error_abort('Either test_id or session_id required')
if session_id is not None:
obj = Session.query.get_or_404(session_id)
else:
obj = Test.query.get_or_404(test_id)
db.session.execute(insert(Entity).values(name=name, type=type).on_conflict_do_nothing())
db.session.commit()
entity = Entity.query.filter_by(name=name, type=type).one()
obj.related_entities.append(entity)
db.session.commit()
def _update_running_test_status(test_id, status, ignore_conflict=False, additional_updates=None, allow_ended=False):
logbook.debug('marking test {} as {}', test_id, status)
updates = {'status': status}
if additional_updates:
updates.update(additional_updates)
query = Test.query.filter(Test.id == test_id, Test.status != statuses.PLANNED)
if not allow_ended:
query = query.filter(Test.status == statuses.RUNNING)
if not query.update(updates):
if Test.query.filter(Test.id == test_id).count():
# we have a test, but it already ended
if not ignore_conflict:
error_abort('Test already ended', requests.codes.conflict)
else:
abort(requests.codes.not_found)
def post_comment(comment: str, session_id: int=None, test_id: int=None):
if not (session_id is not None) ^ (test_id is not None):
error_abort('Either session_id or test_id required')
if session_id is not None:
obj = Session.query.get_or_404(session_id)
else:
obj = Test.query.get_or_404(test_id)
returned = Comment(user_id=current_user.id, comment=comment)
obj.comments.append(returned)
obj.num_comments = type(obj).num_comments + 1
db.session.add(obj)
db.session.commit()
return returned
def add_related_entity(type: str, name: str, test_id: int=None, session_id: int=None):
#pylint: disable=superfluous-parens
if not ((test_id is not None) ^ (session_id is not None)):
error_abort('Either test_id or session_id required')
if session_id is not None:
obj = Session.query.get_or_404(session_id)
else:
obj = Test.query.get_or_404(test_id)
db.session.execute(insert(Entity).values(name=name, type=type).on_conflict_do_nothing())
db.session.commit()
entity = Entity.query.filter_by(name=name, type=type).one()
obj.related_entities.append(entity)
db.session.commit()
def set_preference(preference: str, value: (str, Number)):
user_id = current_user.id
if preference not in current_app.config['DEFAULT_PREFERENCES']:
error_abort('Unknown preference specified: {}'.format(preference))
db.session.execute('''
INSERT INTO user_preference (user_id, preference, value)
VALUES (:user_id, :pref, :value)
ON CONFLICT (user_id, preference) DO UPDATE SET value=EXCLUDED.value
''',
params={
'user_id': user_id,
'pref': preference,
'value': json.dumps({'value': value})
})
db.session.commit()
return value
def report_timing_start(name: str,
session_id: int,
test_id: (int, NoneType) = None): # pylint: disable=bad-whitespace
interval = -flux.current_timeline.time()
try:
db.session.execute(Timing.__table__.insert().values(
session_id=session_id, test_id=test_id, name=name, total=interval))
db.session.commit()
except IntegrityError:
db.session.rollback()
res = db.session.execute(Timing.__table__.update().values(
total=Timing.total + interval).where(
and_(Timing.session_id == session_id,
Timing.test_id == test_id, Timing.name == name,
Timing.total >= 0)))
if res.rowcount != 1:
error_abort(
'Attempted to start measurement on an already started metric',
code=requests.codes.conflict) # pylint: disable=no-member, line-too-long
db.session.commit()
def upload_traceback(error_id):
error = Error.query.get_or_404(error_id)
if error.traceback_url is not None:
error_abort('Error already has an associated traceback', requests.codes.conflict)
temporary_location = os.path.join(current_app.config['TRACEBACK_DIR'], 'incomplete', str(error.id))
_ensure_dir(os.path.dirname(temporary_location))
try:
with _compressed_output(request.stream, temporary_location) as outfile:
shutil.copyfileobj(request.stream, outfile)
url, location = _get_new_traceback_save_location()
_ensure_dir(os.path.dirname(location))
os.rename(temporary_location, location)
error.traceback_url = url
except Exception: # pylint: disable=broad-except
if os.path.exists(temporary_location):
os.unlink(temporary_location)
raise
db.session.add(error)
db.session.commit()
return jsonify({'traceback_url': error.traceback_url})
def report_test_end(id: int, duration: (float, int)=None):
test = Test.query.get(id)
if test is None:
abort(requests.codes.not_found)
if test.end_time is not None:
# we have a test, but it already ended
error_abort('Test already ended', code=requests.codes.conflict)
with updating_session_counters(test):
test.end_time = get_current_time() if duration is None else Test.start_time + \
duration
if test.num_errors:
test.status = statuses.ERROR
elif test.num_failures:
test.status = statuses.FAILURE
elif not test.interrupted and not test.skipped:
test.status = statuses.SUCCESS
db.session.add(test)
db.session.commit()
def upload_traceback(error_id):
error = Error.query.get_or_404(error_id)
if error.traceback_url is not None:
error_abort('Error already has an associated traceback', requests.codes.conflict)
temporary_location = os.path.join(current_app.config['TRACEBACK_DIR'], 'incomplete', str(error.id))
_ensure_dir(os.path.dirname(temporary_location))
try:
with _compressed_output(request.stream, temporary_location) as outfile:
shutil.copyfileobj(request.stream, outfile)
url, location = _get_new_traceback_save_location()
_ensure_dir(os.path.dirname(location))
os.rename(temporary_location, location)
error.traceback_url = url
except Exception: # pylint: disable=broad-except
if os.path.exists(temporary_location):
os.unlink(temporary_location)
raise
error.traceback_size = os.stat(location).st_size
db.session.add(error)
db.session.commit()
return jsonify({'traceback_url': error.traceback_url})
def add_error(message: str, exception_type: (str, NoneType)=None, traceback: (list, NoneType)=None, timestamp: (float, int)=None, test_id: int=None, session_id: int=None, is_failure: bool=False, is_interruption: bool=False): # pylint: disable=bad-whitespace
# pylint: disable=superfluous-parens
if not ((test_id is not None) ^ (session_id is not None)):
error_abort('Either test_id or session_id required')
if is_failure and is_interruption:
error_abort('Interruptions cannot be marked as failures')
if timestamp is None:
timestamp = get_current_time()
if test_id is not None:
cls = Test
object_id = test_id
else:
cls = Session
object_id = session_id
message = message.replace("\x00", "\\x00")
try:
obj = cls.query.filter(cls.id == object_id).one()
if is_failure:
increment_field = cls.num_failures
elif is_interruption:
increment_field = cls.num_interruptions
else:
increment_field = cls.num_errors
cls.query.filter(cls.id == object_id).update(
{increment_field: increment_field + 1})
err = Error(message=message,
exception_type=exception_type,
traceback_url=_normalize_traceback_get_url(traceback),
is_interruption=is_interruption,
is_failure=is_failure,
timestamp=timestamp)
obj.errors.append(err)
if not is_interruption and obj.end_time is not None:
if cls is Test:
if is_failure and obj.status not in (statuses.FAILURE, statuses.ERROR):
obj.status = statuses.FAILURE
obj.session.num_failed_tests = Session.num_failed_tests + 1
elif not is_failure and obj.status != statuses.ERROR:
if obj.status == statuses.FAILURE:
obj.session.num_failed_tests = Session.num_failed_tests - 1
if obj.session.parent:
obj.session.parent.num_failed_tests = obj.session.parent.num_failed_tests + 1
obj.status = statuses.ERROR
obj.session.num_error_tests = Session.num_error_tests + 1
if obj.session.parent:
obj.session.parent.num_error_tests = obj.session.parent.num_error_tests + 1
if cls is Session and obj.parent is not None:
if is_failure:
obj.parent.num_failures = obj.parent.num_failures + 1
else:
obj.parent.num_errors = obj.parent.num_errors + 1
db.session.add(obj)
except NoResultFound:
error_abort('Entity not found', code=requests.codes.not_found)
db.session.commit()
return err
def _login_with_credentials(credentials):
config = get_runtime_config_private_dict()
username = credentials.get('username')
password = credentials.get('password')
email = _fix_email(username, config)
user = User.query.filter_by(email=email).first()
if current_app.config['TESTING']:
if user is None:
user = get_or_create_user({'email': email})
if not login_user(user):
_logger.error('Could not login user {}', email)
error_abort('Login failed', code=requests.codes.unauthorized)
return _make_success_login_response(user)
if user is not None and user.password:
if verify_password(password, user.password):
login_user(user)
return _make_success_login_response(user)
_logger.debug('Could not login user locally (no user or password mismatch)')
return _login_with_ldap(email, password, config)
def reauth():
token = (request.json or {}).get('auth_token')
if token is None:
error_abort('Missing reauth token')
try:
token_data = _get_token_serializer().loads(
token, max_age=_MAX_TOKEN_AGE)
except BadSignature:
error_abort('Reauth token invalid', code=requests.codes.unauthorized)
user = User.query.get_or_404(token_data['user_id'])
if not login_user(user):
error_abort('Login failed', code=requests.codes.unauthorized)
return jsonify({
'auth_token': token,
'user_info': token_data['user_info'],
})
def _get_iterator(self):
args = session_test_query_parser.parse_args()
if not ((args.session_id is not None) ^ (args.test_id is not None)): # pylint: disable=superfluous-parens
error_abort('Either test_id or session_id must be passed to the query')
return models.Comment.query.filter_by(session_id=args.session_id, test_id=args.test_id)
def report_session_start(logical_id: str=None,
parent_logical_id: (NoneType, str)=None,
is_parent_session: bool=False,
child_id: (NoneType, str)=None,
hostname: str=None,
total_num_tests: int=None,
metadata: dict=None,
user_email: str=None,
keepalive_interval: (NoneType, int)=None,
subjects: (list, NoneType)=None,
infrastructure: (str, NoneType)=None,
ttl_seconds: (int, NoneType)=None,
):
if hostname is None:
hostname = request.remote_addr
# fix user identification
if user_email is not None and user_email != g.token_user.email:
if not has_role(g.token_user.id, 'proxy'):
error_abort('User {} is not authorized to run tests on others behalf. Tried running as {}'.format(g.token_user.email, user_email),
code=requests.codes.forbidden)
real_user_id = g.token_user.id
real_user = get_or_create_user({'email': user_email})
user_id = real_user.id
else:
user_id = g.token_user.id
real_user = None
real_user_id = None
if keepalive_interval is None and ttl_seconds is not None:
error_abort("Cannot specify session TTL when keepalive isn't used")
returned = Session(
hostname=hostname,
parent_logical_id=parent_logical_id,
is_parent_session=is_parent_session,
child_id=child_id,
total_num_tests=total_num_tests,
infrastructure=infrastructure,
user_id=user_id,
real_user_id=real_user_id,
status=statuses.RUNNING,
logical_id=logical_id,
keepalive_interval=keepalive_interval,
ttl_seconds=ttl_seconds,
)
if real_user is not None:
real_user.last_activity = flux.current_timeline.time()
returned.mark_started()
returned.update_keepalive()
if subjects:
for subject_data in subjects:
subject_name = subject_data.get('name', None)
if subject_name is None:
error_abort('Missing subject name')
subject = get_or_create_subject_instance(
name=subject_name,
product=subject_data.get('product', None),
version=subject_data.get('version', None),
revision=subject_data.get('revision', None))
returned.subject_instances.append(subject)
db.session.add(subject)
assert list(returned.subject_instances)
returned.notify_subject_activity()
if metadata is not None:
for key, value in metadata.items():
returned.metadata_items.append(SessionMetadata(
session=returned, key=key, metadata_item=value))
db.session.add(returned)
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
if parent_logical_id:
Session.query.filter_by(logical_id=parent_logical_id).first_or_404()
error_abort('Tried to report a session which conflicts with an existing session', code=requests.codes.conflict)
profiling.notify_session_start()
return returned
