def add_relevant_issue_classes(project):
""" Adds relevant issue classes to the given project. Only adds issue classes that were created after the last time
this function was ran, avoiding re-adding issue classes that the user may have removed from the project.
:param project: project to add the issue classes to
:return:
"""
issue_classes = get_relevant_issue_classes(project)
logger.info("Adding %d issue_classes to project %s" %
(len(issue_classes), project.pk))
with backend.transaction():
for issue_class in issue_classes:
project_issue_class_query = {
'project': project,
'issue_class': issue_class
}
try:
backend.get(ProjectIssueClass, project_issue_class_query)
continue
except ProjectIssueClass.DoesNotExist:
project_issue_class = ProjectIssueClass(
project_issue_class_query)
project_issue_class.enabled = True
backend.save(project_issue_class)
# re-sync the issue classes associated with the project with the database
project.revert()
def decorated_function(*args, **kwargs):
def process_anonymously():
request.user = None
request.access_token = None
return f(*args, **kwargs)
def invalid_token(message, status_code=401, cookie_token=False):
response = jsonify({'message': message})
if cookie_token:
response.set_cookie('access_token', '', expires=0)
return response, status_code
cookie_token = False
if request.args.get('access_token'):
access_token_key = request.args['access_token']
elif request.cookies.get('access_token'):
access_token_key = request.cookies['access_token']
cookie_token = True
else:
authorization = request.headers.get('Authorization', '')
match = re.match(r"^bearer\s+([\w\d]+)$", authorization, re.I)
if not match:
if anon_ok:
return process_anonymously()
return {'message': 'Authorization header not valid'}, 401
access_token_key = match.group(1)
try:
access_token = backend.get(AccessToken,
{'token': access_token_key})
except AccessToken.DoesNotExist:
if anon_ok:
return process_anonymously()
# redirect to login
return invalid_token('Invalid / expired access token: %s' %
access_token_key,
cookie_token=cookie_token)
request.access_token = access_token
try:
request.user = backend.get(User, {'pk': access_token.user['pk']},
raw=raw,
only=only,
include=include)
except User.DoesNotExist:
with backend.transaction():
backend.delete(access_token)
return invalid_token('User does not exist',
status_code=404,
cookie_token=cookie_token)
if superuser and not request.user.is_superuser():
return {
'message':
'This endpoint requires super-user privileges. Sorry :/'
}, 401
return f(*args, **kwargs)
def _import_issue_classes():
issue_update_keys = ('severity', 'description', 'title')
analyzer_update_keys = ('language', )
logger.info("Importing issue classes from checkmate...")
analyzers = settings.checkmate_settings.analyzers
for analyzer, params in analyzers.items():
logger.info("Importing issue classes for analyzer {}".format(analyzer))
if not all([key in params for key in analyzer_update_keys]):
logger.warning(
"Skipping analyzer {} as it does not contain a '{}' field".
format(key))
continue
for code, issue_params in params.get('issues_data', {}).items():
if not all([key in issue_params for key in issue_update_keys]):
logger.warning(
"Skipping issue class for code {}, as it does not contain a '{}' field"
.format(code, key))
continue
logger.info("Importing issue class for code {}".format(code))
try:
issue_class = backend.get(IssueClass, {
'analyzer': analyzer,
'code': code
})
except IssueClass.DoesNotExist:
issue_class = IssueClass({
'analyzer': analyzer,
'code': code,
})
for key in issue_update_keys:
issue_class[key] = issue_params[key]
for key in analyzer_update_keys:
issue_class[key] = params[key]
with backend.transaction():
backend.save(issue_class)
issue_class.categories.delete()
for category in issue_params.get('categories', []):
try:
issue_category = backend.get(IssueCategory,
{'name': category})
except IssueCategory.DoesNotExist:
issue_category = IssueCategory({'name': category})
with backend.transaction():
backend.save(issue_category)
issue_class.categories.append(issue_category)
for tag_name in issue_params.get('tags', ['generic']):
try:
tag = backend.get(Tag, {'name': tag_name})
except Tag.DoesNotExist:
tag = Tag({'name': tag_name})
with backend.transaction():
backend.save(tag)
issue_class.tags.append(tag)
with backend.transaction():
backend.save(issue_class)
def decorated_function(*args, **kwargs):
snapshot_a_id = kwargs.get(id_key_a, '')
snapshot_b_id = kwargs.get(id_key_b, '')
try:
snapshot_a = get_snapshot(request.project,
snapshot_a_id,
only=('pk', ))
except Snapshot.DoesNotExist as e:
return {
'message': e.message if e.message else 'invalid snapshot A'
}, 404
try:
snapshot_b = get_snapshot(request.project,
snapshot_b_id,
only=('pk', ))
except Snapshot.DoesNotExist as e:
return {
'message': e.message if e.message else 'invalid snapshot B'
}, 404
try:
diff = backend.get(Diff, {
'snapshot_a.pk': snapshot_a['pk'],
'snapshot_b.pk': snapshot_b['pk']
},
include=include)
except Diff.DoesNotExist as e:
return {'message': e.message if e.message else 'invalid diff'}, 404
setattr(request, store_as, diff)
return f(*args, **kwargs)
def auto_tag(project):
""" First step of the analysis for Git projects. Determines if automatic tags have already been added and, if not
adds them from the requirements.txt file, if one exists in the repository.
:param project: project to auto tag
:return:
"""
logger.debug("Adding automatic tags to {}".format(project.pk))
repository = project.git.eager.repository
# auto tag the project (but only once)
if project.get('automatic_tags_added'):
return
default_branch = project.git.get_default_branch()
if default_branch is None:
return
tags = extract_tags_from_requirements_txt(repository, default_branch)
with backend.transaction():
for tag_name in tags:
try:
tag = backend.get(Tag, {'name': tag_name})
except Tag.DoesNotExist:
tag = Tag({'name': tag_name})
backend.save(tag)
project.tags.append(tag)
project.automatic_tags_added = True
with backend.transaction():
backend.update(project, ['automatic_tags_added'])
def get(self, email_validation_code):
with backend.transaction():
try:
user = backend.get(
User,
{'email_validation_code': email_validation_code}
)
if not user.get('new_email'):
if not user.get('email'):
return {'message': 'no valid e-mail set'}, 403
backend.update(user, {'email_validated': True}, unset_fields=['email_validation_code'])
settings.hooks.call("user.email.validated", user)
logger.warning("Hooray, user {0} has verified his email address".format(user.name))
else:
if backend.filter(User, {'email': user.new_email}):
return {'message': 'A user with this e-mail already exists'}, 403
old_email = user.email
new_email = user.new_email
backend.update(user, {'email_validated': True, 'email': user.new_email},
unset_fields=['new_email', 'email_validation_code'])
settings.hooks.call("user.email.updated", user)
except User.DoesNotExist:
return {'message': 'Unknown validation code'}, 404
return {'message': 'success'}, 200
def post(self):
form = LoginForm(request.form)
if form.validate():
with backend.transaction():
try:
# TODO manually specifying includes is not ideal
user = backend.get(User, {'email': form.email.data.lower()},
include=UserProfile.includes)
if user.delete is True:
return {'message': "Your account is scheduled for deletion. "
"You can sign-up again in a few minutes."}, 403
if not user.check_password(form.password.data):
return ({'message': 'Invalid password'},
403)
access_token = user.get_access_token()
backend.save(access_token)
user_profile = UserProfile.export(user)
response = self.make_response({
'access_token': access_token.token,
'message': 'Success!',
'user': user_profile,
})
expires = (datetime.datetime.now() + datetime.timedelta(days=7)) if form.remember_me.data else None
response.set_cookie('access_token', value=access_token.token, expires=expires)
return response
except User.DoesNotExist:
return {'message': 'Unknown user'}, 404
return {'message': 'Invalid data', 'errors': form.errors}, 403
def delete_project(project_id, task_id=None):
""" Deletes the project with the given project id.
:param project_id: id of the project to delete
:param task_id: id of the task
"""
if not task_id:
task_id = analyze_project.request.id
try:
project = backend.get(Project, {'pk': project_id})
except Project.DoesNotExist:
logger.error(
"Project {} does not exist! Cannot delete it.".format(project_id))
return
try:
with ExclusiveTask(backend, {
'project.pk': project.pk,
'type': {
'$in': ['analysis', 'reset', 'delete']
}
}, {
'type': 'delete',
'project': project
},
task_id,
no_update_on_exit=True) as delete_task:
with TaskLogger(delete_task, backend=backend, ping=True):
_delete_project(project)
logger.info("Deletion of project {} ({}) complete!".format(
project.name, project.pk))
except ExclusiveTask.LockError:
logger.info(
"Unable to acquire lock for deletion of project {} ({}).".format(
project.name, project.pk))
def get_snapshot(project, snapshot_id, raw=True, only=None, include=None):
"""
Returns the snapshot corresponding to a given ID.
Uses providers set by the plugins.
:param snapshot_id: id of the snapshot to retrieve, can be empty??
:param raw: backend param
:param only: backend param
:param include: backend param
:return: snapshot
"""
#we try to retrieve the snapshot by PK...
try:
snapshot = backend.get(Snapshot, {'pk': snapshot_id},
include=include,
only=only,
raw=raw)
return snapshot
except Snapshot.DoesNotExist:
pass
for params in settings.providers['snapshot.resolve']:
if not params['source'] == project.source:
continue #this is not the right provider...
snapshot = params['provider'](project,
snapshot_id,
raw=raw,
only=only,
include=include)
if snapshot is not None:
return snapshot
raise Snapshot.DoesNotExist(
"Could not resolve snapshot ID {}".format(snapshot_id))
def get(self, project_id):
form = ProjectDetailsForm(request.args)
if not form.validate():
return {'message' : 'please correct the errors mentioned below.', 'errors' : form.errors}, 400
data = form.data
exported_project = self.export(request.project)
#we add the analysis queue position of the project
exported_project['analysis_queue_position'] = request.project.get_analysis_queue_position()
exported_project['user_role'] = 'anon'
if request.user is not None:
try:
#we add role information to the project (useful for displaying the correct menus)
user_role = backend.get(UserRole,{'project' : request.project, 'user' : request.user})
exported_project['user_role'] = user_role.role
except UserRole.DoesNotExist:
pass
if data['with_stats']:
exported_project['stats'] = request.project.stats
return {'project': exported_project}, 200
def post(self, password_reset_code):
form = PasswordResetForm(request.form)
if form.validate():
with backend.transaction():
try:
user = backend.get(
User,
{'password_reset_code': password_reset_code}
)
except User.DoesNotExist:
return {'message': 'Unknown user'}, 404
user.set_password(form.password.data)
backend.update(user, ['password', 'password_set'], unset_fields=['password_reset_code'])
access_token = user.get_access_token()
backend.save(access_token)
send_mail(
email_to=user.email,
template="password_reset_successful",
template_context={
"user_name": user.name
}
)
return {'message': 'success'}, 200
return ({'message': 'Invalid data',
'errors': form.errors},
403)
def post(self):
form = PasswordResetRequestForm(request.form)
if form.validate():
with backend.transaction():
try:
user = backend.get(User, {'email': form.email.data})
if 'email_validated' in user and user.email_validated:
backend.update(
user,
{'password_reset_code': uuid.uuid4().hex}
)
reset_url = "{}{}/user/password-reset?reset_code={}".format(
settings.get('url'),
settings.get('frontend.url'),
user.password_reset_code
)
# Reset Email
send_mail(
email_to=form.email.data,
template="reset_password",
template_context={
"user_name": user.name,
"reset_url": reset_url
}
)
return {'message': 'Email with reset link was sent.'}, 200
return {'message': "Your email is not validated, so we cannot send you "
"a password-reset token."}, 403
except User.DoesNotExist:
return {'message': 'Unknown user'}, 404
return {'message': 'Invalid data', 'errors': form.errors}, 403
def _get_user(user_id):
try:
return backend.get(
User,
{'$or': [{'name': user_id},
{'pk': user_id}]}
)
except (User.DoesNotExist, User.MultipleDocumentsReturned):
raise AttributeError()
def get_project(project_id):
if not project_id:
return None
try:
project = backend.get(Project, {'pk': project_id})
if not project.is_authorized(request.user):
return None
return project
except Project.DoesNotExist:
return None
def _get_project_issue_class():
query = {
'project': request.project,
'issue_class': request.issue_class
}
try:
project_issue_class = backend.get(ProjectIssueClass, query)
except ProjectIssueClass.DoesNotExist:
project_issue_class = ProjectIssueClass(query)
return project_issue_class
def decorated_function(*args, **kwargs):
if (snapshot_id_key in kwargs and path_key in kwargs
and kwargs[snapshot_id_key] is not None
and kwargs[path_key] is not None):
try:
snapshot = get_snapshot(request.project,
kwargs[snapshot_id_key],
raw=False)
except Snapshot.DoesNotExist:
return {'message': 'invalid snapshot'}, 404
try:
file_revision = backend.get(FileRevision, {
'snapshots': snapshot,
'path': kwargs[path_key],
})
request.file_revision = file_revision
except (FileRevision.DoesNotExist,
FileRevision.MultipleDocumentsReturned):
# TODO is multipledocumentsreturned a 404?
return {'message': 'invalid file revision'}, 404
elif file_revision_id_key in kwargs:
try:
file_revision = backend.get(
FileRevision, {
'pk': kwargs[file_revision_id_key],
'project': request.project,
})
request.file_revision = file_revision
except FileRevision.DoesNotExist:
return {'message': 'invalid file revision'}, 404
else:
return {
'message':
'you must specify either a snapshot ID and path or a file revision ID'
}, 404
return f(*args, **kwargs)
def delete(self, project_id, user_role_id):
with backend.transaction():
try:
user_role = backend.get(UserRole, {'project': request.project, 'pk': user_role_id})
if user_role.role == 'owner' and user_role.user == request.user:
if len(backend.filter(UserRole, {'project': request.project,'role' : 'owner'})) == 1:
return {'message' : 'You are the last owner of this project, cannot remove you.'}, 400
except UserRole.DoesNotExist:
return {'message': 'invalid role'}, 404
backend.delete(user_role)
return {'message': 'success'}, 200
def delete_user(user_id, task_id=None):
"""
What we need to do here:
-Remove all user data from database, unless user was customer
- Projects
- User profile
- AccessToken
"""
if not task_id:
task_id = delete_user.request.id
try:
user = backend.get(User, {'pk': user_id})
except User.DoesNotExist:
logger.error(
"User {} does not exist! Cannot delete it.".format(user_id))
return
try:
with ExclusiveTask(backend,
{'type': {
'$in': ['delete_{}'.format(user.pk)]
}}, {'type': 'delete_{}'.format(user.pk)},
task_id,
no_update_on_exit=True) as delete_task:
with TaskLogger(delete_task, backend=backend, ping=True):
with backend.transaction():
logger.debug("Starting deletion of user {0} ({1}).".format(
user.name, user.pk))
# Delete all related models
backend.filter(UserRole, {'user': user}).delete()
backend.filter(AccessToken, {'user': user}).delete()
backend.filter(User, {'pk': user.pk}).delete()
# Todo: Delete all of the user's projects that have no "owner" user roles anymore...
logger.info("Deletion of user {0} ({1}) complete!".format(
user.name, user.pk))
except ExclusiveTask.LockError:
pass
except BaseException as err:
logger.error("Error {0}: Can't delete user {1}.".format(
err.__class__.__name__, user))
logger.error(traceback.format_exc())
def post(self, project_id):
form = ProjectTagsForm(request.form)
if not form.validate():
return {'message' : 'please correct the errors mentioned below.', 'errors' : form.errors}, 400
data = form.data
with backend.transaction():
try:
tag = backend.get(Tag, {'name': data['name']})
except Tag.DoesNotExist:
return {'message': 'invalid tag'}, 404
request.project.tags.append(tag)
return {'message': 'success!'}, 200
def get(self, project_id, task_id):
""" Returns the log for the specified project task. Accepts a from request parameter, which seeks into
the log file to return the rest of the file.
:param project_id: id of the project
:param task_id: id of the task
:return: log, its length, offset, and the task status
"""
form = TaskLogForm(request.args)
if not form.validate():
return {
'message': 'please correct the errors mentioned below.'
}, 400
data = form.data
from_chr = data['from_chr']
try:
task = backend.get(Task, {
'project.pk': request.project.pk,
'pk': task_id
},
raw=True)
except Task.DoesNotExist:
return {'message': "unknown task"}, 404
log_path = os.path.join(settings.get('backend.path'),
settings.get('backend.paths.tasks'),
"{}.log".format(task['pk']))
try:
with open(log_path, "r") as task_log:
task_log.seek(from_chr)
content = task_log.read()
data = {
'task_log': content,
'len': len(content),
'from': from_chr,
'task_status':
task['status'] if 'status' in task else "unknown"
}
return data, 200
except IOError:
return {'message': "no log found {}".format(log_path)}, 404
def get(self, project_id, task_id):
""" Get the task with the given task id for the project with the given project id.
:param project_id: id of the project
:param task_id: id of the task
:return: task information
"""
try:
task = backend.get(Task, {
'project.pk': request.project.pk,
'pk': task_id
},
only=self.export_fields,
include=('project', ),
raw=True)
except Task.DoesNotExist:
return {'message': "unknown task"}, 404
return {'task': self.export(task)}, 200
def decorated_function(*args, **kwargs):
if id_key not in kwargs:
return {'message': 'you must specify an issue ID'}, 404
issue_id = kwargs[id_key]
try:
issue = backend.get(Issue, {'pk': issue_id}, include=include)
except Issue.DoesNotExist as e:
return {
'message': e.message if e.message else 'invalid issue'
}, 404
#we make sure the issue belongs to the project for which the user is authenticated
if issue.project != request.project:
return {'message': 'access denied'}, 403
request.issue = issue
return f(*args, **kwargs)
def decorated_function(*args, **kwargs):
if id_key not in kwargs:
return {'message': 'you must specify an issue class ID'}, 404
issue_class_id = kwargs[id_key]
try:
issue_class = backend.get(
IssueClass,
{'$or': [{
'pk': issue_class_id
}, {
'code': issue_class_id
}]},
include=include)
except IssueClass.DoesNotExist as e:
return {
'message': e.message if e.message else 'invalid issue class'
}, 404
request.issue_class = issue_class
return f(*args, **kwargs)
def reset_project(project_id, task_id=None):
if not task_id:
task_id = reset_project.request.id
try:
project = backend.get(Project, {'pk': project_id})
except Project.DoesNotExist:
logger.warning(
"Project %s does not exist and thus cannot be deleted." %
project_id)
return
try:
with ExclusiveTask(
backend, {
'project.pk': project.pk,
'type': {
'$in': ['analysis', 'reset', 'delete']
}
}, {
'type': 'reset',
'project': project
}, task_id) as reset_task:
with TaskLogger(reset_task, backend=backend, ping=True):
_reset_project(project)
except ExclusiveTask.LockError:
# We were unable to acquire a lock for this project.
logger.info(
"Project %s (%s) is currently being processed, aborting..." %
(project.name, project.pk))
with backend.transaction():
backend.update(project, {
'last_reset': {
'dt': datetime.datetime.now(),
'status': 'blocked'
}
})
finally:
logger.info("Done.")
def analyze_project(project_id, task_id=None):
""" Performs an analysis of the project with the given id
:param project_id: primary key of the project
:param task_id: celery task id
:return:
"""
if not task_id:
task_id = analyze_project.request.id
try:
project = backend.get(Project, {'pk': project_id})
except Project.DoesNotExist:
logger.error("Project {} does not exist".format(project_id))
return
try:
with ExclusiveTask(
backend,
{
'project': project,
'type': {
'$in':
(Task.Type.analysis, Task.Type.reset, Task.Type.delete)
}
},
{
'project': project,
'type': Task.Type.analysis
},
task_id,
) as analysis_task:
with TaskLogger(analysis_task, backend=backend, ping=True):
_analyze_project(project)
except ExclusiveTask.LockError:
# We were unable to acquire a lock for this project.
pass
finally:
logger.info("Done.")
def post(self, project_id, role, user_id):
with backend.transaction():
try:
user = self._get_user(user_id)
except AttributeError:
return ({'message': 'invalid user'},
404)
if role not in ('admin', 'collaborator', 'owner'):
return ({'message': 'invalid role: %s' % role},
403)
try:
user_role = backend.get(UserRole, {'project': request.project,
'user': user})
if user_role.role == 'owner' and user_role.user == request.user and role != 'owner':
if len(backend.filter(UserRole, {'project': request.project,'role' : 'owner'})) == 1:
return {'message' : 'You are the last owner of this project, cannot remove you.'}, 400
except UserRole.DoesNotExist:
user_role = UserRole({'project': request.project, 'user': user})
user_role.role = role
backend.save(user_role)
return ProjectRoles.get(self, project_id=project_id)
def decorated_function(*args, **kwargs):
if id_key not in kwargs or kwargs[id_key] is None:
request.project = None
if not optional:
return {'message': 'no project was specified'}, 404
return f(*args, **kwargs)
try:
project_id = kwargs[id_key]
project = backend.get(
Project,
{'$or': [{
'pk': project_id
}, {
'permalink': project_id
}]},
raw=raw,
only=only,
include=include)
if project.get('delete', False):
return {'message': 'project marked for deletion'}, 422
# We get all organizations where the user is an owner
if not private_ok and not (public_ok
and project.get('public', False)):
if request.user is None or not project.is_authorized(
request.user, roles=roles):
return {'message': 'Authorization denied'}, 403
except Project.DoesNotExist:
return {'message': 'Invalid project'}, 404
request.project = project
return f(*args, **kwargs)
def get_file_revision(query):
return backend.get(FileRevision, query, raw=True)
def _find_project(name_or_pk):
try:
return backend.get(Project,{'$or' : [{'name' : name_or_pk}, {'pk' : name_or_pk}]})
except Project.DoesNotExist:
raise click.ClickException("Project does not exist!")
def post(self):
form = SignupForm(request.form)
if form.validate():
with backend.transaction():
email_matches = backend.filter(User, {'email': form.email.data})
if len(email_matches) > 0:
for user in email_matches:
if user.delete is True:
return ({'message': 'Your account is scheduled for deletion. Try again in a few minutes".'},
403)
return {'message': "A user with this e-mail address already exists. "
"Please try resetting your password.",
'resetPasswordLink': True}, 403
try:
user = backend.get(User, {'name': form.name.data.lower()})
return {'errors': {'name': {'message': 'This login has already been chosen by another user'}},
'message': 'Login already in use.'}, 403
except User.DoesNotExist:
pass
except User.MultipleDocumentsReturned:
return {'errors': {'name': {'message': 'This login has already been chosen by another user'}},
'message': 'Login already in use.'}, 403
user = User({
'email': form.email.data.lower(),
'name': form.name.data.lower(),
'email_validated': False,
'email_validation_code': uuid.uuid4().hex,
'terms_accepted': form.terms.data,
'terms_accepted_at': datetime.datetime.utcnow(),
'terms_accepted_from_ip': request.remote_addr,
'email_settings': {
'newsletter': True,
'notifications': True,
},
})
user.set_password(form.password.data)
backend.save(user)
access_token = AccessToken({'user': user,
'token': uuid.uuid4().hex})
backend.save(access_token)
user_profile = UserProfile.export(user)
response = self.make_response({
'access_token': access_token.token,
'message': 'Success!',
'user': user_profile,
})
response.set_cookie(
'access_token',
value=access_token.token,
expires=(datetime.datetime.utcnow() + datetime.timedelta(days=7)),
)
activation_url = u"{}{}/user/validate/{}".format(
settings.get('url'),
settings.get('frontend.url'),
user.email_validation_code,
)
# Activate email
send_mail(
email_to=user.email,
template="verify_email",
template_context={
"user_name": user.name,
"activation_url": activation_url,
})
logger.warning("Hooray, a new user has just signed up: %s" % user.name)
return response
return {'message': 'Invalid data', 'errors': form.errors}, 403
