def auto_tag(project):
""" First step of the analysis for Git projects. Determines if automatic tags have already been added and, if not
adds them from the requirements.txt file, if one exists in the repository.
:param project: project to auto tag
:return:
"""
logger.debug("Adding automatic tags to {}".format(project.pk))
repository = project.git.eager.repository
# auto tag the project (but only once)
if project.get('automatic_tags_added'):
return
default_branch = project.git.get_default_branch()
if default_branch is None:
return
tags = extract_tags_from_requirements_txt(repository, default_branch)
with backend.transaction():
for tag_name in tags:
try:
tag = backend.get(Tag, {'name': tag_name})
except Tag.DoesNotExist:
tag = Tag({'name': tag_name})
backend.save(tag)
project.tags.append(tag)
project.automatic_tags_added = True
with backend.transaction():
backend.update(project, ['automatic_tags_added'])
def _import_issue_classes():
issue_update_keys = ('severity', 'description', 'title')
analyzer_update_keys = ('language', )
logger.info("Importing issue classes from checkmate...")
analyzers = settings.checkmate_settings.analyzers
for analyzer, params in analyzers.items():
logger.info("Importing issue classes for analyzer {}".format(analyzer))
if not all([key in params for key in analyzer_update_keys]):
logger.warning(
"Skipping analyzer {} as it does not contain a '{}' field".
format(key))
continue
for code, issue_params in params.get('issues_data', {}).items():
if not all([key in issue_params for key in issue_update_keys]):
logger.warning(
"Skipping issue class for code {}, as it does not contain a '{}' field"
.format(code, key))
continue
logger.info("Importing issue class for code {}".format(code))
try:
issue_class = backend.get(IssueClass, {
'analyzer': analyzer,
'code': code
})
except IssueClass.DoesNotExist:
issue_class = IssueClass({
'analyzer': analyzer,
'code': code,
})
for key in issue_update_keys:
issue_class[key] = issue_params[key]
for key in analyzer_update_keys:
issue_class[key] = params[key]
with backend.transaction():
backend.save(issue_class)
issue_class.categories.delete()
for category in issue_params.get('categories', []):
try:
issue_category = backend.get(IssueCategory,
{'name': category})
except IssueCategory.DoesNotExist:
issue_category = IssueCategory({'name': category})
with backend.transaction():
backend.save(issue_category)
issue_class.categories.append(issue_category)
for tag_name in issue_params.get('tags', ['generic']):
try:
tag = backend.get(Tag, {'name': tag_name})
except Tag.DoesNotExist:
tag = Tag({'name': tag_name})
with backend.transaction():
backend.save(tag)
issue_class.tags.append(tag)
with backend.transaction():
backend.save(issue_class)
def post(self):
form = LoginForm(request.form)
if form.validate():
with backend.transaction():
try:
# TODO manually specifying includes is not ideal
user = backend.get(User, {'email': form.email.data.lower()},
include=UserProfile.includes)
if user.delete is True:
return {'message': "Your account is scheduled for deletion. "
"You can sign-up again in a few minutes."}, 403
if not user.check_password(form.password.data):
return ({'message': 'Invalid password'},
403)
access_token = user.get_access_token()
backend.save(access_token)
user_profile = UserProfile.export(user)
response = self.make_response({
'access_token': access_token.token,
'message': 'Success!',
'user': user_profile,
})
expires = (datetime.datetime.now() + datetime.timedelta(days=7)) if form.remember_me.data else None
response.set_cookie('access_token', value=access_token.token, expires=expires)
return response
except User.DoesNotExist:
return {'message': 'Unknown user'}, 404
return {'message': 'Invalid data', 'errors': form.errors}, 403
def main(plugin_name):
settings.initialize(backend)
with backend.transaction():
if context.is_offline_mode():
run_migrations_offline(plugin_name)
else:
run_migrations_online(plugin_name)
def get(self, email_validation_code):
with backend.transaction():
try:
user = backend.get(
User,
{'email_validation_code': email_validation_code}
)
if not user.get('new_email'):
if not user.get('email'):
return {'message': 'no valid e-mail set'}, 403
backend.update(user, {'email_validated': True}, unset_fields=['email_validation_code'])
settings.hooks.call("user.email.validated", user)
logger.warning("Hooray, user {0} has verified his email address".format(user.name))
else:
if backend.filter(User, {'email': user.new_email}):
return {'message': 'A user with this e-mail already exists'}, 403
old_email = user.email
new_email = user.new_email
backend.update(user, {'email_validated': True, 'email': user.new_email},
unset_fields=['new_email', 'email_validation_code'])
settings.hooks.call("user.email.updated", user)
except User.DoesNotExist:
return {'message': 'Unknown validation code'}, 404
return {'message': 'success'}, 200
def generate_key_pair(project):
""" Generates a new SSH key pair for the given remote in the given project.
Effect: saves the public and private keys on the appropriate remote in the project object
:param project: project to generate the key pair for
"""
tempdir = tempfile.mkdtemp()
try:
subprocess.call([
"ssh-keygen", "-t", "rsa", "-q", "-f", "testkey", "-N", "", "-C",
project.pk
],
cwd=tempdir)
with open(os.path.join(tempdir, "testkey"), 'r') as private_key_file:
private_key = private_key_file.read()
with open(os.path.join(tempdir, "testkey.pub"),
'r') as public_key_file:
public_key = public_key_file.read()
with backend.transaction():
backend.update(project.git, {
'private_key': private_key,
'public_key': public_key
})
finally:
shutil.rmtree(tempdir)
def create_project(project_data, git_data, user):
"""
Creates a new project with the given data and performs the necessary initialization steps.
:param data: data to use to create the project
:return: object representing the newly created project
"""
project = Project(project_data)
project.pk = uuid.uuid4().hex
settings.hooks.call("project.create.before", project)
if not project.get('permalink'):
project.permalink = project.pk
git = GitRepository(git_data)
git.project = project
project.git = git
#we make the user owner of the project
user_role = UserRole({'project': project, 'user': user, 'role': 'owner'})
with backend.transaction():
backend.save(project)
backend.save(git)
backend.save(user_role)
#we run git-specific initialization tasks
generate_key_pair(project)
initialize_repository(project)
update_remote(project)
settings.hooks.call("project.create.after", project)
return project
def post(self, password_reset_code):
form = PasswordResetForm(request.form)
if form.validate():
with backend.transaction():
try:
user = backend.get(
User,
{'password_reset_code': password_reset_code}
)
except User.DoesNotExist:
return {'message': 'Unknown user'}, 404
user.set_password(form.password.data)
backend.update(user, ['password', 'password_set'], unset_fields=['password_reset_code'])
access_token = user.get_access_token()
backend.save(access_token)
send_mail(
email_to=user.email,
template="password_reset_successful",
template_context={
"user_name": user.name
}
)
return {'message': 'success'}, 200
return ({'message': 'Invalid data',
'errors': form.errors},
403)
def post(self):
form = PasswordResetRequestForm(request.form)
if form.validate():
with backend.transaction():
try:
user = backend.get(User, {'email': form.email.data})
if 'email_validated' in user and user.email_validated:
backend.update(
user,
{'password_reset_code': uuid.uuid4().hex}
)
reset_url = "{}{}/user/password-reset?reset_code={}".format(
settings.get('url'),
settings.get('frontend.url'),
user.password_reset_code
)
# Reset Email
send_mail(
email_to=form.email.data,
template="reset_password",
template_context={
"user_name": user.name,
"reset_url": reset_url
}
)
return {'message': 'Email with reset link was sent.'}, 200
return {'message': "Your email is not validated, so we cannot send you "
"a password-reset token."}, 403
except User.DoesNotExist:
return {'message': 'Unknown user'}, 404
return {'message': 'Invalid data', 'errors': form.errors}, 403
def add_relevant_issue_classes(project):
""" Adds relevant issue classes to the given project. Only adds issue classes that were created after the last time
this function was ran, avoiding re-adding issue classes that the user may have removed from the project.
:param project: project to add the issue classes to
:return:
"""
issue_classes = get_relevant_issue_classes(project)
logger.info("Adding %d issue_classes to project %s" %
(len(issue_classes), project.pk))
with backend.transaction():
for issue_class in issue_classes:
project_issue_class_query = {
'project': project,
'issue_class': issue_class
}
try:
backend.get(ProjectIssueClass, project_issue_class_query)
continue
except ProjectIssueClass.DoesNotExist:
project_issue_class = ProjectIssueClass(
project_issue_class_query)
project_issue_class.enabled = True
backend.save(project_issue_class)
# re-sync the issue classes associated with the project with the database
project.revert()
def get_diff_issue_stats(diffs):
"""
Gets stats for the diffs returned by the given queryset. Determines which commit shas are being diffed,
and issues added and fixed between sha a and sha b.
To do: move this to checkmate.
:param diffs: queryset which specifies the diffs
:return: list of objects with pk, issues_added, issues_fixed, sha_a, sha_b keys
"""
project = request.project
diff_table = diffs.as_table()
diff_issue_occurrence_table = backend.get_table(DiffIssueOccurrence)
# only select issues whose issue classes are enabled for the project
issue_class_table = project.get_issue_classes(enabled=True, backend=backend, only=('analyzer', 'code')).as_table()
issue_occurrence_table = backend.get_table(IssueOccurrence)
issue_table = backend.get_table(Issue)
git_snapshot_table = backend.get_table(GitSnapshot)
git_snapshot_table_a = git_snapshot_table.alias()
git_snapshot_table_b = git_snapshot_table.alias()
table = (diff_table
.join(diff_issue_occurrence_table, diff_issue_occurrence_table.c.diff == diff_table.c.pk)
.join(issue_occurrence_table)
.join(issue_table)
.join(issue_class_table, and_(issue_table.c.analyzer == issue_class_table.c.analyzer,
issue_table.c.code == issue_class_table.c.code,
issue_table.c.ignore == False))
.join(git_snapshot_table_a, git_snapshot_table_a.c.snapshot == diff_table.c.snapshot_a)
.join(git_snapshot_table_b, git_snapshot_table_b.c.snapshot == diff_table.c.snapshot_b))
sha_a = git_snapshot_table_a.c.sha.label('sha_a')
sha_b = git_snapshot_table_b.c.sha.label('sha_b')
s = (select([sha_a, sha_b, diff_table.c.pk, diff_issue_occurrence_table.c.key, func.count('*').label('cnt')])
.select_from(table)
.group_by(diff_issue_occurrence_table.c.key, diff_table.c.pk, sha_a, sha_b))
with backend.transaction():
results = backend.connection.execute(s).fetchall()
diffs_by_pk = {}
for row in results:
if row['pk'] not in diffs_by_pk:
diffs_by_pk[row['pk']] = {'pk': row['pk']}
diffs_by_pk[row['pk']]['issues_{}'.format(row['key'])] = row['cnt']
diffs_by_pk[row['pk']]['sha_a'] = row['sha_a']
diffs_by_pk[row['pk']]['sha_b'] = row['sha_b']
for diff in diffs_by_pk.values():
for key in ('issues_fixed', 'issues_added'):
if key not in diff:
diff[key] = 0
return diffs_by_pk.values()
def delete(self, project_id):
with backend.transaction():
backend.update(
request.project,
{'reset': True,
'reset_requested_at': datetime.datetime.now()})
return ({'message': 'Success, project scheduled for reset. Please be patient.'},
200)
def delete(self, project_id, issue_class_id):
"""
Delete an issue class from a project.
"""
with backend.transaction():
project_issue_class = self._get_project_issue_class()
backend.update(project_issue_class, {'enabled': False})
return {'message': 'success'}, 200
def post(self):
if request.access_token is None:
return {'message': 'already logged out'}, 200
with backend.transaction():
backend.delete(request.access_token)
response = self.make_response({'message': 'success'})
response.set_cookie('access_token', value='', expires=0)
return response
def decorated_function(*args, **kwargs):
def process_anonymously():
request.user = None
request.access_token = None
return f(*args, **kwargs)
def invalid_token(message, status_code=401, cookie_token=False):
response = jsonify({'message': message})
if cookie_token:
response.set_cookie('access_token', '', expires=0)
return response, status_code
cookie_token = False
if request.args.get('access_token'):
access_token_key = request.args['access_token']
elif request.cookies.get('access_token'):
access_token_key = request.cookies['access_token']
cookie_token = True
else:
authorization = request.headers.get('Authorization', '')
match = re.match(r"^bearer\s+([\w\d]+)$", authorization, re.I)
if not match:
if anon_ok:
return process_anonymously()
return {'message': 'Authorization header not valid'}, 401
access_token_key = match.group(1)
try:
access_token = backend.get(AccessToken,
{'token': access_token_key})
except AccessToken.DoesNotExist:
if anon_ok:
return process_anonymously()
# redirect to login
return invalid_token('Invalid / expired access token: %s' %
access_token_key,
cookie_token=cookie_token)
request.access_token = access_token
try:
request.user = backend.get(User, {'pk': access_token.user['pk']},
raw=raw,
only=only,
include=include)
except User.DoesNotExist:
with backend.transaction():
backend.delete(access_token)
return invalid_token('User does not exist',
status_code=404,
cookie_token=cookie_token)
if superuser and not request.user.is_superuser():
return {
'message':
'This endpoint requires super-user privileges. Sorry :/'
}, 401
return f(*args, **kwargs)
def fetch_remote(project,
branch=None,
git_config=None,
git_credentials=None,
report_error=True):
""" Fetches the remote for the given project
:param project:
:param git_config:
:param branch:
:param git_credentials:
:param report_error:
:return:
"""
repository = project.git.eager.repository
remote_name = "origin"
# do not fetch the directory directly but instead use a copy.
# This mitigates the risk of being unable to serve a simultaneous request
# which relies on the git repository, f.e. to get a list of branches.
tmp_repo_path = repository.path + "_temp"
if os.path.exists(tmp_repo_path):
# we should have an exclusive task log on the project, so deleting the temporary copy is ok.
shutil.rmtree(tmp_repo_path)
try:
shutil.copytree(repository.path, tmp_repo_path)
tmp_repository = Repository(tmp_repo_path)
tmp_repository.update_remote_url(remote_name, project.git.url)
# actually fetch the repository
with tempfile.NamedTemporaryFile(delete=False) as tf:
tf.write(project.git.private_key)
tf.close()
rc = tmp_repository.fetch(remote_name,
branch=branch,
ssh_identity_file=tf.name,
git_config=git_config,
git_credentials=git_credentials)
# move the repository back if the fetch was successful
if rc == 0:
shutil.rmtree(repository.path)
shutil.move(tmp_repo_path, repository.path)
finally:
if os.path.isdir(tmp_repo_path):
shutil.rmtree(tmp_repo_path)
with backend.transaction():
backend.update(
project, {
'fetch_status': 'failed' if rc != 0 else 'succeeded',
'fetched_at': datetime.datetime.utcnow(),
'fetch_error': '' if rc == 0 else repository.stderr
})
if rc != 0:
raise IOError("Cannot fetch git repository!")
def put(self):
form = UserProfileForm(request.form)
if not form.validate():
return {u'errors': form.errors}, 403
user = request.user
data = form.data
email = data.get(u'email')
if email:
if (user.get('email_change_requested_at') and
datetime.datetime.utcnow() - user.email_change_requested_at < datetime.timedelta(minutes=30)):
return {'message': "Please wait at least 30 minutes before requesting another e-mail change."}, 403
with backend.transaction():
backend.update(user, {'new_email': email,
'email_validation_code': uuid.uuid4().hex,
'email_change_requested_at': datetime.datetime.utcnow()})
activation_url = "{}{}/user/validate/{}".format(
settings.get('url'),
settings.get('frontend.url'),
request.user.email_validation_code
)
# activate email
send_mail(
email_to=user.new_email,
template="verify_email",
template_context={
"user_name": user.name,
"activation_url": activation_url
}
)
email_settings = data.get(u'email_settings')
with backend.transaction():
if email_settings:
email_settings = user.get('email_settings', {})
email_settings.update(data[u'email_settings'])
user.email_settings = email_settings
backend.update(user, ['email_settings'])
return {'user': self.export(user)}, 200
def project(test, fixtures, name):
project = Project({
'name': name,
'permalink': 'test:{}'.format(name),
'source': 'test'
})
with backend.transaction():
backend.save(project)
return project
def delete(self):
"""
Marks a user for deletion (will be done in a backend task)
"""
logger.warning("Ouch, we lost user {0}".format(request.user.name))
with backend.transaction():
backend.update(request.user, {'delete': True})
return ({'message': 'We\'re sad to see you leave! Your account will be fully deleted within a few minutes.',
'user_id': request.user.pk},
200)
def user(
test,
fixtures,
name,
email,
password,
superuser=False,
):
user = User({'name': name, 'email': email})
user.set_password(password)
user.unencrypted_password = password
with backend.transaction():
backend.save(user)
access_token = AccessToken({'user': user, 'token': uuid.uuid4().hex})
with backend.transaction():
backend.save(access_token)
return user
def _schedule_analysis(project_id, analysis_priority=Project.AnalysisPriority.high):
if not (request.project.get('analyze') and request.project.get('analysis_priority', Project.AnalysisPriority.low) >= analysis_priority):
with backend.transaction():
backend.update(request.project,
{'analyze': True,
'analysis_requested_at': datetime.datetime.now(),
'analysis_priority': analysis_priority})
return ({'message': 'Success, project scheduled for analysis. Please be patient.'},
200)
else:
return ({'message': 'Project was already scheduled for analysis. Please be patient.'},
200)
def update_analysis_status(project, status, extra=None):
""" Updates the analysis status of the given project to the given status.
:param project: project to update the analysis status for
:param status: new analysis status
:param extra: dictionary with extra data to update
"""
extra = extra if extra is not None else {}
with backend.transaction():
data = extra.copy()
data['analyzed_at'] = datetime.datetime.now()
data['analysis_status'] = status
backend.update(project, data)
def post(self, project_id, issue_class_id):
"""
Add a new issue class to a project
"""
with backend.transaction():
project_issue_class = self._get_project_issue_class()
if project_issue_class.pk:
backend.update(project_issue_class, {'enabled': True})
else:
project_issue_class.enabled = True
backend.save(project_issue_class)
return {'message': 'success'}, 201
def _reset_project(project):
try:
settings.hooks.call("project.reset.before", project)
reset_command = ResetCommand(project, settings.checkmate_settings,
backend)
reset_command.run()
with backend.transaction():
backend.update(project, {
'last_reset': {
'dt': datetime.datetime.now(),
'status': 'succeeded'
},
'analyze': True,
'analysis_priority': Project.AnalysisPriority.high,
'analysis_requested_at': datetime.datetime.now()
},
unset_fields=["first_analysis_email_sent"])
settings.hooks.call("project.reset.after", project)
except:
with backend.transaction():
backend.update(
project, {
'last_reset': {
'dt': datetime.datetime.now(),
'status': 'failed'
},
})
logger.error(
"Reset of project {project.name} (pk={project.pk}) failed!".format(
project=project))
raise
finally:
with backend.transaction():
backend.update(project, {'reset': False},
unset_fields=['reset_requested_at'])
def reset_pending_project():
try:
pending_project = backend.filter(Project, {
'reset': True
}).sort('reset_requested_at', 1).limit(1)[0]
except IndexError:
logger.debug("No projects left to reset....")
return
with backend.transaction():
backend.update(pending_project,
{'reset_requested_at': datetime.datetime.now()})
return reset_project(pending_project.pk,
task_id=reset_pending_project.request.id)
def delete(self, project_id, user_role_id):
with backend.transaction():
try:
user_role = backend.get(UserRole, {'project': request.project, 'pk': user_role_id})
if user_role.role == 'owner' and user_role.user == request.user:
if len(backend.filter(UserRole, {'project': request.project,'role' : 'owner'})) == 1:
return {'message' : 'You are the last owner of this project, cannot remove you.'}, 400
except UserRole.DoesNotExist:
return {'message': 'invalid role'}, 404
backend.delete(user_role)
return {'message': 'success'}, 200
def update_project_statistics(project):
""" After analysis hook. Updates the project statistics
:param project: project that is being analyzed
"""
checkmate_settings = settings.checkmate_settings
update_stats_command = UpdateStatsCommand(project, checkmate_settings,
backend)
update_stats_command.run()
if 'stats' in project:
with backend.transaction():
backend.update(project, ['stats'])
def put(self):
""" Enable or disable email notifications
:return: message with status, status code
"""
user = request.user
form = EmailNotificationsForm(request.form)
if form.validate():
with backend.transaction():
email_settings = user.get('email_settings', {})
email_settings.update({'notifications': form.email_notifications_enabled.data})
user.email_settings = email_settings
backend.update(user, ["email_settings"])
return {'message': "success"}, 200
return {'message': "error", 'errors': form.errors}, 400
def put(self, project_id, issue_id):
form = IssueStatusForm(request.form)
if not form.validate():
return {
'message' : 'Please correct the errors mentioned below.',
'errors' : form.errors
}, 400
with backend.transaction():
backend.update(request.issue,form.data)
return {'message' : 'success'}, 200
def delete_user(user_id, task_id=None):
"""
What we need to do here:
-Remove all user data from database, unless user was customer
- Projects
- User profile
- AccessToken
"""
if not task_id:
task_id = delete_user.request.id
try:
user = backend.get(User, {'pk': user_id})
except User.DoesNotExist:
logger.error(
"User {} does not exist! Cannot delete it.".format(user_id))
return
try:
with ExclusiveTask(backend,
{'type': {
'$in': ['delete_{}'.format(user.pk)]
}}, {'type': 'delete_{}'.format(user.pk)},
task_id,
no_update_on_exit=True) as delete_task:
with TaskLogger(delete_task, backend=backend, ping=True):
with backend.transaction():
logger.debug("Starting deletion of user {0} ({1}).".format(
user.name, user.pk))
# Delete all related models
backend.filter(UserRole, {'user': user}).delete()
backend.filter(AccessToken, {'user': user}).delete()
backend.filter(User, {'pk': user.pk}).delete()
# Todo: Delete all of the user's projects that have no "owner" user roles anymore...
logger.info("Deletion of user {0} ({1}) complete!".format(
user.name, user.pk))
except ExclusiveTask.LockError:
pass
except BaseException as err:
logger.error("Error {0}: Can't delete user {1}.".format(
err.__class__.__name__, user))
logger.error(traceback.format_exc())
