def auto_tag(project): """ First step of the analysis for Git projects. Determines if automatic tags have already been added and, if not adds them from the requirements.txt file, if one exists in the repository. :param project: project to auto tag :return: """ logger.debug("Adding automatic tags to {}".format(project.pk)) repository = project.git.eager.repository # auto tag the project (but only once) if project.get('automatic_tags_added'): return default_branch = project.git.get_default_branch() if default_branch is None: return tags = extract_tags_from_requirements_txt(repository, default_branch) with backend.transaction(): for tag_name in tags: try: tag = backend.get(Tag, {'name': tag_name}) except Tag.DoesNotExist: tag = Tag({'name': tag_name}) backend.save(tag) project.tags.append(tag) project.automatic_tags_added = True with backend.transaction(): backend.update(project, ['automatic_tags_added'])
def _import_issue_classes(): issue_update_keys = ('severity', 'description', 'title') analyzer_update_keys = ('language', ) logger.info("Importing issue classes from checkmate...") analyzers = settings.checkmate_settings.analyzers for analyzer, params in analyzers.items(): logger.info("Importing issue classes for analyzer {}".format(analyzer)) if not all([key in params for key in analyzer_update_keys]): logger.warning( "Skipping analyzer {} as it does not contain a '{}' field". format(key)) continue for code, issue_params in params.get('issues_data', {}).items(): if not all([key in issue_params for key in issue_update_keys]): logger.warning( "Skipping issue class for code {}, as it does not contain a '{}' field" .format(code, key)) continue logger.info("Importing issue class for code {}".format(code)) try: issue_class = backend.get(IssueClass, { 'analyzer': analyzer, 'code': code }) except IssueClass.DoesNotExist: issue_class = IssueClass({ 'analyzer': analyzer, 'code': code, }) for key in issue_update_keys: issue_class[key] = issue_params[key] for key in analyzer_update_keys: issue_class[key] = params[key] with backend.transaction(): backend.save(issue_class) issue_class.categories.delete() for category in issue_params.get('categories', []): try: issue_category = backend.get(IssueCategory, {'name': category}) except IssueCategory.DoesNotExist: issue_category = IssueCategory({'name': category}) with backend.transaction(): backend.save(issue_category) issue_class.categories.append(issue_category) for tag_name in issue_params.get('tags', ['generic']): try: tag = backend.get(Tag, {'name': tag_name}) except Tag.DoesNotExist: tag = Tag({'name': tag_name}) with backend.transaction(): backend.save(tag) issue_class.tags.append(tag) with backend.transaction(): backend.save(issue_class)
def post(self): form = LoginForm(request.form) if form.validate(): with backend.transaction(): try: # TODO manually specifying includes is not ideal user = backend.get(User, {'email': form.email.data.lower()}, include=UserProfile.includes) if user.delete is True: return {'message': "Your account is scheduled for deletion. " "You can sign-up again in a few minutes."}, 403 if not user.check_password(form.password.data): return ({'message': 'Invalid password'}, 403) access_token = user.get_access_token() backend.save(access_token) user_profile = UserProfile.export(user) response = self.make_response({ 'access_token': access_token.token, 'message': 'Success!', 'user': user_profile, }) expires = (datetime.datetime.now() + datetime.timedelta(days=7)) if form.remember_me.data else None response.set_cookie('access_token', value=access_token.token, expires=expires) return response except User.DoesNotExist: return {'message': 'Unknown user'}, 404 return {'message': 'Invalid data', 'errors': form.errors}, 403
def main(plugin_name): settings.initialize(backend) with backend.transaction(): if context.is_offline_mode(): run_migrations_offline(plugin_name) else: run_migrations_online(plugin_name)
def get(self, email_validation_code): with backend.transaction(): try: user = backend.get( User, {'email_validation_code': email_validation_code} ) if not user.get('new_email'): if not user.get('email'): return {'message': 'no valid e-mail set'}, 403 backend.update(user, {'email_validated': True}, unset_fields=['email_validation_code']) settings.hooks.call("user.email.validated", user) logger.warning("Hooray, user {0} has verified his email address".format(user.name)) else: if backend.filter(User, {'email': user.new_email}): return {'message': 'A user with this e-mail already exists'}, 403 old_email = user.email new_email = user.new_email backend.update(user, {'email_validated': True, 'email': user.new_email}, unset_fields=['new_email', 'email_validation_code']) settings.hooks.call("user.email.updated", user) except User.DoesNotExist: return {'message': 'Unknown validation code'}, 404 return {'message': 'success'}, 200
def generate_key_pair(project): """ Generates a new SSH key pair for the given remote in the given project. Effect: saves the public and private keys on the appropriate remote in the project object :param project: project to generate the key pair for """ tempdir = tempfile.mkdtemp() try: subprocess.call([ "ssh-keygen", "-t", "rsa", "-q", "-f", "testkey", "-N", "", "-C", project.pk ], cwd=tempdir) with open(os.path.join(tempdir, "testkey"), 'r') as private_key_file: private_key = private_key_file.read() with open(os.path.join(tempdir, "testkey.pub"), 'r') as public_key_file: public_key = public_key_file.read() with backend.transaction(): backend.update(project.git, { 'private_key': private_key, 'public_key': public_key }) finally: shutil.rmtree(tempdir)
def create_project(project_data, git_data, user): """ Creates a new project with the given data and performs the necessary initialization steps. :param data: data to use to create the project :return: object representing the newly created project """ project = Project(project_data) project.pk = uuid.uuid4().hex settings.hooks.call("project.create.before", project) if not project.get('permalink'): project.permalink = project.pk git = GitRepository(git_data) git.project = project project.git = git #we make the user owner of the project user_role = UserRole({'project': project, 'user': user, 'role': 'owner'}) with backend.transaction(): backend.save(project) backend.save(git) backend.save(user_role) #we run git-specific initialization tasks generate_key_pair(project) initialize_repository(project) update_remote(project) settings.hooks.call("project.create.after", project) return project
def post(self, password_reset_code): form = PasswordResetForm(request.form) if form.validate(): with backend.transaction(): try: user = backend.get( User, {'password_reset_code': password_reset_code} ) except User.DoesNotExist: return {'message': 'Unknown user'}, 404 user.set_password(form.password.data) backend.update(user, ['password', 'password_set'], unset_fields=['password_reset_code']) access_token = user.get_access_token() backend.save(access_token) send_mail( email_to=user.email, template="password_reset_successful", template_context={ "user_name": user.name } ) return {'message': 'success'}, 200 return ({'message': 'Invalid data', 'errors': form.errors}, 403)
def post(self): form = PasswordResetRequestForm(request.form) if form.validate(): with backend.transaction(): try: user = backend.get(User, {'email': form.email.data}) if 'email_validated' in user and user.email_validated: backend.update( user, {'password_reset_code': uuid.uuid4().hex} ) reset_url = "{}{}/user/password-reset?reset_code={}".format( settings.get('url'), settings.get('frontend.url'), user.password_reset_code ) # Reset Email send_mail( email_to=form.email.data, template="reset_password", template_context={ "user_name": user.name, "reset_url": reset_url } ) return {'message': 'Email with reset link was sent.'}, 200 return {'message': "Your email is not validated, so we cannot send you " "a password-reset token."}, 403 except User.DoesNotExist: return {'message': 'Unknown user'}, 404 return {'message': 'Invalid data', 'errors': form.errors}, 403
def add_relevant_issue_classes(project): """ Adds relevant issue classes to the given project. Only adds issue classes that were created after the last time this function was ran, avoiding re-adding issue classes that the user may have removed from the project. :param project: project to add the issue classes to :return: """ issue_classes = get_relevant_issue_classes(project) logger.info("Adding %d issue_classes to project %s" % (len(issue_classes), project.pk)) with backend.transaction(): for issue_class in issue_classes: project_issue_class_query = { 'project': project, 'issue_class': issue_class } try: backend.get(ProjectIssueClass, project_issue_class_query) continue except ProjectIssueClass.DoesNotExist: project_issue_class = ProjectIssueClass( project_issue_class_query) project_issue_class.enabled = True backend.save(project_issue_class) # re-sync the issue classes associated with the project with the database project.revert()
def get_diff_issue_stats(diffs): """ Gets stats for the diffs returned by the given queryset. Determines which commit shas are being diffed, and issues added and fixed between sha a and sha b. To do: move this to checkmate. :param diffs: queryset which specifies the diffs :return: list of objects with pk, issues_added, issues_fixed, sha_a, sha_b keys """ project = request.project diff_table = diffs.as_table() diff_issue_occurrence_table = backend.get_table(DiffIssueOccurrence) # only select issues whose issue classes are enabled for the project issue_class_table = project.get_issue_classes(enabled=True, backend=backend, only=('analyzer', 'code')).as_table() issue_occurrence_table = backend.get_table(IssueOccurrence) issue_table = backend.get_table(Issue) git_snapshot_table = backend.get_table(GitSnapshot) git_snapshot_table_a = git_snapshot_table.alias() git_snapshot_table_b = git_snapshot_table.alias() table = (diff_table .join(diff_issue_occurrence_table, diff_issue_occurrence_table.c.diff == diff_table.c.pk) .join(issue_occurrence_table) .join(issue_table) .join(issue_class_table, and_(issue_table.c.analyzer == issue_class_table.c.analyzer, issue_table.c.code == issue_class_table.c.code, issue_table.c.ignore == False)) .join(git_snapshot_table_a, git_snapshot_table_a.c.snapshot == diff_table.c.snapshot_a) .join(git_snapshot_table_b, git_snapshot_table_b.c.snapshot == diff_table.c.snapshot_b)) sha_a = git_snapshot_table_a.c.sha.label('sha_a') sha_b = git_snapshot_table_b.c.sha.label('sha_b') s = (select([sha_a, sha_b, diff_table.c.pk, diff_issue_occurrence_table.c.key, func.count('*').label('cnt')]) .select_from(table) .group_by(diff_issue_occurrence_table.c.key, diff_table.c.pk, sha_a, sha_b)) with backend.transaction(): results = backend.connection.execute(s).fetchall() diffs_by_pk = {} for row in results: if row['pk'] not in diffs_by_pk: diffs_by_pk[row['pk']] = {'pk': row['pk']} diffs_by_pk[row['pk']]['issues_{}'.format(row['key'])] = row['cnt'] diffs_by_pk[row['pk']]['sha_a'] = row['sha_a'] diffs_by_pk[row['pk']]['sha_b'] = row['sha_b'] for diff in diffs_by_pk.values(): for key in ('issues_fixed', 'issues_added'): if key not in diff: diff[key] = 0 return diffs_by_pk.values()
def delete(self, project_id): with backend.transaction(): backend.update( request.project, {'reset': True, 'reset_requested_at': datetime.datetime.now()}) return ({'message': 'Success, project scheduled for reset. Please be patient.'}, 200)
def delete(self, project_id, issue_class_id): """ Delete an issue class from a project. """ with backend.transaction(): project_issue_class = self._get_project_issue_class() backend.update(project_issue_class, {'enabled': False}) return {'message': 'success'}, 200
def post(self): if request.access_token is None: return {'message': 'already logged out'}, 200 with backend.transaction(): backend.delete(request.access_token) response = self.make_response({'message': 'success'}) response.set_cookie('access_token', value='', expires=0) return response
def decorated_function(*args, **kwargs): def process_anonymously(): request.user = None request.access_token = None return f(*args, **kwargs) def invalid_token(message, status_code=401, cookie_token=False): response = jsonify({'message': message}) if cookie_token: response.set_cookie('access_token', '', expires=0) return response, status_code cookie_token = False if request.args.get('access_token'): access_token_key = request.args['access_token'] elif request.cookies.get('access_token'): access_token_key = request.cookies['access_token'] cookie_token = True else: authorization = request.headers.get('Authorization', '') match = re.match(r"^bearer\s+([\w\d]+)$", authorization, re.I) if not match: if anon_ok: return process_anonymously() return {'message': 'Authorization header not valid'}, 401 access_token_key = match.group(1) try: access_token = backend.get(AccessToken, {'token': access_token_key}) except AccessToken.DoesNotExist: if anon_ok: return process_anonymously() # redirect to login return invalid_token('Invalid / expired access token: %s' % access_token_key, cookie_token=cookie_token) request.access_token = access_token try: request.user = backend.get(User, {'pk': access_token.user['pk']}, raw=raw, only=only, include=include) except User.DoesNotExist: with backend.transaction(): backend.delete(access_token) return invalid_token('User does not exist', status_code=404, cookie_token=cookie_token) if superuser and not request.user.is_superuser(): return { 'message': 'This endpoint requires super-user privileges. Sorry :/' }, 401 return f(*args, **kwargs)
def fetch_remote(project, branch=None, git_config=None, git_credentials=None, report_error=True): """ Fetches the remote for the given project :param project: :param git_config: :param branch: :param git_credentials: :param report_error: :return: """ repository = project.git.eager.repository remote_name = "origin" # do not fetch the directory directly but instead use a copy. # This mitigates the risk of being unable to serve a simultaneous request # which relies on the git repository, f.e. to get a list of branches. tmp_repo_path = repository.path + "_temp" if os.path.exists(tmp_repo_path): # we should have an exclusive task log on the project, so deleting the temporary copy is ok. shutil.rmtree(tmp_repo_path) try: shutil.copytree(repository.path, tmp_repo_path) tmp_repository = Repository(tmp_repo_path) tmp_repository.update_remote_url(remote_name, project.git.url) # actually fetch the repository with tempfile.NamedTemporaryFile(delete=False) as tf: tf.write(project.git.private_key) tf.close() rc = tmp_repository.fetch(remote_name, branch=branch, ssh_identity_file=tf.name, git_config=git_config, git_credentials=git_credentials) # move the repository back if the fetch was successful if rc == 0: shutil.rmtree(repository.path) shutil.move(tmp_repo_path, repository.path) finally: if os.path.isdir(tmp_repo_path): shutil.rmtree(tmp_repo_path) with backend.transaction(): backend.update( project, { 'fetch_status': 'failed' if rc != 0 else 'succeeded', 'fetched_at': datetime.datetime.utcnow(), 'fetch_error': '' if rc == 0 else repository.stderr }) if rc != 0: raise IOError("Cannot fetch git repository!")
def put(self): form = UserProfileForm(request.form) if not form.validate(): return {u'errors': form.errors}, 403 user = request.user data = form.data email = data.get(u'email') if email: if (user.get('email_change_requested_at') and datetime.datetime.utcnow() - user.email_change_requested_at < datetime.timedelta(minutes=30)): return {'message': "Please wait at least 30 minutes before requesting another e-mail change."}, 403 with backend.transaction(): backend.update(user, {'new_email': email, 'email_validation_code': uuid.uuid4().hex, 'email_change_requested_at': datetime.datetime.utcnow()}) activation_url = "{}{}/user/validate/{}".format( settings.get('url'), settings.get('frontend.url'), request.user.email_validation_code ) # activate email send_mail( email_to=user.new_email, template="verify_email", template_context={ "user_name": user.name, "activation_url": activation_url } ) email_settings = data.get(u'email_settings') with backend.transaction(): if email_settings: email_settings = user.get('email_settings', {}) email_settings.update(data[u'email_settings']) user.email_settings = email_settings backend.update(user, ['email_settings']) return {'user': self.export(user)}, 200
def project(test, fixtures, name): project = Project({ 'name': name, 'permalink': 'test:{}'.format(name), 'source': 'test' }) with backend.transaction(): backend.save(project) return project
def delete(self): """ Marks a user for deletion (will be done in a backend task) """ logger.warning("Ouch, we lost user {0}".format(request.user.name)) with backend.transaction(): backend.update(request.user, {'delete': True}) return ({'message': 'We\'re sad to see you leave! Your account will be fully deleted within a few minutes.', 'user_id': request.user.pk}, 200)
def user( test, fixtures, name, email, password, superuser=False, ): user = User({'name': name, 'email': email}) user.set_password(password) user.unencrypted_password = password with backend.transaction(): backend.save(user) access_token = AccessToken({'user': user, 'token': uuid.uuid4().hex}) with backend.transaction(): backend.save(access_token) return user
def _schedule_analysis(project_id, analysis_priority=Project.AnalysisPriority.high): if not (request.project.get('analyze') and request.project.get('analysis_priority', Project.AnalysisPriority.low) >= analysis_priority): with backend.transaction(): backend.update(request.project, {'analyze': True, 'analysis_requested_at': datetime.datetime.now(), 'analysis_priority': analysis_priority}) return ({'message': 'Success, project scheduled for analysis. Please be patient.'}, 200) else: return ({'message': 'Project was already scheduled for analysis. Please be patient.'}, 200)
def update_analysis_status(project, status, extra=None): """ Updates the analysis status of the given project to the given status. :param project: project to update the analysis status for :param status: new analysis status :param extra: dictionary with extra data to update """ extra = extra if extra is not None else {} with backend.transaction(): data = extra.copy() data['analyzed_at'] = datetime.datetime.now() data['analysis_status'] = status backend.update(project, data)
def post(self, project_id, issue_class_id): """ Add a new issue class to a project """ with backend.transaction(): project_issue_class = self._get_project_issue_class() if project_issue_class.pk: backend.update(project_issue_class, {'enabled': True}) else: project_issue_class.enabled = True backend.save(project_issue_class) return {'message': 'success'}, 201
def _reset_project(project): try: settings.hooks.call("project.reset.before", project) reset_command = ResetCommand(project, settings.checkmate_settings, backend) reset_command.run() with backend.transaction(): backend.update(project, { 'last_reset': { 'dt': datetime.datetime.now(), 'status': 'succeeded' }, 'analyze': True, 'analysis_priority': Project.AnalysisPriority.high, 'analysis_requested_at': datetime.datetime.now() }, unset_fields=["first_analysis_email_sent"]) settings.hooks.call("project.reset.after", project) except: with backend.transaction(): backend.update( project, { 'last_reset': { 'dt': datetime.datetime.now(), 'status': 'failed' }, }) logger.error( "Reset of project {project.name} (pk={project.pk}) failed!".format( project=project)) raise finally: with backend.transaction(): backend.update(project, {'reset': False}, unset_fields=['reset_requested_at'])
def reset_pending_project(): try: pending_project = backend.filter(Project, { 'reset': True }).sort('reset_requested_at', 1).limit(1)[0] except IndexError: logger.debug("No projects left to reset....") return with backend.transaction(): backend.update(pending_project, {'reset_requested_at': datetime.datetime.now()}) return reset_project(pending_project.pk, task_id=reset_pending_project.request.id)
def delete(self, project_id, user_role_id): with backend.transaction(): try: user_role = backend.get(UserRole, {'project': request.project, 'pk': user_role_id}) if user_role.role == 'owner' and user_role.user == request.user: if len(backend.filter(UserRole, {'project': request.project,'role' : 'owner'})) == 1: return {'message' : 'You are the last owner of this project, cannot remove you.'}, 400 except UserRole.DoesNotExist: return {'message': 'invalid role'}, 404 backend.delete(user_role) return {'message': 'success'}, 200
def update_project_statistics(project): """ After analysis hook. Updates the project statistics :param project: project that is being analyzed """ checkmate_settings = settings.checkmate_settings update_stats_command = UpdateStatsCommand(project, checkmate_settings, backend) update_stats_command.run() if 'stats' in project: with backend.transaction(): backend.update(project, ['stats'])
def put(self): """ Enable or disable email notifications :return: message with status, status code """ user = request.user form = EmailNotificationsForm(request.form) if form.validate(): with backend.transaction(): email_settings = user.get('email_settings', {}) email_settings.update({'notifications': form.email_notifications_enabled.data}) user.email_settings = email_settings backend.update(user, ["email_settings"]) return {'message': "success"}, 200 return {'message': "error", 'errors': form.errors}, 400
def put(self, project_id, issue_id): form = IssueStatusForm(request.form) if not form.validate(): return { 'message' : 'Please correct the errors mentioned below.', 'errors' : form.errors }, 400 with backend.transaction(): backend.update(request.issue,form.data) return {'message' : 'success'}, 200
def delete_user(user_id, task_id=None): """ What we need to do here: -Remove all user data from database, unless user was customer - Projects - User profile - AccessToken """ if not task_id: task_id = delete_user.request.id try: user = backend.get(User, {'pk': user_id}) except User.DoesNotExist: logger.error( "User {} does not exist! Cannot delete it.".format(user_id)) return try: with ExclusiveTask(backend, {'type': { '$in': ['delete_{}'.format(user.pk)] }}, {'type': 'delete_{}'.format(user.pk)}, task_id, no_update_on_exit=True) as delete_task: with TaskLogger(delete_task, backend=backend, ping=True): with backend.transaction(): logger.debug("Starting deletion of user {0} ({1}).".format( user.name, user.pk)) # Delete all related models backend.filter(UserRole, {'user': user}).delete() backend.filter(AccessToken, {'user': user}).delete() backend.filter(User, {'pk': user.pk}).delete() # Todo: Delete all of the user's projects that have no "owner" user roles anymore... logger.info("Deletion of user {0} ({1}) complete!".format( user.name, user.pk)) except ExclusiveTask.LockError: pass except BaseException as err: logger.error("Error {0}: Can't delete user {1}.".format( err.__class__.__name__, user)) logger.error(traceback.format_exc())