def get_background_task_status(object_type, object_id): """Gets the status of a background task which was created for object.""" bg_task = models.BackgroundTask bg_operation = models.BackgroundOperation task = bg_task.query.join( bg_operation, bg_operation.bg_task_id == bg_task.id ).filter( bg_operation.object_type == object_type, bg_operation.object_id == object_id, ).order_by( bg_task.created_at.desc() ).first() if task and task.bg_operation: body = { "status": task.status, "operation": task.bg_operation.bg_operation_type.name, "errors": task.get_content().get("errors", []), } response = app.make_response( (json.dumps(body), 200, [("Content-Type", "application/json")]) ) else: response = app.make_response( ("", 404, [("Content-Type", "application/json")]) ) return response
def decorated_view(*args, **_): """Background task runner. This runner makes sure that the task is called with the task model as the parameter. """ if args and isinstance(args[0], BackgroundTask): task = args[0] else: task_id = request.headers.get("X-Task-Id", request.values.get("task_id")) task = BackgroundTask.query.get(task_id) task.start() try: result = func(task) except: # pylint: disable=bare-except # Bare except is allowed here so that we can respond with the correct # message to all exceptions. logger.exception("Task failed") task.finish("Failure", app.make_response(( traceback.format_exc(), 200, [('Content-Type', 'text/html')]))) # Return 200 so that the task is not retried return app.make_response(( 'failure', 200, [('Content-Type', 'text/html')])) task.finish("Success", result) return result
def make_simple_response(error=None): """Create a response with error message and proper status code.""" from ggrc.app import app if error: return app.make_response(( "Failed with error: {}".format(error), 500, [("Content-Type", "text/html")] )) return app.make_response(("Success", 200, [("Content-Type", "text/html")]))
def admin_reindex(): """Simple re-index of all indexable objects """ if not permissions.is_allowed_read("/admin", 1): raise Forbidden() from ggrc.fulltext import get_indexer from ggrc.fulltext.recordbuilder import fts_record_for indexer = get_indexer() indexer.delete_all_records(False) from ggrc.models import all_models from ggrc.app import db # Find all models then remove base classes models = set(all_models.all_models) -\ set([all_models.Directive, all_models.SystemOrProcess]) for model in models: mapper_class = model._sa_class_manager.mapper.base_mapper.class_ query = model.query.options( db.undefer_group(mapper_class.__name__+'_complete'), ) for instance in query.all(): indexer.create_record(fts_record_for(instance), False) db.session.commit() return app.make_response(( 'success', 200, [('Content-Type', 'text/html')]))
def update_cad_related_objects(task): """Update CAD related objects""" event_id = task.parameters.get("event_id") model_name = task.parameters.get("model_name") need_revisions = task.parameters.get("need_revisions") modified_by_id = task.parameters.get("modified_by_id") event = models.all_models.Event.query.filter_by(id=event_id).first() cad = models.all_models.CustomAttributeDefinition.query.filter_by( id=event.resource_id ).first() model = models.get_model(model_name) query = db.session.query(model if need_revisions else model.id) objects_count = query.count() handled_objects = 0 for chunk in ggrc_utils.generate_query_chunks(query): handled_objects += chunk.count() logger.info( "Updating CAD related objects: %s/%s", handled_objects, objects_count ) if need_revisions: for obj in chunk: obj.updated_at = datetime.datetime.utcnow() obj.modified_by_id = modified_by_id else: model.bulk_record_update_for([obj_id for obj_id, in chunk]) log_event.log_event(db.session, cad, event=event) db.session.commit() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def make_response(self, default=None): if self.result is None: return default from ggrc.app import app return app.make_response((self.result['content'], self.result['status_code'], self.result['headers']))
def send_event_job(): with benchmark("POST /admin/compute_attributes"): if request.data: revision_ids = request.get_json().get("revision_ids", []) else: revision_ids = "all_latest" start_compute_attributes(revision_ids) return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def admin_reindex(): """Calls a webhook that reindexes indexable objects """ if not permissions.is_allowed_read("/admin", None, 1): raise Forbidden() task_queue = create_task("reindex", url_for(reindex.__name__), reindex) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def send_event_job(): """Trigger background task on every event for computed attributes.""" with benchmark("POST /admin/compute_attributes"): if flask.request.data: revision_ids = flask.request.get_json().get("revision_ids", []) else: revision_ids = "all_latest" start_compute_attributes(revision_ids=revision_ids) return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def compute_attributes(args): """Web hook to update the full text search index.""" with benchmark("Run compute_attributes background task"): from ggrc.data_platform import computed_attributes if str(args.parameters["revision_ids"]) == "all_latest": revision_ids = "all_latest" else: revision_ids = [id_ for id_ in args.parameters["revision_ids"]] computed_attributes.compute_attributes(revision_ids) return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def decorated_view(*args, **kwargs): if len(args) > 0 and isinstance(args[0], BackgroundTask): task = args[0] else: task = BackgroundTask.query.get(request.values.get("task_id")) task.start() try: result = func(task) except: import traceback logger.exception("Task failed") task.finish("Failure", app.make_response(( traceback.format_exc(), 200, [('Content-Type', 'text/html')]))) # Return 200 so that the task is not retried return app.make_response(( 'failure', 200, [('Content-Type', 'text/html')])) task.finish("Success", result) return result
def task_scheduled_response(self): """Create success response with info about scheduled task.""" from ggrc.app import app return self.make_response( app.make_response(( json.dumps("scheduled %s" % self.name), 200, [("Content-Type", "application/json")] )) )
def update_audit_issues(args): """Web hook to update the issues associated with an audit.""" audit_id = args.parameters['audit_id'] message = args.parameters['message'] if not audit_id or not message: logger.warning( 'One or more of required parameters (audit_id, message) is empty.') return app.make_response(( 'Parameters audit_id and message are required.', 400, [('Content-Type', 'text/html')])) issue_tracker = all_models.IssuetrackerIssue relationships = all_models.Relationship query = db.session.query( issue_tracker.enabled, issue_tracker.issue_id, issue_tracker.object_id ).join( relationships, relationships.source_id == issue_tracker.object_id ).filter( relationships.source_type == 'Assessment', relationships.destination_type == 'Audit', relationships.destination_id == audit_id ) cli = issues.Client() issue_params = { 'comment': message, } for enabled, issue_id, assessment_id in query.all(): if not enabled: continue try: cli.update_issue(issue_id, issue_params) except integrations_errors.Error as error: logger.error( 'Unable to update IssueTracker issue ID=%s ' 'for Assessment ID=%s while archiving/unarchiving Audit ID=%s: %s', issue_id, assessment_id, audit_id, error) return app.make_response(('success', 200, [('Content-Type', 'text/html')]))
def admin_find_empty_revisions(): """Process all revisions and find empty.""" bg_task = background_task.create_task( name="find_empty_revisions", url=flask.url_for(find_empty_revisions.__name__), queued_callback=find_empty_revisions, ) db.session.commit() return bg_task.make_response( app.make_response(("scheduled %s" % bg_task.name, 200, [('Content-Type', 'text/html')])))
def admin_create_missing_revisions(): """Create revisions for new objects""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if get_current_user().email not in admins: raise exceptions.Forbidden() task_queue = create_task("create_missing_revisions", url_for( create_missing_revisions.__name__), create_missing_revisions) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def generate_wf_tasks_notifs(): """Generate notifications for updated wf cycle tasks.""" bg_task = background_task.create_task( name="generate_wf_tasks_notifications", url=flask.url_for(generate_wf_tasks_notifications.__name__), queued_callback=generate_wf_tasks_notifications, ) db.session.commit() return bg_task.make_response( app.make_response(("scheduled %s" % bg_task.name, 200, [('Content-Type', 'text/html')])))
def admin_propagate_acl(): """Propagates all ACL entries""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if get_current_user().email not in admins: raise exceptions.Forbidden() task_queue = create_task("propagate_acl", url_for( propagate_acl.__name__), propagate_acl) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def admin_refresh_revisions(): """Calls a webhook that refreshes revision content.""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if get_current_user().email not in admins: raise Forbidden() task_queue = create_task("refresh_revisions", url_for( refresh_revisions.__name__), refresh_revisions) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def admin_reindex_snapshots(): """Calls a webhook that reindexes indexable objects """ task_queue = create_task( name="reindex_snapshots", url=url_for(reindex_snapshots.__name__), queued_callback=reindex_snapshots, ) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def admin_full_reindex(): """Calls a webhook that reindexes all indexable objects """ task_queue = create_task( name="full_reindex", url=url_for(full_reindex.__name__), queued_callback=full_reindex ) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def admin_full_reindex(): """Calls a webhook that reindexes all indexable objects """ bg_task = background_task.create_task( name="full_reindex", url=flask.url_for(full_reindex.__name__), queued_callback=full_reindex ) db.session.commit() return bg_task.make_response( app.make_response(("scheduled %s" % bg_task.name, 200, [('Content-Type', 'text/html')])))
def decorated_view(*args, **kwargs): if len(args) > 0 and isinstance(args[0], BackgroundTask): task = args[0] else: task = BackgroundTask.query.get(request.values.get("task_id")) task.start() try: result = func(task) except: import traceback logger.exception("Task failed") task.finish( "Failure", app.make_response((traceback.format_exc(), 200, [('Content-Type', 'text/html')]))) # Return 200 so that the task is not retried return app.make_response( ('failure', 200, [('Content-Type', 'text/html')])) task.finish("Success", result) return result
def admin_refresh_revisions(): """Calls a webhook that refreshes revision content.""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if get_current_user().email not in admins: raise exceptions.Forbidden() task_queue = create_task("refresh_revisions", url_for(refresh_revisions.__name__), refresh_revisions) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def admin_create_missing_revisions(): """Create revisions for new objects""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if get_current_user().email not in admins: raise exceptions.Forbidden() task_queue = create_task("create_missing_revisions", url_for(create_missing_revisions.__name__), create_missing_revisions) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def admin_reindex(): """Calls a webhook that reindexes indexable objects """ bg_task = background_task.create_task( name="reindex", url=flask.url_for(reindex.__name__), queued_callback=reindex, ) db.session.commit() return bg_task.make_response( app.make_response(("scheduled %s" % bg_task.name, 200, [('Content-Type', 'text/html')])))
def admin_reindex(): """Calls a webhook that reindexes indexable objects """ if not permissions.is_allowed_read("/admin", None, 1): raise Forbidden() task_queue = create_task( name="reindex", url=url_for(reindex.__name__), queued_callback=reindex ) return task_queue.make_response( app.make_response(("scheduled %s" % task_queue.name, 200, [('Content-Type', 'text/html')])))
def get_background_task_status(object_type, object_id): """Gets the status of a background task which was created for object.""" bg_task = models.BackgroundTask bg_operation = models.BackgroundOperation task = bg_task.query.join(bg_operation, bg_operation.bg_task_id == bg_task.id).filter( bg_operation.object_type == object_type, bg_operation.object_id == object_id, ).order_by(bg_task.created_at.desc()).first() if task and task.bg_operation: body = { "status": task.status, "operation": task.bg_operation.bg_operation_type.name, "errors": task.get_content().get("errors", []), } response = app.make_response( (json.dumps(body), 200, [("Content-Type", "application/json")])) else: response = app.make_response( ("", 404, [("Content-Type", "application/json")])) return response
def update_audit_issues(args): """Web hook to update the issues associated with an audit.""" audit_id = args.parameters['audit_id'] message = args.parameters['message'] if not audit_id or not message: logger.warning( 'One or more of required parameters (audit_id, message) is empty.') return app.make_response( ('Parameters audit_id and message are required.', 400, [('Content-Type', 'text/html')])) issue_tracker = models.all_models.IssuetrackerIssue relationships = models.all_models.Relationship query = db.session.query( issue_tracker.enabled, issue_tracker.issue_id, issue_tracker.object_id).join( relationships, relationships.source_id == issue_tracker.object_id).filter( relationships.source_type == 'Assessment', relationships.destination_type == 'Audit', relationships.destination_id == audit_id) cli = issues.Client() issue_params = { 'comment': message, } for enabled, issue_id, assessment_id in query.all(): if not enabled: continue try: cli.update_issue(issue_id, issue_params) except integrations_errors.Error as error: logger.error( 'Unable to update IssueTracker issue ID=%s ' 'for Assessment ID=%s while archiving/unarchiving Audit ID=%s: %s', issue_id, assessment_id, audit_id, error) return app.make_response(('success', 200, [('Content-Type', 'text/html')]))
def run_bulk_verify(): """Call bulk verify job""" data = flask.request.json parameters = {"data": data} bg_task = background_task.create_task(name="bulk_verify", url=flask.url_for( bulk_verify.__name__), queued_callback=bulk_verify, parameters=parameters) db.session.commit() return bg_task.make_response( app.make_response( (utils.as_json(bg_task), 200, [('Content-Type', "text/json")])))
def run_bulk_complete(): """Call bulk complete job""" data = flask.request.json parameters = {"data": data} if _detect_files(data): try: gdrive.get_http_auth() except gdrive.GdriveUnauthorized: response = app.make_response( ("auth", 401, [("Content-Type", "text/html")])) return response parameters["credentials"] = flask.session.get('credentials') bg_task = background_task.create_task(name="bulk_complete", url=flask.url_for( bulk_complete.__name__), queued_callback=bulk_complete, parameters=parameters) db.session.commit() return bg_task.make_response( app.make_response( (utils.as_json(bg_task), 200, [('Content-Type', "text/json")])))
def compute_attributes(args): """Web hook to update the full text search index.""" with benchmark("Run compute_attributes background task"): from ggrc.data_platform import computed_attributes if args.parameters["event_id"] and not args.parameters["revision_ids"]: rows = db.session.query(Revision.id).filter_by( event_id=args.parameters["event_id"],).all() revision_ids = [revision_id for revision_id, in rows] elif str(args.parameters["revision_ids"]) == "all_latest": revision_ids = "all_latest" else: revision_ids = [id_ for id_ in args.parameters["revision_ids"]] computed_attributes.compute_attributes(revision_ids) return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def make_response(errors): """Create response with provided body and status. Args: errors: List with errors to return in response. Returns: Created response. """ error_list = [] for obj, err in errors: error_list.append((obj.type, obj.id, err)) return app.make_response((utils.as_json({"errors": error_list}), 200, [("Content-Type", "application/json")]))
def admin_propagate_acl(): """Propagates all ACL entries""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if login.get_current_user().email not in admins: raise exceptions.Forbidden() bg_task = background_task.create_task( name="propagate_acl", url=flask.url_for(propagate_acl.__name__), queued_callback=propagate_acl, ) db.session.commit() return bg_task.make_response( app.make_response(("scheduled %s" % bg_task.name, 200, [('Content-Type', 'text/html')])))
def admin_create_missing_revisions(): """Create revisions for new objects""" admins = getattr(settings, "BOOTSTRAP_ADMIN_USERS", []) if login.get_current_user().email not in admins: raise exceptions.Forbidden() bg_task = background_task.create_task( name="create_missing_revisions", url=flask.url_for(create_missing_revisions.__name__), queued_callback=create_missing_revisions, ) db.session.commit() return bg_task.make_response( app.make_response(("scheduled %s" % bg_task.name, 200, [('Content-Type', 'text/html')])))
def compute_attributes(args): """Web hook to update the full text search index.""" with benchmark("Run compute_attributes background task"): from ggrc.data_platform import computed_attributes if args.parameters["event_id"] and not args.parameters["revision_ids"]: rows = db.session.query(Revision.id).filter_by( event_id=args.parameters["event_id"], ).all() revision_ids = [revision_id for revision_id, in rows] elif str(args.parameters["revision_ids"]) == "all_latest": revision_ids = "all_latest" else: revision_ids = [id_ for id_ in args.parameters["revision_ids"]] computed_attributes.compute_attributes(revision_ids) return app.make_response( ("success", 200, [("Content-Type", "text/html")]))
def compute_attributes(*_, **kwargs): """Web hook to update the full text search index.""" with benchmark("Run compute_attributes background task"): event_id = utils.get_task_attr("event_id", kwargs) revision_ids = utils.get_task_attr("revision_ids", kwargs) if event_id and not revision_ids: rows = db.session.query(Revision.id).filter_by(event_id=event_id).all() revision_ids = [revision_id for revision_id, in rows] elif str(revision_ids) == "all_latest": revision_ids = "all_latest" else: revision_ids = list(revision_ids) from ggrc.data_platform import computed_attributes computed_attributes.compute_attributes(revision_ids) return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def make_response(errors): """Create response with provided body and status. Args: errors: List with errors to return in response. Returns: Created response. """ error_list = [] for obj, err in errors: error_list.append((obj.type, obj.id, err)) return app.make_response(( utils.as_json({"errors": error_list}), 200, [("Content-Type", "application/json")] ))
def compute_attributes(task): """Web hook to update the full text search index.""" with benchmark("Run compute_attributes background task"): event_id = task.parameters.get("event_id") revision_ids = task.parameters.get("revision_ids") if event_id and not revision_ids: rows = db.session.query( revision.Revision.id).filter_by(event_id=event_id).all() revision_ids = [revision_id for revision_id, in rows] elif str(revision_ids) == "all_latest": revision_ids = "all_latest" else: revision_ids = list(revision_ids) from ggrc.data_platform import computed_attributes computed_attributes.compute_attributes(revision_ids) return app.make_response( ("success", 200, [("Content-Type", "text/html")]))
def bulk_verify(task): """Process bulk complete""" with benchmark("Create CsvBuilder"): builder = csvbuilder.CsvBuilder(task.parameters.get("data", {})) with benchmark("Prepare import data for verification"): update_data = builder.assessments_verify_to_csv() with benchmark("Verify assessments"): verify_assmts = converters.make_import(csv_data=update_data, dry_run=False, bulk_import=True) _log_import(verify_assmts["data"]) verify_errors = set(verify_assmts["failed_slugs"]) bulk_notifications.send_notification(update_errors=verify_errors, partial_errors={}, asmnt_ids=builder.assessment_ids) return app.make_response(('success', 200, [("Content-Type", "text/json")]))
def task_scheduled_response(self): """Create success response with info about scheduled task.""" from ggrc.app import app return self.make_response( app.make_response((json.dumps("scheduled %s" % self.name), 200, [("Content-Type", "application/json")])))
def full_reindex(_): """Web hook to update the full text search index for all models.""" do_full_reindex() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def find_empty_revisions(_): """Web hook to find empty revisions.""" empty_revisions.find_empty_revisions() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def reindex_snapshots(_): """Web hook to update the full text search index.""" logger.info("Updating index for: %s", "Snapshot") with benchmark("Create records for %s" % "Snapshot"): snapshot_indexer.reindex() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def create_missing_revisions(_): """Web hook to create revisions for new objects.""" revisions.do_missing_revisions() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def propagate_acl(_): """Web hook to update revision content.""" models.hooks.acl.propagation.propagate_all() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def bg_update_ft_records(task): """Background indexing endpoint""" fulltext.listeners.update_ft_records(task.parameters.get("models_ids", {}), task.parameters.get("chunk_size")) db.session.plain_commit() return app.make_response(('success', 200, [('Content-Type', 'text/html')]))
def onetime_back_sync(_): """Onetime synchronization of IssueTrackerIssue component_id/hotlist_id.""" one_time_back_sync.update_synced_issues() return app.make_response(('success', 200, [("Content-Type", "text/json")]))
def refresh_revisions(_): """Web hook to update revision content.""" revisions.do_refresh_revisions() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def run_reindex_pairs_bg(task): """Run reindexation of snapshots specified in pairs.""" pairs = task.parameters.get("pairs") reindex_pairs(pairs) return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def generate_wf_tasks_notifications(_): """Generate notifications for wf cycle tasks.""" common.generate_cycle_tasks_notifs() return app.make_response(("success", 200, [("Content-Type", "text/html")]))
def reindex(_): """Web hook to update the full text search index.""" do_reindex() return app.make_response(("success", 200, [("Content-Type", "text/html")]))