def run_export(objects, ie_id, user_id, url_root): """Run export""" with app.app_context(): try: user = person.Person.query.get(user_id) setattr(g, '_current_user', user) ie = import_export.get(ie_id) check_for_previous_run() content, _ = make_export(objects) db.session.refresh(ie) if ie.status == "Stopped": return ie.status = "Finished" ie.content = content db.session.commit() job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email, url_root, ie.title) except Exception as e: # pylint: disable=broad-except logger.exception("Export failed: %s", e.message) try: ie.status = "Failed" ie.end_date = datetime.now() db.session.commit() job_emails.send_email(job_emails.EXPORT_FAILED, user.email, url_root) except Exception as e: # pylint: disable=broad-except logger.exception("Failed to set job status: %s", e.message)
def handle_export_stop(**kwargs): """Handle export stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status == "In Progress": ie_job.status = "Stopped" # Stop tasks only on non local instance if getattr(settings, "APPENGINE_INSTANCE", "local") != "local": stop_ie_bg_tasks(ie_job) db.session.commit() expire_ie_cache(ie_job) return make_import_export_response(ie_job.log_json()) if ie_job.status == "Stopped": raise models_exceptions.ExportStoppedException() if ie_job.status == "Finished": raise models_exceptions.ExportFinishedException() except wzg_exceptions.Forbidden: raise except models_exceptions.ExportStoppedException: raise wzg_exceptions.BadRequest(app_errors.EXPORT_STOPPED_WARNING) except models_exceptions.ExportFinishedException: raise wzg_exceptions.BadRequest(app_errors.EXPORT_FINISHED_WARNING) except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Export")) raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def handle_import_stop(**kwargs): """Handle import stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status in ("Analysis", "In Progress", "Blocked"): ie_job.status = "Stopped" ie_job.end_at = datetime.utcnow() # Stop tasks only on non local instance if getattr(settings, "APPENGINE_INSTANCE", "local") != "local": stop_ie_bg_tasks(ie_job) db.session.commit() expire_ie_cache(ie_job) return make_import_export_response(ie_job.log_json()) if ie_job.status == "Stopped": raise models_exceptions.ImportStoppedException() except wzg_exceptions.Forbidden: raise except models_exceptions.ImportStoppedException: raise wzg_exceptions.BadRequest(app_errors.IMPORT_STOPPED_WARNING) except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import")) # Need to implement a better solution in order to identify specific # errors like here raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def run_import_phases(ie_id, user_id, url_root): # noqa: ignore=C901 """Execute import phases""" with app.app_context(): try: user = person.Person.query.get(user_id) setattr(g, '_current_user', user) ie_job = import_export.get(ie_id) check_for_previous_run() csv_data = read_csv_file(StringIO(ie_job.content.encode("utf-8"))) if ie_job.status == "Analysis": info = make_import(csv_data, True) db.session.rollback() db.session.refresh(ie_job) if ie_job.status == "Stopped": return ie_job.results = json.dumps(info) for block_info in info: if block_info["block_errors"] or block_info["row_errors"]: ie_job.status = "Analysis Failed" db.session.commit() job_emails.send_email(job_emails.IMPORT_FAILED, user.email, url_root, ie_job.title) return for block_info in info: if block_info["block_warnings"] or block_info["row_warnings"]: ie_job.status = "Blocked" db.session.commit() job_emails.send_email(job_emails.IMPORT_BLOCKED, user.email, url_root, ie_job.title) return ie_job.status = "In Progress" db.session.commit() if ie_job.status == "In Progress": info = make_import(csv_data, False) ie_job.results = json.dumps(info) for block_info in info: if block_info["block_errors"] or block_info["row_errors"]: ie_job.status = "Analysis Failed" job_emails.send_email(job_emails.IMPORT_FAILED, user.email, url_root, ie_job.title) db.session.commit() return ie_job.status = "Finished" db.session.commit() job_emails.send_email(job_emails.IMPORT_COMPLETED, user.email, url_root, ie_job.title) except Exception as e: # pylint: disable=broad-except logger.exception("Import failed: %s", e.message) try: ie_job.status = "Failed" ie_job.end_date = datetime.now() db.session.commit() job_emails.send_email(job_emails.IMPORT_FAILED, user.email, url_root, ie_job.title) except Exception as e: # pylint: disable=broad-except logger.exception("Failed to set job status: %s", e.message)
def handle_file_download(id2): """ Download file """ try: export_to = request.args.get("export_to") ie = import_export.get(id2) return export_file(export_to, ie.title, ie.content.encode("utf-8")) except (Forbidden, NotFound, Unauthorized): raise except Exception as e: logger.exception("Download failed due incorrect request data: %s", e.message) raise BadRequest("Download failed due incorrect request data")
def handle_file_download(id2): """ Download file """ try: export_to = request.args.get("export_to") ie = import_export.get(id2) return export_file(export_to, ie.title, ie.content.encode("utf-8")) except (Forbidden, NotFound, Unauthorized): raise except Exception as e: logger.exception(e.message) raise BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Download"))
def handle_file_download(id2): """ Download file """ try: export_to = request.args.get("export_to") ie = import_export.get(id2) return export_file(export_to, ie.title, ie.content.encode("utf-8")) except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound, wzg_exceptions.Unauthorized): raise except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Download"))
def handle_delete(**kwargs): """ Delete import_export entry """ check_import_export_headers() try: ie = import_export.get(kwargs["id2"]) db.session.delete(ie) db.session.commit() return make_import_export_response("OK") except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound): raise except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import/Export"))
def handle_delete(**kwargs): """ Delete import_export entry """ check_import_export_headers() try: ie = import_export.get(kwargs["id2"]) db.session.delete(ie) db.session.commit() return make_import_export_response("OK") except (Forbidden, NotFound): raise except Exception as e: logger.exception("Import/Export failed due incorrect request data: %s", e.message) raise BadRequest("Import/Export failed due incorrect request data")
def run_export(task): """Run export""" user = get_current_user() ie_id = task.parameters.get("ie_id") objects = task.parameters.get("objects") exportable_objects = task.parameters.get("exportable_objects") try: ie = import_export.get(ie_id) check_for_previous_run() content, _ = make_export(objects, exportable_objects) db.session.refresh(ie) if ie.status == "Stopped": return utils.make_simple_response() ie.status = "Finished" ie.end_at = datetime.utcnow() ie.content = content db.session.commit() job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email, ie.title, ie_id) except Exception as e: # pylint: disable=broad-except logger.exception("Export failed: %s", e.message) ie = import_export.get(ie_id) try: ie.status = "Failed" ie.end_at = datetime.utcnow() db.session.commit() job_emails.send_email(job_emails.EXPORT_FAILED, user.email) return utils.make_simple_response(e.message) except Exception as e: # pylint: disable=broad-except logger.exception("%s: %s", app_errors.STATUS_SET_FAILED, e.message) return utils.make_simple_response(e.message) return utils.make_simple_response()
def run_export(task): """Run export""" user = login.get_current_user() ie_id = task.parameters.get("ie_id") objects = task.parameters.get("objects") exportable_objects = task.parameters.get("exportable_objects") try: ie_job = import_export.get(ie_id) content, _ = make_export(objects, exportable_objects, ie_job) db.session.refresh(ie_job) if ie_job.status == "Stopped": return utils.make_simple_response() ie_job.status = "Finished" ie_job.end_at = datetime.utcnow() ie_job.content = content db.session.commit() job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email, ie_job.title, ie_id) except models_exceptions.ExportStoppedException: logger.info("Export was stopped by user.") except Exception as e: # pylint: disable=broad-except logger.exception("Export failed: %s", e.message) ie_job = import_export.get(ie_id) try: ie_job.status = "Failed" ie_job.end_at = datetime.utcnow() db.session.commit() job_emails.send_email(job_emails.EXPORT_CRASHED, user.email) return utils.make_simple_response(e.message) except Exception as e: # pylint: disable=broad-except logger.exception("%s: %s", app_errors.STATUS_SET_FAILED, e.message) return utils.make_simple_response(e.message) return utils.make_simple_response()
def handle_export_stop(**kwargs): """Handle export stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status == "In Progress": ie_job.status = "Stopped" db.session.commit() return make_import_export_response(ie_job.log_json()) except Forbidden: raise except Exception as e: logger.exception(e.message) raise BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Export")) raise BadRequest(app_errors.WRONG_STATUS)
def handle_export_stop(**kwargs): """Handle export stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status == "In Progress": ie_job.status = "Stopped" db.session.commit() return make_import_export_response(ie_job.log_json()) except Forbidden: raise except Exception as e: logger.exception("Export stop failed due incorrect request data: %s", e.message) raise BadRequest("Export stop failed due incorrect request data") raise BadRequest("Wrong status")
def handle_import_stop(**kwargs): """Handle import stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status in ("Analysis", "Blocked"): ie_job.status = "Stopped" db.session.commit() return make_import_export_response(ie_job.log_json()) except Forbidden: raise except Exception as e: logger.exception("Import stop failed due incorrect request data: %s", e.message) raise BadRequest("Import stop failed due incorrect request data") # Need to implement a better solution in order to identify specific # errors like here raise BadRequest("Wrong status")
def handle_import_stop(**kwargs): """Handle import stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status in ("Analysis", "Blocked"): ie_job.status = "Stopped" db.session.commit() return make_import_export_response(ie_job.log_json()) except wzg_exceptions.Forbidden: raise except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import")) # Need to implement a better solution in order to identify specific # errors like here raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def handle_get(id2, command, job_type): """Handle simple get and collection get""" check_import_export_headers() if command: if command != "download": BadRequest("Unknown command") return handle_file_download(id2) try: if id2: res = import_export.get(id2).log_json() else: ids = request.args.get("id__in") res = import_export.get_jobs(job_type, ids.split(",") if ids else None) except (Forbidden, NotFound): raise except Exception as e: logger.exception("%s failed due incorrect request data: %s", job_type, e.message) raise BadRequest("%s failed due incorrect request data" % job_type) return make_import_export_response(res)
def handle_get(id2, command, job_type): """Handle simple get and collection get""" check_import_export_headers() if command: if command != "download": wzg_exceptions.BadRequest("Unknown command") return handle_file_download(id2) try: if id2: res = import_export.get(id2).log_json() else: ids = request.args.get("id__in") res = import_export.get_jobs(job_type, ids.split(",") if ids else None) except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound): raise except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type=job_type)) return make_import_export_response(res)
def handle_import_put(**kwargs): """Handle import put""" command = kwargs.get("command2") ie_id = kwargs.get("id2") user = get_current_user() if user.system_wide_role == 'No Access': raise Forbidden() if not ie_id or not command or command not in ("start", "stop"): raise BadRequest("Import failed due incorrect request data") try: ie_job = import_export.get(ie_id) except (Forbidden, NotFound): raise except Exception as e: logger.exception("Import failed due incorrect request data: %s", e.message) raise BadRequest("Import failed due incorrect request data") if command == 'start': return handle_start(ie_job, user.id) elif command == "stop": return handle_import_stop(**kwargs) raise BadRequest("Bad params")
def handle_import_put(**kwargs): """Handle import put""" command = kwargs.get("command2") ie_id = kwargs.get("id2") user = login.get_current_user() if user.system_wide_role == 'No Access': raise wzg_exceptions.Forbidden() if not ie_id or not command or command not in ("start", "stop"): raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import")) try: ie_job = import_export.get(ie_id) except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound): raise except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import")) if command == 'start': return handle_start(ie_job) elif command == "stop": return handle_import_stop(**kwargs) raise wzg_exceptions.BadRequest(app_errors.BAD_PARAMS)
def handle_export_stop(**kwargs): """Handle export stop""" try: ie_job = import_export.get(kwargs["id2"]) if ie_job.status == "In Progress": ie_job.status = "Stopped" # Stop tasks only on non local instance if getattr(settings, "APPENGINE_INSTANCE", "local") != "local": stop_ie_bg_tasks(ie_job) db.session.commit() expire_ie_cache(ie_job) return make_import_export_response(ie_job.log_json()) if ie_job.status == "Stopped": raise models_exceptions.ExportStoppedException() except wzg_exceptions.Forbidden: raise except models_exceptions.ExportStoppedException: raise wzg_exceptions.BadRequest(app_errors.STOPPED_WARNING) except Exception as e: logger.exception(e.message) raise wzg_exceptions.BadRequest( app_errors.INCORRECT_REQUEST_DATA.format(job_type="Export")) raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def run_import_phases(task): """Execute import phases""" ie_id = task.parameters.get("ie_id") user = login.get_current_user() try: ie_job = import_export.get(ie_id) csv_data = import_helper.read_csv_file( StringIO(ie_job.content.encode("utf-8")) ) if ie_job.status == "Analysis": info = make_import(csv_data, True, ie_job) db.session.rollback() db.session.refresh(ie_job) if ie_job.status == "Stopped": return utils.make_simple_response() ie_job.results = json.dumps(info) for block_info in info: if block_info["block_errors"] or block_info["row_errors"]: ie_job.status = "Analysis Failed" ie_job.end_at = datetime.utcnow() db.session.commit() job_emails.send_email(job_emails.IMPORT_FAILED, user.email, ie_job.title) return utils.make_simple_response() for block_info in info: if block_info["block_warnings"] or block_info["row_warnings"]: ie_job.status = "Blocked" db.session.commit() job_emails.send_email(job_emails.IMPORT_BLOCKED, user.email, ie_job.title) return utils.make_simple_response() ie_job.status = "In Progress" db.session.commit() if ie_job.status == "In Progress": info = make_import(csv_data, False, ie_job) ie_job.results = json.dumps(info) for block_info in info: if block_info["block_errors"] or block_info["row_errors"]: ie_job.status = "Analysis Failed" ie_job.end_at = datetime.utcnow() job_emails.send_email(job_emails.IMPORT_FAILED, user.email, ie_job.title) db.session.commit() return utils.make_simple_response() ie_job.status = "Finished" ie_job.end_at = datetime.utcnow() db.session.commit() job_emails.send_email(job_emails.IMPORT_COMPLETED, user.email, ie_job.title) except Exception as e: # pylint: disable=broad-except logger.exception(e.message) ie_job = import_export.get(ie_id) try: ie_job.status = "Failed" ie_job.end_at = datetime.utcnow() db.session.commit() job_emails.send_email(job_emails.IMPORT_FAILED, user.email, ie_job.title) return utils.make_simple_response(e.message) except Exception as e: # pylint: disable=broad-except logger.exception("%s: %s", app_errors.STATUS_SET_FAILED, e.message) return utils.make_simple_response(e.message) return utils.make_simple_response()