def run_export(objects, ie_id, user_id, url_root):
"""Run export"""
with app.app_context():
try:
user = person.Person.query.get(user_id)
setattr(g, '_current_user', user)
ie = import_export.get(ie_id)
check_for_previous_run()
content, _ = make_export(objects)
db.session.refresh(ie)
if ie.status == "Stopped":
return
ie.status = "Finished"
ie.content = content
db.session.commit()
job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email,
url_root, ie.title)
except Exception as e: # pylint: disable=broad-except
logger.exception("Export failed: %s", e.message)
try:
ie.status = "Failed"
ie.end_date = datetime.now()
db.session.commit()
job_emails.send_email(job_emails.EXPORT_FAILED, user.email,
url_root)
except Exception as e: # pylint: disable=broad-except
logger.exception("Failed to set job status: %s", e.message)
def handle_export_stop(**kwargs):
"""Handle export stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status == "In Progress":
ie_job.status = "Stopped"
# Stop tasks only on non local instance
if getattr(settings, "APPENGINE_INSTANCE", "local") != "local":
stop_ie_bg_tasks(ie_job)
db.session.commit()
expire_ie_cache(ie_job)
return make_import_export_response(ie_job.log_json())
if ie_job.status == "Stopped":
raise models_exceptions.ExportStoppedException()
if ie_job.status == "Finished":
raise models_exceptions.ExportFinishedException()
except wzg_exceptions.Forbidden:
raise
except models_exceptions.ExportStoppedException:
raise wzg_exceptions.BadRequest(app_errors.EXPORT_STOPPED_WARNING)
except models_exceptions.ExportFinishedException:
raise wzg_exceptions.BadRequest(app_errors.EXPORT_FINISHED_WARNING)
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Export"))
raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def run_export(objects, ie_id, user_id, url_root):
"""Run export"""
with app.app_context():
try:
user = person.Person.query.get(user_id)
setattr(g, '_current_user', user)
ie = import_export.get(ie_id)
check_for_previous_run()
content, _ = make_export(objects)
db.session.refresh(ie)
if ie.status == "Stopped":
return
ie.status = "Finished"
ie.content = content
db.session.commit()
job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email, url_root,
ie.title)
except Exception as e: # pylint: disable=broad-except
logger.exception("Export failed: %s", e.message)
try:
ie.status = "Failed"
ie.end_date = datetime.now()
db.session.commit()
job_emails.send_email(job_emails.EXPORT_FAILED, user.email, url_root)
except Exception as e: # pylint: disable=broad-except
logger.exception("Failed to set job status: %s", e.message)
def handle_import_stop(**kwargs):
"""Handle import stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status in ("Analysis", "In Progress", "Blocked"):
ie_job.status = "Stopped"
ie_job.end_at = datetime.utcnow()
# Stop tasks only on non local instance
if getattr(settings, "APPENGINE_INSTANCE", "local") != "local":
stop_ie_bg_tasks(ie_job)
db.session.commit()
expire_ie_cache(ie_job)
return make_import_export_response(ie_job.log_json())
if ie_job.status == "Stopped":
raise models_exceptions.ImportStoppedException()
except wzg_exceptions.Forbidden:
raise
except models_exceptions.ImportStoppedException:
raise wzg_exceptions.BadRequest(app_errors.IMPORT_STOPPED_WARNING)
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import"))
# Need to implement a better solution in order to identify specific
# errors like here
raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def run_import_phases(ie_id, user_id, url_root): # noqa: ignore=C901
"""Execute import phases"""
with app.app_context():
try:
user = person.Person.query.get(user_id)
setattr(g, '_current_user', user)
ie_job = import_export.get(ie_id)
check_for_previous_run()
csv_data = read_csv_file(StringIO(ie_job.content.encode("utf-8")))
if ie_job.status == "Analysis":
info = make_import(csv_data, True)
db.session.rollback()
db.session.refresh(ie_job)
if ie_job.status == "Stopped":
return
ie_job.results = json.dumps(info)
for block_info in info:
if block_info["block_errors"] or block_info["row_errors"]:
ie_job.status = "Analysis Failed"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
url_root, ie_job.title)
return
for block_info in info:
if block_info["block_warnings"] or block_info["row_warnings"]:
ie_job.status = "Blocked"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_BLOCKED, user.email,
url_root, ie_job.title)
return
ie_job.status = "In Progress"
db.session.commit()
if ie_job.status == "In Progress":
info = make_import(csv_data, False)
ie_job.results = json.dumps(info)
for block_info in info:
if block_info["block_errors"] or block_info["row_errors"]:
ie_job.status = "Analysis Failed"
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
url_root, ie_job.title)
db.session.commit()
return
ie_job.status = "Finished"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_COMPLETED, user.email,
url_root, ie_job.title)
except Exception as e: # pylint: disable=broad-except
logger.exception("Import failed: %s", e.message)
try:
ie_job.status = "Failed"
ie_job.end_date = datetime.now()
db.session.commit()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
url_root, ie_job.title)
except Exception as e: # pylint: disable=broad-except
logger.exception("Failed to set job status: %s", e.message)
def run_import_phases(ie_id, user_id, url_root): # noqa: ignore=C901
"""Execute import phases"""
with app.app_context():
try:
user = person.Person.query.get(user_id)
setattr(g, '_current_user', user)
ie_job = import_export.get(ie_id)
check_for_previous_run()
csv_data = read_csv_file(StringIO(ie_job.content.encode("utf-8")))
if ie_job.status == "Analysis":
info = make_import(csv_data, True)
db.session.rollback()
db.session.refresh(ie_job)
if ie_job.status == "Stopped":
return
ie_job.results = json.dumps(info)
for block_info in info:
if block_info["block_errors"] or block_info["row_errors"]:
ie_job.status = "Analysis Failed"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
url_root, ie_job.title)
return
for block_info in info:
if block_info["block_warnings"] or block_info["row_warnings"]:
ie_job.status = "Blocked"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_BLOCKED, user.email,
url_root, ie_job.title)
return
ie_job.status = "In Progress"
db.session.commit()
if ie_job.status == "In Progress":
info = make_import(csv_data, False)
ie_job.results = json.dumps(info)
for block_info in info:
if block_info["block_errors"] or block_info["row_errors"]:
ie_job.status = "Analysis Failed"
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
url_root, ie_job.title)
db.session.commit()
return
ie_job.status = "Finished"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_COMPLETED, user.email,
url_root, ie_job.title)
except Exception as e: # pylint: disable=broad-except
logger.exception("Import failed: %s", e.message)
try:
ie_job.status = "Failed"
ie_job.end_date = datetime.now()
db.session.commit()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
url_root, ie_job.title)
except Exception as e: # pylint: disable=broad-except
logger.exception("Failed to set job status: %s", e.message)
def handle_file_download(id2):
""" Download file """
try:
export_to = request.args.get("export_to")
ie = import_export.get(id2)
return export_file(export_to, ie.title, ie.content.encode("utf-8"))
except (Forbidden, NotFound, Unauthorized):
raise
except Exception as e:
logger.exception("Download failed due incorrect request data: %s",
e.message)
raise BadRequest("Download failed due incorrect request data")
def handle_file_download(id2):
""" Download file """
try:
export_to = request.args.get("export_to")
ie = import_export.get(id2)
return export_file(export_to, ie.title, ie.content.encode("utf-8"))
except (Forbidden, NotFound, Unauthorized):
raise
except Exception as e:
logger.exception(e.message)
raise BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Download"))
def handle_file_download(id2):
""" Download file """
try:
export_to = request.args.get("export_to")
ie = import_export.get(id2)
return export_file(export_to, ie.title, ie.content.encode("utf-8"))
except (Forbidden, NotFound, Unauthorized):
raise
except Exception as e:
logger.exception("Download failed due incorrect request data: %s",
e.message)
raise BadRequest("Download failed due incorrect request data")
def handle_file_download(id2):
""" Download file """
try:
export_to = request.args.get("export_to")
ie = import_export.get(id2)
return export_file(export_to, ie.title, ie.content.encode("utf-8"))
except (wzg_exceptions.Forbidden,
wzg_exceptions.NotFound,
wzg_exceptions.Unauthorized):
raise
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Download"))
def handle_delete(**kwargs):
""" Delete import_export entry """
check_import_export_headers()
try:
ie = import_export.get(kwargs["id2"])
db.session.delete(ie)
db.session.commit()
return make_import_export_response("OK")
except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound):
raise
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import/Export"))
def handle_delete(**kwargs):
""" Delete import_export entry """
check_import_export_headers()
try:
ie = import_export.get(kwargs["id2"])
db.session.delete(ie)
db.session.commit()
return make_import_export_response("OK")
except (Forbidden, NotFound):
raise
except Exception as e:
logger.exception("Import/Export failed due incorrect request data: %s",
e.message)
raise BadRequest("Import/Export failed due incorrect request data")
def handle_delete(**kwargs):
""" Delete import_export entry """
check_import_export_headers()
try:
ie = import_export.get(kwargs["id2"])
db.session.delete(ie)
db.session.commit()
return make_import_export_response("OK")
except (Forbidden, NotFound):
raise
except Exception as e:
logger.exception("Import/Export failed due incorrect request data: %s",
e.message)
raise BadRequest("Import/Export failed due incorrect request data")
def run_export(task):
"""Run export"""
user = get_current_user()
ie_id = task.parameters.get("ie_id")
objects = task.parameters.get("objects")
exportable_objects = task.parameters.get("exportable_objects")
try:
ie = import_export.get(ie_id)
check_for_previous_run()
content, _ = make_export(objects, exportable_objects)
db.session.refresh(ie)
if ie.status == "Stopped":
return utils.make_simple_response()
ie.status = "Finished"
ie.end_at = datetime.utcnow()
ie.content = content
db.session.commit()
job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email,
ie.title, ie_id)
except Exception as e: # pylint: disable=broad-except
logger.exception("Export failed: %s", e.message)
ie = import_export.get(ie_id)
try:
ie.status = "Failed"
ie.end_at = datetime.utcnow()
db.session.commit()
job_emails.send_email(job_emails.EXPORT_FAILED, user.email)
return utils.make_simple_response(e.message)
except Exception as e: # pylint: disable=broad-except
logger.exception("%s: %s", app_errors.STATUS_SET_FAILED, e.message)
return utils.make_simple_response(e.message)
return utils.make_simple_response()
def run_export(task):
"""Run export"""
user = login.get_current_user()
ie_id = task.parameters.get("ie_id")
objects = task.parameters.get("objects")
exportable_objects = task.parameters.get("exportable_objects")
try:
ie_job = import_export.get(ie_id)
content, _ = make_export(objects, exportable_objects, ie_job)
db.session.refresh(ie_job)
if ie_job.status == "Stopped":
return utils.make_simple_response()
ie_job.status = "Finished"
ie_job.end_at = datetime.utcnow()
ie_job.content = content
db.session.commit()
job_emails.send_email(job_emails.EXPORT_COMPLETED, user.email,
ie_job.title, ie_id)
except models_exceptions.ExportStoppedException:
logger.info("Export was stopped by user.")
except Exception as e: # pylint: disable=broad-except
logger.exception("Export failed: %s", e.message)
ie_job = import_export.get(ie_id)
try:
ie_job.status = "Failed"
ie_job.end_at = datetime.utcnow()
db.session.commit()
job_emails.send_email(job_emails.EXPORT_CRASHED, user.email)
return utils.make_simple_response(e.message)
except Exception as e: # pylint: disable=broad-except
logger.exception("%s: %s", app_errors.STATUS_SET_FAILED, e.message)
return utils.make_simple_response(e.message)
return utils.make_simple_response()
def handle_export_stop(**kwargs):
"""Handle export stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status == "In Progress":
ie_job.status = "Stopped"
db.session.commit()
return make_import_export_response(ie_job.log_json())
except Forbidden:
raise
except Exception as e:
logger.exception(e.message)
raise BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Export"))
raise BadRequest(app_errors.WRONG_STATUS)
def handle_export_stop(**kwargs):
"""Handle export stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status == "In Progress":
ie_job.status = "Stopped"
db.session.commit()
return make_import_export_response(ie_job.log_json())
except Forbidden:
raise
except Exception as e:
logger.exception("Export stop failed due incorrect request data: %s",
e.message)
raise BadRequest("Export stop failed due incorrect request data")
raise BadRequest("Wrong status")
def handle_export_stop(**kwargs):
"""Handle export stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status == "In Progress":
ie_job.status = "Stopped"
db.session.commit()
return make_import_export_response(ie_job.log_json())
except Forbidden:
raise
except Exception as e:
logger.exception("Export stop failed due incorrect request data: %s",
e.message)
raise BadRequest("Export stop failed due incorrect request data")
raise BadRequest("Wrong status")
def handle_import_stop(**kwargs):
"""Handle import stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status in ("Analysis", "Blocked"):
ie_job.status = "Stopped"
db.session.commit()
return make_import_export_response(ie_job.log_json())
except Forbidden:
raise
except Exception as e:
logger.exception("Import stop failed due incorrect request data: %s",
e.message)
raise BadRequest("Import stop failed due incorrect request data")
# Need to implement a better solution in order to identify specific
# errors like here
raise BadRequest("Wrong status")
def handle_import_stop(**kwargs):
"""Handle import stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status in ("Analysis", "Blocked"):
ie_job.status = "Stopped"
db.session.commit()
return make_import_export_response(ie_job.log_json())
except wzg_exceptions.Forbidden:
raise
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import"))
# Need to implement a better solution in order to identify specific
# errors like here
raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def handle_import_stop(**kwargs):
"""Handle import stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status in ("Analysis", "Blocked"):
ie_job.status = "Stopped"
db.session.commit()
return make_import_export_response(ie_job.log_json())
except Forbidden:
raise
except Exception as e:
logger.exception("Import stop failed due incorrect request data: %s",
e.message)
raise BadRequest("Import stop failed due incorrect request data")
# Need to implement a better solution in order to identify specific
# errors like here
raise BadRequest("Wrong status")
def handle_get(id2, command, job_type):
"""Handle simple get and collection get"""
check_import_export_headers()
if command:
if command != "download":
BadRequest("Unknown command")
return handle_file_download(id2)
try:
if id2:
res = import_export.get(id2).log_json()
else:
ids = request.args.get("id__in")
res = import_export.get_jobs(job_type, ids.split(",") if ids else None)
except (Forbidden, NotFound):
raise
except Exception as e:
logger.exception("%s failed due incorrect request data: %s",
job_type, e.message)
raise BadRequest("%s failed due incorrect request data" % job_type)
return make_import_export_response(res)
def handle_get(id2, command, job_type):
"""Handle simple get and collection get"""
check_import_export_headers()
if command:
if command != "download":
BadRequest("Unknown command")
return handle_file_download(id2)
try:
if id2:
res = import_export.get(id2).log_json()
else:
ids = request.args.get("id__in")
res = import_export.get_jobs(job_type, ids.split(",") if ids else None)
except (Forbidden, NotFound):
raise
except Exception as e:
logger.exception("%s failed due incorrect request data: %s",
job_type, e.message)
raise BadRequest("%s failed due incorrect request data" % job_type)
return make_import_export_response(res)
def handle_get(id2, command, job_type):
"""Handle simple get and collection get"""
check_import_export_headers()
if command:
if command != "download":
wzg_exceptions.BadRequest("Unknown command")
return handle_file_download(id2)
try:
if id2:
res = import_export.get(id2).log_json()
else:
ids = request.args.get("id__in")
res = import_export.get_jobs(job_type, ids.split(",") if ids else None)
except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound):
raise
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type=job_type))
return make_import_export_response(res)
def handle_import_put(**kwargs):
"""Handle import put"""
command = kwargs.get("command2")
ie_id = kwargs.get("id2")
user = get_current_user()
if user.system_wide_role == 'No Access':
raise Forbidden()
if not ie_id or not command or command not in ("start", "stop"):
raise BadRequest("Import failed due incorrect request data")
try:
ie_job = import_export.get(ie_id)
except (Forbidden, NotFound):
raise
except Exception as e:
logger.exception("Import failed due incorrect request data: %s",
e.message)
raise BadRequest("Import failed due incorrect request data")
if command == 'start':
return handle_start(ie_job, user.id)
elif command == "stop":
return handle_import_stop(**kwargs)
raise BadRequest("Bad params")
def handle_import_put(**kwargs):
"""Handle import put"""
command = kwargs.get("command2")
ie_id = kwargs.get("id2")
user = get_current_user()
if user.system_wide_role == 'No Access':
raise Forbidden()
if not ie_id or not command or command not in ("start", "stop"):
raise BadRequest("Import failed due incorrect request data")
try:
ie_job = import_export.get(ie_id)
except (Forbidden, NotFound):
raise
except Exception as e:
logger.exception("Import failed due incorrect request data: %s",
e.message)
raise BadRequest("Import failed due incorrect request data")
if command == 'start':
return handle_start(ie_job, user.id)
elif command == "stop":
return handle_import_stop(**kwargs)
raise BadRequest("Bad params")
def handle_import_put(**kwargs):
"""Handle import put"""
command = kwargs.get("command2")
ie_id = kwargs.get("id2")
user = login.get_current_user()
if user.system_wide_role == 'No Access':
raise wzg_exceptions.Forbidden()
if not ie_id or not command or command not in ("start", "stop"):
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import"))
try:
ie_job = import_export.get(ie_id)
except (wzg_exceptions.Forbidden, wzg_exceptions.NotFound):
raise
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Import"))
if command == 'start':
return handle_start(ie_job)
elif command == "stop":
return handle_import_stop(**kwargs)
raise wzg_exceptions.BadRequest(app_errors.BAD_PARAMS)
def handle_export_stop(**kwargs):
"""Handle export stop"""
try:
ie_job = import_export.get(kwargs["id2"])
if ie_job.status == "In Progress":
ie_job.status = "Stopped"
# Stop tasks only on non local instance
if getattr(settings, "APPENGINE_INSTANCE", "local") != "local":
stop_ie_bg_tasks(ie_job)
db.session.commit()
expire_ie_cache(ie_job)
return make_import_export_response(ie_job.log_json())
if ie_job.status == "Stopped":
raise models_exceptions.ExportStoppedException()
except wzg_exceptions.Forbidden:
raise
except models_exceptions.ExportStoppedException:
raise wzg_exceptions.BadRequest(app_errors.STOPPED_WARNING)
except Exception as e:
logger.exception(e.message)
raise wzg_exceptions.BadRequest(
app_errors.INCORRECT_REQUEST_DATA.format(job_type="Export"))
raise wzg_exceptions.BadRequest(app_errors.WRONG_STATUS)
def run_import_phases(task):
"""Execute import phases"""
ie_id = task.parameters.get("ie_id")
user = login.get_current_user()
try:
ie_job = import_export.get(ie_id)
csv_data = import_helper.read_csv_file(
StringIO(ie_job.content.encode("utf-8"))
)
if ie_job.status == "Analysis":
info = make_import(csv_data, True, ie_job)
db.session.rollback()
db.session.refresh(ie_job)
if ie_job.status == "Stopped":
return utils.make_simple_response()
ie_job.results = json.dumps(info)
for block_info in info:
if block_info["block_errors"] or block_info["row_errors"]:
ie_job.status = "Analysis Failed"
ie_job.end_at = datetime.utcnow()
db.session.commit()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
ie_job.title)
return utils.make_simple_response()
for block_info in info:
if block_info["block_warnings"] or block_info["row_warnings"]:
ie_job.status = "Blocked"
db.session.commit()
job_emails.send_email(job_emails.IMPORT_BLOCKED, user.email,
ie_job.title)
return utils.make_simple_response()
ie_job.status = "In Progress"
db.session.commit()
if ie_job.status == "In Progress":
info = make_import(csv_data, False, ie_job)
ie_job.results = json.dumps(info)
for block_info in info:
if block_info["block_errors"] or block_info["row_errors"]:
ie_job.status = "Analysis Failed"
ie_job.end_at = datetime.utcnow()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
ie_job.title)
db.session.commit()
return utils.make_simple_response()
ie_job.status = "Finished"
ie_job.end_at = datetime.utcnow()
db.session.commit()
job_emails.send_email(job_emails.IMPORT_COMPLETED, user.email,
ie_job.title)
except Exception as e: # pylint: disable=broad-except
logger.exception(e.message)
ie_job = import_export.get(ie_id)
try:
ie_job.status = "Failed"
ie_job.end_at = datetime.utcnow()
db.session.commit()
job_emails.send_email(job_emails.IMPORT_FAILED, user.email,
ie_job.title)
return utils.make_simple_response(e.message)
except Exception as e: # pylint: disable=broad-except
logger.exception("%s: %s", app_errors.STATUS_SET_FAILED, e.message)
return utils.make_simple_response(e.message)
return utils.make_simple_response()
