def api_v1_delete_job():
# Check, whether we got all the required data from the user.
required_fields = set([
"job_id"
])
request_form_keys=request.form.keys()
if len(set.intersection(required_fields, request_form_keys)) != len(required_fields):
raise BadRequest("Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join(
required_fields.difference(request_form_keys)))
#get oozie_job_id for the job
jobs = DatabaseManager.meta_connector.get_jobs(**{"job_id": request.form["job_id"]})
result = None
if len(jobs)>0:
job=jobs[0]
runner = OozieRunner()
status_code = runner.manage(job,'kill')
DatabaseManager.meta_connector.delete_job(job)
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id, "kill status_code": status_code}
else:
result = {"message": "job does not exist"}
return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
def api_v1_delete_job():
# Check, whether we got all the required data from the user.
required_fields = set(["job_id"])
request_form_keys = request.form.keys()
if len(set.intersection(required_fields,
request_form_keys)) != len(required_fields):
raise BadRequest(
"Required fields: " + ', '.join(required_fields) +
", missing fields: " +
', '.join(required_fields.difference(request_form_keys)))
#get oozie_job_id for the job
jobs = DatabaseManager.meta_connector.get_jobs(
**{"job_id": request.form["job_id"]})
result = None
if len(jobs) > 0:
job = jobs[0]
runner = OozieRunner()
status_code = runner.manage(job, 'kill')
DatabaseManager.meta_connector.delete_job(job)
result = {
"job_id": job.job_id,
"oozie_job_id": job.oozie_job_id,
"kill status_code": status_code
}
else:
result = {"message": "job does not exist"}
return Response(json.dumps(result,
indent=4,
default=json_datetime_handler_default),
mimetype="application/json")
def api_v1_post_job():
"""
Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job.
:return: the inserted job
"""
# Check, whether we got all the required data from the user.
required_fields = set([
"source_server_id", "source_schema_name", "source_object_name",
"destination_cluster_id", "destination_path", "owner_username",
"workflow_suffix", "sqoop_direct"
])
request_form_keys = request.form.keys()
if len(set.intersection(required_fields,
request_form_keys)) != len(required_fields):
raise BadRequest(
"Required fields: " + ', '.join(required_fields) +
", missing fields: " +
', '.join(required_fields.difference(request_form_keys)))
# Check, whether the user could access the schema.
source_server_id_ = int(request.form["source_server_id"])
server = DatabaseManager.meta_connector.get_servers(
**{"server_id": source_server_id_})
if len(server) < 1:
raise BadRequest("Server with ID " + source_server_id_ + " not found.")
server = server[0]
source_schema_name_ = request.form["source_schema_name"]
owner_username = request.form["owner_username"]
# TODO
# if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema):
if not DatabaseManager.auth_connector.can_user_access_schema(
get_username("CERN\\" + owner_username), server.server_name,
source_schema_name_):
abort(403)
# Check, whether the selected schema has the provided object
source_object_name_ = request.form["source_object_name"]
if source_object_name_.upper() not in (
obj.upper()
for obj in DatabaseManager.auth_connector.get_available_objects(
server.server_name, source_schema_name_)):
raise BadRequest("Object with name " + source_object_name_ +
" not found.")
# cluster exists
destination_cluster_id_ = int(request.form["destination_cluster_id"])
cluster = DatabaseManager.meta_connector.get_clusters(
**{"cluster_id": destination_cluster_id_})
if len(cluster) < 1:
raise BadRequest("Cluster with ID " + destination_cluster_id_ +
" not found.")
cluster = cluster[0]
# destination path valid
destination_path_ = request.form["destination_path"]
if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None:
raise BadRequest(
"The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters."
)
# TODO check start time
# TODO check interval
# insert new job
job = DatabaseManager.meta_connector.create_job()
job.source_server_id = source_server_id_
job.source_schema_name = source_schema_name_
job.source_object_name = source_object_name_
job.destination_cluster_id = destination_cluster_id_
job.destination_path = destination_path_
# TODO
# job.owner_username = get_username(request.remote_user)
job.owner_username = get_username("CERN\\" + owner_username)
job.workflow_suffix = request.form["workflow_suffix"]
if 'coordinator_suffix' in request_form_keys:
job.coordinator_suffix = request.form["coordinator_suffix"]
if 'sqoop_nmap' in request_form_keys:
job.sqoop_nmap = request.form["sqoop_nmap"]
if 'sqoop_splitting_column' in request_form_keys:
job.sqoop_splitting_column = request.form["sqoop_splitting_column"]
if 'sqoop_incremental_method' in request_form_keys:
job.sqoop_incremental_method = request.form["sqoop_incremental_method"]
job.sqoop_direct = request.form["sqoop_direct"]
if 'start_time' in request_form_keys:
job.start_time = request.form["start_time"]
if 'end_time' in request_form_keys:
job.end_time = request.form["end_time"]
if 'interval' in request_form_keys:
job.interval = request.form["interval"]
if 'job_last_update' in request_form_keys:
job.job_last_update = request.form["job_last_update"]
runner = OozieRunner()
job.oozie_job_id = runner.submit(job)
DatabaseManager.meta_connector.add_job(job)
result = None
if job.coordinator_suffix is None:
status_code = runner.manage(job, 'start')
result = {
"job_id": job.job_id,
"oozie_job_id": job.oozie_job_id,
"start status_code": status_code
}
else:
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id}
return Response(json.dumps(result,
indent=4,
default=json_datetime_handler_default),
mimetype="application/json")
def api_v1_post_job():
"""
Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job.
:return: the inserted job
"""
# Check, whether we got all the required data from the user.
required_fields = set([
"source_server_id",
"source_schema_name",
"source_object_name",
"destination_cluster_id",
"destination_path",
"owner_username",
"workflow_suffix",
"sqoop_direct"
])
request_form_keys=request.form.keys()
if len(set.intersection(required_fields, request_form_keys)) != len(required_fields):
raise BadRequest("Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join(
required_fields.difference(request_form_keys)))
# Check, whether the user could access the schema.
source_server_id_ = int(request.form["source_server_id"])
server = DatabaseManager.meta_connector.get_servers(**{"server_id": source_server_id_})
if len(server) < 1:
raise BadRequest("Server with ID " + source_server_id_ + " not found.")
server = server[0]
source_schema_name_ = request.form["source_schema_name"]
owner_username = request.form["owner_username"]
# TODO
# if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema):
if not DatabaseManager.auth_connector.can_user_access_schema(get_username("CERN\\"+owner_username), server.server_name,
source_schema_name_):
abort(403)
# Check, whether the selected schema has the provided object
source_object_name_ = request.form["source_object_name"]
if source_object_name_.upper() not in (
obj.upper() for obj
in DatabaseManager.auth_connector.get_available_objects(server.server_name, source_schema_name_)
):
raise BadRequest("Object with name " + source_object_name_ + " not found.")
# cluster exists
destination_cluster_id_ = int(request.form["destination_cluster_id"])
cluster = DatabaseManager.meta_connector.get_clusters(**{"cluster_id": destination_cluster_id_})
if len(cluster) < 1:
raise BadRequest("Cluster with ID " + destination_cluster_id_ + " not found.")
cluster = cluster[0]
# destination path valid
destination_path_ = request.form["destination_path"]
if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None:
raise BadRequest("The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters.")
# TODO check start time
# TODO check interval
# insert new job
job = DatabaseManager.meta_connector.create_job()
job.source_server_id = source_server_id_
job.source_schema_name = source_schema_name_
job.source_object_name = source_object_name_
job.destination_cluster_id = destination_cluster_id_
job.destination_path = destination_path_
# TODO
# job.owner_username = get_username(request.remote_user)
job.owner_username = get_username("CERN\\"+owner_username)
job.workflow_suffix = request.form["workflow_suffix"]
if 'coordinator_suffix' in request_form_keys:
job.coordinator_suffix = request.form["coordinator_suffix"]
if 'sqoop_nmap' in request_form_keys:
job.sqoop_nmap = request.form["sqoop_nmap"]
if 'sqoop_splitting_column' in request_form_keys:
job.sqoop_splitting_column = request.form["sqoop_splitting_column"]
if 'sqoop_incremental_method' in request_form_keys:
job.sqoop_incremental_method = request.form["sqoop_incremental_method"]
job.sqoop_direct = request.form["sqoop_direct"]
if 'start_time' in request_form_keys:
job.start_time = request.form["start_time"]
if 'end_time' in request_form_keys:
job.end_time = request.form["end_time"]
if 'interval' in request_form_keys:
job.interval = request.form["interval"]
if 'job_last_update' in request_form_keys:
job.job_last_update = request.form["job_last_update"]
runner = OozieRunner()
job.oozie_job_id=runner.submit(job)
DatabaseManager.meta_connector.add_job(job)
result = None
if job.coordinator_suffix is None:
status_code = runner.manage(job,'start')
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id, "start status_code": status_code}
else:
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id}
return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
