def api_v1_schemas():
# TODO
# auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(request.remote_user))
"""
:return: Schemas in the DB
"""
owner_username = request.args["owner_username"]
if owner_username is None:
raise BadRequest("Indicate the owner_username parameter")
auth = DatabaseManager.auth_connector.get_servers_for_user(
get_username("CERN\\" + owner_username))
meta = DatabaseManager.meta_connector.get_servers()
meta_aliases = list(map(lambda server: server.server_name, meta))
available = list(
filter(lambda key: key if key["database"] in meta_aliases else None,
auth["databases"]))
unavailable = list(
filter(
lambda key: key if key["database"] not in meta_aliases else None,
auth["databases"]))
result = {"schemas": {"available": available, "unavailable": unavailable}}
return Response(json.dumps(result, indent=4), mimetype="application/json")
def api_v1_schemas():
# TODO
# auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(request.remote_user))
"""
:return: Schemas in the DB
"""
owner_username=request.args["owner_username"]
if owner_username is None:
raise BadRequest("Indicate the owner_username parameter")
auth = DatabaseManager.auth_connector.get_servers_for_user(get_username("CERN\\"+owner_username))
meta = DatabaseManager.meta_connector.get_servers()
meta_aliases = list(map(lambda server: server.server_name, meta))
available = list(filter(lambda key: key if key["database"] in meta_aliases else None, auth["databases"]))
unavailable = list(filter(lambda key: key if key["database"] not in meta_aliases else None, auth["databases"]))
result = {
"schemas": {
"available": available,
"unavailable": unavailable
}
}
return Response(json.dumps(result, indent=4), mimetype="application/json")
def api_v1_schemas():
# TODO
# auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(request.remote_user))
"""
:return: Schemas in the DB
"""
auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(r"CERN\kdziedzi"))
meta = DatabaseManager.meta_connector.get_servers()
meta_aliases = map(lambda server: server.server_name, meta)
available = filter(lambda key: key if key["database"] in meta_aliases else None, auth["databases"])
unavailable = filter(lambda key: key if key["database"] not in meta_aliases else None, auth["databases"])
result = {
"schemas": {
"available": available,
"unavailable": unavailable
}
}
return Response(json.dumps(result, indent=4), mimetype="application/json")
def api_v1_get_jobs():
# TODO if the user is an administrator for the system, allow them to see every job
# kwargs = {"owner_username": get_username(request.remote_user)}
kwargs = {"owner_username": get_username(r"CERN\kdziedzi")}
jobs = DatabaseManager.meta_connector.get_jobs(**kwargs)
filter_key_list = [
"job_id",
"source_server_id",
"source_schema_name",
"source_object_name",
"destination_cluster_id",
"destination_path",
"owner_username",
"sqoop_nmap",
"sqoop_splitting_column",
"sqoop_incremental_method",
"sqoop_direct",
"start_time",
"interval",
]
result = {"jobs": []}
for job in jobs:
j = {}
for key in filter_key_list:
j.update({key: getattr(job, key, None)})
j.update({"source_server_alias": job.source_server.server_name})
j.update({"destination_cluster_alias": job.destination_cluster.cluster_name})
result["jobs"].append(j)
return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
def api_v1_post_job():
"""
Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job.
:return: the inserted job
"""
# Check, whether we got all the required data from the user.
required_fields = set([
"source_server_id", "source_schema_name", "source_object_name",
"destination_cluster_id", "destination_path", "owner_username",
"workflow_suffix", "sqoop_direct"
])
request_form_keys = request.form.keys()
if len(set.intersection(required_fields,
request_form_keys)) != len(required_fields):
raise BadRequest(
"Required fields: " + ', '.join(required_fields) +
", missing fields: " +
', '.join(required_fields.difference(request_form_keys)))
# Check, whether the user could access the schema.
source_server_id_ = int(request.form["source_server_id"])
server = DatabaseManager.meta_connector.get_servers(
**{"server_id": source_server_id_})
if len(server) < 1:
raise BadRequest("Server with ID " + source_server_id_ + " not found.")
server = server[0]
source_schema_name_ = request.form["source_schema_name"]
owner_username = request.form["owner_username"]
# TODO
# if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema):
if not DatabaseManager.auth_connector.can_user_access_schema(
get_username("CERN\\" + owner_username), server.server_name,
source_schema_name_):
abort(403)
# Check, whether the selected schema has the provided object
source_object_name_ = request.form["source_object_name"]
if source_object_name_.upper() not in (
obj.upper()
for obj in DatabaseManager.auth_connector.get_available_objects(
server.server_name, source_schema_name_)):
raise BadRequest("Object with name " + source_object_name_ +
" not found.")
# cluster exists
destination_cluster_id_ = int(request.form["destination_cluster_id"])
cluster = DatabaseManager.meta_connector.get_clusters(
**{"cluster_id": destination_cluster_id_})
if len(cluster) < 1:
raise BadRequest("Cluster with ID " + destination_cluster_id_ +
" not found.")
cluster = cluster[0]
# destination path valid
destination_path_ = request.form["destination_path"]
if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None:
raise BadRequest(
"The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters."
)
# TODO check start time
# TODO check interval
# insert new job
job = DatabaseManager.meta_connector.create_job()
job.source_server_id = source_server_id_
job.source_schema_name = source_schema_name_
job.source_object_name = source_object_name_
job.destination_cluster_id = destination_cluster_id_
job.destination_path = destination_path_
# TODO
# job.owner_username = get_username(request.remote_user)
job.owner_username = get_username("CERN\\" + owner_username)
job.workflow_suffix = request.form["workflow_suffix"]
if 'coordinator_suffix' in request_form_keys:
job.coordinator_suffix = request.form["coordinator_suffix"]
if 'sqoop_nmap' in request_form_keys:
job.sqoop_nmap = request.form["sqoop_nmap"]
if 'sqoop_splitting_column' in request_form_keys:
job.sqoop_splitting_column = request.form["sqoop_splitting_column"]
if 'sqoop_incremental_method' in request_form_keys:
job.sqoop_incremental_method = request.form["sqoop_incremental_method"]
job.sqoop_direct = request.form["sqoop_direct"]
if 'start_time' in request_form_keys:
job.start_time = request.form["start_time"]
if 'end_time' in request_form_keys:
job.end_time = request.form["end_time"]
if 'interval' in request_form_keys:
job.interval = request.form["interval"]
if 'job_last_update' in request_form_keys:
job.job_last_update = request.form["job_last_update"]
runner = OozieRunner()
job.oozie_job_id = runner.submit(job)
DatabaseManager.meta_connector.add_job(job)
result = None
if job.coordinator_suffix is None:
status_code = runner.manage(job, 'start')
result = {
"job_id": job.job_id,
"oozie_job_id": job.oozie_job_id,
"start status_code": status_code
}
else:
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id}
return Response(json.dumps(result,
indent=4,
default=json_datetime_handler_default),
mimetype="application/json")
def api_v1_post_job():
"""
Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job.
:return: the inserted job
"""
# Check, whether we got all the required data from the user.
required_fields = set([
"source_server_id",
"source_schema_name",
"source_object_name",
"destination_cluster_id",
"destination_path",
"owner_username",
"workflow_suffix",
"sqoop_direct"
])
request_form_keys=request.form.keys()
if len(set.intersection(required_fields, request_form_keys)) != len(required_fields):
raise BadRequest("Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join(
required_fields.difference(request_form_keys)))
# Check, whether the user could access the schema.
source_server_id_ = int(request.form["source_server_id"])
server = DatabaseManager.meta_connector.get_servers(**{"server_id": source_server_id_})
if len(server) < 1:
raise BadRequest("Server with ID " + source_server_id_ + " not found.")
server = server[0]
source_schema_name_ = request.form["source_schema_name"]
owner_username = request.form["owner_username"]
# TODO
# if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema):
if not DatabaseManager.auth_connector.can_user_access_schema(get_username("CERN\\"+owner_username), server.server_name,
source_schema_name_):
abort(403)
# Check, whether the selected schema has the provided object
source_object_name_ = request.form["source_object_name"]
if source_object_name_.upper() not in (
obj.upper() for obj
in DatabaseManager.auth_connector.get_available_objects(server.server_name, source_schema_name_)
):
raise BadRequest("Object with name " + source_object_name_ + " not found.")
# cluster exists
destination_cluster_id_ = int(request.form["destination_cluster_id"])
cluster = DatabaseManager.meta_connector.get_clusters(**{"cluster_id": destination_cluster_id_})
if len(cluster) < 1:
raise BadRequest("Cluster with ID " + destination_cluster_id_ + " not found.")
cluster = cluster[0]
# destination path valid
destination_path_ = request.form["destination_path"]
if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None:
raise BadRequest("The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters.")
# TODO check start time
# TODO check interval
# insert new job
job = DatabaseManager.meta_connector.create_job()
job.source_server_id = source_server_id_
job.source_schema_name = source_schema_name_
job.source_object_name = source_object_name_
job.destination_cluster_id = destination_cluster_id_
job.destination_path = destination_path_
# TODO
# job.owner_username = get_username(request.remote_user)
job.owner_username = get_username("CERN\\"+owner_username)
job.workflow_suffix = request.form["workflow_suffix"]
if 'coordinator_suffix' in request_form_keys:
job.coordinator_suffix = request.form["coordinator_suffix"]
if 'sqoop_nmap' in request_form_keys:
job.sqoop_nmap = request.form["sqoop_nmap"]
if 'sqoop_splitting_column' in request_form_keys:
job.sqoop_splitting_column = request.form["sqoop_splitting_column"]
if 'sqoop_incremental_method' in request_form_keys:
job.sqoop_incremental_method = request.form["sqoop_incremental_method"]
job.sqoop_direct = request.form["sqoop_direct"]
if 'start_time' in request_form_keys:
job.start_time = request.form["start_time"]
if 'end_time' in request_form_keys:
job.end_time = request.form["end_time"]
if 'interval' in request_form_keys:
job.interval = request.form["interval"]
if 'job_last_update' in request_form_keys:
job.job_last_update = request.form["job_last_update"]
runner = OozieRunner()
job.oozie_job_id=runner.submit(job)
DatabaseManager.meta_connector.add_job(job)
result = None
if job.coordinator_suffix is None:
status_code = runner.manage(job,'start')
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id, "start status_code": status_code}
else:
result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id}
return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
def api_v1_post_job():
"""
Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job.
:return: the inserted job
"""
# Check, whether we got all the required data from the user.
required_fields = set([
"source_server_id",
"source_schema_name",
"source_object_name",
"destination_cluster_id",
"destination_path",
"start_time",
"interval",
])
if len(set.intersection(required_fields, request.form.keys())) != len(required_fields):
raise BadRequest("Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join(
required_fields.difference(request.form.keys())))
# Check, whether the user could access the schema.
source_server_id_ = int(request.form["source_server_id"])
server = DatabaseManager.meta_connector.get_servers(**{"server_id": source_server_id_})
if len(server) < 1:
raise BadRequest("Server with ID " + source_server_id_ + " not found.")
server = server[0]
source_schema_name_ = request.form["source_schema_name"]
# TODO
# if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema):
if not DatabaseManager.auth_connector.can_user_access_schema(get_username(r"CERN\kdziedzi"), server.server_name,
source_schema_name_):
abort(403)
# Check, whether the selected schema has the provided object
source_object_name_ = request.form["source_object_name"]
if source_object_name_.upper() not in (
obj.upper() for obj
in DatabaseManager.auth_connector.get_available_objects(server.server_name, source_schema_name_)
):
raise BadRequest("Object with name " + source_object_name_ + " not found.")
# cluster exists
destination_cluster_id_ = int(request.form["destination_cluster_id"])
cluster = DatabaseManager.meta_connector.get_clusters(**{"cluster_id": destination_cluster_id_})
if len(cluster) < 1:
raise BadRequest("Cluster with ID " + destination_cluster_id_ + " not found.")
cluster = cluster[0]
# destination path valid
destination_path_ = request.form["destination_path"]
if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None:
raise BadRequest("The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters.")
# TODO check start time
# TODO check interval
# insert new job
job = DatabaseManager.meta_connector.create_job()
job.source_server_id = source_server_id_
job.source_schema_name = source_schema_name_
job.source_object_name = source_object_name_
job.destination_cluster_id = destination_cluster_id_
job.destination_path = destination_path_
# TODO
# job.owner_username = get_username(request.remote_user)
job.owner_username = get_username(r"CERN\kdziedzi")
job.sqoop_direct = True
# TODO set start time and interval
job_id = DatabaseManager.meta_connector.add_job(job)
result = {"job_id": job_id}
return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
