def api_v1_schemas(): # TODO # auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(request.remote_user)) """ :return: Schemas in the DB """ owner_username = request.args["owner_username"] if owner_username is None: raise BadRequest("Indicate the owner_username parameter") auth = DatabaseManager.auth_connector.get_servers_for_user( get_username("CERN\\" + owner_username)) meta = DatabaseManager.meta_connector.get_servers() meta_aliases = list(map(lambda server: server.server_name, meta)) available = list( filter(lambda key: key if key["database"] in meta_aliases else None, auth["databases"])) unavailable = list( filter( lambda key: key if key["database"] not in meta_aliases else None, auth["databases"])) result = {"schemas": {"available": available, "unavailable": unavailable}} return Response(json.dumps(result, indent=4), mimetype="application/json")
def api_v1_schemas(): # TODO # auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(request.remote_user)) """ :return: Schemas in the DB """ owner_username=request.args["owner_username"] if owner_username is None: raise BadRequest("Indicate the owner_username parameter") auth = DatabaseManager.auth_connector.get_servers_for_user(get_username("CERN\\"+owner_username)) meta = DatabaseManager.meta_connector.get_servers() meta_aliases = list(map(lambda server: server.server_name, meta)) available = list(filter(lambda key: key if key["database"] in meta_aliases else None, auth["databases"])) unavailable = list(filter(lambda key: key if key["database"] not in meta_aliases else None, auth["databases"])) result = { "schemas": { "available": available, "unavailable": unavailable } } return Response(json.dumps(result, indent=4), mimetype="application/json")
def api_v1_schemas(): # TODO # auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(request.remote_user)) """ :return: Schemas in the DB """ auth = DatabaseManager.auth_connector.get_servers_for_user(get_username(r"CERN\kdziedzi")) meta = DatabaseManager.meta_connector.get_servers() meta_aliases = map(lambda server: server.server_name, meta) available = filter(lambda key: key if key["database"] in meta_aliases else None, auth["databases"]) unavailable = filter(lambda key: key if key["database"] not in meta_aliases else None, auth["databases"]) result = { "schemas": { "available": available, "unavailable": unavailable } } return Response(json.dumps(result, indent=4), mimetype="application/json")
def api_v1_get_jobs(): # TODO if the user is an administrator for the system, allow them to see every job # kwargs = {"owner_username": get_username(request.remote_user)} kwargs = {"owner_username": get_username(r"CERN\kdziedzi")} jobs = DatabaseManager.meta_connector.get_jobs(**kwargs) filter_key_list = [ "job_id", "source_server_id", "source_schema_name", "source_object_name", "destination_cluster_id", "destination_path", "owner_username", "sqoop_nmap", "sqoop_splitting_column", "sqoop_incremental_method", "sqoop_direct", "start_time", "interval", ] result = {"jobs": []} for job in jobs: j = {} for key in filter_key_list: j.update({key: getattr(job, key, None)}) j.update({"source_server_alias": job.source_server.server_name}) j.update({"destination_cluster_alias": job.destination_cluster.cluster_name}) result["jobs"].append(j) return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
def api_v1_post_job(): """ Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job. :return: the inserted job """ # Check, whether we got all the required data from the user. required_fields = set([ "source_server_id", "source_schema_name", "source_object_name", "destination_cluster_id", "destination_path", "owner_username", "workflow_suffix", "sqoop_direct" ]) request_form_keys = request.form.keys() if len(set.intersection(required_fields, request_form_keys)) != len(required_fields): raise BadRequest( "Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join(required_fields.difference(request_form_keys))) # Check, whether the user could access the schema. source_server_id_ = int(request.form["source_server_id"]) server = DatabaseManager.meta_connector.get_servers( **{"server_id": source_server_id_}) if len(server) < 1: raise BadRequest("Server with ID " + source_server_id_ + " not found.") server = server[0] source_schema_name_ = request.form["source_schema_name"] owner_username = request.form["owner_username"] # TODO # if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema): if not DatabaseManager.auth_connector.can_user_access_schema( get_username("CERN\\" + owner_username), server.server_name, source_schema_name_): abort(403) # Check, whether the selected schema has the provided object source_object_name_ = request.form["source_object_name"] if source_object_name_.upper() not in ( obj.upper() for obj in DatabaseManager.auth_connector.get_available_objects( server.server_name, source_schema_name_)): raise BadRequest("Object with name " + source_object_name_ + " not found.") # cluster exists destination_cluster_id_ = int(request.form["destination_cluster_id"]) cluster = DatabaseManager.meta_connector.get_clusters( **{"cluster_id": destination_cluster_id_}) if len(cluster) < 1: raise BadRequest("Cluster with ID " + destination_cluster_id_ + " not found.") cluster = cluster[0] # destination path valid destination_path_ = request.form["destination_path"] if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None: raise BadRequest( "The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters." ) # TODO check start time # TODO check interval # insert new job job = DatabaseManager.meta_connector.create_job() job.source_server_id = source_server_id_ job.source_schema_name = source_schema_name_ job.source_object_name = source_object_name_ job.destination_cluster_id = destination_cluster_id_ job.destination_path = destination_path_ # TODO # job.owner_username = get_username(request.remote_user) job.owner_username = get_username("CERN\\" + owner_username) job.workflow_suffix = request.form["workflow_suffix"] if 'coordinator_suffix' in request_form_keys: job.coordinator_suffix = request.form["coordinator_suffix"] if 'sqoop_nmap' in request_form_keys: job.sqoop_nmap = request.form["sqoop_nmap"] if 'sqoop_splitting_column' in request_form_keys: job.sqoop_splitting_column = request.form["sqoop_splitting_column"] if 'sqoop_incremental_method' in request_form_keys: job.sqoop_incremental_method = request.form["sqoop_incremental_method"] job.sqoop_direct = request.form["sqoop_direct"] if 'start_time' in request_form_keys: job.start_time = request.form["start_time"] if 'end_time' in request_form_keys: job.end_time = request.form["end_time"] if 'interval' in request_form_keys: job.interval = request.form["interval"] if 'job_last_update' in request_form_keys: job.job_last_update = request.form["job_last_update"] runner = OozieRunner() job.oozie_job_id = runner.submit(job) DatabaseManager.meta_connector.add_job(job) result = None if job.coordinator_suffix is None: status_code = runner.manage(job, 'start') result = { "job_id": job.job_id, "oozie_job_id": job.oozie_job_id, "start status_code": status_code } else: result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id} return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
def api_v1_post_job(): """ Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job. :return: the inserted job """ # Check, whether we got all the required data from the user. required_fields = set([ "source_server_id", "source_schema_name", "source_object_name", "destination_cluster_id", "destination_path", "owner_username", "workflow_suffix", "sqoop_direct" ]) request_form_keys=request.form.keys() if len(set.intersection(required_fields, request_form_keys)) != len(required_fields): raise BadRequest("Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join( required_fields.difference(request_form_keys))) # Check, whether the user could access the schema. source_server_id_ = int(request.form["source_server_id"]) server = DatabaseManager.meta_connector.get_servers(**{"server_id": source_server_id_}) if len(server) < 1: raise BadRequest("Server with ID " + source_server_id_ + " not found.") server = server[0] source_schema_name_ = request.form["source_schema_name"] owner_username = request.form["owner_username"] # TODO # if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema): if not DatabaseManager.auth_connector.can_user_access_schema(get_username("CERN\\"+owner_username), server.server_name, source_schema_name_): abort(403) # Check, whether the selected schema has the provided object source_object_name_ = request.form["source_object_name"] if source_object_name_.upper() not in ( obj.upper() for obj in DatabaseManager.auth_connector.get_available_objects(server.server_name, source_schema_name_) ): raise BadRequest("Object with name " + source_object_name_ + " not found.") # cluster exists destination_cluster_id_ = int(request.form["destination_cluster_id"]) cluster = DatabaseManager.meta_connector.get_clusters(**{"cluster_id": destination_cluster_id_}) if len(cluster) < 1: raise BadRequest("Cluster with ID " + destination_cluster_id_ + " not found.") cluster = cluster[0] # destination path valid destination_path_ = request.form["destination_path"] if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None: raise BadRequest("The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters.") # TODO check start time # TODO check interval # insert new job job = DatabaseManager.meta_connector.create_job() job.source_server_id = source_server_id_ job.source_schema_name = source_schema_name_ job.source_object_name = source_object_name_ job.destination_cluster_id = destination_cluster_id_ job.destination_path = destination_path_ # TODO # job.owner_username = get_username(request.remote_user) job.owner_username = get_username("CERN\\"+owner_username) job.workflow_suffix = request.form["workflow_suffix"] if 'coordinator_suffix' in request_form_keys: job.coordinator_suffix = request.form["coordinator_suffix"] if 'sqoop_nmap' in request_form_keys: job.sqoop_nmap = request.form["sqoop_nmap"] if 'sqoop_splitting_column' in request_form_keys: job.sqoop_splitting_column = request.form["sqoop_splitting_column"] if 'sqoop_incremental_method' in request_form_keys: job.sqoop_incremental_method = request.form["sqoop_incremental_method"] job.sqoop_direct = request.form["sqoop_direct"] if 'start_time' in request_form_keys: job.start_time = request.form["start_time"] if 'end_time' in request_form_keys: job.end_time = request.form["end_time"] if 'interval' in request_form_keys: job.interval = request.form["interval"] if 'job_last_update' in request_form_keys: job.job_last_update = request.form["job_last_update"] runner = OozieRunner() job.oozie_job_id=runner.submit(job) DatabaseManager.meta_connector.add_job(job) result = None if job.coordinator_suffix is None: status_code = runner.manage(job,'start') result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id, "start status_code": status_code} else: result = {"job_id": job.job_id, "oozie_job_id": job.oozie_job_id} return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")
def api_v1_post_job(): """ Add a new job to the database. If the user is allowed to access the given schema, put it in as a new job. :return: the inserted job """ # Check, whether we got all the required data from the user. required_fields = set([ "source_server_id", "source_schema_name", "source_object_name", "destination_cluster_id", "destination_path", "start_time", "interval", ]) if len(set.intersection(required_fields, request.form.keys())) != len(required_fields): raise BadRequest("Required fields: " + ', '.join(required_fields) + ", missing fields: " + ', '.join( required_fields.difference(request.form.keys()))) # Check, whether the user could access the schema. source_server_id_ = int(request.form["source_server_id"]) server = DatabaseManager.meta_connector.get_servers(**{"server_id": source_server_id_}) if len(server) < 1: raise BadRequest("Server with ID " + source_server_id_ + " not found.") server = server[0] source_schema_name_ = request.form["source_schema_name"] # TODO # if not DatabaseManager.auth_connector.can_user_access_schema(get_username(request.remote_user), database, schema): if not DatabaseManager.auth_connector.can_user_access_schema(get_username(r"CERN\kdziedzi"), server.server_name, source_schema_name_): abort(403) # Check, whether the selected schema has the provided object source_object_name_ = request.form["source_object_name"] if source_object_name_.upper() not in ( obj.upper() for obj in DatabaseManager.auth_connector.get_available_objects(server.server_name, source_schema_name_) ): raise BadRequest("Object with name " + source_object_name_ + " not found.") # cluster exists destination_cluster_id_ = int(request.form["destination_cluster_id"]) cluster = DatabaseManager.meta_connector.get_clusters(**{"cluster_id": destination_cluster_id_}) if len(cluster) < 1: raise BadRequest("Cluster with ID " + destination_cluster_id_ + " not found.") cluster = cluster[0] # destination path valid destination_path_ = request.form["destination_path"] if re.match("^[a-zA-Z0-9-_/]*$", destination_path_) is None: raise BadRequest("The relative path is not valid. Please only use the a-z, A-Z, 0-9, -, _, / characters.") # TODO check start time # TODO check interval # insert new job job = DatabaseManager.meta_connector.create_job() job.source_server_id = source_server_id_ job.source_schema_name = source_schema_name_ job.source_object_name = source_object_name_ job.destination_cluster_id = destination_cluster_id_ job.destination_path = destination_path_ # TODO # job.owner_username = get_username(request.remote_user) job.owner_username = get_username(r"CERN\kdziedzi") job.sqoop_direct = True # TODO set start time and interval job_id = DatabaseManager.meta_connector.add_job(job) result = {"job_id": job_id} return Response(json.dumps(result, indent=4, default=json_datetime_handler_default), mimetype="application/json")