def post_job(scheduled_jobs_object, et_scanner_sqs, request_body_str): """ the request body should be a dictionary (so \*\*request_body are kwargs), and it with the following required keys: * redshift_id * log_name * log_schema_version * start_date * s3_path * contact_emails :param scheduled_jobs_object: the ScheduledJobs to which we post jobs :type scheduled_jobs_object: an instance of ScheduledJobs :param et_scanner_sqs: the scanner sqs to send message to :type et_scanner_sqs: SQSWrapper object :param request_body_str: a string version of the request body dict :type request_body_str: string :param scheduled_jobs_object: an instance of ScheduledJobs :type scheduled_jobs_object: ScheduledJobs :returns: success :rtype: boolean :raises S3ResponseError: if the bytes written don't match the length of the content """ request_body_dict = simplejson.loads(request_body_str) if "contact_emails" in request_body_dict and request_body_dict["contact_emails"] is not None: request_body_dict["contact_emails"] = set(request_body_dict["contact_emails"]) _check_required_args(request_body_dict, "post") s_date = datetime.datetime.strptime(request_body_dict["start_date"], "%Y-%m-%d") if "end_date" not in request_body_dict: request_body_dict["end_date"] = None elif request_body_dict["end_date"] is not None: e_date = datetime.datetime.strptime(request_body_dict["end_date"], "%Y-%m-%d") if s_date > e_date: raise ValueError("start date should not be greater than end date") if not request_body_dict.get("log_format"): request_body_dict["log_format"] = "json" request_body_dict["additional_arguments"] = _validate_additional_args(request_body_dict) # check that redshift cluster exists, throws ItemNotFound list_cluster_by_name(TableConnection.get_connection("RedshiftClusters"), request_body_dict["redshift_id"]) request_body_dict["hash_key"] = _create_hash_key(request_body_dict) request_body_dict["uuid"] = _get_uuid(scheduled_jobs_object) request_body_dict["et_status"] = NULL ret = scheduled_jobs_object.put(**request_body_dict) if ret: dummy_message = {"message": "dummy"} # TODO: use meaningful message instead of dummy et_scanner_sqs.write_message_to_queue(dummy_message) return {"post_accepted": {"result": ret, "uuid": request_body_dict["uuid"]}}
def cluster_by_name(request): """ cluster_by_name returns a dictionary with key "clusters" and value of a list of clusters with one entry -- the cluster specified in the route **GET /v1/clusters/**\ *{string: cluster_name}* **Query Parameters:** * **cluster_name** - the name of the log for which we want to see clusters Example: ``/v1/clusters/cluster-1`` *Example Response* :: { 'clusters': [ { 'redshift_id': 'cluster-1', 'port': 5439, 'host': 'cluster-1.account.region.redshift.amazonaws.com', 'db_schema': 'public', 'groups': ['search_infra', 'biz'] }] } ============ =========== Status Code Description ============ =========== **200** Success **404** invalid cluster_name **500** unknown exception ============ =========== * **Encoding type:** *application/json* """ cluster_name = request.matchdict.get('cluster_name') try: return 200, list_cluster_by_name( TableConnection.get_connection('RedshiftClusters'), cluster_name) except ValueError as e: return 404, {'error': repr(e)} except Exception as unknown_exception: return 500, {'error': repr(unknown_exception)}
def _get_redshift_cluster_details(self, rs_id): """ Get the host and port for a particular redshift id :param rs_id: redshift id of a cluster e.g., cluster-1 :type rs_id: string :returns: a 2 tuple containing a redshift host name and redshift port :rtype: tuple """ cluster = list_cluster_by_name( TableConnection.get_connection('RedshiftClusters'), rs_id) if cluster: port = cluster['port'] host = cluster['host'] schema = cluster['db_schema'] else: raise ValueError("No cluster named: {0}".format(rs_id)) return host, port, schema
def test_list_cluster_by_name_bad_cluster_name(self, cluster_name): with pytest.raises(ValueError) as e: list_cluster_by_name(None, cluster_name) expected_error = "ValueError: invalid cluster_name: {0}".format(cluster_name) assert e.exconly() == expected_error
def test_list_cluster_by_name(self, redshift_clusters): return_value = list_cluster_by_name(redshift_clusters, SAMPLE_REDSHIFT_ID) assert return_value['redshift_id'] == SAMPLE_REDSHIFT_ID
def test_list_cluster_by_name_bad_cluster_name(self, cluster_name): with pytest.raises(ValueError) as e: list_cluster_by_name(None, cluster_name) expected_error = "ValueError: invalid cluster_name: {0}".format( cluster_name) assert e.exconly() == expected_error
def post_job(scheduled_jobs_object, et_scanner_sqs, request_body_str): """ the request body should be a dictionary (so \*\*request_body are kwargs), and it with the following required keys: * redshift_id * log_name * log_schema_version * start_date * s3_path * contact_emails :param scheduled_jobs_object: the ScheduledJobs to which we post jobs :type scheduled_jobs_object: an instance of ScheduledJobs :param et_scanner_sqs: the scanner sqs to send message to :type et_scanner_sqs: SQSWrapper object :param request_body_str: a string version of the request body dict :type request_body_str: string :param scheduled_jobs_object: an instance of ScheduledJobs :type scheduled_jobs_object: ScheduledJobs :returns: success :rtype: boolean :raises S3ResponseError: if the bytes written don't match the length of the content """ request_body_dict = simplejson.loads(request_body_str) if 'contact_emails' in request_body_dict and request_body_dict[ 'contact_emails'] is not None: request_body_dict['contact_emails'] = set( request_body_dict['contact_emails']) _check_required_args(request_body_dict, 'post') s_date = datetime.datetime.strptime(request_body_dict['start_date'], "%Y-%m-%d") if 'end_date' not in request_body_dict: request_body_dict['end_date'] = None elif request_body_dict['end_date'] is not None: e_date = datetime.datetime.strptime(request_body_dict['end_date'], "%Y-%m-%d") if s_date > e_date: raise ValueError("start date should not be greater than end date") if not request_body_dict.get('log_format'): request_body_dict['log_format'] = 'json' request_body_dict['additional_arguments'] = _validate_additional_args( request_body_dict) # check that redshift cluster exists, throws ItemNotFound list_cluster_by_name(TableConnection.get_connection('RedshiftClusters'), request_body_dict['redshift_id']) request_body_dict['hash_key'] = _create_hash_key(request_body_dict) request_body_dict['uuid'] = _get_uuid(scheduled_jobs_object) request_body_dict['et_status'] = NULL ret = scheduled_jobs_object.put(**request_body_dict) if ret: dummy_message = { 'message': 'dummy' } # TODO: use meaningful message instead of dummy et_scanner_sqs.write_message_to_queue(dummy_message) return { 'post_accepted': { 'result': ret, 'uuid': request_body_dict['uuid'] } }