def post_job(scheduled_jobs_object, et_scanner_sqs, request_body_str):
"""
the request body should be a dictionary (so \*\*request_body are kwargs),
and it with the following required keys:
* redshift_id
* log_name
* log_schema_version
* start_date
* s3_path
* contact_emails
:param scheduled_jobs_object: the ScheduledJobs to which we post jobs
:type scheduled_jobs_object: an instance of ScheduledJobs
:param et_scanner_sqs: the scanner sqs to send message to
:type et_scanner_sqs: SQSWrapper object
:param request_body_str: a string version of the request body dict
:type request_body_str: string
:param scheduled_jobs_object: an instance of ScheduledJobs
:type scheduled_jobs_object: ScheduledJobs
:returns: success
:rtype: boolean
:raises S3ResponseError: if the bytes written don't match the length of
the content
"""
request_body_dict = simplejson.loads(request_body_str)
if "contact_emails" in request_body_dict and request_body_dict["contact_emails"] is not None:
request_body_dict["contact_emails"] = set(request_body_dict["contact_emails"])
_check_required_args(request_body_dict, "post")
s_date = datetime.datetime.strptime(request_body_dict["start_date"], "%Y-%m-%d")
if "end_date" not in request_body_dict:
request_body_dict["end_date"] = None
elif request_body_dict["end_date"] is not None:
e_date = datetime.datetime.strptime(request_body_dict["end_date"], "%Y-%m-%d")
if s_date > e_date:
raise ValueError("start date should not be greater than end date")
if not request_body_dict.get("log_format"):
request_body_dict["log_format"] = "json"
request_body_dict["additional_arguments"] = _validate_additional_args(request_body_dict)
# check that redshift cluster exists, throws ItemNotFound
list_cluster_by_name(TableConnection.get_connection("RedshiftClusters"), request_body_dict["redshift_id"])
request_body_dict["hash_key"] = _create_hash_key(request_body_dict)
request_body_dict["uuid"] = _get_uuid(scheduled_jobs_object)
request_body_dict["et_status"] = NULL
ret = scheduled_jobs_object.put(**request_body_dict)
if ret:
dummy_message = {"message": "dummy"} # TODO: use meaningful message instead of dummy
et_scanner_sqs.write_message_to_queue(dummy_message)
return {"post_accepted": {"result": ret, "uuid": request_body_dict["uuid"]}}
def cluster_by_name(request):
"""
cluster_by_name returns a dictionary with key "clusters" and value of a
list of clusters with one entry -- the cluster specified in the route
**GET /v1/clusters/**\ *{string: cluster_name}*
**Query Parameters:**
* **cluster_name** - the name of the log for which we want to see clusters
Example: ``/v1/clusters/cluster-1``
*Example Response* ::
{ 'clusters': [
{
'redshift_id': 'cluster-1',
'port': 5439,
'host': 'cluster-1.account.region.redshift.amazonaws.com',
'db_schema': 'public',
'groups': ['search_infra', 'biz']
}]
}
============ ===========
Status Code Description
============ ===========
**200** Success
**404** invalid cluster_name
**500** unknown exception
============ ===========
* **Encoding type:** *application/json*
"""
cluster_name = request.matchdict.get('cluster_name')
try:
return 200, list_cluster_by_name(
TableConnection.get_connection('RedshiftClusters'), cluster_name)
except ValueError as e:
return 404, {'error': repr(e)}
except Exception as unknown_exception:
return 500, {'error': repr(unknown_exception)}
def _get_redshift_cluster_details(self, rs_id):
""" Get the host and port for a particular redshift id
:param rs_id: redshift id of a cluster e.g., cluster-1
:type rs_id: string
:returns: a 2 tuple containing a redshift host name and redshift port
:rtype: tuple
"""
cluster = list_cluster_by_name(
TableConnection.get_connection('RedshiftClusters'),
rs_id)
if cluster:
port = cluster['port']
host = cluster['host']
schema = cluster['db_schema']
else:
raise ValueError("No cluster named: {0}".format(rs_id))
return host, port, schema
def test_list_cluster_by_name_bad_cluster_name(self, cluster_name):
with pytest.raises(ValueError) as e:
list_cluster_by_name(None, cluster_name)
expected_error = "ValueError: invalid cluster_name: {0}".format(cluster_name)
assert e.exconly() == expected_error
def test_list_cluster_by_name(self, redshift_clusters):
return_value = list_cluster_by_name(redshift_clusters, SAMPLE_REDSHIFT_ID)
assert return_value['redshift_id'] == SAMPLE_REDSHIFT_ID
def test_list_cluster_by_name_bad_cluster_name(self, cluster_name):
with pytest.raises(ValueError) as e:
list_cluster_by_name(None, cluster_name)
expected_error = "ValueError: invalid cluster_name: {0}".format(
cluster_name)
assert e.exconly() == expected_error
def test_list_cluster_by_name(self, redshift_clusters):
return_value = list_cluster_by_name(redshift_clusters,
SAMPLE_REDSHIFT_ID)
assert return_value['redshift_id'] == SAMPLE_REDSHIFT_ID
def post_job(scheduled_jobs_object, et_scanner_sqs, request_body_str):
"""
the request body should be a dictionary (so \*\*request_body are kwargs),
and it with the following required keys:
* redshift_id
* log_name
* log_schema_version
* start_date
* s3_path
* contact_emails
:param scheduled_jobs_object: the ScheduledJobs to which we post jobs
:type scheduled_jobs_object: an instance of ScheduledJobs
:param et_scanner_sqs: the scanner sqs to send message to
:type et_scanner_sqs: SQSWrapper object
:param request_body_str: a string version of the request body dict
:type request_body_str: string
:param scheduled_jobs_object: an instance of ScheduledJobs
:type scheduled_jobs_object: ScheduledJobs
:returns: success
:rtype: boolean
:raises S3ResponseError: if the bytes written don't match the length of
the content
"""
request_body_dict = simplejson.loads(request_body_str)
if 'contact_emails' in request_body_dict and request_body_dict[
'contact_emails'] is not None:
request_body_dict['contact_emails'] = set(
request_body_dict['contact_emails'])
_check_required_args(request_body_dict, 'post')
s_date = datetime.datetime.strptime(request_body_dict['start_date'],
"%Y-%m-%d")
if 'end_date' not in request_body_dict:
request_body_dict['end_date'] = None
elif request_body_dict['end_date'] is not None:
e_date = datetime.datetime.strptime(request_body_dict['end_date'],
"%Y-%m-%d")
if s_date > e_date:
raise ValueError("start date should not be greater than end date")
if not request_body_dict.get('log_format'):
request_body_dict['log_format'] = 'json'
request_body_dict['additional_arguments'] = _validate_additional_args(
request_body_dict)
# check that redshift cluster exists, throws ItemNotFound
list_cluster_by_name(TableConnection.get_connection('RedshiftClusters'),
request_body_dict['redshift_id'])
request_body_dict['hash_key'] = _create_hash_key(request_body_dict)
request_body_dict['uuid'] = _get_uuid(scheduled_jobs_object)
request_body_dict['et_status'] = NULL
ret = scheduled_jobs_object.put(**request_body_dict)
if ret:
dummy_message = {
'message': 'dummy'
} # TODO: use meaningful message instead of dummy
et_scanner_sqs.write_message_to_queue(dummy_message)
return {
'post_accepted': {
'result': ret,
'uuid': request_body_dict['uuid']
}
}
