def create_topic(client, topic):
"""
Check if topix exists if not create it.
:param client:
:param topic:
:return:
"""
project = 'projects/{}'.format(utils.get_project_id())
dest_topic = project + '/topics/' + topic
@backoff.on_exception(backoff.expo,
HttpError,
max_tries=3,
giveup=utils.fatal_code)
def _do_get_request():
return client.projects().topics().get(topic=dest_topic).execute()
@backoff.on_exception(backoff.expo,
HttpError,
max_tries=3,
giveup=utils.fatal_code)
def _do_create_request():
client.projects().topics().create(name=dest_topic, body={}).execute()
try:
_do_get_request()
except HttpError as e:
if e.resp.status == 404:
_do_create_request()
else:
logging.error(e)
raise PubSubException(e)
def __init__(self, cluster_name):
self.monitorservice = googleapiclient.discovery. \
build('monitoring',
'v3',
credentials=CREDENTIALS)
self.project_id = utils.get_project_id()
self.project_resource = "projects/{0}".format(self.project_id)
self.metric_domain = 'dataproc.googleapis.com'
self.cluster_name = cluster_name
def __init__(self, cluster_name):
self.dataproc = googleapiclient.discovery. \
build('dataproc', 'v1',
credentials=CREDENTIALS)
self.cluster_name = cluster_name
self.project_id = utils.get_project_id()
s = settings.get_cluster_settings(cluster_name)
if s.count(1) == 1:
for st in s:
self.cluster_settings = st
else:
raise DataProcException('Cluster not found!')
def __init__(self, cluster_name):
self.monitorservice = googleapiclient.discovery. \
build('monitoring',
'v3',
credentials=CREDENTIALS)
self.project_id = utils.get_project_id()
self.project_resource = "projects/{0}".format(self.project_id)
self.metric_domain = 'custom.googleapis.com'
self.cluster_name = cluster_name
self.metrics = [
'ContainerPendingRatio', 'YARNMemoryAvailablePercentage',
'YarnNodes', 'Workers', 'PreemptibleWorkers'
]
def get_regions():
"""
Get all available regions.
:return: all regions
"""
compute = googleapiclient.discovery.build('compute', 'v1')
request = compute.regions().list(project=utils.get_project_id())
response = request.execute()
rg = []
for region in response['items']:
rg.append(region['description'])
return rg
def get_zones():
"""
Get all available zones.
:return: all regions
"""
compute = googleapiclient.discovery.build("compute",
"v1",
cache_discovery=False)
request = compute.zones().list(project=utils.get_project_id())
response = request.execute()
zones = []
for region in response["items"]:
zones.append(region["description"])
return zones
def publish(client, body, topic):
"""Publish a message to a Pub/Sub topic."""
project = 'projects/{}'.format(utils.get_project_id())
dest_topic = project + '/topics/' + topic
@backoff.on_exception(backoff.expo,
HttpError,
max_tries=3,
giveup=utils.fatal_code)
def _do_request():
client.projects().topics().publish(topic=dest_topic,
body=body).execute()
try:
_do_request()
except HttpError as e:
logging.error(e)
raise PubSubException(e)
def pull(client, sub, endpoint):
"""Register a listener endpoint."""
subscription = get_full_subscription_name(utils.get_project_id(), sub)
body = {'pushConfig': {'pushEndpoint': endpoint}}
@backoff.on_exception(backoff.expo,
HttpError,
max_tries=3,
giveup=utils.fatal_code)
def _do_request():
client.projects().subscriptions().modifyPushConfig(
subscription=subscription, body=body).execute()
try:
_do_request()
except HttpError as e:
logging.error(e)
return 'Error', 500
return 'ok, 204'
def create_subscriptions(client, sub, topic):
"""
Create a subscription in pub/sub.
:param client:
:param sub:
:param topic:
:return:
"""
project = 'projects/{}'.format(utils.get_project_id())
dest_sub = project + '/subscriptions/' + sub
dest_topic = project + '/topics/' + topic
body = {'topic': dest_topic}
@backoff.on_exception(backoff.expo,
HttpError,
max_tries=3,
giveup=utils.fatal_code)
def _do_get_request():
return client.projects().subscriptions().get(
subscription=dest_sub).execute()
@backoff.on_exception(backoff.expo,
HttpError,
max_tries=3,
giveup=utils.fatal_code)
def _do_create_request():
client.projects().subscriptions().create(name=dest_sub,
body=body).execute()
try:
_do_get_request()
except HttpError as e:
if e.resp.status == 404:
_do_create_request()
else:
logging.error(e)
raise PubSubException(e)
def policy_checker(name):
"""
Check if there is a need to take an action for a policy.
Args:
name: policy name
Returns:
"""
client = ndb.Client()
with client.context():
policy = PolicyModel.query(PolicyModel.Name == name).get()
if not policy:
logging.error("Policy %s not found!", name)
return "not found", 404
schedule = SchedulesModel.query(
SchedulesModel.Name == policy.Schedule).get()
if not schedule:
logging.error("Schedule %s not found!", policy.Schedule)
return "not found", 404
logging.debug(
"Time at Timezone %s is %s",
schedule.Timezone,
tz.get_time_at_timezone(schedule.Timezone),
)
day, hour = tz.convert_time_to_index(
tz.get_time_at_timezone(schedule.Timezone))
logging.debug("Working on day %s hour %s", day, hour)
arr = np.asarray(schedule.Schedule["__ndarray__"],
dtype=np.int).flatten()
matrix_size = schedule.Schedule["Shape"][0] * schedule.Schedule[
"Shape"][1]
prev = utils.get_prev_idx(day * 24 + hour, matrix_size)
now = arr[day * 24 + hour]
prev = arr[prev]
logging.debug("Previous state %s current %s", prev, now)
if now == prev:
# do nothing
logging.info("Nothing should be done for %s", name)
return "ok", 200
else:
# stop/start
logging.info("State is changing for %s to %s", name, now)
# for each tag lets do it
task_client = tasks_v2.CloudTasksClient()
task = {
"app_engine_http_request": { # Specify the type of request.
"http_method": "POST",
"relative_uri": "/tasks/change_state",
}
}
parent = task_client.queue_path(
queue="zorya-tasks",
project=utils.get_project_id(),
location=location.get_location(),
)
for tag in policy.Tags:
for project in policy.Projects:
payload = {
"project": project,
"tagkey": next(iter(tag)),
"tagvalue": tag[next(iter(tag))],
"action": str(now),
}
task["app_engine_http_request"]["body"] = (
json.dumps(payload)).encode()
response = task_client.create_task(parent, task)
logging.debug("Task %s enqueued", response.name)
return "ok", 200
def _do_request():
return self.dataproc.projects().regions().clusters().get(
projectId=utils.get_project_id(),
region=self.cluster_settings.Region,
clusterName=self.cluster_name).execute()