def create_topic(client, topic): """ Check if topix exists if not create it. :param client: :param topic: :return: """ project = 'projects/{}'.format(utils.get_project_id()) dest_topic = project + '/topics/' + topic @backoff.on_exception(backoff.expo, HttpError, max_tries=3, giveup=utils.fatal_code) def _do_get_request(): return client.projects().topics().get(topic=dest_topic).execute() @backoff.on_exception(backoff.expo, HttpError, max_tries=3, giveup=utils.fatal_code) def _do_create_request(): client.projects().topics().create(name=dest_topic, body={}).execute() try: _do_get_request() except HttpError as e: if e.resp.status == 404: _do_create_request() else: logging.error(e) raise PubSubException(e)
def __init__(self, cluster_name): self.monitorservice = googleapiclient.discovery. \ build('monitoring', 'v3', credentials=CREDENTIALS) self.project_id = utils.get_project_id() self.project_resource = "projects/{0}".format(self.project_id) self.metric_domain = 'dataproc.googleapis.com' self.cluster_name = cluster_name
def __init__(self, cluster_name): self.dataproc = googleapiclient.discovery. \ build('dataproc', 'v1', credentials=CREDENTIALS) self.cluster_name = cluster_name self.project_id = utils.get_project_id() s = settings.get_cluster_settings(cluster_name) if s.count(1) == 1: for st in s: self.cluster_settings = st else: raise DataProcException('Cluster not found!')
def __init__(self, cluster_name): self.monitorservice = googleapiclient.discovery. \ build('monitoring', 'v3', credentials=CREDENTIALS) self.project_id = utils.get_project_id() self.project_resource = "projects/{0}".format(self.project_id) self.metric_domain = 'custom.googleapis.com' self.cluster_name = cluster_name self.metrics = [ 'ContainerPendingRatio', 'YARNMemoryAvailablePercentage', 'YarnNodes', 'Workers', 'PreemptibleWorkers' ]
def get_regions(): """ Get all available regions. :return: all regions """ compute = googleapiclient.discovery.build('compute', 'v1') request = compute.regions().list(project=utils.get_project_id()) response = request.execute() rg = [] for region in response['items']: rg.append(region['description']) return rg
def get_zones(): """ Get all available zones. :return: all regions """ compute = googleapiclient.discovery.build("compute", "v1", cache_discovery=False) request = compute.zones().list(project=utils.get_project_id()) response = request.execute() zones = [] for region in response["items"]: zones.append(region["description"]) return zones
def publish(client, body, topic): """Publish a message to a Pub/Sub topic.""" project = 'projects/{}'.format(utils.get_project_id()) dest_topic = project + '/topics/' + topic @backoff.on_exception(backoff.expo, HttpError, max_tries=3, giveup=utils.fatal_code) def _do_request(): client.projects().topics().publish(topic=dest_topic, body=body).execute() try: _do_request() except HttpError as e: logging.error(e) raise PubSubException(e)
def pull(client, sub, endpoint): """Register a listener endpoint.""" subscription = get_full_subscription_name(utils.get_project_id(), sub) body = {'pushConfig': {'pushEndpoint': endpoint}} @backoff.on_exception(backoff.expo, HttpError, max_tries=3, giveup=utils.fatal_code) def _do_request(): client.projects().subscriptions().modifyPushConfig( subscription=subscription, body=body).execute() try: _do_request() except HttpError as e: logging.error(e) return 'Error', 500 return 'ok, 204'
def create_subscriptions(client, sub, topic): """ Create a subscription in pub/sub. :param client: :param sub: :param topic: :return: """ project = 'projects/{}'.format(utils.get_project_id()) dest_sub = project + '/subscriptions/' + sub dest_topic = project + '/topics/' + topic body = {'topic': dest_topic} @backoff.on_exception(backoff.expo, HttpError, max_tries=3, giveup=utils.fatal_code) def _do_get_request(): return client.projects().subscriptions().get( subscription=dest_sub).execute() @backoff.on_exception(backoff.expo, HttpError, max_tries=3, giveup=utils.fatal_code) def _do_create_request(): client.projects().subscriptions().create(name=dest_sub, body=body).execute() try: _do_get_request() except HttpError as e: if e.resp.status == 404: _do_create_request() else: logging.error(e) raise PubSubException(e)
def policy_checker(name): """ Check if there is a need to take an action for a policy. Args: name: policy name Returns: """ client = ndb.Client() with client.context(): policy = PolicyModel.query(PolicyModel.Name == name).get() if not policy: logging.error("Policy %s not found!", name) return "not found", 404 schedule = SchedulesModel.query( SchedulesModel.Name == policy.Schedule).get() if not schedule: logging.error("Schedule %s not found!", policy.Schedule) return "not found", 404 logging.debug( "Time at Timezone %s is %s", schedule.Timezone, tz.get_time_at_timezone(schedule.Timezone), ) day, hour = tz.convert_time_to_index( tz.get_time_at_timezone(schedule.Timezone)) logging.debug("Working on day %s hour %s", day, hour) arr = np.asarray(schedule.Schedule["__ndarray__"], dtype=np.int).flatten() matrix_size = schedule.Schedule["Shape"][0] * schedule.Schedule[ "Shape"][1] prev = utils.get_prev_idx(day * 24 + hour, matrix_size) now = arr[day * 24 + hour] prev = arr[prev] logging.debug("Previous state %s current %s", prev, now) if now == prev: # do nothing logging.info("Nothing should be done for %s", name) return "ok", 200 else: # stop/start logging.info("State is changing for %s to %s", name, now) # for each tag lets do it task_client = tasks_v2.CloudTasksClient() task = { "app_engine_http_request": { # Specify the type of request. "http_method": "POST", "relative_uri": "/tasks/change_state", } } parent = task_client.queue_path( queue="zorya-tasks", project=utils.get_project_id(), location=location.get_location(), ) for tag in policy.Tags: for project in policy.Projects: payload = { "project": project, "tagkey": next(iter(tag)), "tagvalue": tag[next(iter(tag))], "action": str(now), } task["app_engine_http_request"]["body"] = ( json.dumps(payload)).encode() response = task_client.create_task(parent, task) logging.debug("Task %s enqueued", response.name) return "ok", 200
def _do_request(): return self.dataproc.projects().regions().clusters().get( projectId=utils.get_project_id(), region=self.cluster_settings.Region, clusterName=self.cluster_name).execute()