Ejemplo n.º 1
0
def create_topic(client, topic):
    """
    Check if topix exists if not create it.

    :param client:
    :param topic:
    :return:
    """
    project = 'projects/{}'.format(utils.get_project_id())
    dest_topic = project + '/topics/' + topic

    @backoff.on_exception(backoff.expo,
                          HttpError,
                          max_tries=3,
                          giveup=utils.fatal_code)
    def _do_get_request():
        return client.projects().topics().get(topic=dest_topic).execute()

    @backoff.on_exception(backoff.expo,
                          HttpError,
                          max_tries=3,
                          giveup=utils.fatal_code)
    def _do_create_request():
        client.projects().topics().create(name=dest_topic, body={}).execute()

    try:
        _do_get_request()
    except HttpError as e:
        if e.resp.status == 404:
            _do_create_request()
        else:
            logging.error(e)
            raise PubSubException(e)
Ejemplo n.º 2
0
 def __init__(self, cluster_name):
     self.monitorservice = googleapiclient.discovery. \
         build('monitoring',
               'v3',
               credentials=CREDENTIALS)
     self.project_id = utils.get_project_id()
     self.project_resource = "projects/{0}".format(self.project_id)
     self.metric_domain = 'dataproc.googleapis.com'
     self.cluster_name = cluster_name
Ejemplo n.º 3
0
 def __init__(self, cluster_name):
     self.dataproc = googleapiclient.discovery. \
         build('dataproc', 'v1',
               credentials=CREDENTIALS)
     self.cluster_name = cluster_name
     self.project_id = utils.get_project_id()
     s = settings.get_cluster_settings(cluster_name)
     if s.count(1) == 1:
         for st in s:
             self.cluster_settings = st
     else:
         raise DataProcException('Cluster not found!')
Ejemplo n.º 4
0
 def __init__(self, cluster_name):
     self.monitorservice = googleapiclient.discovery. \
         build('monitoring',
               'v3',
               credentials=CREDENTIALS)
     self.project_id = utils.get_project_id()
     self.project_resource = "projects/{0}".format(self.project_id)
     self.metric_domain = 'custom.googleapis.com'
     self.cluster_name = cluster_name
     self.metrics = [
         'ContainerPendingRatio', 'YARNMemoryAvailablePercentage',
         'YarnNodes', 'Workers', 'PreemptibleWorkers'
     ]
Ejemplo n.º 5
0
def get_regions():
    """
    Get all available regions.

    :return: all regions
    """
    compute = googleapiclient.discovery.build('compute', 'v1')

    request = compute.regions().list(project=utils.get_project_id())

    response = request.execute()
    rg = []
    for region in response['items']:
        rg.append(region['description'])
    return rg
Ejemplo n.º 6
0
def get_zones():
    """
    Get all available zones.

    :return: all regions
    """
    compute = googleapiclient.discovery.build("compute",
                                              "v1",
                                              cache_discovery=False)

    request = compute.zones().list(project=utils.get_project_id())

    response = request.execute()
    zones = []
    for region in response["items"]:
        zones.append(region["description"])
    return zones
Ejemplo n.º 7
0
def publish(client, body, topic):
    """Publish a message to a Pub/Sub topic."""
    project = 'projects/{}'.format(utils.get_project_id())
    dest_topic = project + '/topics/' + topic

    @backoff.on_exception(backoff.expo,
                          HttpError,
                          max_tries=3,
                          giveup=utils.fatal_code)
    def _do_request():
        client.projects().topics().publish(topic=dest_topic,
                                           body=body).execute()

    try:
        _do_request()
    except HttpError as e:
        logging.error(e)
        raise PubSubException(e)
Ejemplo n.º 8
0
def pull(client, sub, endpoint):
    """Register a listener endpoint."""
    subscription = get_full_subscription_name(utils.get_project_id(), sub)
    body = {'pushConfig': {'pushEndpoint': endpoint}}

    @backoff.on_exception(backoff.expo,
                          HttpError,
                          max_tries=3,
                          giveup=utils.fatal_code)
    def _do_request():
        client.projects().subscriptions().modifyPushConfig(
            subscription=subscription, body=body).execute()

    try:
        _do_request()
    except HttpError as e:

        logging.error(e)
        return 'Error', 500
    return 'ok, 204'
Ejemplo n.º 9
0
def create_subscriptions(client, sub, topic):
    """
    Create a subscription in pub/sub.

    :param client:
    :param sub:
    :param topic:
    :return:
    """
    project = 'projects/{}'.format(utils.get_project_id())
    dest_sub = project + '/subscriptions/' + sub
    dest_topic = project + '/topics/' + topic
    body = {'topic': dest_topic}

    @backoff.on_exception(backoff.expo,
                          HttpError,
                          max_tries=3,
                          giveup=utils.fatal_code)
    def _do_get_request():
        return client.projects().subscriptions().get(
            subscription=dest_sub).execute()

    @backoff.on_exception(backoff.expo,
                          HttpError,
                          max_tries=3,
                          giveup=utils.fatal_code)
    def _do_create_request():
        client.projects().subscriptions().create(name=dest_sub,
                                                 body=body).execute()

    try:
        _do_get_request()
    except HttpError as e:
        if e.resp.status == 404:
            _do_create_request()
        else:
            logging.error(e)
            raise PubSubException(e)
Ejemplo n.º 10
0
def policy_checker(name):
    """
    Check if there is a need to take an action for a policy.
    Args:
        name: policy name

    Returns:

    """
    client = ndb.Client()
    with client.context():
        policy = PolicyModel.query(PolicyModel.Name == name).get()
        if not policy:
            logging.error("Policy %s not found!", name)
            return "not found", 404
        schedule = SchedulesModel.query(
            SchedulesModel.Name == policy.Schedule).get()
        if not schedule:
            logging.error("Schedule %s not found!", policy.Schedule)
            return "not found", 404
        logging.debug(
            "Time at Timezone %s is %s",
            schedule.Timezone,
            tz.get_time_at_timezone(schedule.Timezone),
        )
        day, hour = tz.convert_time_to_index(
            tz.get_time_at_timezone(schedule.Timezone))
        logging.debug("Working on day  %s hour  %s", day, hour)
        arr = np.asarray(schedule.Schedule["__ndarray__"],
                         dtype=np.int).flatten()
        matrix_size = schedule.Schedule["Shape"][0] * schedule.Schedule[
            "Shape"][1]
        prev = utils.get_prev_idx(day * 24 + hour, matrix_size)
        now = arr[day * 24 + hour]
        prev = arr[prev]
        logging.debug("Previous state %s current %s", prev, now)
        if now == prev:
            # do nothing
            logging.info("Nothing should be done for %s", name)
            return "ok", 200
        else:
            # stop/start
            logging.info("State is changing for %s to %s", name, now)
            # for each tag lets do it

            task_client = tasks_v2.CloudTasksClient()
            task = {
                "app_engine_http_request": {  # Specify the type of request.
                    "http_method": "POST",
                    "relative_uri": "/tasks/change_state",
                }
            }
            parent = task_client.queue_path(
                queue="zorya-tasks",
                project=utils.get_project_id(),
                location=location.get_location(),
            )
            for tag in policy.Tags:
                for project in policy.Projects:
                    payload = {
                        "project": project,
                        "tagkey": next(iter(tag)),
                        "tagvalue": tag[next(iter(tag))],
                        "action": str(now),
                    }
                    task["app_engine_http_request"]["body"] = (
                        json.dumps(payload)).encode()
                    response = task_client.create_task(parent, task)
                    logging.debug("Task %s enqueued", response.name)
        return "ok", 200
Ejemplo n.º 11
0
 def _do_request():
     return self.dataproc.projects().regions().clusters().get(
         projectId=utils.get_project_id(),
         region=self.cluster_settings.Region,
         clusterName=self.cluster_name).execute()