Python get_logger Examples

Example #1

File: settings_default.py Project: mikebin/inviso

import socket
import requests
from urllib import urlencode
from elasticsearch import Elasticsearch

from inviso import util
from inviso.monitor import Cluster
from inviso.handler import IndexHandler
from inviso.publish import DirectPublisher

from snakebite import channel
from snakebite.channel import log_protobuf_message
from snakebite.protobuf.IpcConnectionContext_pb2 import IpcConnectionContextProto

log = util.get_logger("inviso-settings")

# A monkey patch to make us run as hadoop
def create_hadoop_connection_context(self):
    '''Creates and seriazlies a IpcConnectionContextProto (not delimited)'''
    context = IpcConnectionContextProto()
    context.userInfo.effectiveUser = '******'
    context.protocol = "org.apache.hadoop.hdfs.protocol.ClientProtocol"

    s_context = context.SerializeToString()
    log_protobuf_message("RequestContext (len: %d)" % len(s_context), context)
    return s_context
channel.SocketRpcChannel.create_connection_context = create_hadoop_connection_context


def genie_lookup(job_id):
    path = '/genie/v0/jobs/' + job_id

Example #2

#!/usr/bin/python

import sys
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
from inviso.publish import DirectPublisher
from inviso import util
from inviso.monitor import HdfsMr2LogMonitor

log = util.get_logger('inviso.jes')

try:
    import settings
except Exception as e:
    log.error('settings.py not found')
    log.exception(e)
    sys.exit(1)


def main():
    publisher = settings.publisher

    monitors = []

    for cluster in settings.clusters:
        monitors.append(
            HdfsMr2LogMonitor(jobflow=cluster.id,
                              cluster_id=cluster.id,
                              cluster_name=cluster.name,
                              host=cluster.host,
                              port=cluster.namenode_rpc_port,

Example #3

import sys

from elasticsearch.helpers import bulk

import re
import requests
import arrow
from inviso.util import get_logger
import settings
import boto.emr

log = get_logger('inviso.cluster')
es_index = 'inviso-cluster'


def index_apps(es, cluster, info):
    apps = requests.get('http://%s:%s/ws/v1/cluster/apps?state=RUNNING' %
                        (cluster.host, cluster.resourcemanager_webapp_port),
                        headers={
                            'ACCEPT': 'application/json'
                        }).json().get('apps')

    if not apps:
        log.info(cluster.name + ': no applications running.')
        return

    apps = apps['app']

    documents = [{
        '_op_type': 'index',
        '_index': es_index,

Example #4

File: settings_default.py Project: s905060/inviso

import socket
import requests
from urllib import urlencode
from elasticsearch import Elasticsearch

from inviso import util
from inviso.monitor import Cluster
from inviso.handler import IndexHandler
from inviso.publish import DirectPublisher

from snakebite import channel
from snakebite.channel import log_protobuf_message
from snakebite.protobuf.IpcConnectionContext_pb2 import IpcConnectionContextProto

log = util.get_logger("inviso-settings")

# A monkey patch to make us run as hadoop
def create_hadoop_connection_context(self):
    '''Creates and seriazlies a IpcConnectionContextProto (not delimited)'''
    context = IpcConnectionContextProto()
    context.userInfo.effectiveUser = '******'
    context.protocol = "org.apache.hadoop.hdfs.protocol.ClientProtocol"

    s_context = context.SerializeToString()
    log_protobuf_message("RequestContext (len: %d)" % len(s_context), context)
    return s_context
channel.SocketRpcChannel.create_connection_context = create_hadoop_connection_context


def genie_lookup(job_id):
    path = '/genie/v0/jobs/' + job_id

Example #5

File: index_cluster_stats.py Project: othman-essabir/inviso

import sys

from elasticsearch.helpers import bulk

import re
import requests
import arrow
from inviso.util import get_logger
import settings
import boto.emr


log = get_logger('inviso.cluster')
es_index = 'inviso-cluster'

def index_apps(es, cluster, info):
    apps = requests.get('http://%s:%s/ws/v1/cluster/apps?state=RUNNING' % (cluster.host, cluster.port), headers = {'ACCEPT':'application/json'}).json().get('apps')
    
    if not apps:
        log.info(cluster.name + ': no applications running.')
        return
    
    apps = apps['app']
    
    documents = [
        {
            '_op_type': 'index',
            '_index': es_index,
            '_type': 'applications',
            '_id':  '_heartbeat_%s' % (info['timestamp'],),
            '_ttl': '30d',

Example #6

File: jes.py Project: Pivotal-Field-Engineering/inviso

#!/usr/bin/python

import sys
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
from inviso.publish import DirectPublisher
from inviso import util
from inviso.monitor import HdfsMr2LogMonitor

log = util.get_logger('inviso.jes')

try:
    import settings
except Exception as e:
    log.error('settings.py not found')
    log.exception(e)
    sys.exit(1)


def main():
    publisher = settings.publisher

    monitors = []
    
    for cluster in settings.clusters:
        monitors.append(HdfsMr2LogMonitor(jobflow=cluster.id,
                                          cluster_id=cluster.id, 
                                          cluster_name=cluster.name, 
                                          host=cluster.host,
                                          publisher=publisher,
                                          elasticsearch=settings.elasticsearch))