Python get_logger Examples

Example #1

File: restresource.py Project: LogicalSpark/indextank-service

 def __init__(self):
     self.__name__ = self.__class__.__name__
     self._account = None
     self.permitted_methods = []
     if hasattr(self, 'GET'):
         self.permitted_methods.append('GET')
     if hasattr(self, 'POST'):
         self.permitted_methods.append('POST')
     if hasattr(self, 'PUT'):
         self.permitted_methods.append('PUT')
         if not hasattr(self, 'POST'):
             self.permitted_methods.append('POST')
             self.POST = self.PUT
     if hasattr(self, 'DELETE'):
         self.permitted_methods.append('DELETE')
     self.logger = FakeLogger(get_logger(self.__name__), self)
     self.error_logger = FakeLogger(get_logger('Errors'), self, compname=self.__name__)
     self.executing = True

Example #2

File: monitor.py Project: LogicalSpark/indextank-service

 def __init__(self, pagerduty_email='*****@*****.**'):
     super(Monitor, self).__init__()
     self.name = self.__class__.__name__
     self.statuses = shelve.open('/data/monitor-%s.shelf' % self.name)
     self.logger = flaptor_logging.get_logger(self.name)
     self.failure_threshold = 1
     self.fatal_failure_threshold = 0
     self.severity = 'WARNING'
     self.title_template = '%s::%s: [%s] %s'
     self.pagerduty_email = pagerduty_email

Example #3

File: monitor.py Project: yiyinianhua/indextank-service

 def __init__(self, pagerduty_email='*****@*****.**'):
     super(Monitor, self).__init__()
     self.name = self.__class__.__name__
     self.statuses = shelve.open('/data/monitor-%s.shelf' % self.name)
     self.logger = flaptor_logging.get_logger(self.name)
     self.failure_threshold = 1
     self.fatal_failure_threshold = 0
     self.severity = 'WARNING'
     self.title_template = '%s::%s: [%s] %s'
     self.pagerduty_email = pagerduty_email

Example #4

 def __init__(self):
     self.__name__ = self.__class__.__name__
     self._account = None
     self.permitted_methods = []
     if hasattr(self, 'GET'):
         self.permitted_methods.append('GET')
     if hasattr(self, 'POST'):
         self.permitted_methods.append('POST')
     if hasattr(self, 'PUT'):
         self.permitted_methods.append('PUT')
         if not hasattr(self, 'POST'):
             self.permitted_methods.append('POST')
             self.POST = self.PUT
     if hasattr(self, 'DELETE'):
         self.permitted_methods.append('DELETE')
     self.logger = FakeLogger(get_logger(self.__name__), self)
     self.error_logger = FakeLogger(get_logger('Errors'),
                                    self,
                                    compname=self.__name__)
     self.executing = True

Example #5

File: populator.py Project: LogicalSpark/indextank-service

from lib.monitor import Monitor
from nebu.models import Index, IndexPopulation
from lib.indextank.client import ApiClient
from django.utils import simplejson as json

from lib import flaptor_logging

batch_size = 1000
dataset_files_path = './'

logger = flaptor_logging.get_logger('Populator')

class IndexPopulator(Monitor):
    
    def __init__(self):
        super(IndexPopulator, self).__init__()
        self.failure_threshold = 5
        self.fatal_failure_threshold = 20
        self.period = 30

    def iterable(self):
        return IndexPopulation.objects.exclude(status=IndexPopulation.Statuses.finished)
        
    def monitor(self, population):
        client = ApiClient(population.index.account.get_private_apiurl())
        index = client.get_index(population.index.name)
        
        if index.has_started():
            if population.status == IndexPopulation.Statuses.created:
                logger.info('Populating index ' + population.index.code + ' with dataset "' + population.dataset.name + '"')
    

Example #6

import signal
import systemutils

import flaptor.indextank.rpc.Controller as TController
from flaptor.indextank.rpc.ttypes import WorkerMountStats, WorkerLoadStats, IndexStats

from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
import sys
import commands
from lib import flaptor_logging
import simplejson as json

logger = flaptor_logging.get_logger('Controller')


def _get_working_directory(index_code, base_port):
    return "/data/indexes/%s-%d" % (index_code, base_port)  # XXX /data?


class Controller:
    """ Controls a Host"""
    def __init__(self):
        pass

    def __create_indexengine_environment(self, working_directory):
        shutil.copytree('indextank_lib', working_directory + "/lib")
        shutil.copy('startIndex.py', working_directory)
        shutil.copy('log4j.properties', working_directory)

Example #7

File: api_linked_rpc.py Project: LogicalSpark/indextank-service

    DeployManager, Controller, FrontendManager
    
from flaptor.indextank.rpc.ttypes import NebuException, IndextankException

''' =========================== 
           THRIFT STUFF
    =========================== '''
from thrift.transport import TSocket, TTransport
from thrift.protocol import TBinaryProtocol
from lib import flaptor_logging, exceptions
from thrift.transport.TTransport import TTransportException
from socket import socket
from socket import error as SocketError


logger = flaptor_logging.get_logger('RPC')

# Missing a way to close transport 
def getThriftControllerClient(host, timeout_ms=None):
    protocol, transport = __getThriftProtocolTransport(host,19010, timeout_ms)
    client = Controller.Client(protocol)
    transport.open()
    return client

# Missing a way to close transport 
def getThriftIndexerClient(host, base_port, timeout_ms=None):
    protocol, transport = __getThriftProtocolTransport(host, base_port + 1, timeout_ms)
    client = Indexer.Client(protocol)
    transport.open()
    return client
    

Example #8

File: deploy_manager.py Project: LogicalSpark/indextank-service

from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer

from lib import flaptor_logging, mail
import simplejson as json
import rpc
import sys
import random
from traceback import format_tb
from django.db import transaction
from datetime import datetime, timedelta

logger = flaptor_logging.get_logger('DeployMgr')

INITIAL_XMX = 100
INITIAL_XMX_THRESHOLD = 1000
MAXIMUM_PARALLEL_MOVES = 10
timeout_ms = 1000

def logerrors(func):
    def decorated(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception, e:
            logger.exception("Failed while executing %s", func.__name__)
            raise e
    return decorated

Example #9

import traceback
from lib import flaptor_logging
from django.http import HttpResponse

logger = flaptor_logging.get_logger('error_logging')


class ViewErrorLoggingMiddleware:
    def process_view(self, request, view_func, view_args, view_kwargs):
        self.view_name = view_func.__name__

    def process_exception(self, request, exception):
        logger.error('UNEXPECTED EXCEPTION in view "%s". Exception is: %s',
                     self.view_name, repr(traceback.print_exc()))
        return HttpResponse(
            '{"status":"ERROR", "message":"Unexpected error."}')

Example #10

from flaptor.indextank.rpc import Indexer, Searcher, Suggestor, Storage, LogWriter, WorkerManager,\
    DeployManager, Controller, FrontendManager

from flaptor.indextank.rpc.ttypes import NebuException, IndextankException
''' =========================== 
           THRIFT STUFF
    =========================== '''
from thrift.transport import TSocket, TTransport
from thrift.protocol import TBinaryProtocol
from lib import flaptor_logging, exceptions
from thrift.transport.TTransport import TTransportException
from socket import socket
from socket import error as SocketError

logger = flaptor_logging.get_logger('RPC')


# Missing a way to close transport
def getThriftControllerClient(host, timeout_ms=None):
    protocol, transport = __getThriftProtocolTransport(host, 19010, timeout_ms)
    client = Controller.Client(protocol)
    transport.open()
    return client


# Missing a way to close transport
def getThriftIndexerClient(host, base_port, timeout_ms=None):
    protocol, transport = __getThriftProtocolTransport(host, base_port + 1,
                                                       timeout_ms)
    client = Indexer.Client(protocol)
    transport.open()

Example #11

File: controller.py Project: MechanisM/indextank-service

import subprocess

import flaptor.indextank.rpc.Controller as TController
from flaptor.indextank.rpc.ttypes import WorkerMountStats, WorkerLoadStats, IndexStats


from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
import sys
import commands
from lib import flaptor_logging
import simplejson as json

logger = flaptor_logging.get_logger('Controller')

def _get_working_directory(index_code,base_port):
    return "/data/indexes/%s-%d"% ( index_code, base_port) # XXX /data?

class Controller:
    """ Controls a Host"""

    def __init__(self):
        pass

    def __create_indexengine_environment(self,working_directory):
        shutil.copytree('indextank_lib', working_directory + "/lib")
        shutil.copy('startIndex.py', working_directory)
        shutil.copy('log4j.properties', working_directory)
        

Example #12

File: error_logging.py Project: LogicalSpark/indextank-service

import traceback
from lib import flaptor_logging
from django.http import HttpResponse

logger = flaptor_logging.get_logger('error_logging')

class ViewErrorLoggingMiddleware:
    
    def process_view(self, request, view_func, view_args, view_kwargs):
        self.view_name = view_func.__name__
    def process_exception(self, request, exception):
        logger.error('UNEXPECTED EXCEPTION in view "%s". Exception is: %s', self.view_name, repr(traceback.print_exc()))
        return HttpResponse('{"status":"ERROR", "message":"Unexpected error."}')

Example #13

from flaptor.indextank.rpc import WorkerManager as TWorkerManager
from nebu.models import Worker

from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer

import boto, time
import socket
from lib import flaptor_logging, mail
import rpc

IMAGE_ID = 'ami-c6fa07af'

logger = flaptor_logging.get_logger('WorkerMgr')


def logerrors(func):
    def decorated(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception, e:
            logger.exception("Failed while executing %s", func.__name__)
            raise e

    return decorated


class WorkerManager:
    @logerrors

Example #14

File: api_linked_models.py Project: LogicalSpark/indextank-service

from lib.indextank.client import ApiClient, IndexAlreadyExists
from lib.authorizenet import AuthorizeNet, BillingException

from django.db import models
from django.contrib.auth.models import User
from django.utils import simplejson as json
from django.db import IntegrityError
from django.db.models.aggregates import Sum, Count

from lib import encoder, flaptor_logging

from django.conf import settings
from datetime import datetime

logger = flaptor_logging.get_logger('Models')

# idea taken from https://www.grc.com/passwords.htm
def generate_apikey(id):
    key = "2A1A8AE7CAEFAC47D6F74920CE4B0CE46430CDA6CF03D254C1C29402D727E570"
    while True:
        hash = hashlib.md5()
        hash.update('%d' % id)
        hash.update(key)
        hash.update('%d' % random.randint(0,1000000))
        random_part = binascii.b2a_base64(hash.digest())[:14]
        if not '/' in random_part:
            break
       
    unique_part = encoder.to_key(id)  
    

Example #15

from lib.indextank.client import ApiClient, IndexAlreadyExists
from lib.authorizenet import AuthorizeNet, BillingException

from django.db import models
from django.contrib.auth.models import User
from django.utils import simplejson as json
from django.db import IntegrityError
from django.db.models.aggregates import Sum, Count

from lib import encoder, flaptor_logging

from django.conf import settings
from datetime import datetime

logger = flaptor_logging.get_logger('Models')


# idea taken from https://www.grc.com/passwords.htm
def generate_apikey(id):
    key = settings.APIKEY_KEY
    while True:
        hash = hashlib.md5()
        hash.update('%d' % id)
        hash.update(key)
        hash.update('%d' % random.randint(0, sys.maxint))
        random_part = binascii.b2a_base64(hash.digest())[:14]
        if not '/' in random_part:
            break

    unique_part = encoder.to_key(id)

Example #16

File: controller.py Project: rvijax/indextank-service

import systemutils

import flaptor.indextank.rpc.Controller as TController
from flaptor.indextank.rpc.ttypes import WorkerMountStats, WorkerLoadStats, IndexStats


from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
import sys
import commands
from lib import flaptor_logging
import simplejson as json

logger = flaptor_logging.get_logger("Controller")


def _get_working_directory(index_code, base_port):
    return "/data/indexes/%s-%d" % (index_code, base_port)  # XXX /data?


class Controller:
    """ Controls a Host"""

    def __init__(self):
        pass

    def __create_indexengine_environment(self, working_directory):
        shutil.copytree("indextank_lib", working_directory + "/lib")
        shutil.copy("startIndex.py", working_directory)

Example #17

File: worker_manager.py Project: LogicalSpark/indextank-service

from flaptor.indextank.rpc import WorkerManager as TWorkerManager
from nebu.models import Worker

from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer

import boto, time
import socket
from lib import flaptor_logging, mail
import rpc

IMAGE_ID = 'ami-c6fa07af'

logger = flaptor_logging.get_logger('WorkerMgr')

def logerrors(func):
    def decorated(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception, e:
            logger.exception("Failed while executing %s", func.__name__)
            raise e
    return decorated


class WorkerManager:
    
    @logerrors
    def ec2_connection(self):

Example #18

File: models.py Project: rvijax/indextank-service

from lib.indextank.client import ApiClient, IndexAlreadyExists
from lib.authorizenet import AuthorizeNet, BillingException

from django.db import models
from django.contrib.auth.models import User
from django.utils import simplejson as json
from django.db import IntegrityError
from django.db.models.aggregates import Sum, Count

from lib import encoder, flaptor_logging

from django.conf import settings
from datetime import datetime

logger = flaptor_logging.get_logger("Models")

# idea taken from https://www.grc.com/passwords.htm
def generate_apikey(id):
    key = settings.APIKEY_KEY
    while True:
        hash = hashlib.md5()
        hash.update("%d" % id)
        hash.update(key)
        hash.update("%d" % random.randint(0, sys.maxint))
        random_part = binascii.b2a_base64(hash.digest())[:14]
        if not "/" in random_part:
            break

    unique_part = encoder.to_key(id)

Example #19

from lib.monitor import Monitor
from nebu.models import Index, IndexPopulation
from lib.indextank.client import ApiClient
from django.utils import simplejson as json

from lib import flaptor_logging

batch_size = 1000
dataset_files_path = './'

logger = flaptor_logging.get_logger('Populator')


class IndexPopulator(Monitor):
    def __init__(self):
        super(IndexPopulator, self).__init__()
        self.failure_threshold = 5
        self.fatal_failure_threshold = 20
        self.period = 30

    def iterable(self):
        return IndexPopulation.objects.exclude(
            status=IndexPopulation.Statuses.finished)

    def monitor(self, population):
        client = ApiClient(population.index.account.get_private_apiurl())
        index = client.get_index(population.index.name)

        if index.has_started():
            if population.status == IndexPopulation.Statuses.created:
                logger.info('Populating index ' + population.index.code +