예제 #1
0
    def fail(self, request, response, content):
        if self.fail_limit is None or self.fail_period is None or self.blackout_wait is None:
            return

        now = time.time()

        ### Was getting deque() corruption error when the server was uploaded with requests.  This is to help prevent that.
        self.__semaphore.acquire()

        self.__fails.append(self.FailLog(request.url, request.body, request.headers, response.status, content))

        cutoff = now - self.fail_period
        count = 0

        while len(self.__fails) > 0:
            if self.__fails[0].timestamp < cutoff:
                self.__fails.popleft()
            else:
                break
        count = len(self.__fails)

        self.__semaphore.release()

        if count > self.fail_limit:
            logs.warning("hit fail limit for service '%s'" % self.__service_name)
            self.blackout_start = time.time()

            # Email dev if a fail limit was reached
            if is_ec2():
                self.sendFailLogEmail()
예제 #2
0
파일: SimpleDB.py 프로젝트: Stamped/Stamped
    def __init__(self, domain=None):
        self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        self.domain_name = None
        self.domains = {}

        if domain is None and is_ec2():
            stack = get_stack()
            stack_name = str(stack['instance']['stack'])
            self.domain_name = stack_name
        elif domain is not None:
            self.domain_name = domain
예제 #3
0
def parseCommandLine():
    usage   = "Usage: %prog [options] command [args]"
    parser  = OptionParser(usage=usage)
    
    parser.add_option("-f", "--form", action="store_true", dest="show_form", 
        default=False, help="Include user-submitted input form in results")
    
    parser.add_option("-s", "--stack", default=None, type="string",
        action="store", dest="stack", help="stack to pull logs from")
    
    parser.add_option("-u", "--user-id", dest="user_id", 
        default=None, type="string", help="Filter results on user id")
    
    parser.add_option("-m", "--method", dest="method", 
        default=None, type="string", help="Request method (POST, GET, or ASYNC)")
    
    parser.add_option("-l", "--limit", dest="limit", 
        default=10, type="int", help="Limit number of results returned")
    
    parser.add_option("-p", "--path", dest="path", 
        default=None, type="string", help="Filter results on url path")
    
    parser.add_option("-c", "--code", dest="code", 
        default=None, type="int", help="Filter results on status code")

    parser.add_option("-S", "--slow", dest="slow", action="store",
        default=None, type="int", help="Filter results that took longer than n seconds")


    (options, args) = parser.parse_args()
    
    if is_ec2() or options.stack:
        logger_nodes = get_db_nodes('logger')
        if len(logger_nodes) == 0:
            print("Warning: No instances exist on stack 'logger'")
            return options
        if options.stack is None:
            options.stack = 'bowser'

    return options
예제 #4
0
    def __init__(self):
        ASourceContainer.__init__(self)
        ASourceController.__init__(self)
        self.__now = None
        self.__global_priorities = {}
        self.__group_priorities = {}
        self.__groups = {}
        self.__sources = []
        self.__default_max_iterations = 10
        self.__group_max_ages = {}
        if is_prod_stack():
            self.__global_max_age = timedelta(7)
        elif is_ec2():
            self.__global_max_age = timedelta(2)
        else:
            self.__global_max_age = timedelta(minutes=0)

        for group in allGroups:
            self.addGroup(group())
        self.setGlobalPriority("seed", 100)
        self.setGlobalPriority("manual", 10000)
        self.setGlobalPriority("derived", -100)
        self.setMaxAge("stamped", "nemeses", timedelta())
예제 #5
0
파일: settings.py 프로젝트: Stamped/Stamped
# Django settings for httpapi project.

import Globals
import logs
from libs.ec2_utils import is_ec2

DEBUG = (not is_ec2())
TEMPLATE_DEBUG = DEBUG

logs.info("Django DEBUG=%s" % DEBUG)

ADMINS = (
    # ('Your Name', '*****@*****.**'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
예제 #6
0
파일: logger.py 프로젝트: Stamped/Stamped
def parseCommandLine():
    usage = "Usage: %prog [options] command [args]"
    version = "%prog " + __version__
    parser = OptionParser(usage=usage, version=version)

    parser.add_option(
        "-d",
        "--headers",
        action="store_true",
        dest="show_headers",
        default=False,
        help="Include HTTP headers in results",
    )

    parser.add_option(
        "-f",
        "--form",
        action="store_true",
        dest="show_form",
        default=False,
        help="Include user-submitted input form in results",
    )

    parser.add_option(
        "-o", "--output", action="store_true", dest="show_output", default=False, help="Include JSON result in results"
    )

    parser.add_option(
        "-e", "--errors", action="store_true", dest="show_errors", default=False, help="Only display errors in results"
    )

    parser.add_option(
        "-v", "--verbose", action="store_true", dest="verbose", default=False, help="Set verbosity of logs"
    )

    parser.add_option(
        "-s", "--stack", default=None, type="string", action="store", dest="stack", help="stack to pull logs from"
    )

    parser.add_option("-u", "--user-id", dest="user_id", default=None, type="string", help="Filter results on user id")

    parser.add_option("-i", "--request-id", dest="request_id", default=None, type="string", help="Filter on request id")

    parser.add_option(
        "-m", "--method", dest="method", default=None, type="string", help="Request method (POST, GET, or ASYNC)"
    )

    parser.add_option("-l", "--limit", dest="limit", default=10, type="int", help="Limit number of results returned")

    parser.add_option("-p", "--path", dest="path", default=None, type="string", help="Filter results on url path")

    parser.add_option(
        "-t",
        "--severity",
        dest="severity",
        default=None,
        type="string",
        help="Filter results on severity (debug, info, warning, error, critical)",
    )

    parser.add_option("-c", "--code", dest="code", default=None, type="int", help="Filter results on status code")

    parser.add_option(
        "-S",
        "--slow",
        dest="slow",
        action="store",
        default=None,
        type="int",
        help="Filter results that took longer than n seconds",
    )

    (options, args) = parser.parse_args()

    if is_ec2() or options.stack:
        logger_nodes = get_db_nodes("logger")
        if len(logger_nodes) == 0:
            print ("Warning: No instances exist on stack 'logger'")
            return options
        instance_name = "%s.%s" % (logger_nodes[0]["stack"], logger_nodes[0]["name"])
        utils.init_log_db_config(instance_name)
        if options.stack is None:
            options.stack = get_stack().instance.stack

    return options
예제 #7
0
파일: logger.py 프로젝트: Stamped/Stamped
def main():
    # parse commandline
    options = parseCommandLine()
    options = options.__dict__

    user_id = options.pop("user_id", None)
    request_id = options.pop("request_id", None)
    limit = options.pop("limit", 10)
    errors = options.pop("show_errors", False)
    path = options.pop("path", False)
    severity = options.pop("severity", None)
    verbose = options.pop("verbose", False)
    method = options.pop("method", None)
    code = options.pop("code", None)
    stack = options.pop("stack", None)
    slow = options.pop("slow", None)

    if severity not in ["debug", "info", "warning", "error", "critical"]:
        severity = None

    if verbose:
        levels = ["debug", "info", "warning", "error", "critical"]
    else:
        levels = ["info", "warning", "error", "critical"]

    logsCollection = MongoLogsCollection(stack)
    if not logsCollection.isCapped:
        size = LOG_COLLECTION_SIZE if is_ec2() else LOG_LOCAL_COLLECTION_SIZE
        logsCollection.convertToCapped(size)
    logs = logsCollection.getLogs(
        userId=user_id,
        requestId=request_id,
        limit=limit,
        errors=errors,
        path=path,
        severity=severity,
        method=method,
        code=code,
        slow=slow,
    )
    for i in xrange(len(logs)):
        print
        print

        if "path" in logs[i] and "method" in logs[i]:
            node = ""
            if "node" in logs[i]:
                node = "(%s)" % logs[i]["node"]
            print u"%-10s %s %s %s" % (i + 1, logs[i]["method"], logs[i]["path"], node)
        else:
            print i + 1

        if "result" in logs[i] and logs[i]["result"] != "200":
            print u"%-10s %s ERROR" % ("", logs[i]["result"])
        print "-" * 40

        if "request_id" in logs[i]:
            print u"%-10s %s" % ("ID:", logs[i]["request_id"])

        if "begin" in logs[i]:
            begin_utc = logs[i]["begin"]
            begin_est = begin_utc - timedelta(hours=4)
            print u"%-10s %s" % ("Begin:", begin_est.strftime("%a %b %d %H:%M:%S.%f"))

            if "finish" in logs[i]:
                duration = logs[i]["finish"] - begin_utc
                print u"%-10s %s" % ("Duration:", duration)

        if "token" in logs[i]:
            print u"%-10s %s" % ("Token:", logs[i]["token"])

        if "client_id" in logs[i]:
            print u"%-10s %s" % ("Client:", logs[i]["client_id"])

        if "user_id" in logs[i]:
            print u"%-10s %s" % ("User:"******"user_id"])

        if "headers" in logs[i] and options["show_headers"]:
            print u"%-10s %s" % ("Headers:", logs[i]["headers"])

        if "form" in logs[i] and options["show_form"]:
            j = json.dumps(logs[i]["form"], indent=4)
            prefix = "Form:"
            for line in j.splitlines():
                print u"%-10s %s" % (prefix, line)
                prefix = ""

        if "output" in logs[i] and options["show_output"]:
            j = json.dumps(json.loads(logs[i]["output"]), indent=4)
            prefix = "Output:"
            for line in j.splitlines():
                print u"%-10s %s" % (prefix, line)
                prefix = ""

        if "log" in logs[i]:
            prefix = "Logs:"
            for log in logs[i]["log"]:
                if log[1] in levels:
                    try:
                        # TODO: eventually get rid of the if condition and always assume len() == 6.  Must wait for
                        # old logs to flush out
                        if len(log) == 6:
                            print u"{0:10} {1} | {2:25}:{3:>5} | {4:25} | {5}".format(
                                prefix, log[0].strftime("%H:%M:%S"), log[2], int(log[3]), log[4], log[5]
                            )
                        elif len(log) == 5:
                            print u"%-10s %s | %-30s | %-5s | %s" % (
                                prefix,
                                log[0].strftime("%H:%M:%S"),
                                log[2],
                                log[3],
                                log[4],
                            )
                        else:
                            print u"%-10s %s | %-30s | %s" % (prefix, log[0].strftime("%H:%M:%S"), log[2], log[3])

                    except:
                        utils.printException()

                    prefix = ""

        if "stack_trace" in logs[i]:
            print "-" * 40
            print logs[i]["stack_trace"]

    print