def load_settings(settings): headers=dict() for k in settings.keys(): if k.startswith('Header_'): key = k[7:].replace('_','-') headers[key] = settings[k] del settings[k] if k.startswith('easycrawl_'): define(k, settings[k]) settings['headers'] = headers site_name = settings['site_name'] rules_name = "%s.rules.rules" % site_name rules = import_object(rules_name) settings['rules'] = rules parsers_name = "%s.spider.parsers" % site_name parsers = import_object(parsers_name) settings['parsers'] = parsers
#!/usr/bin/env python import platform import socket import threading import time import statemon from options import options, define define("carbon_server", default="127.0.0.1", help="Montioring server", type=str) define("carbon_port", default=8090, help="Monitoring port", type=int) define("service_name", default=None, help="Two+ services running at once step on statemon, this solves it", type=str) define("sleep_interval", default=60, help="time between stats", type=int) def send_data(name, value): ''' Format metric name/val pair and send the data to the carbon server This is a best effort send ''' node = platform.node().replace('.', '-') timestamp = int(time.time()) if options.service_name: data = 'system.%s.%s.%s %s %d\n' % (node, options.service_name, name, value, timestamp) else: data = 'system.%s.%s %s %d\n' % (node,
import sys import time import random import logging from tornado import ioloop from tornado import httpclient from tornado import process from utils import setup_settings # use a modified options based on tornado's options from options import define, options, parse_command_line, parse_config_file define('use_curl', type=bool, default=False, help='use pycurl as AsyncHTTPClient backend') define('multi_processes', type=int, default=-1, help='run as multi-processes, 0 for cpu count') define('max_clients', type=int, default=10, help='max concurrent clients') define('time_len', type=int, default=60, help='time length of the test') define('timeout', type=float, default=1.0, help='request timeout') define('follow_redirects', type=bool, default=True, help='request follow redirects') define('validate_cert', type=bool, default=True, help='request validate cert') define('urls_file', help='request urls file') define('url_template', help='request url template, if not "", build with values in urls_file') define('log_info_format', help='info log format') define('log_warning_format', help='warning log format') define('log_error_format', help='error log format') parse_config_file(setup_settings()) requests = parse_command_line()
import logging from tornado import process try: import matplotlib.pyplot as plt from matplotlib import rcParams rcParams['savefig.dpi'] = options.dpi except ImportError: plt = None from utils import setup_settings from options import define, options, parse_command_line, parse_config_file define('multi_processes', type=int, default=-1, help='benched as multi-processes') define('max_clients', type=int, default=10, help='max concurrent clients') define('dpi', type=int, default=72, help='figure DPI') define('figure_file', default='log.png', help='figure file') parse_config_file(setup_settings()) log_files = parse_command_line() REGEX = re.compile(r'\[(?P<level>[IWE]) (?P<date>\d+) (?P<time>[\d:\.]+)' r' (?P<file>\w+):(?P<lineno>\d+)\]' r' cost:\[(?P<cost>[\d\.]+)\]' r' code:\[(?P<code>\d+)\]' r' url:\[(?P<url>\S+)\]' r'( error:\[(?P<error>[^\]]+)\])?')
#!/usr/bin/env python import platform import socket import threading import time import statemon from options import options, define define("carbon_server", default="127.0.0.1", help="Montioring server", type=str) define("carbon_port", default=8090, help="Monitoring port", type=int) define("service_name", default=None, help="Two+ services running at once step on statemon, this solves it", type=str) define("sleep_interval", default=60, help="time between stats", type=int) def send_data(name, value): ''' Format metric name/val pair and send the data to the carbon server This is a best effort send ''' node = platform.node().replace('.', '-') timestamp = int(time.time()) if options.service_name: