def load_settings(settings):
headers=dict()
for k in settings.keys():
if k.startswith('Header_'):
key = k[7:].replace('_','-')
headers[key] = settings[k]
del settings[k]
if k.startswith('easycrawl_'):
define(k, settings[k])
settings['headers'] = headers
site_name = settings['site_name']
rules_name = "%s.rules.rules" % site_name
rules = import_object(rules_name)
settings['rules'] = rules
parsers_name = "%s.spider.parsers" % site_name
parsers = import_object(parsers_name)
settings['parsers'] = parsers
#!/usr/bin/env python
import platform
import socket
import threading
import time
import statemon
from options import options, define
define("carbon_server", default="127.0.0.1", help="Montioring server", type=str)
define("carbon_port", default=8090, help="Monitoring port", type=int)
define("service_name", default=None, help="Two+ services running at once step on statemon, this solves it", type=str)
define("sleep_interval", default=60, help="time between stats", type=int)
def send_data(name, value):
'''
Format metric name/val pair and send the data to the carbon server
This is a best effort send
'''
node = platform.node().replace('.', '-')
timestamp = int(time.time())
if options.service_name:
data = 'system.%s.%s.%s %s %d\n' % (node,
options.service_name,
name,
value,
timestamp)
else:
data = 'system.%s.%s %s %d\n' % (node,
import sys
import time
import random
import logging
from tornado import ioloop
from tornado import httpclient
from tornado import process
from utils import setup_settings
# use a modified options based on tornado's options
from options import define, options, parse_command_line, parse_config_file
define('use_curl', type=bool, default=False, help='use pycurl as AsyncHTTPClient backend')
define('multi_processes', type=int, default=-1, help='run as multi-processes, 0 for cpu count')
define('max_clients', type=int, default=10, help='max concurrent clients')
define('time_len', type=int, default=60, help='time length of the test')
define('timeout', type=float, default=1.0, help='request timeout')
define('follow_redirects', type=bool, default=True, help='request follow redirects')
define('validate_cert', type=bool, default=True, help='request validate cert')
define('urls_file', help='request urls file')
define('url_template', help='request url template, if not "", build with values in urls_file')
define('log_info_format', help='info log format')
define('log_warning_format', help='warning log format')
define('log_error_format', help='error log format')
parse_config_file(setup_settings())
requests = parse_command_line()
import logging
from tornado import process
try:
import matplotlib.pyplot as plt
from matplotlib import rcParams
rcParams['savefig.dpi'] = options.dpi
except ImportError:
plt = None
from utils import setup_settings
from options import define, options, parse_command_line, parse_config_file
define('multi_processes', type=int, default=-1, help='benched as multi-processes')
define('max_clients', type=int, default=10, help='max concurrent clients')
define('dpi', type=int, default=72, help='figure DPI')
define('figure_file', default='log.png', help='figure file')
parse_config_file(setup_settings())
log_files = parse_command_line()
REGEX = re.compile(r'\[(?P<level>[IWE]) (?P<date>\d+) (?P<time>[\d:\.]+)'
r' (?P<file>\w+):(?P<lineno>\d+)\]'
r' cost:\[(?P<cost>[\d\.]+)\]'
r' code:\[(?P<code>\d+)\]'
r' url:\[(?P<url>\S+)\]'
r'( error:\[(?P<error>[^\]]+)\])?')
#!/usr/bin/env python
import platform
import socket
import threading
import time
import statemon
from options import options, define
define("carbon_server",
default="127.0.0.1",
help="Montioring server",
type=str)
define("carbon_port", default=8090, help="Monitoring port", type=int)
define("service_name",
default=None,
help="Two+ services running at once step on statemon, this solves it",
type=str)
define("sleep_interval", default=60, help="time between stats", type=int)
def send_data(name, value):
'''
Format metric name/val pair and send the data to the carbon server
This is a best effort send
'''
node = platform.node().replace('.', '-')
timestamp = int(time.time())
if options.service_name:
import logging
from tornado import process
try:
import matplotlib.pyplot as plt
from matplotlib import rcParams
rcParams['savefig.dpi'] = options.dpi
except ImportError:
plt = None
from utils import setup_settings
from options import define, options, parse_command_line, parse_config_file
define('multi_processes',
type=int,
default=-1,
help='benched as multi-processes')
define('max_clients', type=int, default=10, help='max concurrent clients')
define('dpi', type=int, default=72, help='figure DPI')
define('figure_file', default='log.png', help='figure file')
parse_config_file(setup_settings())
log_files = parse_command_line()
REGEX = re.compile(r'\[(?P<level>[IWE]) (?P<date>\d+) (?P<time>[\d:\.]+)'
r' (?P<file>\w+):(?P<lineno>\d+)\]'
r' cost:\[(?P<cost>[\d\.]+)\]'
r' code:\[(?P<code>\d+)\]'
r' url:\[(?P<url>\S+)\]'
r'( error:\[(?P<error>[^\]]+)\])?')
