def init(self, binary=False, behaviors=None):
try:
memcached_nodes = []
if utils.is_ec2():
stack = ec2_utils.get_stack()
for node in stack.nodes:
if 'mem' in node.roles:
memcached_nodes.append(node.private_ip_address)
if 0 == len(memcached_nodes):
raise Exception("[%s] unable to any find memcached servers" % self)
else:
# running locally so default to localhost
memcached_nodes.append('127.0.0.1')
self._client = pylibmc.Client(memcached_nodes, binary=binary, behaviors=behaviors)
# Verify it works
self._client.set('test', 'test', time=0)
except Exception, e:
logs.warning("[%s] unable to initialize memcached (%s)" % (self, e))
self._client = None
return False
def ensure_index(self, key_or_list, **kwargs):
if self._debug:
print("Mongo 'ensure_index'")
num_retries = 0
max_retries = 5
# NOTE (travis): this method should never throw an error locally if connected to
# a non-master DB node that can't ensure_index because the conn doesn't have
# write permissions
while True:
try:
ret = self._collection.ensure_index(key_or_list, **kwargs)
return ret
except AutoReconnect as e:
if not utils.is_ec2():
return
num_retries += 1
if num_retries > max_retries:
msg = "Unable to ensure_index after %d retries (%s)" % \
(max_retries, self._parent.__class__.__name__)
logs.warning(msg)
raise
logs.info("Retrying ensure_index (%s)" % (self._parent.__class__.__name__))
time.sleep(0.25)
def devOnlyCachedFn(memberFn=True, schemaClasses=[]):
"""
Decorator that wraps a function in a Mongo-based caching layer only if running on a development machine.
Meanings of memberFn and schemaClasses are as with cachedFn decorator above.
"""
if utils.is_ec2():
return lambda wrappedFn: wrappedFn
else:
return mongoCachedFn(maxStaleness=datetime.timedelta(0, ttl), memberFn=memberFn, schemaClasses=schemaClasses)
def decoratorFn(userFn):
if devOnly and utils.is_ec2():
return userFn
@functools.wraps(userFn)
def wrappedFn(*args, **kwargs):
global functionCounts
fnName = name if name is not None else userFn.__name__
functionCounts[fnName] += 1
return userFn(*args, **kwargs)
return wrappedFn
def _wrapper(request, *args, **kwargs):
import servers.web2.error.views as web_error
try:
logs.begin(saveLog=stampedAPIProxy.api._logsDB.saveLog,
saveStat=stampedAPIProxy.api._statsDB.addStat,
requestData=request,
nodeName=stampedAPIProxy.api.node_name)
logs.info("%s %s" % (request.method, request.path))
subkwargs = kwargs
if schema is not None:
parse_kwargs = parse_request_kwargs or {}
django_kwargs = {}
if parse_django_kwargs:
django_kwargs = kwargs or {}
subkwargs = {}
result = parse_request(request, schema(), django_kwargs, overflow=ignore_extra_params, **parse_kwargs)
subkwargs['schema'] = result
response = fn(request, *args, **subkwargs)
logs.info("End request: Success")
if no_cache:
expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime()
cache_control = 'no-cache'
elif utils.is_ec2():
expires = (dt.datetime.utcnow() + dt.timedelta(minutes=60)).ctime()
cache_control = 'max-age=600'
else:
# disable caching for local development / debugging
expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime()
cache_control = 'max-age=0'
response['Expires'] = expires
response['Cache-Control'] = cache_control
return response
except urllib2.HTTPError, e:
logs.warning("%s Error: %s" % (e.code, e))
logs.warning(utils.getFormattedException())
if e.code == 404:
return web_error.error_404(request)
elif e.code >= 500:
return web_error.error_500(request)
raise # invoke django's default 500 handler
def cachedFn(ttl=ONE_WEEK, memberFn=True, schemaClasses=[]):
"""
Decorator that wraps a function in a caching layer, using either Memcache if operating in production or MongoCache
if running on a development machine.
ttl determines how long, in seconds, results will be stored in the cache before being invalidated.
memberFn should be set to False when the wrapped function is not a member function.
schemaClasses should be a list of the actual schema classes that we expect to be returned within the response. So,
for instance, you might decorate a function with @cachedFn(schemaClasses=[User,Entity]) if you expected it to
return a user and a list of entities on their to-do list.
"""
if utils.is_ec2():
return memcached_function(time=ttl)
else:
return mongoCachedFn(maxStaleness=datetime.timedelta(0, ttl), memberFn=memberFn, schemaClasses=schemaClasses)
def connection(self):
if self._connection:
return self._connection
with self._connection_lock:
if self._connection:
return self._connection
reinitialized = False
max_delay = 16
delay = 1
if utils.is_ec2():
replicaset = 'stamped-dev-01'
else:
replicaset = None
while True:
try:
hosts = ','.join(map(lambda x: "%s:%s" % (x[0], x[1]), self.hosts))
logs.info("Connecting to MongoDB: %s" % hosts)
if replicaset:
self._connection = pymongo.ReplicaSetConnection(hosts,
read_preference=pymongo.ReadPreference.SECONDARY,
replicaset=replicaset,
use_greenlets=True)
else:
self._connection = pymongo.Connection(hosts,
read_preference=pymongo.ReadPreference.SECONDARY,
use_greenlets=True)
return self._connection
except AutoReconnect as e:
if delay > max_delay:
if reinitialized:
raise
# attempt to reinitialize our MongoDB configuration and retry
self._init()
delay = 1
reinitialized = True
logs.warning("Retrying to connect to host: %s (delay %d)" % (str(e), delay))
time.sleep(delay)
delay *= 2
def _get_servers(self):
servers = [ ]
port = 9200
if utils.is_ec2():
stack = ec2_utils.get_stack()
for node in stack.nodes:
if 'search' in node.roles:
servers.append("%s:%d" % (node.private_ip_address, port))
if 0 == len(servers):
raise Exception("[%s] unable to any find search servers" % self)
else:
# running locally so default to localhost
servers.append("localhost:%d" % port)
return servers
def __init__(self, fail_limit, fail_period, blackout_wait):
self.__local_rlservice = None
self.__request_fails = 0
self.__fails = deque()
self.__fail_limit = fail_limit
self.__fail_period = fail_period
self.__blackout_start = None
self.__blackout_wait = blackout_wait
self.__is_ec2 = utils.is_ec2()
self.__service_init_semaphore = Semaphore()
stack_info = libs.ec2_utils.get_stack()
self.__stack_name = 'localhost'
self.__node_name = 'localhost'
if stack_info is not None:
self.__stack_name = stack_info.instance.stack
self.__node_name = stack_info.instance.name
self.__last_email_time = 0
self.__emails = deque()
# determine the private ip address of the ratelimiter instance for this stack
self._getHost()
print('### host: %s' % self.__host)
def parseCommandLine():
usage = "Usage: %prog [options] query"
version = "%prog " + __version__
parser = OptionParser(usage=usage, version=version)
parser.add_option(
"-s",
"--stack",
default=None,
action="store",
type="string",
help="stack to monitor (defaults to whatever stack the local AWS instance belongs to)",
)
parser.add_option("-t", "--time", default=10, action="store", type="int", help="polling interval (in seconds)")
parser.add_option(
"-n",
"--noop",
default=False,
action="store_true",
help="don't send any notifications; just monitor and report to stdout",
)
parser.add_option("-v", "--verbose", default=False, action="store_true", help="enable verbose logging")
(options, args) = parser.parse_args()
if options.time < 0:
utils.log("invalid time parameter")
parser.print_help()
sys.exit(1)
if options.stack is None and not utils.is_ec2():
utils.log("error: if this program isn't run from an EC2 instance, you must specify a stack to monitor")
parser.print_help()
sys.exit(1)
return (options, args)
def _init(self):
if utils.is_ec2():
dbNodes = libs.ec2_utils.get_db_nodes()
hosts = []
for dbNode in dbNodes:
hosts.append((dbNode['private_ip_address'], 27017))
if len(hosts) > 0:
self.config['mongodb'] = {
"hosts" : hosts
}
if not 'mongodb' in self.config:
self.config = AttributeDict({
"mongodb" : {
"hosts" : [("localhost", 27017)]
}
})
logs.info("MongoDB connection defaulting to %s" %
(self.config.mongodb.hosts))
def _init(self):
# NOTE: disabling StatsD for V2 launch
return
time.sleep(15)
logs.info("initializing StatsD")
host, port = "localhost", 8125
if utils.is_ec2():
done = False
sleep = 1
while not done:
try:
stack_info = libs.ec2_utils.get_stack()
if stack_info is None:
raise
for node in stack_info.nodes:
if 'monitor' in node.roles:
host, port = node.private_ip_address, 8125
done = True
break
except:
utils.printException()
sleep *= 2
time.sleep(sleep)
if sleep > 32:
logs.warning("ERROR initializing StatsD!!!")
return
else:
return
logs.info("initializing StatsD at %s:%d" % (host, port))
self.statsd = StatsD(host, port)
def main():
options, args = parseCommandLine()
api = MongoStampedAPI(lite_mode=True)
db = api._entityDB._collection._database
# if we're on prod, instruct pymongo to perform integrity checks on
# secondaries to reduce load in primary
if utils.is_ec2() and libs.ec2_utils.is_prod_stack():
db.read_preference = pymongo.ReadPreference.SECONDARY
checks = integrity.checks
for check_cls in checks:
if options.check is None or options.check.lower() in check_cls.__name__.lower():
utils.log("Running %s" % check_cls.__name__)
check = check_cls(api, db, options)
try:
check.run()
except:
utils.printException()
utils.log("Done running %s" % check_cls.__name__)
utils.log()
def render(self, template_name, context):
less = self._renderer.render(self.templates[template_name][1], context)
proxy = ".%s.%s.less" % (template_name, threading.currentThread().getName())
if utils.is_ec2():
prog = "/stamped/node_modules/less/bin/lessc"
else:
prog = "lessc"
cmd = "%s %s" % (prog, proxy)
with open(proxy, 'w') as fp:
fp.write(less)
fp.close()
pp = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
output = pp.stdout.read().strip()
error = pp.stderr.read().strip()
status = pp.wait()
if 0 != status:
raise Exception("lessc error %d: %s\n%s" % (status, output, error))
return output
#!/usr/bin/env python __author__ = "Stamped ([email protected])" __version__ = "1.0" __copyright__ = "Copyright (c) 2011-2012 Stamped.com" __license__ = "TODO" import Globals import inspect, os from utils import is_ec2 from distutils.core import setup, Extension debug_macro = [] if is_ec2() else [('NDEBUG', '1')] script_dir = os.path.dirname(inspect.getfile(inspect.currentframe())) fast_compare_module = Extension('fastcompare', define_macros=debug_macro, sources=[os.path.join(script_dir, 'fastcompare.c')]) setup(name="FastCompare", ext_modules=[fast_compare_module])
def mongoCachedFn(maxStaleness=ONE_WEEK, memberFn=True, schemaClasses=[]):
# Don't use Mongo caching in production.
assert(not utils.is_ec2())
schemaClassesMap = {}
for schemaClass in schemaClasses:
schemaClassesMap[schemaClass.__name__] = schemaClass
def decoratingFn(userFunction):
userFnName = userFunction.func_name
@functools.wraps(userFunction)
def wrappedFn(*args, **kwargs):
global cacheTableError
if cacheTableError is not None:
# We haven't been able to connect to the cache. MongoDB may not be running. Just issue the call.
return userFunction(*args, **kwargs)
now = datetime.datetime.now()
fullArgs = args
if memberFn == True:
self = args[0]
fnName = '%s.%s' % (self.__class__.__name__, userFnName)
args = args[1:]
else:
fnName = userFnName
assertCallIsSerializable(args, kwargs)
callHash = hashFunctionCall(fnName, args, kwargs)
force_recalculate = kwargs.pop('force_recalculate', False)
try:
connection = MongoDBConfig.getInstance().connection
dbname = MongoDBConfig.getInstance().database_name
table = getattr(getattr(connection, dbname), cacheTableName)
result = table.find_one({'_id':callHash})
except AutoReconnect as exc:
cacheTableError = exc
logs.warning("Couldn't connect to Mongo cache table; disabling Mongo cache.")
return userFunction(*fullArgs, **kwargs)
if result and result['expiration'] is None and not disableStaleness:
raise ValueError('We should never be using non-expiring cache entries outside of test fixtures!')
if result and result['expiration'] is not None and disableStaleness:
raise ValueError('We should never be using expiring cache entries inside of test fixtures!')
if result and (disableStaleness or (result['expiration'] > now)) and not force_recalculate:
# We hit the cache and the result isn't stale! Woo!
return deserializeValue(result['value'], schemaClassesMap)
elif exceptionOnCacheMiss:
raise CacheMissException(fnName)
expiration = None if disableStaleness else now + maxStaleness
result = userFunction(*fullArgs, **kwargs)
cacheEntry = {'_id':callHash,
'func_name': fnName,
'value': serializeValue(result, schemaClassesMap),
'expiration':expiration}
table.update({'_id':callHash}, cacheEntry, upsert=True)
return result
return wrappedFn
return decoratingFn
def ec2(self):
return utils.is_ec2()
#!/usr/bin/env python __author__ = "Stamped ([email protected])" __version__ = "1.0" __copyright__ = "Copyright (c) 2011-2012 Stamped.com" __license__ = "TODO" import Globals import libs.ec2_utils, logs, re, sys, time, urllib2, utils from libs.notify import StampedNotificationHandler from optparse import OptionParser from pprint import pprint is_ec2 = utils.is_ec2() class MonitorException(Exception): def __init__(self, desc, detail=None, email=True, sms=False): Exception.__init__(self, desc) self.detail = None self.email = email self.sms = sms class Monitor(object): def __init__(self, options=None): self.handler = StampedNotificationHandler() self.status = {} self._info = None
def get_db_instances():
if utils.is_ec2():
stack = ec2_utils.get_stack()
members = filter(lambda m: 'db' in m.roles, stack.members)
def __init__(self):
self._prod = IS_PROD
self._ec2 = utils.is_ec2()
self.api = globalMongoStampedAPI()
self._cache = globalMemcache()
def call(queue, key, payload, **options):
# Naming convention is "namespace::function"
# assert '::' in key
# Run synchronously if not on EC2
# if not utils.is_ec2():
# logs.warning("Local - Running synchronously")
# raise Exception
global __errors
global __cooldown
maxErrors = 5
numErrors = len(__errors)
# Initiate cooldown period if number of errors is too high
if numErrors > maxErrors:
msg = "Number of errors (%s) exceeds maximum (%s): cooling down" % (numErrors, maxErrors)
logs.warning(msg)
__cooldown = datetime.utcnow()
__errors = []
# Fail if in cooldown period
if __cooldown is not None and datetime.utcnow() - timedelta(minutes=5) < __cooldown:
msg = "Cooling down until %s" % __cooldown
logs.warning(msg)
raise Exception(msg)
# Build payload
data = {
'task_id': str(ObjectId()),
'key': key,
'data': payload,
'timestamp': datetime.utcnow()
}
maxRetries = 5
numRetries = 0
while True:
try:
# Submit job
logs.info("Submitting task: %s" % data)
client = getClient()
client.submit_job(queue, pickle.dumps(data), background=True, wait_until_complete=False)
# Reset errors
__errors = []
__cooldown = None
return True
except Exception as e:
# Retry
numRetries += 1
if numRetries <= maxRetries:
time.sleep(0.1)
continue
logs.warning("Task failed: %s (%s)" % (type(e), e))
# Reset client
resetClient()
# Cap the number of errors logged
__errors.append(e)
__errors = __errors[-25:]
numErrors = len(__errors)
# Initiate a cool down period
if numErrors >= maxErrors:
__cooldown = datetime.utcnow()
# Send an email alert the first time this happens
if numErrors == maxErrors:
msg = "Unable to connect to task broker: %s" % getHosts()
logs.warning(msg)
if utils.is_ec2():
# Send email
email = {}
try:
stack_info = libs.ec2_utils.get_stack()
email['subject'] = "%s.%s - Unable to connect to task broker" % \
(stack_info.instance.stack, stack_info.instance.name)
except Exception:
email['subject'] = msg
email['body'] = "All async tasks running locally\n\n%s\n\n" % msg
for error in __errors:
email['body'] += str(error)
email['body'] += '\n'
email['from'] = 'Stamped <*****@*****.**>'
email['to'] = '*****@*****.**'
utils.sendEmail(email)
raise
import libs.ec2_utils
import multiprocessing, os, sys
from datetime import timedelta
from celery.schedules import crontab
# List of modules to import when celery starts.
CELERY_IMPORTS = ("tasks", )
## Result store settings.
CELERY_RESULT_BACKEND = "amqp"
host, port = "localhost", 5672
user, password, vhost = "guest", "guest", "/"
if utils.is_ec2():
stack = libs.ec2_utils.get_stack()
for node in stack.nodes:
if 'monitor' in node.roles:
host = node.private_ip_address
break
if 'work-enrich' in stack.instance.roles:
CELERY_ACKS_LATE = True
CELERYD_CONCURRENCY = 5
## Broker settings.
BROKER_URL = "pyamqp://%s:%s@%s:%s/%s" % (user, password, host, port, vhost)
BROKER_HEARTBEAT = 10
logs.info('BROKER_URL: %s' % BROKER_URL)
# Django settings for www project.
import Globals
import utils, os, libs.ec2_utils
IS_PROD = libs.ec2_utils.is_prod_stack()
DEBUG = (not utils.is_ec2())
STAMPED_DEBUG = DEBUG
TEMPLATE_DEBUG = DEBUG
PROJ_ROOT = os.path.abspath(os.path.dirname(__file__))
STAMPED_ASSET_VERSION = utils.shell("cd %s && make version" % PROJ_ROOT)[0]
STAMPED_DOWNLOAD_APP_LINK = "http://itunes.apple.com/us/app/stamped/id467924760?mt=8&uo=4"
utils.log("Django DEBUG=%s ROOT=%s VERSION=%s" % (DEBUG, PROJ_ROOT, STAMPED_ASSET_VERSION))
ADMINS = (
('Travis', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
'NAME': '', # Or path to database file if using sqlite3.
'USER': '', # Not used with sqlite3.
'PASSWORD': '', # Not used with sqlite3.
'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
'PORT': '', # Set to empty string for default. Not used with sqlite3.
}
