def init(self, binary=False, behaviors=None): try: memcached_nodes = [] if utils.is_ec2(): stack = ec2_utils.get_stack() for node in stack.nodes: if 'mem' in node.roles: memcached_nodes.append(node.private_ip_address) if 0 == len(memcached_nodes): raise Exception("[%s] unable to any find memcached servers" % self) else: # running locally so default to localhost memcached_nodes.append('127.0.0.1') self._client = pylibmc.Client(memcached_nodes, binary=binary, behaviors=behaviors) # Verify it works self._client.set('test', 'test', time=0) except Exception, e: logs.warning("[%s] unable to initialize memcached (%s)" % (self, e)) self._client = None return False
def ensure_index(self, key_or_list, **kwargs): if self._debug: print("Mongo 'ensure_index'") num_retries = 0 max_retries = 5 # NOTE (travis): this method should never throw an error locally if connected to # a non-master DB node that can't ensure_index because the conn doesn't have # write permissions while True: try: ret = self._collection.ensure_index(key_or_list, **kwargs) return ret except AutoReconnect as e: if not utils.is_ec2(): return num_retries += 1 if num_retries > max_retries: msg = "Unable to ensure_index after %d retries (%s)" % \ (max_retries, self._parent.__class__.__name__) logs.warning(msg) raise logs.info("Retrying ensure_index (%s)" % (self._parent.__class__.__name__)) time.sleep(0.25)
def devOnlyCachedFn(memberFn=True, schemaClasses=[]): """ Decorator that wraps a function in a Mongo-based caching layer only if running on a development machine. Meanings of memberFn and schemaClasses are as with cachedFn decorator above. """ if utils.is_ec2(): return lambda wrappedFn: wrappedFn else: return mongoCachedFn(maxStaleness=datetime.timedelta(0, ttl), memberFn=memberFn, schemaClasses=schemaClasses)
def decoratorFn(userFn): if devOnly and utils.is_ec2(): return userFn @functools.wraps(userFn) def wrappedFn(*args, **kwargs): global functionCounts fnName = name if name is not None else userFn.__name__ functionCounts[fnName] += 1 return userFn(*args, **kwargs) return wrappedFn
def _wrapper(request, *args, **kwargs): import servers.web2.error.views as web_error try: logs.begin(saveLog=stampedAPIProxy.api._logsDB.saveLog, saveStat=stampedAPIProxy.api._statsDB.addStat, requestData=request, nodeName=stampedAPIProxy.api.node_name) logs.info("%s %s" % (request.method, request.path)) subkwargs = kwargs if schema is not None: parse_kwargs = parse_request_kwargs or {} django_kwargs = {} if parse_django_kwargs: django_kwargs = kwargs or {} subkwargs = {} result = parse_request(request, schema(), django_kwargs, overflow=ignore_extra_params, **parse_kwargs) subkwargs['schema'] = result response = fn(request, *args, **subkwargs) logs.info("End request: Success") if no_cache: expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime() cache_control = 'no-cache' elif utils.is_ec2(): expires = (dt.datetime.utcnow() + dt.timedelta(minutes=60)).ctime() cache_control = 'max-age=600' else: # disable caching for local development / debugging expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime() cache_control = 'max-age=0' response['Expires'] = expires response['Cache-Control'] = cache_control return response except urllib2.HTTPError, e: logs.warning("%s Error: %s" % (e.code, e)) logs.warning(utils.getFormattedException()) if e.code == 404: return web_error.error_404(request) elif e.code >= 500: return web_error.error_500(request) raise # invoke django's default 500 handler
def cachedFn(ttl=ONE_WEEK, memberFn=True, schemaClasses=[]): """ Decorator that wraps a function in a caching layer, using either Memcache if operating in production or MongoCache if running on a development machine. ttl determines how long, in seconds, results will be stored in the cache before being invalidated. memberFn should be set to False when the wrapped function is not a member function. schemaClasses should be a list of the actual schema classes that we expect to be returned within the response. So, for instance, you might decorate a function with @cachedFn(schemaClasses=[User,Entity]) if you expected it to return a user and a list of entities on their to-do list. """ if utils.is_ec2(): return memcached_function(time=ttl) else: return mongoCachedFn(maxStaleness=datetime.timedelta(0, ttl), memberFn=memberFn, schemaClasses=schemaClasses)
def connection(self): if self._connection: return self._connection with self._connection_lock: if self._connection: return self._connection reinitialized = False max_delay = 16 delay = 1 if utils.is_ec2(): replicaset = 'stamped-dev-01' else: replicaset = None while True: try: hosts = ','.join(map(lambda x: "%s:%s" % (x[0], x[1]), self.hosts)) logs.info("Connecting to MongoDB: %s" % hosts) if replicaset: self._connection = pymongo.ReplicaSetConnection(hosts, read_preference=pymongo.ReadPreference.SECONDARY, replicaset=replicaset, use_greenlets=True) else: self._connection = pymongo.Connection(hosts, read_preference=pymongo.ReadPreference.SECONDARY, use_greenlets=True) return self._connection except AutoReconnect as e: if delay > max_delay: if reinitialized: raise # attempt to reinitialize our MongoDB configuration and retry self._init() delay = 1 reinitialized = True logs.warning("Retrying to connect to host: %s (delay %d)" % (str(e), delay)) time.sleep(delay) delay *= 2
def _get_servers(self): servers = [ ] port = 9200 if utils.is_ec2(): stack = ec2_utils.get_stack() for node in stack.nodes: if 'search' in node.roles: servers.append("%s:%d" % (node.private_ip_address, port)) if 0 == len(servers): raise Exception("[%s] unable to any find search servers" % self) else: # running locally so default to localhost servers.append("localhost:%d" % port) return servers
def __init__(self, fail_limit, fail_period, blackout_wait): self.__local_rlservice = None self.__request_fails = 0 self.__fails = deque() self.__fail_limit = fail_limit self.__fail_period = fail_period self.__blackout_start = None self.__blackout_wait = blackout_wait self.__is_ec2 = utils.is_ec2() self.__service_init_semaphore = Semaphore() stack_info = libs.ec2_utils.get_stack() self.__stack_name = 'localhost' self.__node_name = 'localhost' if stack_info is not None: self.__stack_name = stack_info.instance.stack self.__node_name = stack_info.instance.name self.__last_email_time = 0 self.__emails = deque() # determine the private ip address of the ratelimiter instance for this stack self._getHost() print('### host: %s' % self.__host)
def parseCommandLine(): usage = "Usage: %prog [options] query" version = "%prog " + __version__ parser = OptionParser(usage=usage, version=version) parser.add_option( "-s", "--stack", default=None, action="store", type="string", help="stack to monitor (defaults to whatever stack the local AWS instance belongs to)", ) parser.add_option("-t", "--time", default=10, action="store", type="int", help="polling interval (in seconds)") parser.add_option( "-n", "--noop", default=False, action="store_true", help="don't send any notifications; just monitor and report to stdout", ) parser.add_option("-v", "--verbose", default=False, action="store_true", help="enable verbose logging") (options, args) = parser.parse_args() if options.time < 0: utils.log("invalid time parameter") parser.print_help() sys.exit(1) if options.stack is None and not utils.is_ec2(): utils.log("error: if this program isn't run from an EC2 instance, you must specify a stack to monitor") parser.print_help() sys.exit(1) return (options, args)
def _init(self): if utils.is_ec2(): dbNodes = libs.ec2_utils.get_db_nodes() hosts = [] for dbNode in dbNodes: hosts.append((dbNode['private_ip_address'], 27017)) if len(hosts) > 0: self.config['mongodb'] = { "hosts" : hosts } if not 'mongodb' in self.config: self.config = AttributeDict({ "mongodb" : { "hosts" : [("localhost", 27017)] } }) logs.info("MongoDB connection defaulting to %s" % (self.config.mongodb.hosts))
def _init(self): # NOTE: disabling StatsD for V2 launch return time.sleep(15) logs.info("initializing StatsD") host, port = "localhost", 8125 if utils.is_ec2(): done = False sleep = 1 while not done: try: stack_info = libs.ec2_utils.get_stack() if stack_info is None: raise for node in stack_info.nodes: if 'monitor' in node.roles: host, port = node.private_ip_address, 8125 done = True break except: utils.printException() sleep *= 2 time.sleep(sleep) if sleep > 32: logs.warning("ERROR initializing StatsD!!!") return else: return logs.info("initializing StatsD at %s:%d" % (host, port)) self.statsd = StatsD(host, port)
def main(): options, args = parseCommandLine() api = MongoStampedAPI(lite_mode=True) db = api._entityDB._collection._database # if we're on prod, instruct pymongo to perform integrity checks on # secondaries to reduce load in primary if utils.is_ec2() and libs.ec2_utils.is_prod_stack(): db.read_preference = pymongo.ReadPreference.SECONDARY checks = integrity.checks for check_cls in checks: if options.check is None or options.check.lower() in check_cls.__name__.lower(): utils.log("Running %s" % check_cls.__name__) check = check_cls(api, db, options) try: check.run() except: utils.printException() utils.log("Done running %s" % check_cls.__name__) utils.log()
def render(self, template_name, context): less = self._renderer.render(self.templates[template_name][1], context) proxy = ".%s.%s.less" % (template_name, threading.currentThread().getName()) if utils.is_ec2(): prog = "/stamped/node_modules/less/bin/lessc" else: prog = "lessc" cmd = "%s %s" % (prog, proxy) with open(proxy, 'w') as fp: fp.write(less) fp.close() pp = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) output = pp.stdout.read().strip() error = pp.stderr.read().strip() status = pp.wait() if 0 != status: raise Exception("lessc error %d: %s\n%s" % (status, output, error)) return output
#!/usr/bin/env python __author__ = "Stamped ([email protected])" __version__ = "1.0" __copyright__ = "Copyright (c) 2011-2012 Stamped.com" __license__ = "TODO" import Globals import inspect, os from utils import is_ec2 from distutils.core import setup, Extension debug_macro = [] if is_ec2() else [('NDEBUG', '1')] script_dir = os.path.dirname(inspect.getfile(inspect.currentframe())) fast_compare_module = Extension('fastcompare', define_macros=debug_macro, sources=[os.path.join(script_dir, 'fastcompare.c')]) setup(name="FastCompare", ext_modules=[fast_compare_module])
def mongoCachedFn(maxStaleness=ONE_WEEK, memberFn=True, schemaClasses=[]): # Don't use Mongo caching in production. assert(not utils.is_ec2()) schemaClassesMap = {} for schemaClass in schemaClasses: schemaClassesMap[schemaClass.__name__] = schemaClass def decoratingFn(userFunction): userFnName = userFunction.func_name @functools.wraps(userFunction) def wrappedFn(*args, **kwargs): global cacheTableError if cacheTableError is not None: # We haven't been able to connect to the cache. MongoDB may not be running. Just issue the call. return userFunction(*args, **kwargs) now = datetime.datetime.now() fullArgs = args if memberFn == True: self = args[0] fnName = '%s.%s' % (self.__class__.__name__, userFnName) args = args[1:] else: fnName = userFnName assertCallIsSerializable(args, kwargs) callHash = hashFunctionCall(fnName, args, kwargs) force_recalculate = kwargs.pop('force_recalculate', False) try: connection = MongoDBConfig.getInstance().connection dbname = MongoDBConfig.getInstance().database_name table = getattr(getattr(connection, dbname), cacheTableName) result = table.find_one({'_id':callHash}) except AutoReconnect as exc: cacheTableError = exc logs.warning("Couldn't connect to Mongo cache table; disabling Mongo cache.") return userFunction(*fullArgs, **kwargs) if result and result['expiration'] is None and not disableStaleness: raise ValueError('We should never be using non-expiring cache entries outside of test fixtures!') if result and result['expiration'] is not None and disableStaleness: raise ValueError('We should never be using expiring cache entries inside of test fixtures!') if result and (disableStaleness or (result['expiration'] > now)) and not force_recalculate: # We hit the cache and the result isn't stale! Woo! return deserializeValue(result['value'], schemaClassesMap) elif exceptionOnCacheMiss: raise CacheMissException(fnName) expiration = None if disableStaleness else now + maxStaleness result = userFunction(*fullArgs, **kwargs) cacheEntry = {'_id':callHash, 'func_name': fnName, 'value': serializeValue(result, schemaClassesMap), 'expiration':expiration} table.update({'_id':callHash}, cacheEntry, upsert=True) return result return wrappedFn return decoratingFn
def ec2(self): return utils.is_ec2()
#!/usr/bin/env python __author__ = "Stamped ([email protected])" __version__ = "1.0" __copyright__ = "Copyright (c) 2011-2012 Stamped.com" __license__ = "TODO" import Globals import libs.ec2_utils, logs, re, sys, time, urllib2, utils from libs.notify import StampedNotificationHandler from optparse import OptionParser from pprint import pprint is_ec2 = utils.is_ec2() class MonitorException(Exception): def __init__(self, desc, detail=None, email=True, sms=False): Exception.__init__(self, desc) self.detail = None self.email = email self.sms = sms class Monitor(object): def __init__(self, options=None): self.handler = StampedNotificationHandler() self.status = {} self._info = None
def get_db_instances(): if utils.is_ec2(): stack = ec2_utils.get_stack() members = filter(lambda m: 'db' in m.roles, stack.members)
def __init__(self): self._prod = IS_PROD self._ec2 = utils.is_ec2() self.api = globalMongoStampedAPI() self._cache = globalMemcache()
def call(queue, key, payload, **options): # Naming convention is "namespace::function" # assert '::' in key # Run synchronously if not on EC2 # if not utils.is_ec2(): # logs.warning("Local - Running synchronously") # raise Exception global __errors global __cooldown maxErrors = 5 numErrors = len(__errors) # Initiate cooldown period if number of errors is too high if numErrors > maxErrors: msg = "Number of errors (%s) exceeds maximum (%s): cooling down" % (numErrors, maxErrors) logs.warning(msg) __cooldown = datetime.utcnow() __errors = [] # Fail if in cooldown period if __cooldown is not None and datetime.utcnow() - timedelta(minutes=5) < __cooldown: msg = "Cooling down until %s" % __cooldown logs.warning(msg) raise Exception(msg) # Build payload data = { 'task_id': str(ObjectId()), 'key': key, 'data': payload, 'timestamp': datetime.utcnow() } maxRetries = 5 numRetries = 0 while True: try: # Submit job logs.info("Submitting task: %s" % data) client = getClient() client.submit_job(queue, pickle.dumps(data), background=True, wait_until_complete=False) # Reset errors __errors = [] __cooldown = None return True except Exception as e: # Retry numRetries += 1 if numRetries <= maxRetries: time.sleep(0.1) continue logs.warning("Task failed: %s (%s)" % (type(e), e)) # Reset client resetClient() # Cap the number of errors logged __errors.append(e) __errors = __errors[-25:] numErrors = len(__errors) # Initiate a cool down period if numErrors >= maxErrors: __cooldown = datetime.utcnow() # Send an email alert the first time this happens if numErrors == maxErrors: msg = "Unable to connect to task broker: %s" % getHosts() logs.warning(msg) if utils.is_ec2(): # Send email email = {} try: stack_info = libs.ec2_utils.get_stack() email['subject'] = "%s.%s - Unable to connect to task broker" % \ (stack_info.instance.stack, stack_info.instance.name) except Exception: email['subject'] = msg email['body'] = "All async tasks running locally\n\n%s\n\n" % msg for error in __errors: email['body'] += str(error) email['body'] += '\n' email['from'] = 'Stamped <*****@*****.**>' email['to'] = '*****@*****.**' utils.sendEmail(email) raise
import libs.ec2_utils import multiprocessing, os, sys from datetime import timedelta from celery.schedules import crontab # List of modules to import when celery starts. CELERY_IMPORTS = ("tasks", ) ## Result store settings. CELERY_RESULT_BACKEND = "amqp" host, port = "localhost", 5672 user, password, vhost = "guest", "guest", "/" if utils.is_ec2(): stack = libs.ec2_utils.get_stack() for node in stack.nodes: if 'monitor' in node.roles: host = node.private_ip_address break if 'work-enrich' in stack.instance.roles: CELERY_ACKS_LATE = True CELERYD_CONCURRENCY = 5 ## Broker settings. BROKER_URL = "pyamqp://%s:%s@%s:%s/%s" % (user, password, host, port, vhost) BROKER_HEARTBEAT = 10 logs.info('BROKER_URL: %s' % BROKER_URL)
# Django settings for www project. import Globals import utils, os, libs.ec2_utils IS_PROD = libs.ec2_utils.is_prod_stack() DEBUG = (not utils.is_ec2()) STAMPED_DEBUG = DEBUG TEMPLATE_DEBUG = DEBUG PROJ_ROOT = os.path.abspath(os.path.dirname(__file__)) STAMPED_ASSET_VERSION = utils.shell("cd %s && make version" % PROJ_ROOT)[0] STAMPED_DOWNLOAD_APP_LINK = "http://itunes.apple.com/us/app/stamped/id467924760?mt=8&uo=4" utils.log("Django DEBUG=%s ROOT=%s VERSION=%s" % (DEBUG, PROJ_ROOT, STAMPED_ASSET_VERSION)) ADMINS = ( ('Travis', '*****@*****.**'), ) MANAGERS = ADMINS DATABASES = { 'default': { 'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 'NAME': '', # Or path to database file if using sqlite3. 'USER': '', # Not used with sqlite3. 'PASSWORD': '', # Not used with sqlite3. 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 'PORT': '', # Set to empty string for default. Not used with sqlite3. }