def get_cluster(self, region, stack, site): """Return the right cluster for given region and tags TODO: wrap in some kind of clusterfinder object? """ tags = self.site_tags[stack][site] if not tags: tags = self.stack_tags[stack] if not tags: return self.default_by_region[region] try: key = frozenset(tags.items()) return self.es_cluster_lookup_by_region[region][key] except KeyError: logger.info("Creating runner for region: %s, stack: %s, tags: %s", region, stack, tags) runner = amz.locator_for_region(region, self.aws_access, self.aws_secret, tags) self.es_cluster_lookup_by_region[region][key] = runner return runner
def main(argv=sys.argv): global logger if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) settings = get_appsettings(config_uri) logger = logging.getLogger('scripts') ca = settings.get('mysql.ca') or None aws_access = settings.get('aws.access') or None aws_secret = settings.get('aws.secret') or None user = settings['es.user'] pasw = settings['es.pass'] mysqluser = settings['mysql.user'] mysqlpass = settings['mysql.pass'] default_west_runner = amz.locator_for_region('us-west-1', aws_access, aws_secret, tags={'app':'west_es','env':'prod'}) default_east_runner = amz.locator_for_region('us-east-1', aws_access, aws_secret, tags={'app':'east_es','env':'prod'}) default_cluster_for_region = {'us-east-1': default_east_runner, 'us-west-1': default_west_runner} default_west_runner_logs = amz.locator_for_region('us-west-1', aws_access, aws_secret, tags={'app':'west_es_c2','env':'prod'}) default_east_runner_logs = amz.locator_for_region('us-east-1', aws_access, aws_secret, tags={'app':'east_es_c2','env':'prod'}) default_cluster_for_region_logs = {'us-east-1': default_east_runner_logs, 'us-west-1': default_west_runner_logs} region_map = amz.get_stacks(aws_access, aws_secret) logger.debug("Region map: %s", region_map) stack_map = cf.build_stack_map(settings) logger.debug("Stack map: %s", stack_map) site_tags = cf.build_site_tags(settings) logger.debug("Site tags: %s", site_tags) stack_tags = cf.build_stack_tags(settings) logger.debug("Stack tags: %s", stack_tags) site_tags_logs = cf.build_site_tags(settings, prefix='sitetags.logs') logger.debug("Site tags logs: %s", site_tags_logs) stack_tags_logs = cf.build_stack_tags(settings, prefix='stacktags.logs') logger.debug("Stack tags logs: %s", stack_tags_logs) dbs_map = cf.build_dbs_map(settings) logger.debug("DBs Map: %s", dbs_map) ri = amz.get_rds_instances(aws_access, aws_secret) logger.debug("RDS instances: %s", ri) clusterlookup = ClusterLookup(default_cluster_for_region, stack_tags, site_tags, aws_access, aws_secret) # for activities and logs we use a different cluster :( clusterlookup_logs = ClusterLookup(default_cluster_for_region_logs, stack_tags_logs, site_tags_logs, aws_access, aws_secret) # eg [stack][site]['counts'][name] bigguy = defaultdict(lambda: defaultdict(dict)) # will hold the db conns by instance id dbs = {} while True: try: for region, stacks in region_map.items(): for stack in stacks: # so timestamp the time we do the site for site in stack_map[stack]: try: counts = {} try: rdsinstance_id = dbs_map[stack][site][0] except IndexError: logger.warn("No db mapping configured for stack %s and site %s", stack, site) continue try: db = dbs[rdsinstance_id] except KeyError: db = amz.get_connection_for_rdsname(rdsinstance_id, mysqluser, mysqlpass, ca=ca, ri=ri) dbs[rdsinstance_id] = db esrunner = clusterlookup.get_cluster(region, stack, site) esrunner_logs = clusterlookup_logs.get_cluster(region, stack, site) # switch to the right db c = db.cursor() c.execute("use %s" % site) # get names of libraries to count def get_library_names(): c = db.cursor() c.execute("select library_name, display_name from library order by display_name") rows = c.fetchall() return [(row['library_name'], row['display_name']) for row in rows] library_names = get_library_names() logger.info("listing region: %s, stack: %s, site: %s", region, stack, site ) logger.debug("libraries: %s", region, stack, site, library_names) def simple_db_count(entityname): c = db.cursor() c.execute("select count(*) as cnt from %s" % (entityname)) row = c.fetchone() return row['cnt'] def simple_es_count(entityname, prefix=''): # bad hack if stack in ['d8', 'd15']: entityname = entityname.lower() index = "%s_%s" % (site, "live") else: index = "%s_%s" % (stack, site) if entityname in ['log', 'report_activity']: host = esrunner_logs.random_instance().dns_name else: host = esrunner.random_instance().dns_name if entityname == 'report_activity': name = 'activity' else: name = entityname tipe = "%s%s" % (prefix, name) url = "https://%s/%s/%s/_count?q=*" % (host, index, tipe) auth = HTTPBasicAuth(user, pasw) r = requests.get(url, auth=auth, verify=False) if r.status_code != 200: raise Exception(r.text) j = json.loads(r.content) return j['count'] def compare_simple(names): label, dbname, esname = names db_count = simple_db_count(dbname) es_count = simple_es_count(esname) logger.debug("found: %s, %s, %s", entityname, db_count, es_count) counts[label] = {'db': db_count, 'es': es_count} def compare_library(libraryname, displayname): db_count = get_db_library_count(libraryname, db) es_count = simple_es_count(libraryname, prefix='library_') logger.debug("found: %s, %s, %s",libraryname, db_count, es_count) counts[displayname] = {'db': db_count, 'es': es_count} for entityname in [('Patient', 'patient','patient'), ('Activity', 'activity','report_activity'), ('Log', 'log','log')]: compare_simple(entityname) for libraryname, displayname in library_names: compare_library(libraryname, displayname) db.commit() logger.info("Counts: %s", counts) bigguy[stack][site]['counts'] = counts except ConnectionError as e: logger.warn( "Unable to get stats for region %s, dbname %s, stack %s and " \ "site %s: %s, attempting to update instances", region, rdsinstance_id, stack, site, e, exc_info=1) esrunner.update() esrunner_logs.update() except Exception as e: logger.warn( "Unable to get stats for region %s, dbname %s, stack %s and " \ "site %s: %s", region, rdsinstance_id, stack, site, e, exc_info=1) continue auth = HTTPDigestAuth('warmachine', 'w@rm@ch1n3') requests.put('https://warmachine.devero.com/merge/bigguy.json', data=json.dumps(bigguy), auth=auth, verify=False) logger.info('Flushed info to warmachine master') except KeyboardInterrupt: print "Exiting now" sys.exit(0)