def main(): define("console", default=False, type=bool) define("mongo_host", default='localhost') parse_command_line() basicConfig(options=options) country_db = CountryDB(options.mongo_host) company_db = CompanyDB(options.mongo_host) stats_db = StatsDB(options.mongo_host) #companies = company_db.find({}, fields={'offices': 1}, count=None) ##for company in adv_enumerate(companies): ## for office in company['offices']: ## country_db.increment(office['country_code']) ##print company_db #_create_stats(stats_db, company_db) _create_csv(stats_db, company_db) return -1 categories = defaultdict(int) year_month = defaultdict(int) year = defaultdict(int) for stats in adv_enumerate(stats_db.find()): categories[get_dotted(stats, 'data.category_code')] += 1 founded_at = get_dotted(stats, 'data.founded_at') if not founded_at: continue #if founded_at.year < 1995: # print stats year_month[(founded_at.year, founded_at.month)] += 1 year[founded_at.year] += 1 print sum(categories.values()) print year print sorted(year_month.items())
def find(self, query=None, *args, **kwargs): end_ts = datetime(1995, 1, 1) query = query or {} query['data.founded_at'] = {'$gte': end_ts} query['data.category_code'] = {'$ne': None} query['data.number_of_employees'] = {'$gt': 0} query['data.total_money_raised'] = {'$gt': 0} for i in self._db.find(query, *args, **kwargs): if get_dotted(i, 'data.founded_at') >= end_ts: yield i raise StopIteration
def get(self): count = int(self.get_argument("count", 1000)) format = self.get_argument("format", "html") human = bool(self.get_argument("human", None)) data = {} stats = [] stats_cursor = self._db.find({}, None) for idx, stat in enumerate(stats_cursor): if count and idx > count: break item = {} for k, v in DB_API_MAP.items(): set_dotted(item, v, get_dotted(stat, k)) stats.append(item) if format == "html": return self._get_html(data) self.set_header("Content-Type", "application/json; charset=UTF-8") data["count"] = len(stats) data["stats"] = make_jsonable(stats, human) data = ujson.encode(data) self.write(data)