def _process_events(self, events): ''' Function to process expected, maybe later unexpected event types to classify them events: List of column types Returns: number of events added to database ''' for n, event in enumerate(events): events[n]['Date'] = calendar.timegm(dateutil.parser.parse(event['Date']).timetuple()) #events[n]['Date'] = int(ed.USAA2Timestamp(event['Date'])) oldest = min(e['Date'] for e in events) newest = max(e['Date'] for e in events) history = self._db[self._name].aggregate([ {'$project': {'etime': '$details.etime', 'ticker': '$details.ticker'}}, {'$match': {'etime': {'$gte': oldest, '$lte': newest}}} ])['result'] ## Remove Duplicates for event in events[:]: for h in history: if event['Date'] == h['etime'] and event['Symbol'] == h['ticker']: events.remove(event) break ## Shape documents events = self._shape_events(events) if events: self._db[self._name].insert(events) company.ensureCompanies(self._name) print 'Adding {} items to DB'.format(len(events)) else: print 'Nothing to add to DB'
def __init__(self, name="main"): """Select an optional save file name. save: optional string specifying a portfolio name. Defaults to 'store' """ self._name = name self._db = db_connect.get_db() self._update() company.ensureCompanies(self._name)