def testBuildFullArrayFlat(self): '''Build a full FLATTENED array from a cursor result''' st = datetime.datetime.utcnow() # A keyword that went in yesterday creates a timeseries yesterday nowDt = datetime.datetime(year=2011,month=1,day=12,hour=11,minute=1,second=1) oneDay= datetime.timedelta(days=1) # Get a db handle c, dbh = mdb.getHandle() dbh = mdb.setupCollections(dbh, dropCollections=True) # Set up collections # Build a keyword kword = kw(keyword='keyword1', timeStamp=nowDt-oneDay, lat=34.4, lon=45.5, text='this text contained the hashtag #keyword1', tweetID=346664, userID=4444, source='twitter') # New timeseries object ts = timeSeries() ts.importData(kword) success = ts.insertBlankDoc() # Insert 2ND DOC IN THE COLLECTION kword.timeStamp = nowDt ts = timeSeries() ts.importData(kword) success = ts.insertBlankDoc() nowDate = nowDt.replace(hour=0,minute=0,second=0,microsecond=0) # Last 1 weeks worth of documents resultSet = bl.getResultsPerCell(dbh, '38SND4595706622', 'keyword1', nowDate, 168) # Close the connection mdb.close(c, dbh) # Inputs period = datetime.timedelta(days=7) dates, data = bl.buildFullArray(resultSet, nowDate, period, 1) firstDay = dates[0] lastDay = dates[-1] self.assertEquals(data.shape[0], 11520) self.assertEquals(firstDay, nowDate - period) self.assertEquals(lastDay, nowDate)
def testBuildFullArray(self): '''Build a full array from a cursor result''' # Get a db handle c, dbh = mdb.getHandle() dbh = mdb.setupCollections(dbh, dropCollections=True) # Set up collections # Build a keyword kword = kw(keyword='keyword1', timeStamp=datetime.datetime(2011,1,2,12,1,1), lat=34.4, lon=45.5, text='this text contained the hashtag #keyword1', tweetID=346664, userID=4444, source='twitter') # New timeseries object ts = timeSeries() ts.importData(kword) success = ts.insertBlankDoc() # Insert the doc now that its been modified kword.timeStamp = datetime.datetime(2011,1,1,12,1,1) ts = timeSeries() ts.importData(kword) success = ts.insertBlankDoc() # Last 1 weeks worth of documents resultSet = bl.getResultsPerCell(dbh, '38SND4595706622', 'keyword1', datetime.datetime(2011,1,2), 168) # Inputs inDate = datetime.datetime(2011, 1, 2, 0, 0) period = datetime.timedelta(days=7) flat = None dates, data = bl.buildFullArray(resultSet, inDate, period, flat) self.assertEquals(len(dates), 8) self.assertEquals(len(data), 8) # Close the connection mdb.close(c, dbh)
# Query based on a keyword only keyword = 'sick' query = {'keyword':keyword, 'start':{'$gte' : queryStart}, 'start':{'$lte' : queryEnd}} if mgrs: query['mgrs'] = mgrs if mgrsPrecision: query['mgrsPrecision'] = mgrsPrecision res = collHandle.find(query).sort({'start':1}) resultSet = [r for r in res] arr = buildFullArray(resultSet, queryEnd, lookback, blankDay, flat=1) fftDev(arr) """ TESTS/EXPERIMENTS 1. Check the array coming back is sensible 2. See what an FFT looks like for a keyword 3. See what an FFT looks like for all documents - need to write some aggregation bits in here. 4. What does the periodogram look like? 5. Build a JTF plot of the frequency data """