def innerLoopActivity(self, item): """ This is where the actual ETL activity is called for the given compare item. """ # Write unit data to the target repository: if self.itemCount == 0: config.createUnitDataAccessor(self.storageTgt).store(self.unitData) # Put together the site file: self._insertSiteFile(item, item.provItem.payload) # Obtain the raw count data archive: countsFilePath = self.gsProvider.resolvePayload(item) if not countsFilePath: return 0 # Write raw count data archive to storage: print("%s -> %s" % (item.label, self.storageTgt.repository)) catalogElement = self.storageTgt.createCatalogElement( item.identifier.base, item.identifier.ext, item.identifier.date, self.processingDate) self.storageTgt.writeFile(countsFilePath, catalogElement, cacheCatalogFlag=True) # Clean up: os.remove(countsFilePath) # Performance metrics: self.perfmet.recordCollect(item.identifier.date, representsDay=True) return 1
def innerLoopActivity(self, item): """ This is where the actual ETL activity is called for the given compare item. """ # Write unit data to the target repository: if self.itemCount == 0: config.createUnitDataAccessor(self.storageTgt).store(self.unitData) # Read in the file and call the transformation code. print("%s: %s -> %s" % (item.label, self.storageSrc.repository, self.storageTgt.repository)) filepathSrc = self.storageSrc.retrieveFilePath(item.label) outJSON, perfWork = wtStandardize(item, filepathSrc, self.storageTgt.makeFilename(item.identifier.base, "json", item.identifier.date), self.processingDate) # Clean up: os.remove(filepathSrc) # Prepare for writing to the target: catalogElement = self.storageTgt.createCatalogElement(item.identifier.base, "json", item.identifier.date, self.processingDate) self.storageTgt.writeJSON(outJSON, catalogElement) # Final stages: self.perfmet.recordCollect(item.identifier.date, representsDay=True) for sensor, rec in perfWork.items(): self.perfmet.recordSensorObs(sensor, "Vehicle Counts", perfmet.SensorObs(observation=rec[0], expected=None, collectionDate=item.identifier.date, minTimestamp=rec[1], maxTimestamp=rec[2])) self.perfmet.writeSensorObs() return 1
def innerLoopActivity(self, item): """ This is where the actual ETL activity is called for the given compare item. """ # Commit old sensor observations: if self.prevDate and item.identifier.date > self.prevDate: # Write out uncommitted sensor performance metric observations for the prior date: self.perfmet.writeSensorObs() # Get site file: siteFileCatElem, newSiteFlag = self.siteFileCatElems.getForPrevDate( item.identifier.base, item.identifier.date, forceValid=True) if not siteFileCatElem: print("ERROR: No site file is found for '%s' for date %s." % (item.identifier.base, str(item.identifier.date))) return 0 if not newSiteFlag: siteFile = self.siteFileCache[item.identifier.base] else: # Get site file from repository if needed: siteFile = json.loads( self.storageSrc.retrieveBuffer(siteFileCatElem["pointer"])) self.siteFileCache[item.identifier.base] = siteFile # Obtain unit data, and write it to the target repository if it's new: unitData = self.unitDataProv.retrieve(item.identifier.date) if unitData != self.prevUnitData: config.createUnitDataAccessor(self.storageTgt).store(unitData) print("%s: %s -> %s" % (item.label, self.storageSrc.repository, self.storageTgt.repository)) worker = GSJSONStandard(item, siteFile, self.storageSrc, self.storageTgt, self.processingDate) if not worker.jsonize(): return 0 # Write the site file if it is a new one: if newSiteFlag: catalogElement = self.storageTgt.createCatalogElement( item.identifier.base, "site.json", siteFileCatElem["collection_date"], self.processingDate) self.storageTgt.writeJSON(siteFile, catalogElement) # Performance metrics logging: self.perfmet.recordCollect(item.identifier.date, representsDay=True) self.perfmet.recordSensorObs( item.identifier.base, "Vehicle Counts", perfmet.SensorObs(observation=worker.perfWork[0], expected=None, collectionDate=item.identifier.date, minTimestamp=worker.perfWork[1], maxTimestamp=worker.perfWork[2])) return 1
def innerLoopActivity(self, item): """ This is where the actual ETL activity is called for the given compare item. """ if item.identifier.ext not in ("traf_match_summary.txt", "matched.txt", "unmatched.txt"): print("WARNING: Unsupported file type or extension: %s" % item.identifier.ext) return 0 # Write unit data to the target repository: if self.itemCount == 0: config.createUnitDataAccessor(self.storageTgt).store(self.unitData) # Read in the file and call the transformation code. print("%s: %s -> %s" % (item.label, self.storageSrc.repository, self.storageTgt.repository)) filepathSrc = self.storageSrc.retrieveFilePath(item.label) fileType = item.identifier.ext.split(".")[ 0] # Get string up to the file type extension. outJSON, perfWork = btStandardize( item, filepathSrc, self.storageTgt.makeFilename(item.identifier.base, fileType + ".json", item.identifier.date), fileType, self.processingDate) # Clean up: os.remove(filepathSrc) # Prepare for writing to the target: catalogElement = self.storageTgt.createCatalogElement( item.identifier.base, fileType + ".json", item.identifier.date, self.processingDate) self.storageTgt.writeJSON(outJSON, catalogElement) # Final stages: self.perfmet.recordCollect(item.identifier.date, representsDay=True) if fileType == "unmatched": for sensor, rec in perfWork.items(): self.perfmet.recordSensorObs( sensor, "Unmatched Entries", perfmet.SensorObs(observation=rec[0], expected=None, collectionDate=item.identifier.date, minTimestamp=rec[1], maxTimestamp=rec[2])) # TODO: One issue we have with this is that there isn't a definitive way to know if a BT sensor is dead # without it totally missing from the list. We need to add in zero-entries for sensors we expect to be working. # Or, calculate expectations using prior records. self.perfmet.writeSensorObs() return 1
def etlActivity(self): """ This performs the main ETL processing. @return count: A general number of records processed """ # First, get the unit data for GRIDSMART: self.unitDataProv = config.createUnitDataAccessor( self.storageSrc).prepare(self.startDate, self.endDate) # Prepare to get site files: self.siteFileCatElems = self.storageSrc.catalog.getSearchableQueryDict( self.storageSrc.repository, base=None, ext="site.json", earlyDate=self.startDate, lateDate=self.endDate) # Configure the source and target repositories and start the compare loop: self.bases.clear() self.curDate = None count = self.doCompareLoop( last_update.LastUpdStorageCatProv(self.storageSrc, extFilter="%%.json"), last_update.LastUpdStorageCatProv(self.storageTgt), baseExtKey=False) # Process the last day's worth of records: count += self._processDay(self.curDate) print("Records processed: %d" % count) return count
def etlActivity(self): """ This performs the main ETL processing. @return count: A general number of records processed """ # First, get the unit data for Wavetronix: unitDataProv = config.createUnitDataAccessor(self.dataSource) self.unitData = unitDataProv.retrieve() # Configure the source and target repositories and start the compare loop: count = self.doCompareLoop(last_update.LastUpdStorageCatProv(self.storageSrc), last_update.LastUpdStorageCatProv(self.storageTgt), baseExtKey=False) print("Records processed: %d" % count) return count
def etlActivity(self): """ This performs the main ETL processing. @return count: A general number of records processed """ # First, get the unit data for GRIDSMART: unitDataProv = config.createUnitDataAccessor(self.dataSource) self.unitData = unitDataProv.retrieve() deviceLogreaders = gs_support.getDevicesLogreaders( self.unitData, self.deviceFilter) # Configure the source and target repositories and start the compare loop: self.gsProvider = last_upd_gs.LastUpdGSProv(deviceLogreaders, self.tempDir) count = self.doCompareLoop(self.gsProvider, last_update.LastUpdStorageCatProv( self.storageTgt), baseExtKey=False) print("Records processed: %d" % count) return count