def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# First, get the unit data for GRIDSMART:
self.unitDataProv = config.createUnitDataAccessor(
self.storageSrc).prepare(self.startDate, self.endDate)
# Prepare to get site files:
self.siteFileCatElems = self.storageSrc.catalog.getSearchableQueryDict(
self.storageSrc.repository,
base=None,
ext="site.json",
earlyDate=self.startDate,
lateDate=self.endDate)
# Configure the source and target repositories and start the compare loop:
self.bases.clear()
self.curDate = None
count = self.doCompareLoop(
last_update.LastUpdStorageCatProv(self.storageSrc,
extFilter="%%.json"),
last_update.LastUpdStorageCatProv(self.storageTgt),
baseExtKey=False)
# Process the last day's worth of records:
count += self._processDay(self.curDate)
print("Records processed: %d" % count)
return count
def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# Configure the source and target repositories and start the compare loop:
count = self.doCompareLoop(last_update.LastUpdStorageCatProv(self.storageSrc, extFilter="counts.json"),
last_update.LastUpdStorageCatProv(self.storageTgt, extFilter="agg%d.json" % self.args.agg),
baseExtKey=False)
print("Records processed: %d" % count)
return count
def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# First, get the unit data for Wavetronix:
unitDataProv = config.createUnitDataAccessor(self.dataSource)
self.unitData = unitDataProv.retrieve()
# Configure the source and target repositories and start the compare loop:
count = self.doCompareLoop(last_update.LastUpdStorageCatProv(self.storageSrc),
last_update.LastUpdStorageCatProv(self.storageTgt),
baseExtKey=False)
print("Records processed: %d" % count)
return count
def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# Configure the source and target repositories and start the compare loop:
count = self.doCompareLoop(BTLastUpdateProv(self.sourceDir, DIR_DEFS),
last_update.LastUpdStorageCatProv(self.storageTgt),
baseExtKey=True)
print("Records processed: %d" % count)
return count
def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# First, get the unit data for GRIDSMART:
unitDataProv = config.createUnitDataAccessor(self.dataSource)
self.unitData = unitDataProv.retrieve()
deviceLogreaders = gs_support.getDevicesLogreaders(
self.unitData, self.deviceFilter)
# Configure the source and target repositories and start the compare loop:
self.gsProvider = last_upd_gs.LastUpdGSProv(deviceLogreaders,
self.tempDir)
count = self.doCompareLoop(self.gsProvider,
last_update.LastUpdStorageCatProv(
self.storageTgt),
baseExtKey=False)
print("Records processed: %d" % count)
return count
def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# Establish the publishers:
self.publishers = {"traf_match_summary": config.createPublisher("bt", "traf_match_summary", self.storageSrc.catalog,
simulationMode=self.simulationMode,
writeFilePath=self.writeFilePath),
"matched": config.createPublisher("bt", "matched", self.storageSrc.catalog,
simulationMode=self.simulationMode,
writeFilePath=self.writeFilePath),
"unmatched": config.createPublisher("bt", "unmatched", self.storageSrc.catalog,
simulationMode=self.simulationMode,
writeFilePath=self.writeFilePath)}
# Configure the source and target repositories and start the compare loop:
count = self.doCompareLoop(last_update.LastUpdStorageCatProv(self.storageSrc),
last_update.LastUpdCatProv(self.storageSrc.catalog, config.getRepository("public")),
baseExtKey=False)
print("Records processed: %d" % count)
return count
def etlActivity(self):
"""
This performs the main ETL processing.
@return count: A general number of records processed
"""
# Establish the publishers:
self.publisher = config.createPublisher(
"gs",
None,
self.storageSrc.catalog,
simulationMode=self.simulationMode,
writeFilePath=self.writeFilePath)
# Configure the source and target repositories and start the compare loop:
count = self.doCompareLoop(last_update.LastUpdStorageCatProv(
self.storageSrc, extFilter="agg%d.json" % self.args.agg),
last_update.LastUpdCatProv(
self.storageSrc.catalog,
config.getRepository("public")),
baseExtKey=False)
print("Records processed: %d" % count)
return count