def execute_request_at(serverHost, serverPort, serverProtocol, mhsid, siteID,
ancf, bncf, xmtScript):
#--------------------------------------------------------------------
# Create a message - pickled
# (MHSsiteID, mySiteID, listOfVTECMergeSites, countDict, issueTime)
# Note that VTEC_MERGE_SITES does not contain our site or SPC, TPC.
#--------------------------------------------------------------------
# determine my 4 letter siteid
if siteID in ['SJU']:
mysite4 = "TJSJ"
elif siteID in ['AFG', 'AJK', 'HFO', 'GUM']:
mysite4 = "P" + siteID
elif siteID in ['AER', 'ALU']:
mysite4 = "PAFC"
else:
mysite4 = "K" + siteID
otherSites = [mysite4]
tpcSites = JUtil.javaObjToPyVal(
JavaVTECPartners.getInstance(siteID).getTpcSites())
spcSites = JUtil.javaObjToPyVal(
JavaVTECPartners.getInstance(siteID).getSpcSites())
otherSites.extend(tpcSites)
otherSites.extend(spcSites)
# determine the MHS WMO id for this message
wmoid = "TTAA00 " + mysite4
wmoid += " " + time.strftime("%d%H%M", time.gmtime(time.time()))
# connect to ifpServer and retrieve active table
actTab = ActiveTable.getActiveTable(mysite4, ActiveTableMode.OPERATIONAL)
# analyze active table to get counts
countDict = {}
issueTime = 0
for i in xrange(actTab.size()):
rec = actTab.get(i)
# only care about our own sites that we merge
if rec.getOfficeid() not in VTECPartners.VTEC_MERGE_SITES and \
rec.getOfficeid() not in otherSites:
continue
recIssueTime = float(rec.getIssueTime().getTime() /
TimeUtil.MILLIS_PER_SECOND)
#track latest
issueTime = max(recIssueTime, issueTime)
cnt = countDict.get(rec.getOfficeid(), 0) #get old count
countDict[rec.getOfficeid()] = cnt + 1
data = (mhsid, siteID, VTECPartners.VTEC_MERGE_SITES, countDict, issueTime)
logger.info("Data: " + repr(data))
tempdir = os.path.join(siteConfig.GFESUITE_HOME, "products", "ATBL")
with tempfile.NamedTemporaryFile(suffix='.reqat',
dir=tempdir,
delete=False) as fp:
fname = fp.name
buf = cPickle.dumps(data)
fp.write(buf)
sourceServer = {
'mhsid': mhsid,
'host': serverHost,
'port': serverPort,
'protocol': serverProtocol,
'site': siteID
}
destSites = VTECPartners.VTEC_TABLE_REQUEST_SITES
if not destSites:
raise Exception('No destSites defined for VTEC_TABLE_REQUEST_SITES')
irt = IrtAccess.IrtAccess(ancf, bncf, logger=logger)
msgSendDest, xml = irt.createDestinationXML(destSites, sourceServer)
# create the XML file
with tempfile.NamedTemporaryFile(suffix='.xml', dir=tempdir,
delete=False) as fd:
fnameXML = fd.name
fd.write(ElementTree.tostring(xml))
#--------------------------------------------------------------------
# Now send the message
#--------------------------------------------------------------------
irt.transmitFiles("GET_ACTIVE_TABLE2", msgSendDest, mhsid,
[fname, fnameXML], xmtScript)
def __init__(self,
activeTable,
activeTableMode,
newRecords,
offsetSecs=0.0,
makeBackups=True,
logger=None,
atChangeLog=None):
# activeTable - current activeTable records
# activeTableMode - which table is being modified--OPERATIONAL or PRACTICE
# newRecords - records to merge in to activeTable
# inputIsGZIP (0,1) - remote input file is gzipped
# offsetSecs - Number of seconds +/- current time
# makeBackups (False, True) - make backups of previous table
# logger - python logging object to send all logs to
if logger is not None:
self._log = logger
else:
self._log = self.__initLogging()
# get our site
siteid = siteConfig.GFESUITE_SITEID
self._ourSite = self._get4ID(siteid)
# create a dummy name to simplify the file access code in VTECTableUtil
filePath = os.path.join(EDEXUtil.getEdexData(), "activetable", siteid)
fileName = os.path.join(filePath, activeTableMode + ".tbl")
# to ensure time calls are based on Zulu
os.environ['TZ'] = "GMT0"
self._time = time.time() + offsetSecs #present time
self._makeBackups = makeBackups
VTECTableUtil.VTECTableUtil.__init__(self, fileName)
# get the SPC site id from the configuration file
self._spcSite = JUtil.javaObjToPyVal(
JavaVTECPartners.getInstance(siteid).getSpcSites("KWNS"))
self._tpcSite = JUtil.javaObjToPyVal(
JavaVTECPartners.getInstance(siteid).getTpcSites("KNHC"))
self._siteFilter = self._getFilterSites()
self._log.info("MergeVTEC Starting")
self._log.info("localFN= " + self._activeTableFilename + " sites= " +
repr(self._siteFilter))
#read table to merge
otherTable = newRecords
self._log.info("Remote Table size: %d", len(otherTable))
#read active table
self._log.info("Active Table size: %d", len(activeTable))
#save a copy for later backup purposes
oldActiveTable = copy.deepcopy(activeTable)
#delete "obsolete" records from our table and the other table
vts = VTECTableSqueeze.VTECTableSqueeze(self._time)
activeTable, tossRecords = vts.squeeze(activeTable)
self._log.info("Active Table squeezed size: %d", len(activeTable))
self._log.info("Other Table size: %d", len(otherTable))
otherTable, tossRecordsOther = vts.squeeze(otherTable)
self._log.info("Other Table squeezed size: %d", len(otherTable))
#merge the tables
updatedTable, toDelete, changes = self._mergeTable(
activeTable, otherTable, atChangeLog)
self._log.info("Updated Active Table size: %d", len(updatedTable))
updatedTable, tossRecordsMerged = vts.squeeze(updatedTable)
self._log.info("Updated Active Table squeeze size: %d",
len(updatedTable))
del vts
if atChangeLog is not None:
atChangeLog.info("Entries tossed after merge: " +
self.printActiveTable(tossRecordsMerged, 1))
# determine final changes
finalChanges = []
for rec in updatedTable:
if rec['state'] != 'Existing':
item = (rec['officeid'], rec['pil'], rec['phensig'],
rec['xxxid'])
if item not in finalChanges:
finalChanges.append(item)
changes = finalChanges
if atChangeLog is not None:
atChangeLog.info("Table Changes: " + str(changes))
self._updatedTable = []
self._purgedTable = []
self._changes = []
#notify the ifpServer of changes, save a backup copy
if tossRecords or tossRecordsMerged or changes:
self._log.debug("#tossRecords: %d", len(tossRecords))
self._log.debug("#tossRecordsMerged: %d", len(tossRecordsMerged))
self._log.debug("#changes: %d", len(changes))
# save lists for later retrieval
self._updatedTable = updatedTable
self._purgedTable.extend(tossRecords)
self._purgedTable.extend(toDelete)
self._purgedTable.extend(
[rec for rec in tossRecordsMerged if rec in oldActiveTable])
self._changes = changes
#save backup copy
if self._makeBackups:
oldActiveTable = self._convertTableToPurePython(
oldActiveTable, siteid)
self.saveOldActiveTable(oldActiveTable)
pTime = getattr(VTECPartners, "VTEC_BACKUP_TABLE_PURGE_TIME",
168)
self.purgeOldSavedTables(pTime)
self._log.info("MergeVTEC Finished")
def execute_send_at(myServerHost, myServerPort, myServerProtocol,
myServerMHSID, myServerSite, sites, filterSites, mhsSites,
issueTime, countDict, fname, xmlIncoming, xmtScript):
logger.info('reqSite= ' + repr(sites))
logger.info('filterSites= ' + repr(filterSites))
logger.info('mhsSite= ' + repr(mhsSites))
logger.info('reqCountDict= ' + repr(countDict))
if issueTime is None:
logger.info('reqIssueTime= None')
else:
logger.info('reqIssueTime= ' + str(issueTime) + ' ' +
time.asctime(time.gmtime(issueTime)))
irt = IrtAccess.IrtAccess("")
myServer = {
'mhsid': myServerMHSID,
'host': myServerHost,
'port': myServerPort,
'protocol': myServerProtocol,
'site': myServerSite
}
logger.info('MyServer: ' + irt.printServerInfo(myServer))
#--------------------------------------------------------------------
# Prepare the file for sending
#--------------------------------------------------------------------
with open(fname, 'rb') as fd:
buf = fd.read()
os.remove(fname)
table = cPickle.loads(buf) #unpickle it
logger.info("Local Table Length= " + str(len(table)))
filtTable = []
# filter by sites listing
if filterSites:
tpcSites = JUtil.javaObjToPyVal(
JavaVTECPartners.getInstance(myServerSite).getTpcSites())
spcSites = JUtil.javaObjToPyVal(
JavaVTECPartners.getInstance(myServerSite).getSpcSites())
for t in table:
if t['oid'] in filterSites or t['oid'][1:4] in sites or \
t['oid'] in tpcSites or t['oid'] in spcSites:
filtTable.append(t)
else:
filtTable = table #no filtering
logger.info("Site Filtered Table Length= " + str(len(filtTable)))
# eliminate obsolete records
ctime = time.time() #now time
vts = VTECTableSqueeze.VTECTableSqueeze(ctime)
filtTable = rename_fields_for_A2(filtTable)
actTable, tossRecords = vts.squeeze(filtTable)
actTable = rename_fields_for_A1(actTable)
logger.info("Squeezed Table Length= " + str(len(actTable)))
# check issuance time - any times newer in remote table (this table) than
# the local table (requesting site)?
if issueTime is not None:
newerRec = False
newestRec = 0.0
for t in actTable:
if newestRec < t['issueTime']:
newestRec = t['issueTime']
if issueTime < newestRec:
newerRec = True
logger.info("NewestFound= " + str(newestRec) + ' ' +
time.asctime(time.gmtime(newestRec)))
logger.info("IssueTime check. Newer record found= " + str(newerRec))
else:
newerRec = True #just assume there are newer records
# check "counts" for number of records. Any more records means that
# we may be missing some records.
if countDict:
missingRec = False
localCountDict = {}
for t in actTable:
if localCountDict.has_key(t['oid']):
localCountDict[t['oid']] = localCountDict[t['oid']] + 1
else:
localCountDict[t['oid']] = 1
for site in localCountDict:
reqCount = countDict.get(site, 0) #number in requesting site
if reqCount != localCountDict[
site]: #records different in request site
missingRec = True
break
logger.info("MissingRec check. Missing record found= " +
str(missingRec))
logger.info("lclCountBySite= " + repr(localCountDict))
logger.info("reqCountBySite= " + repr(countDict))
else:
missingRec = True #just assume there are
#should we send?
if missingRec or newerRec or FORCE_SEND:
sendIt = True
else:
sendIt = False
if sendIt:
actTablePickled = cPickle.dumps(actTable) #repickle it
rawSize = len(actTablePickled)
#output it as gzipped
fname = fname + ".gz"
with gzip.open(fname, 'wb', 6) as fd:
fd.write(actTablePickled)
gzipSize = os.stat(fname)[stat.ST_SIZE]
logger.info('#dataSize: ' + str(rawSize) + ', #gzipSize: ' +
str(gzipSize))
#--------------------------------------------------------------------
# Create the destination XML file
#--------------------------------------------------------------------
iscOut = ElementTree.Element('isc')
irt.addSourceXML(iscOut, myServer)
destServers = []
if xmlIncoming is not None:
with open(xmlIncoming, 'rb') as fd:
xml = fd.read()
os.remove(xmlIncoming)
reqTree = ElementTree.ElementTree(ElementTree.XML(xml))
sourceE = reqTree.find('source')
for addressE in sourceE.getchildren():
destServer = irt.decodeXMLAddress(addressE)
if destServer is None:
continue
destServers.append(destServer)
break
# no XML received on request, this is probably from an older site.
# create a dummy response XML file. Try all combinations. Only
# the mhsid is important for older sites.
else:
servers = []
for mhss in mhsSites:
for port in xrange(98000000, 98000002):
for site in sites:
for host in ['dx4f', 'px3']:
destServer = {
'mhsid': mhss,
'host': host,
'port': port,
'protocol': "20070723",
'site': site
}
destServers.append(destServer)
irt.addDestinationXML(iscOut, destServers) #add the dest server xml
# print out destinations
s = "Destinations:"
for destServer in destServers:
s += "\n" + irt.printServerInfo(destServer)
logger.info(s)
# create XML file
tempdir = os.path.join(siteConfig.GFESUITE_HOME, "products", "ATBL")
with tempfile.NamedTemporaryFile(suffix='.xml',
dir=tempdir,
delete=False) as fd:
fnameXML = fd.name
fd.write(ElementTree.tostring(iscOut))
#--------------------------------------------------------------------
# Send it
#--------------------------------------------------------------------
irt.transmitFiles("PUT_ACTIVE_TABLE2", mhsSites, myServerSite,
[fname, fnameXML], xmtScript)
else:
logger.info("Send has been skipped")