def handle_request(self, src_addr, msg):
try:
req = Serialization.decode(msg)
print 'handle_request: %r' % (req,)
req = self.on_request(req, src_addr)
out = Serialization.encode(req)
if len(out) != 0 and req is not None:
print 'replying to %d: %r' % (src_addr, req)
self.RIOSend(src_addr, Protocol.CHITTER_RPC_REPLY, out)
except Serialization.DecodingException as e:
print 'failed to decode RPC request'
except Serialization.EncodingException as e:
print 'failed to encode RPC request'
def onRIODrop(self, payload):
try:
req = Serialization.decode(payload)
self.send_queue.put(req)
self.pump_send_queue()
except Serialization.DecodingException as e:
print "Failed to decode dropped RPC request"
def on_paxos_consensus(self, success, proposal):
pid = proposal['pid']
if pid not in self.pending_proposals:
# Ignore for now
return
memo = self.pending_proposals.pop(pid)
req, src_addr = memo['req'], memo['src_addr']
snapshot, sid = memo['snapshot'], memo['sid']
if success:
snapshot.commit(self)
cookie = (src_addr, req['tid'])
self.completed_transactions.add(cookie)
self.txn_journal.push(cookie)
self.last_sid = sid
req['result'] = True
else:
req['error'] = 'Paxos proposal rejected'
out = Serialization.encode(req)
print 'replying to %d: %r' % (src_addr, req)
self.RIOSend(src_addr, Protocol.CHITTER_RPC_REPLY, out)
def _serialize_json(self, file):
d = self._get_saved_attr()
d['serialize_obj'] = {}
for key, obj in self.serialize_obj.iteritems():
skwargs = self.serialize_kwargs.get(key, {})
d['serialize_obj'][key] = obj._get_saved_attr(**skwargs)
s = Serialization.to_json(d, self.json_preset)
file.write(s)
def send_msg(self, dest_addr, msg, error_str="Message sending failed"):
try:
byte_msg = Serialization.encode(msg)
except Serialization.EncodingException as e:
print error_str
return
self.RIOSend(dest_addr, Protocol.PAXOS, byte_msg)
def _serialize_json(self, file):
d = self._get_saved_attr()
d['serialize_obj'] = {}
for key, obj in self.serialize_obj.iteritems():
skwargs = self.serialize_kwargs.get(key, {})
d['serialize_obj'][key] = obj._get_saved_attr(**skwargs)
s = Serialization.to_json(d, self.json_preset)
file.write(s)
def handle_reply(self, src_addr, msg):
print 'handle_reply'
try:
req = Serialization.decode(msg)
self.recv_queue.put(req)
self.pump_recv_queue()
except Serialization.DecodingException as e:
print 'failed to decode RPC reply'
def _deserialize_json(self, file):
js = ''
for line in file:
js += line
d = Serialization.from_json(js)
self._load_saved_attr(d)
for key, objdict in d['serialize_obj'].iteritems():
obj = self.serialize_obj.get(key)
if not obj:
continue
dskwargs = self.deserialize_kwargs.get(key, {})
obj._load_saved_attr(objdict, **dskwargs)
def _deserialize_json(self, file):
js = ''
for line in file:
js += line
d = Serialization.from_json(js)
self._load_saved_attr(d)
for key, objdict in d['serialize_obj'].iteritems():
obj = self.serialize_obj.get(key)
if not obj:
continue
dskwargs = self.deserialize_kwargs.get(key, {})
obj._load_saved_attr(objdict, **dskwargs)
def pump_send_queue(self):
if self.send_queue.empty():
return
print 'send_queue: %r' % (self.send_queue,)
req = self.send_queue.get()
try:
payload = Serialization.encode(req)
if len(payload) != 0:
self.RIOSend(req['dest'], Protocol.CHITTER_RPC_REQUEST, payload)
except Serialization.EncodingException as e:
print 'failed to encode request'
def runRoutesOverDays(s, dates, routesFile):
messageLimit = 10
warningLimit = 10
rawInstructions = Serialization.loadInstructionsFromFile(routesFile)
# if Array.length parseWarnings > 0 then
# printfn "%d instruction parse warnings" (Array.length parseWarnings)
routes = InstructionParser.ConvertRawInstructions(s.projection, rawInstructions)
flightCountsAndCosts = {}
for date in dates:
# weather = s.weather[date]
flights = s.flights[date]
airports = s.airports #.[date]
airspace = s.airspace[date]
print "Simulating " + str(date)
# airports = None
# print routes
# raw_input('routes')
weather = None
results = Simulator.simulateFlights(s.simulationParameters,
airports,
airspace,
weather,
flights,
routes)
# return a map of FlightId to
# print results
# raw_input('next')
# countArrived = [ x.ReachedDestination for x in results]
# cost = [x.Cost for x in results]
flightCosts = {}
cost = []
countArrived = 0
for k, v in results.iteritems():
flightCosts[k] = v['CostDetail']
countArrived += 1 if v['ReachedDestination'] else 0
cost.append(v['Cost'])
print 'countArrived'
print countArrived
print 'cost'
print cost
flightCountsAndCosts[date] = {
'date' : date,
'flightCount' : countArrived,
'meanCost' : sum(cost) / len(cost),
#flightLogs = logs
'flightCosts' : flightCosts
}
return flightCountsAndCosts
def runRoutesOverDays(s, dates, routesFile):
messageLimit = 10
warningLimit = 10
rawInstructions = Serialization.loadInstructionsFromFile(routesFile)
# if Array.length parseWarnings > 0 then
# printfn "%d instruction parse warnings" (Array.length parseWarnings)
routes = InstructionParser.ConvertRawInstructions(s.projection,
rawInstructions)
flightCountsAndCosts = {}
for date in dates:
# weather = s.weather[date]
flights = s.flights[date]
airports = s.airports #.[date]
airspace = s.airspace[date]
print "Simulating " + str(date)
# airports = None
# print routes
# raw_input('routes')
weather = None
results = Simulator.simulateFlights(s.simulationParameters, airports,
airspace, weather, flights, routes)
# return a map of FlightId to
# print results
# raw_input('next')
# countArrived = [ x.ReachedDestination for x in results]
# cost = [x.Cost for x in results]
flightCosts = {}
cost = []
countArrived = 0
for k, v in results.iteritems():
flightCosts[k] = v['CostDetail']
countArrived += 1 if v['ReachedDestination'] else 0
cost.append(v['Cost'])
print 'countArrived'
print countArrived
print 'cost'
print cost
flightCountsAndCosts[date] = {
'date': date,
'flightCount': countArrived,
'meanCost': sum(cost) / len(cost),
#flightLogs = logs
'flightCosts': flightCosts
}
return flightCountsAndCosts
def _on_osc_child_update(self, **kwargs):
self.updating_child_from_osc = True
values = kwargs.get('values')
mode = values[0]
key = values[1]
i = values[2]
if mode == 'add':
js = values[3]
d = self._get_saved_attr(saved_child_objects=[])
d['saved_children']['indexed_items'] = {i:Serialization.from_json(js)}
self._load_saved_attr(d)
elif mode == 'remove':
child = self.get(key)
if child is not None:
self.del_child(child)
self.updating_child_from_osc = False
def onRIOReceive(self, src_addr, protocol, byte_msg):
if protocol != Protocol.PAXOS:
return
try:
msg = Serialization.decode(byte_msg)
except Serialization.DecodingException as e:
# TODO(sumanvyj): better logging
print "Failed to decode RPC reply."
return
try:
handler = getattr(self, msg.kind.lower())
handler(src_addr, msg)
except AttributeError:
print "Invalid paxos message kind:", msg.kind
return
def _on_osc_child_update(self, **kwargs):
self.updating_child_from_osc = True
values = kwargs.get('values')
mode = values[0]
key = values[1]
i = values[2]
if mode == 'add':
js = values[3]
d = self._get_saved_attr(saved_child_objects=[])
d['saved_children']['indexed_items'] = {
i: Serialization.from_json(js)
}
self._load_saved_attr(d)
elif mode == 'remove':
child = self.get(key)
if child is not None:
self.del_child(child)
self.updating_child_from_osc = False
def retrieveSavedExportsList(mainControl, wikiData, continuousExport):
unifNames = wikiData.getDataBlockUnifNamesStartingWith(u"savedexport/")
result = []
suppExTypes = PluginManager.getSupportedExportTypes(mainControl,
None, continuousExport)
for un in unifNames:
name = un[12:]
content = wikiData.retrieveDataBlock(un)
xmlDoc = minidom.parseString(content)
xmlNode = xmlDoc.firstChild
etype = Serialization.serFromXmlUnicode(xmlNode, u"exportTypeName")
if etype not in suppExTypes:
# Export type of saved export not supported
continue
result.append((name, xmlNode))
mainControl.getCollator().sortByFirst(result)
return result
def chit(self, username, text):
"""chit username text"""
chit = Chit(text)
content = None
try:
content = Serialization.encode(chit)
except Serialization.EncodingException as e:
yield False
content.append(ord("\n"))
tweets_fn = "tweets:" + username
with Transaction() as fs:
yield fs.begin()
if not (yield fs.append(tweets_fn, content)):
yield False
yield (yield fs.commit())
def main(argv=None):
# // Fetch config settings
basePath = config.settings["basePath"]
configFile = basePath + config.settings["configFile"]
projectionFile = basePath + config.settings["projectionFile"]
airportsFile = basePath + config.settings["airportsFile"]
landingFile = basePath + config.settings["landingFile"]
taxiFile = basePath + config.settings["taxiFile"]
restrictedAirspaceFile = basePath + config.settings["restrictedZonesFile"]
# let weatherPattern = basePath + appSetting "weatherFiles"
flightPattern = basePath + config.settings["flightFiles"]
# let groundConditionPattern = basePath + appSetting "groundConditionFiles"
# let actualTakeoffsPattern = basePath + appSetting "actualTakeoffFiles"
# let actualLandingPattern = basePath + appSetting "actualLandingFiles"
# let writeFlightLogStr = appSetting "writeFlightLog"
# let writeFlightLog = ref false;
# Boolean.TryParse(writeFlightLogStr, writeFlightLog) |> ignore
# let flightLogFiles = basePath + appSetting "flightLogFiles"
# let flightCostFile = basePath + appSetting "flightCostsFile"
turbulentZonesPattern = basePath + config.settings["turbulentZonesFiles"]
dateList = config.settings["dates"]
dates = dateList.split(',')
# // Start timer
# let sw = Diagnostics.Stopwatch.StartNew()
# timeStampPrint sw "Loading configuration, projection, airports"
simulationParameters = Serialization.loadConfigurationFromFile(configFile)
restrictedAirspace = Serialization.loadZonesFromFile(restrictedAirspaceFile)
projection = Serialization.loadProjectionFromFile(projectionFile)
airports = Serialization.loadAirportsFromFile(airportsFile) #, landingFile, taxiFile)
# timeStampPrint sw "Loading date-specific info "
(flightsDates, airspaceDates) = Serialization.loadDateMaps(
flightPattern, restrictedAirspace, turbulentZonesPattern, dates)
# // config contains references to pre-loaded objects/maps
configL = config.Config(projection, airports,
flightsDates, airspaceDates, simulationParameters)
# print flightsDates
# print airspaceDates
# raw_input("Press Enter to continue...")
# timeStampPrint sw "Done with pre-simulation loads"
# printfn "-----------------------------------------------"
print 'Finished pre-simulation loads'
# let mutable moreSubmissions = true
subfiles = config.settings['submissions'].split(',')
for submission in subfiles:
print submission
print 'runRoutesOverDays'
dayResults = runRoutesOverDays(configL, dates, basePath + '/' + submission)
# Calculate final weighted cost
totalFlights = 0.0
finalScore = 0.0
for k, day in dayResults.iteritems():
totalFlights += day['flightCount']
finalScore += day['flightCount'] * day['meanCost']
print finalScore / totalFlights
def create_client(self, clientSocket , clientName, clientPassword):
return Manipulate_users.authorize_client(clientName , clientPassword)
def _content_to_chits(self, content):
try:
return [Serialization.decode(line) for line in lines(content)]
except Serialization.DecodingException as e:
e.printStackTrace()
return None
def _runSavedExport(self, pWiki, savedExportName, continuousExport):
import Serialization, PluginManager, Exporters, SearchAndReplace
exportList = Exporters.retrieveSavedExportsList(pWiki,
pWiki.getWikiData(), continuousExport)
xmlNode = None
for exportName, xn in exportList:
if exportName == savedExportName:
xmlNode = xn
break
if xmlNode is None:
self.showCmdLineUsage(pWiki,
_(u"Saved export '%s' is unknown.") % savedExportName + u"\n\n")
return
# TODO: Refactor, it is based on AdditionalDialogs.ExportDialog._showExportProfile
try:
etypeProfile = Serialization.serFromXmlUnicode(xmlNode,
u"exportTypeName")
try:
exporter, etype, desc, panel = PluginManager.getSupportedExportTypes(
pWiki, None, continuousExport)[etypeProfile]
except KeyError:
self.showCmdLineUsage(pWiki,
_(u"Export type '%s' of saved export is not supported") %
etypeProfile + u"\n\n")
return
addOptXml = Serialization.findXmlElementFlat(xmlNode,
u"additionalOptions")
addOptVersion = int(addOptXml.getAttribute(u"version"))
if addOptVersion != exporter.getAddOptVersion():
self.showCmdLineUsage(pWiki,
_(u"Saved export uses different version for additional "
"options than current export\nExport type: '%s'\n"
"Saved export version: %i\nCurrent export version: %i") %
(etypeProfile, addOptVersion, exporter.getAddOptVersion()) +
u"\n\n")
return
if addOptXml.getAttribute(u"type") != u"simpleTuple":
self.showCmdLineUsage(pWiki,
_(u"Type of additional option storage ('%s') is unknown") %
addOptXml.getAttribute(u"type") + u"\n\n")
return
pageSetXml = Serialization.findXmlElementFlat(xmlNode, u"pageSet")
sarOp = SearchAndReplace.SearchReplaceOperation()
sarOp.serializeFromXml(pageSetXml)
addOpt = Serialization.convertTupleFromXml(addOptXml)
# self.ctrls.chSelectedSet.SetSelection(3)
# self.ctrls.chExportTo.SetSelection(sel)
# exporter.setAddOpt(addOpt, panel)
exportDest = Serialization.serFromXmlUnicode(xmlNode,
u"destinationPath")
# self._refreshForEtype()
except SerializationException, e:
self.showCmdLineUsage(pWiki, _(u"Error during retrieving "
"saved export: ") + e.message + u"\n\n")
def main(argv=None):
# // Fetch config settings
basePath = config.settings["basePath"]
configFile = basePath + config.settings["configFile"]
projectionFile = basePath + config.settings["projectionFile"]
airportsFile = basePath + config.settings["airportsFile"]
landingFile = basePath + config.settings["landingFile"]
taxiFile = basePath + config.settings["taxiFile"]
restrictedAirspaceFile = basePath + config.settings["restrictedZonesFile"]
# let weatherPattern = basePath + appSetting "weatherFiles"
flightPattern = basePath + config.settings["flightFiles"]
# let groundConditionPattern = basePath + appSetting "groundConditionFiles"
# let actualTakeoffsPattern = basePath + appSetting "actualTakeoffFiles"
# let actualLandingPattern = basePath + appSetting "actualLandingFiles"
# let writeFlightLogStr = appSetting "writeFlightLog"
# let writeFlightLog = ref false;
# Boolean.TryParse(writeFlightLogStr, writeFlightLog) |> ignore
# let flightLogFiles = basePath + appSetting "flightLogFiles"
# let flightCostFile = basePath + appSetting "flightCostsFile"
turbulentZonesPattern = basePath + config.settings["turbulentZonesFiles"]
dateList = config.settings["dates"]
dates = dateList.split(',')
# // Start timer
# let sw = Diagnostics.Stopwatch.StartNew()
# timeStampPrint sw "Loading configuration, projection, airports"
simulationParameters = Serialization.loadConfigurationFromFile(configFile)
restrictedAirspace = Serialization.loadZonesFromFile(
restrictedAirspaceFile)
projection = Serialization.loadProjectionFromFile(projectionFile)
airports = Serialization.loadAirportsFromFile(
airportsFile) #, landingFile, taxiFile)
# timeStampPrint sw "Loading date-specific info "
(flightsDates,
airspaceDates) = Serialization.loadDateMaps(flightPattern,
restrictedAirspace,
turbulentZonesPattern, dates)
# // config contains references to pre-loaded objects/maps
configL = config.Config(projection, airports, flightsDates, airspaceDates,
simulationParameters)
# print flightsDates
# print airspaceDates
# raw_input("Press Enter to continue...")
# timeStampPrint sw "Done with pre-simulation loads"
# printfn "-----------------------------------------------"
print 'Finished pre-simulation loads'
# let mutable moreSubmissions = true
subfiles = config.settings['submissions'].split(',')
for submission in subfiles:
print submission
print 'runRoutesOverDays'
dayResults = runRoutesOverDays(configL, dates,
basePath + '/' + submission)
# Calculate final weighted cost
totalFlights = 0.0
finalScore = 0.0
for k, day in dayResults.iteritems():
totalFlights += day['flightCount']
finalScore += day['flightCount'] * day['meanCost']
print finalScore / totalFlights
basePath = config.settings["basePath"]
configFile = basePath + config.settings["configFile"]
projectionFile = basePath + config.settings["projectionFile"]
airportsFile = basePath + config.settings["airportsFile"]
landingFile = basePath + config.settings["landingFile"]
taxiFile = basePath + config.settings["taxiFile"]
restrictedAirspaceFile = basePath + config.settings["restrictedZonesFile"]
# let weatherPattern = basePath + appSetting "weatherFiles"
flightPattern = basePath + config.settings["flightFiles"]
turbulentZonesPattern = basePath + config.settings["turbulentZonesFiles"]
dateList = config.settings["dates"]
#dates = dateList.split(',')
dates = dateList
simulationParameters = Serialization.loadConfigurationFromFile(configFile)
restrictedAirspace = Serialization.loadZonesFromFile(restrictedAirspaceFile)
projection = Serialization.loadProjectionFromFile(projectionFile)
airports = Serialization.loadAirportsFromFile(airportsFile) #, landingFile, taxiFile)
# timeStampPrint sw "Loading date-specific info "
(flightsDates, airspaceDates) = Serialization.loadDateMaps(
flightPattern, restrictedAirspace, turbulentZonesPattern, dates)
# // config contains references to pre-loaded objects/maps
configL = config.Config(projection, airports,
flightsDates, airspaceDates, simulationParameters)
submission = config.settings['submissions']
print submission
routesFile = basePath + '/' + submission
def _runSavedExport(self, pWiki, savedExportName, continuousExport):
import Serialization, PluginManager, Exporters, SearchAndReplace
exportList = Exporters.retrieveSavedExportsList(
pWiki, pWiki.getWikiData(), continuousExport)
xmlNode = None
for exportName, xn in exportList:
if exportName == savedExportName:
xmlNode = xn
break
if xmlNode is None:
self.showCmdLineUsage(
pWiki,
_(u"Saved export '%s' is unknown.") % savedExportName +
u"\n\n")
return
# TODO: Refactor, it is based on AdditionalDialogs.ExportDialog._showExportProfile
try:
etypeProfile = Serialization.serFromXmlUnicode(
xmlNode, u"exportTypeName")
try:
exporter, etype, desc, panel = PluginManager.getSupportedExportTypes(
pWiki, None, continuousExport)[etypeProfile]
except KeyError:
self.showCmdLineUsage(
pWiki,
_(u"Export type '%s' of saved export is not supported") %
etypeProfile + u"\n\n")
return
addOptXml = Serialization.findXmlElementFlat(
xmlNode, u"additionalOptions")
addOptVersion = int(addOptXml.getAttribute(u"version"))
if addOptVersion != exporter.getAddOptVersion():
self.showCmdLineUsage(
pWiki,
_(u"Saved export uses different version for additional "
"options than current export\nExport type: '%s'\n"
"Saved export version: %i\nCurrent export version: %i") %
(etypeProfile, addOptVersion, exporter.getAddOptVersion())
+ u"\n\n")
return
if addOptXml.getAttribute(u"type") != u"simpleTuple":
self.showCmdLineUsage(
pWiki,
_(u"Type of additional option storage ('%s') is unknown") %
addOptXml.getAttribute(u"type") + u"\n\n")
return
pageSetXml = Serialization.findXmlElementFlat(xmlNode, u"pageSet")
sarOp = SearchAndReplace.SearchReplaceOperation()
sarOp.serializeFromXml(pageSetXml)
addOpt = Serialization.convertTupleFromXml(addOptXml)
# self.ctrls.chSelectedSet.SetSelection(3)
# self.ctrls.chExportTo.SetSelection(sel)
# exporter.setAddOpt(addOpt, panel)
exportDest = Serialization.serFromXmlUnicode(
xmlNode, u"destinationPath")
# self._refreshForEtype()
except SerializationException, e:
self.showCmdLineUsage(
pWiki,
_(u"Error during retrieving "
"saved export: ") + e.message + u"\n\n")
