def executeFetch(self, uuid, modality):
"""Generate traces and load appropriate filehandler to read a trace file in the cache."""
try:
if self.datasourcehandlerconnection is None:
self.generateScenario(uuid, modality)
module_ = importlib.import_module(
"contextmonkey.tracelayer.handlers.file.FileRequestHandlerFactory"
)
class_ = getattr(module_, "FileRequestHandlerFactory")
self.datasourcehandlerconnection = class_(
**{
"tracefile": modality.datasource.filepath,
"formattype": modality.datasource.datasourceformat
})
self.datasourcehandlerconnection.build()
d = Deferred()
d.addCallback(self.datasourcehandlerconnection.executeFetch,
modality)
d.callback(uuid)
tracelayer.log(
"TWOPHASEMODELREQUESTHANDLER-executeFetch-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return d
except Exception as e:
traceback.print_exc()
def filterTrace(self, tracedata, **kwargs):
"""Remove unwanted attributes from the OpenSignal trace."""
try:
tracelayer.log("OPENSIGNALNETWORKMODIFY-filterTrace","In filter trace")
tracelayer.log("OPENSIGNALNETWORKMODIFY-filterTrace",str(tracedata))
begin=time.time()
tempdict = {}
kwargs={'carrierName':'Telekom','carrierType':'type3G','attributes':['networkName','averageRssiDb','downloadSpeed','uploadSpeed','pingTime']}
b = json.loads(tracedata)
carrierlist = b['networkRank']
networkattributes = None
for carrier in carrierlist:
networklist = carrierlist[carrier]
if kwargs['carrierType'] in networklist:
networkattributes = networklist[kwargs['carrierType']]
if kwargs['carrierName'] in networkattributes['networkName']:
for key in kwargs['attributes']:
tempdict.update({key:networkattributes[key]})
end=time.time()-begin
tracelayer.log("OPENSIGNALNETWORKMODIFY-TraceProcessing",str(end*1000))
tracelayer.log("OPENSIGNALNETWORKMODIFY-filterTrace",str(tempdict))
return tempdict
raise FilterTraceError("Network name missing")
except:
raise FilterTraceError("Unable to process trace")
tracelayer.log("OPENSIGNALNETWORKMODIFY-filterTrace",traceback.print_exc())
def processTrace(self,tracedata, **kwargs):
"""Perform Unit conversions from kbps to bps."""
tracelayer.log("OPENSIGNALNETWORKMODIFY-processTrace",str(tracedata))
tracedata['uploadSpeed']= float(tracedata['uploadSpeed'])*1024*8 #in bits/s
tracedata['downloadSpeed']= float(tracedata['downloadSpeed'])*1024*8
tracelayer.log("OPENSIGNALNETWORKMODIFY-processTrace",str(tracedata))
return tracedata
def success(self, dummy=None, uuid=None, modality=None):
"""Handle file line read success."""
d = Deferred()
d.addCallback(DataSourceManager.filter_data, uuid, modality)
tracelayer.log(
"TEXTFileRequestHandler-success-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
d.callback(dummy)
def success(self, dummy=None, uuid=None, modality=None):
"""Forward the trace for filtering and processing."""
d = Deferred()
d.addCallback(DataSourceManager.filter_data, uuid, modality)
d.callback(dummy)
tracelayer.log(
"HTTPSGetRequestHandler-success-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
def executeFetch(self, uuid, modality):
"""Forward trace fetch request to appropriate source handler factory."""
if self.datasourcehandlerconnection is not None:
d = Deferred()
d.addCallback(self.datasourcehandlerconnection.executeFetch, modality)
d.callback(uuid)
tracelayer.log("DATASOURCEHANDLER-executeFetch-timestamp:",str(modality.name)+" "+str("%0.20f" % time.time())+" "+str(uuid))
return d
else:
raise SystemError("Handler connection not initialized")
def getContextTraceValue(self, uuid, modality):
"""Provide API to the context layer to get trace."""
d = Deferred()
d.addCallback(self.executeFetch, modality)
d.callback(uuid)
tracelayer.log(
"DATASOURCEMANAGER-getContextTraceValue-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return d
def bodyFormat(self, response, uuid, modality):
"""Extract HTTPS trace response."""
d = readBody(response)
d.addCallback(self.getBody)
d.addCallback(self.success, uuid, modality)
d.addErrback(self.failure)
tracelayer.log(
"HTTPSGetRequestHandler-bodyFormat-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return self.body
def processTrace(self, tracedatalist, uuid, **kwargs):
"""Perform trace processing."""
tracelayer.log("CABSPOTTINGLOCATIONMODIFY-processTrace",
str(tracedatalist))
tracedata = tracedatalist[0].split(' ')
tempdict = {
'latitude': float(tracedata[0]),
'longitude': float(tracedata[1]),
'altitude': float(tracedata[2])
}
return tempdict
def executeFetch(self,uuid, modality):
"""Handle a trace fetch request."""
filename = modality.datasource.filepath
fileiterator = modality.datasource.fileiterator
d = Deferred()
d.addCallback(self.fileread,fileiterator)
d.addErrback(self.failure)
d.addCallback(self.success,uuid,modality)
d.callback(filename)
tracelayer.log("XMLFileRequestHandler-fileread-timestamp:",str(modality.name)+" "+str("%0.20f" % time.time())+" "+str(uuid))
return d
def handlefailure(self, dummy=None):
"""Handle failure of fetching OpenSingal trace, by generating default values."""
tracelayer.log("OPENSIGNALNETWORKMODIFY-handlefailure",
"A failure occured")
tempdict = {
'uploadSpeed': 637.44,
'averageRssiDb': 22,
'downloadSpeed': 1253.12,
'pingTime': 181
} #KB/s ms
return tempdict
def executeFetch(self,uuid,modality):
"""Send HTTPS GET request for downloading trace file from a database service."""
try:
global cachepath
self.url = modality.datasource.url+"?"+urllib.urlencode(modality.datasource.queryparameters)
self.extension=modality.datasource.extension
self.filename=cachepath+'/'+str(uuid)+'.'+self.extension
self.deferred = downloadPage(self.url,self.filename,contextFactory = self.contextFactory).addCallback(self.success,uuid,modality)
tracelayer.log("HTTPSFileDownloadRequestHandler-executeFetch-timestamp:",str(modality.name)+" "+str("%0.20f" % time.time())+" "+str(uuid))
return defer.gatherResults([self.deferred])
except:
traceback.print_exc()
def executeFetch(self, uuid, modality):
"""Handle trace fetch request from a database."""
if self.currentprotocol == None:
raise SystemError("No appropriate protocol loaded")
d = Deferred()
d.addCallback(self.currentprotocol.executeFetch, modality)
d.addErrback(self.loadProtocolFailed)
d.callback(uuid)
tracelayer.log(
"DATABASEREQUESTHANDLERFACTORY-executeFetch-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return d
def executeFetch(self, uuid, modality):
"""Send HTTPS GET request for fetching trace from a database service."""
try:
self.url = modality.datasource.url + "?" + urllib.urlencode(
modality.datasource.queryparameters)
d = self.agent.request('GET', self.url, self.headers)
d.addErrback(self.failure)
d.addCallback(self.bodyFormat, uuid, modality)
tracelayer.log(
"HTTPSGetRequestHandler-executeFetch-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return d
except:
tracelayer.log("HTTPSGetRequestHandler-executeFetch:",
traceback.print_exc())
def loadProtocol(self, dummy=None, **kwargs):
"""Load appropriate database handler protocol."""
try:
fetchtype = kwargs['fetchtype'] ##secure, normal
formattype = kwargs['formattype'] ### json, binary, xml
proto = self.protocoltype[fetchtype]
module_ = importlib.import_module(
"contextmonkey.tracelayer.handlers.database." +
proto[formattype])
class_ = getattr(module_, proto[formattype])
self.currentprotocol = class_()
tracelayer.log("DATABASEREQUESTHANDLERFACTORY-LOADPROTOCOL",
str(self.currentprotocol))
except:
tracelayer.log("DATABASEREQUESTHANDLERFACTORY-LOADPROTOCOL",
traceback.format_exc())
def processTrace(self,tracedata, uuid, **kwargs):
"""Perform image format conversion."""
try:
tracelayer.log("StreetViewModify-processTrace1-timestamp:",str("%0.20f" % time.time())+" "+str(uuid))
global cachepath
tracedata=cachepath+'/'+str(tracedata)+'.jpg'
print tracedata
filename,extension = path.splitext(tracedata)
command = 'ffmpeg -loglevel panic -i '+tracedata+' -s 640x400 -pix_fmt yuv422p -thread 0 -f v4l2 /dev/video0'
subprocess.call(command.split(),shell=False)
with open(tracedata, "rb") as binary_file:
data = binary_file.read()
tracelayer.log("StreetViewModify-processTrace2-timestamp:",str("%0.20f" % time.time())+" "+str(uuid))
return {'image':data,'encoded':len(data),'original':len(data),'camera_type':'back'}
except:
raise ContextMonkeyTraceProcessingFailedError(traceback.print_exc())
def filter_data(tracedata, uuid, modality):
"""Filter & process, and forward trace for format conversion."""
module_ = importlib.import_module(
"contextmonkey.tracelayer.processingfilters.%s" %
(modality.traceprocessing))
class_ = getattr(module_, "%s" % modality.traceprocessing)
ostm = class_()
d = Deferred()
d.addCallback(ostm.filterTrace, uuid)
d.addErrback(ostm.handlefailure)
d.addCallback(ostm.processTrace, uuid)
d.addCallback(DataSourceManager.traceformatmanager.toProtobuf,
modality)
d.addCallback(receiveValue, uuid, modality)
d.callback(tracedata)
tracelayer.log(
"DataSourceManager-filterData-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return d
def processTrace(self,tracedatalist, uuid, **kwargs):
"""Perform trace processing."""
print tracedatalist
tracelayer.log("COMPASSMODIFYTEXT-processTrace",str(tracedatalist))
tempdict=tracedatalist.copy()
return tempdict
def failure(self, reason):
"""Handle trace fetching failure."""
tracelayer.log("HTTPSGETREQUESTHANDLER-FAILURE",
"Request failed " + str(reason))
def failure(self,reason,reason1):
"""Handle file download failure."""
tracelayer.log("HTTPSFileDownloadRequestHandler-failure","Request failed "+ str(reason))
def getBody(self, body):
"""Get response body of the HTTPS based trace fetching request."""
self.body = body
tracelayer.log("HTTPSGetRequestHandler-getBody",
unicode(self.body, errors='ignore'))
return self.body
def executeFetch(self, uuid, modality):
"""Send trace fetching request to the appropriate source handler."""
global modalitylist
#########DATABASE##########
d = Deferred()
if modality.datasource.datasourcetype == 'database':
try:
print modality.name + "type found"
d.addCallback(
self.databasehandlers[currentmodality.name].executeFetch,
modality=modality)
tracelayer.log(
"DATASOURCEMANAGER-executeFetch-database-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) +
" " + str(uuid))
except Exception as e:
self.databasehandlers[modality.name] = DataSourceHandler(
'database',
fetchtype=modality.datasource.fetchtype,
formattype=modality.datasource.datasourceformat)
d.addCallback(
self.databasehandlers[modality.name].executeFetch,
modality=modality)
tracelayer.log(
"DATASOURCEMANAGER-executeFetch-database-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) +
" " + str(uuid))
#########FILE##########
elif modality.datasource.datasourcetype == 'file':
modality.datasource.fileiterator = self.circular_increment(
modality.datasource.fileiterator, modality.datasource.length)
try:
d.addCallback(self.filehandlers[modality.name].executeFetch,
modality=modality)
tracelayer.log(
"DATASOURCEMANAGER-executeFetch-file-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) +
" " + str(uuid))
except Exception as e:
self.filehandlers[modality.name] = DataSourceHandler(
'file',
tracefile=modality.datasource.filepath,
formattype=modality.datasource.datasourceformat)
tracelayer.log(
"DATASOURCEMANAGER-executeFetch-file-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) +
" " + str(uuid))
d.addCallback(self.filehandlers[modality.name].executeFetch,
modality=modality)
#########MODEL##########
elif modality.datasource.datasourcetype == 'model':
modality.datasource.fileiterator = self.circular_increment(
modality.datasource.fileiterator, 10)
try:
d.addCallback(self.modelhandlers[modality.name].executeFetch,
modality=modality)
tracelayer.log(
"DATASOURCEMANAGER-executeFetch-model-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) +
" " + str(uuid))
except Exception as e:
self.modelhandlers[modality.name] = DataSourceHandler(
'model',
modeltype=modality.datasource.modeltype,
formattype=modality.datasource.datasourceformat)
tracelayer.log(
"DATASOURCEMANAGER-executeFetch-model-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) +
" " + str(uuid))
d.addCallback(self.modelhandlers[modality.name].executeFetch,
modality=modality)
else:
pass
d.callback(uuid)
return uuid
def filterTrace(self, tracedata, uuid, **kwargs):
"""Remove unwanted attributes from the tracedata."""
tracelayer.log("StreetViewModify-filterTrace-timestamp:",str("%0.20f" % time.time())+" "+str(uuid))
return tracedata
def handletracefailure(self, reason):
"""Handle trace fetching failures."""
print reason
tracelayer.log("DATASOURCEMANAGER TRACEFAILURE", reason)
def failure(self, reason):
"""Handle file read failure."""
tracelayer.log("XMLFileRequestHandler-failure-","Request failed "+str(reason))
