def testReportParser_parse_unicode(self):
"""The JSON deserialiser returns strings as unicode objects. Check that they are correctly parsed in URIs."""
fullReport = """{"remote-addr": "XXX", "policy-type": "regular", "http-user-agent":""" \
+ """ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0",""" \
+ """ "timestamp-utc": "2014-03-01 12:13:14.156789", "csp-report": {"violated-directive":""" \
+ """ "img-src 'none'", "referrer": "http://handbook5.com/a/a-security-analysis-of-amazon%E2%80%99s-elastic-compute-cloud-service-w14847.html",""" \
+ """ "blocked-uri": "http://www.iseclab.org/images/anr.png", "document-uri":""" \
+ """ "http://www.iseclab.org/?p"}, "header-type": "standard"}"""
expected = Report({
"violated-directive":
Directive("img-src", ()),
"referrer":
URI(
"http", "handbook5.com", None,
u"/a/a-security-analysis-of-amazon’s-elastic-compute-cloud-service-w14847.html"
),
"blocked-uri":
URI("http", "www.iseclab.org", None, u"/images/anr.png"),
"document-uri":
URI("http", "www.iseclab.org", None, u"/", "p")
})
parser = ReportParser(requiredKeys=[])
jsonReport = json.loads(fullReport)
cspReport = parser.parseJsonDict(jsonReport["csp-report"])
assert cspReport == expected
def testReportParser_parse_stringVsJSON(self):
"""Ensure the string parsing returns the same result as JSON parsing."""
reportJSON = {"something": 123, "uri": "http://seclab.nu"}
reportString = """{"something": 123, "uri": "http://seclab.nu"}"""
expected = Report({"something": 123, "uri": ReportTest.sampleURI1a})
parser = ReportParser(uriKeys=["uri"], requiredKeys=[], strict=True)
parsedFromJSON = parser.parseJsonDict(reportJSON)
parsedFromString = parser.parseString(reportString)
print parsedFromJSON
print parsedFromString
assert parsedFromJSON == parsedFromString
assert parsedFromJSON == expected
def testReportParser_parse_stringVsJSON(self):
"""Ensure the string parsing returns the same result as JSON parsing."""
reportJSON = {"something": 123, "uri": "http://seclab.nu"}
reportString = """{"something": 123, "uri": "http://seclab.nu"}"""
expected = Report({"something": 123, "uri": ReportTest.sampleURI1a})
parser = ReportParser(uriKeys=["uri"], requiredKeys=[], strict=True)
parsedFromJSON = parser.parseJsonDict(reportJSON)
parsedFromString = parser.parseString(reportString)
print parsedFromJSON
print parsedFromString
assert parsedFromJSON == parsedFromString
assert parsedFromJSON == expected
def testReportParser_parse_unicode(self):
"""The JSON deserialiser returns strings as unicode objects. Check that they are correctly parsed in URIs."""
fullReport = """{"remote-addr": "XXX", "policy-type": "regular", "http-user-agent":""" \
+ """ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0",""" \
+ """ "timestamp-utc": "2014-03-01 12:13:14.156789", "csp-report": {"violated-directive":""" \
+ """ "img-src 'none'", "referrer": "http://handbook5.com/a/a-security-analysis-of-amazon%E2%80%99s-elastic-compute-cloud-service-w14847.html",""" \
+ """ "blocked-uri": "http://www.iseclab.org/images/anr.png", "document-uri":""" \
+ """ "http://www.iseclab.org/?p"}, "header-type": "standard"}"""
expected = Report({"violated-directive": Directive("img-src", ()),
"referrer": URI("http", "handbook5.com", None, u"/a/a-security-analysis-of-amazon’s-elastic-compute-cloud-service-w14847.html"),
"blocked-uri": URI("http", "www.iseclab.org", None, u"/images/anr.png"),
"document-uri": URI("http", "www.iseclab.org", None, u"/", "p")
})
parser = ReportParser(requiredKeys=[])
jsonReport = json.loads(fullReport)
cspReport = parser.parseJsonDict(jsonReport["csp-report"])
assert cspReport == expected
class LogEntryParser(object):
"""
Pre-configured object that parses strings or JSON dictionaries into LogEntries.
"""
def __init__(self,
uriKeys=('blocked-uri', 'document-uri', 'referrer'),
directiveKeys=('violated-directive',),
policyKeys=("original-policy",),
keyNameReplacements={'document-url': 'document-uri'},
requiredKeys=('blocked-uri', 'violated-directive', 'document-uri'),
strict=True,
addSchemeToURIs=False,
defaultURIScheme="http",
addPortToURIs=False,
defaultURIPort=80,
schemePortMappings={'http': 80, 'https': 443, 'data': None, 'chrome-extension': None,
'safari-extension': None, 'chromenull': None, 'chromeinvoke': None,
'chromeinvokeimmediate': None, 'mx': None, 'moz-icon': None,
'about': None, 'view-source': None},
portSchemeMappings={80: 'http', 443: 'https', 8080: 'http'},
directiveTypeTranslations={"xhr-src": "connect-src"},
allowedDirectiveTypes=("base-uri", "child-src", "connect-src", "default-src",
"font-src", "form-action", "frame-ancestors", "frame-src",
"img-src", "media-src", "object-src", "script-src", "style-src"),
ignoredDirectiveTypes=("plugin-types", "referrer", "reflected-xss", "report-uri", "sandbox"),
expandDefaultSrc=False,
defaultSrcTypes=("child-src", "connect-src", "font-src", "img-src", "media-src",
"object-src", "script-src", "style-src")):
"""
Creates a new LogEntryParser object configured with the following parameters:
'uriKeys': an iterable of key (entry) names of which the corresponding values, if present,
will be parsed and replaced with an URI object.
'directiveKeys': an iterable of key (entry) names of which the corresponding values, if present,
will be parsed and replaced with a Directive object.
'policyKeys': an iterable of key (entry) names of which the corresponding values, if present,
will be parsed and replaced with a Policy object.
The 'uriKeys', 'directiveKeys', and 'policyKeys' lists are mutually exclusive (a key may appear
in at most one of these lists.)
'keyNameReplacements': a dictionary of old key (entry) names mapped to the new name to be given
to them before any further parsing. This can be used to adjust to renamed
fields in the violation reports generated by different browser versions.
'requiredKeys': an iterable of key (entry) names that are mandatory and must appear in the report
with a valid value (if parsed, cannot be URI.INVALID()/Directive.INVALID()/
Policy.INVALID()). If this constraint is violated, the parsing result will be
Report.INVALID() (independent of the 'strict' setting). This restriction is
applied after performing key name replacement.
'strict': whether a parsing error of a child element should be ignored if it can be fixed (if
set to False, invalid children will be skipped), or if any parsing error
should cause the Report to become Report.INVALID() (if set to True).
'addSchemeToURIs': [for parsed URIs] whether to add the scheme. (See URIParser for details.)
'defaultURIScheme': [for parsed URIs] if the scheme should be added, the default scheme to be
assumed if nothing can be inferred from the port. (See URIParser for details.)
'addPortToURIs': [for parsed URIs] whether to add the port. (See URIParser for details.)
'defaultURIPort': [for parsed URIs] if the port should be added, the default port to be assumed
if nothing can be inferred from the scheme. (See URIParser for details.)
'schemePortMappings': [for parsed URIs and policy/directive parsing] A map from scheme names to the
corresponding default port, or None if the scheme does not use ports. Any scheme
that may appear inside an URI, source expression, directive or policy should be
listed. (See URIParser and SourceExpressionParser for details.)
'portSchemeMappings': [for parsed URIs] A map from port numbers to scheme names (only for "real" ports).
See URIParser for details.
'directiveTypeTranslations': [for parsed directives and policies] A map from the old directive name to
the new name to be used. (See DirectiveParser for details.)
'allowedDirectiveTypes': [for parsed directives and policies] a list of directive types that are allowed.
(See DirectiveParser or PolicyParser for details.)
'ignoredDirectiveTypes': [for parsed policies] a list of directive types that are ignored when parsing
policies. (See PolicyParser for details.)
'expandDefaultSrc': [for parsed policies] if set to True, each "default-src" directive in a parsed policy
will be expanded to the corresponding elementary directive types, if not yet present.
(See PolicyParser for details.)
'defaultSrcTypes': [for parsed policies] when "default-src" is expanded, the elementary directive types
that will be added to replace the default policy. (See PolicyParser for details.)
"""
self._strict = strict
self._reportParser = ReportParser(uriKeys, directiveKeys, policyKeys,
keyNameReplacements, requiredKeys, strict, addSchemeToURIs,
defaultURIScheme, addPortToURIs, defaultURIPort, schemePortMappings,
portSchemeMappings, directiveTypeTranslations, allowedDirectiveTypes,
ignoredDirectiveTypes, expandDefaultSrc, defaultSrcTypes)
def parseString(self, stringLogEntry):
"""
Parses the given 'stringLogEntry' according to the parameters set in the constructor of this LogEntryParser
and returns a LogEntry object. 'stringLogEntry' is expected to be a JSON-serialised map. If 'stringLogEntry'
cannot be parsed because it is syntactically invalid (or empty), LogEntry.INVALID() will be returned.
"""
try:
jsonDict = json.loads(stringLogEntry)
return self.parseJsonDict(jsonDict)
except ValueError:
return LogEntry.INVALID()
def parseJsonDict(self, jsonLogEntry):
"""
Parses the given 'jsonLogEntry' according to the parameters set in the constructor of this LogEntryParser
and returns a LogEntry object. 'jsonLogEntry' is expected to be a Python dict object. If 'jsonLogEntry'
cannot be parsed because it is syntactically invalid (or empty), LogEntry.INVALID() will be returned.
"""
# TODO: could also parse the timestamp string etc.
if "csp-report" in jsonLogEntry:
jsonLogEntry["csp-report"] = self._reportParser.parseJsonDict(jsonLogEntry["csp-report"])
if self._strict and jsonLogEntry["csp-report"] == Report.INVALID():
return LogEntry.INVALID()
else:
return LogEntry(jsonLogEntry)
else:
return LogEntry.INVALID()
class LogEntryParser(object):
"""
Pre-configured object that parses strings or JSON dictionaries into LogEntries.
"""
def __init__(
self,
uriKeys=('blocked-uri', 'document-uri', 'referrer'),
directiveKeys=('violated-directive', ),
policyKeys=("original-policy", ),
keyNameReplacements={'document-url': 'document-uri'},
requiredKeys=('blocked-uri', 'violated-directive', 'document-uri'),
strict=True,
addSchemeToURIs=False,
defaultURIScheme="http",
addPortToURIs=False,
defaultURIPort=80,
schemePortMappings={
'http': 80,
'https': 443,
'data': None,
'chrome-extension': None,
'safari-extension': None,
'chromenull': None,
'chromeinvoke': None,
'chromeinvokeimmediate': None,
'mx': None,
'moz-icon': None,
'about': None,
'view-source': None
},
portSchemeMappings={
80: 'http',
443: 'https',
8080: 'http'
},
directiveTypeTranslations={"xhr-src": "connect-src"},
allowedDirectiveTypes=("base-uri", "child-src", "connect-src",
"default-src", "font-src", "form-action",
"frame-ancestors", "frame-src", "img-src",
"media-src", "object-src", "script-src",
"style-src"),
ignoredDirectiveTypes=("plugin-types", "referrer", "reflected-xss",
"report-uri", "sandbox"),
expandDefaultSrc=False,
defaultSrcTypes=("child-src", "connect-src", "font-src", "img-src",
"media-src", "object-src", "script-src",
"style-src")):
"""
Creates a new LogEntryParser object configured with the following parameters:
'uriKeys': an iterable of key (entry) names of which the corresponding values, if present,
will be parsed and replaced with an URI object.
'directiveKeys': an iterable of key (entry) names of which the corresponding values, if present,
will be parsed and replaced with a Directive object.
'policyKeys': an iterable of key (entry) names of which the corresponding values, if present,
will be parsed and replaced with a Policy object.
The 'uriKeys', 'directiveKeys', and 'policyKeys' lists are mutually exclusive (a key may appear
in at most one of these lists.)
'keyNameReplacements': a dictionary of old key (entry) names mapped to the new name to be given
to them before any further parsing. This can be used to adjust to renamed
fields in the violation reports generated by different browser versions.
'requiredKeys': an iterable of key (entry) names that are mandatory and must appear in the report
with a valid value (if parsed, cannot be URI.INVALID()/Directive.INVALID()/
Policy.INVALID()). If this constraint is violated, the parsing result will be
Report.INVALID() (independent of the 'strict' setting). This restriction is
applied after performing key name replacement.
'strict': whether a parsing error of a child element should be ignored if it can be fixed (if
set to False, invalid children will be skipped), or if any parsing error
should cause the Report to become Report.INVALID() (if set to True).
'addSchemeToURIs': [for parsed URIs] whether to add the scheme. (See URIParser for details.)
'defaultURIScheme': [for parsed URIs] if the scheme should be added, the default scheme to be
assumed if nothing can be inferred from the port. (See URIParser for details.)
'addPortToURIs': [for parsed URIs] whether to add the port. (See URIParser for details.)
'defaultURIPort': [for parsed URIs] if the port should be added, the default port to be assumed
if nothing can be inferred from the scheme. (See URIParser for details.)
'schemePortMappings': [for parsed URIs and policy/directive parsing] A map from scheme names to the
corresponding default port, or None if the scheme does not use ports. Any scheme
that may appear inside an URI, source expression, directive or policy should be
listed. (See URIParser and SourceExpressionParser for details.)
'portSchemeMappings': [for parsed URIs] A map from port numbers to scheme names (only for "real" ports).
See URIParser for details.
'directiveTypeTranslations': [for parsed directives and policies] A map from the old directive name to
the new name to be used. (See DirectiveParser for details.)
'allowedDirectiveTypes': [for parsed directives and policies] a list of directive types that are allowed.
(See DirectiveParser or PolicyParser for details.)
'ignoredDirectiveTypes': [for parsed policies] a list of directive types that are ignored when parsing
policies. (See PolicyParser for details.)
'expandDefaultSrc': [for parsed policies] if set to True, each "default-src" directive in a parsed policy
will be expanded to the corresponding elementary directive types, if not yet present.
(See PolicyParser for details.)
'defaultSrcTypes': [for parsed policies] when "default-src" is expanded, the elementary directive types
that will be added to replace the default policy. (See PolicyParser for details.)
"""
self._strict = strict
self._reportParser = ReportParser(
uriKeys, directiveKeys, policyKeys, keyNameReplacements,
requiredKeys, strict, addSchemeToURIs, defaultURIScheme,
addPortToURIs, defaultURIPort, schemePortMappings,
portSchemeMappings, directiveTypeTranslations,
allowedDirectiveTypes, ignoredDirectiveTypes, expandDefaultSrc,
defaultSrcTypes)
def parseString(self, stringLogEntry):
"""
Parses the given 'stringLogEntry' according to the parameters set in the constructor of this LogEntryParser
and returns a LogEntry object. 'stringLogEntry' is expected to be a JSON-serialised map. If 'stringLogEntry'
cannot be parsed because it is syntactically invalid (or empty), LogEntry.INVALID() will be returned.
"""
try:
jsonDict = json.loads(stringLogEntry)
return self.parseJsonDict(jsonDict)
except ValueError:
return LogEntry.INVALID()
def parseJsonDict(self, jsonLogEntry):
"""
Parses the given 'jsonLogEntry' according to the parameters set in the constructor of this LogEntryParser
and returns a LogEntry object. 'jsonLogEntry' is expected to be a Python dict object. If 'jsonLogEntry'
cannot be parsed because it is syntactically invalid (or empty), LogEntry.INVALID() will be returned.
"""
# TODO: could also parse the timestamp string etc.
if "csp-report" in jsonLogEntry:
jsonLogEntry["csp-report"] = self._reportParser.parseJsonDict(
jsonLogEntry["csp-report"])
if self._strict and jsonLogEntry["csp-report"] == Report.INVALID():
return LogEntry.INVALID()
else:
return LogEntry(jsonLogEntry)
else:
return LogEntry.INVALID()