def try_parse_and_handle_directive(ctx):
from directive_impl import same_as_file
from directive_impl import file_same_as_stdout
try:
line = ctx.doc_file.next_non_empty_line()
except RuntimeError as e:
# Do not raise exception when next non-empty line
# does not exist. Instead, return failure.
if str(e) != "Enf of file.":
raise
return failure()
# Register all directives.
all_directives: List[Directive] = [
Directive(same_as_file.ext_to_patterns, [generic_config_parser, same_as_file.parse], same_as_file.handle),
Directive(file_same_as_stdout.ext_to_patterns, [generic_config_parser], file_same_as_stdout.handle),
]
for directive in all_directives:
directive_config = []
if succeeded(directive.try_parse_directive(line, ctx.doc_file_ext(), directive_config)):
directive.handle(directive_config.pop(), ctx)
return success(directive_config)
return failure()
def parse(self, stringPolicy):
"""
Parses the given 'stringPolicy' according to the parameters set in the constructor of this PolicyParser
and returns a Policy object. If 'stringPolicy' cannot be parsed because it is syntactically invalid (or empty),
Policy.INVALID() will be returned. (A policy cannot consist of only whitespace.)
Depending on the configuration of this PolicyParser object, may perform internal translation of the type
and filter certain directive types.
"""
directiveStrings = stringPolicy.split(";")
directives = {} # type -> Directive
for directiveString in directiveStrings:
if self._isIgnoredDirectiveType(directiveString):
continue
directive = self._directiveParser.parse(directiveString)
if directive == Directive.INVALID():
if self._strict:
return Policy.INVALID()
else:
continue
if directive.getType() in directives:
continue # could emit a warning here; the standard says to ignore subsequent definitions
directives[directive.getType()] = directive
if self._expandDefaultSrc:
self._normaliseDirectivesMap(directives)
if len(directives) == 0:
return Policy.INVALID()
return Policy(directives.values())
def run_parser(body):
child = Node()
child.name = command.Run
items = body.split("&&")
directive = Directive()
for item in items:
item = item.strip()
install_flag = False
for prefix in thesaurus.INSTALL_PREFIX.keys():
item = purify_code(item)
match = re.search(prefix, item)
if match: # 说明这一行安装了软件
tmp = re.sub(prefix, constants.SEP, item).strip()
if not tmp.startswith(constants.SEP):
break
item = re.sub(prefix, '', item).strip()
install_flag = True
install_list = item.split(" ")
for install in install_list:
install_pattern = re.compile("^\w.*$")
if install_pattern.match(install):
install = purify_install(install)
if len(install.strip()) == 0:
continue
directive.add_install(install)
# else:
# print >> sys.stderr, "invalid format: %s" % install
break
if not install_flag:
directive.add_directive(item)
child.directives = directive
return child
def _normaliseDirectivesMap(self, directivesMap):
"""
Removes any 'default-src' directive and expands it to all basic directive types that are not yet
specified (in place).
"""
if not "default-src" in directivesMap:
return
defaultDirective = directivesMap['default-src']
for otherType in self._defaultSrcTypes:
if not otherType in directivesMap:
directivesMap[otherType] = Directive(
otherType,
defaultDirective.getWhitelistedSourceExpressions())
del directivesMap["default-src"]
def try_parse_and_handle_directive(ctx):
from directive_impl import same_as_file
# Register all directives.
all_directives: List[Directive] = [
Directive(same_as_file.ext_to_patterns, [generic_config_parser, same_as_file.parse], same_as_file.handle)
]
for directive in all_directives:
directive_config = []
if succeeded(directive.try_parse_directive(ctx, directive_config)):
directive.handle(directive_config.pop(), ctx)
return success(directive_config)
return failure()
def parseJsonDict(self, jsonReport):
"""
Parses the given 'jsonReport' according to the parameters set in the constructor of this ReportParser
and returns a Report object. 'jsonReport' is expected to be a Python dict object with attribute names
and values corresponding to the definition of CSP violation reports. If 'jsonReport' cannot be parsed
because it is syntactically invalid (or empty), Report.INVALID() will be returned.
Depending on the configuration of this ReportParser object, some attributes will be parsed to replace their
plain string values with a more high-level object representation.
"""
# replace names
renamedReport = dict(map(lambda (key, val): (self._replaceName(key), val), jsonReport.iteritems()))
# convert data in report
convertedReport = {}
deferredSelfURIs = set([]) # all key names that have URIs that are exactly 'self' (handle after parsing everything else)
for (key, value) in renamedReport.iteritems():
if key in self._uriKeys:
if value.lower().strip() == "self":
deferredSelfURIs.add(key)
continue
else:
value = self._uriParser.parse(value)
elif key in self._directiveKeys:
value = self._directiveParser.parse(value)
elif key in self._policyKeys:
value = self._policyParser.parse(value)
if value in (URI.INVALID(), Directive.INVALID(), Policy.INVALID()):
if self._strict:
return Report.INVALID()
else:
continue
convertedReport[key] = value
# handle deferred parsing of 'self' URIs (they represent the document-uri)
for key in deferredSelfURIs:
if "document-uri" in self._uriKeys and "document-uri" in convertedReport:
convertedReport[key] = convertedReport["document-uri"]
elif self._strict:
return Report.INVALID()
for requiredKey in self._requiredKeys:
if not requiredKey in convertedReport:
return Report.INVALID()
return Report(convertedReport)
def combinedPolicy(self, otherPolicy):
"""
Returns a new Policy that is the recursive combination of the Directives contained in this
Policy and 'otherPolicy' (combining Directives of the same type, and adding Directives
of a type that is missing in one of the policies). This extends the whitelists and results
in a more permissive overall Policy. This method can be used when generating policies from
violation reports, but should not be used to enforce multiple policies at the same time
(the semantics are different).
If this Policy or 'otherPolicy' is Policy.INVALID(), or if the combination of any of the
contained Directives is Directive.INVALID(), this method returns Policy.INVALID().
Because of the special meaning of 'default-src', two policies containing 'default-src'
(in one or both policies) can be combined only if 'default-src' is the only directive
type in both policies (else Policy.INVALID() is returned).
"""
if (self == Policy.INVALID() or otherPolicy == Policy.INVALID()):
return Policy.INVALID()
myDirectives = dict(
map(lambda x: (x.getType(), x), self.getDirectives()))
otherDirectives = dict(
map(lambda x: (x.getType(), x), otherPolicy.getDirectives()))
allDirectiveTypes = set(myDirectives.keys()) | set(
otherDirectives.keys())
if ('default-src' in allDirectiveTypes and
((not 'default-src' in myDirectives and len(myDirectives) > 0) or
('default-src' in myDirectives and len(myDirectives) > 1) or
(not 'default-src' in otherDirectives
and len(otherDirectives) > 0) or
('default-src' in otherDirectives and len(otherDirectives) > 1))):
return Policy.INVALID()
combinedDirectives = set([])
for directiveType in allDirectiveTypes:
if directiveType not in myDirectives:
combinedDirectives.add(otherDirectives[directiveType])
elif directiveType not in otherDirectives:
combinedDirectives.add(myDirectives[directiveType])
else:
combined = myDirectives[directiveType].combinedDirective(
otherDirectives[directiveType])
if combined == Directive.INVALID():
return Policy.INVALID()
else:
combinedDirectives.add(combined)
return Policy(combinedDirectives)
def generatePolicy(self, reportType):
"""
Generates a new basic policy that allows exactly the kind of event that caused this CSP violation report,
assuming the given 'reportType' (permitted values are 'regular', 'eval', and 'inline').
This assumes that this report contains a specific violated-directive field (it may not be 'default-src').
If any inconsistent reports are used to generate policies, the policies themselves will be inconsistent.
Policies should be collected with CSP headers like these:
Content-Security-Policy-Report-Only: default-src 'none'; script-src 'unsafe-eval' 'unsafe-inline'; object-src 'none'; style-src 'unsafe-inline'; img-src 'none'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'; report-uri /csp.cgi?type=regular
Content-Security-Policy-Report-Only: default-src *; script-src * 'unsafe-inline'; style-src * 'unsafe-inline'; report-uri /csp.cgi?type=eval
Content-Security-Policy-Report-Only: default-src *; script-src * 'unsafe-eval'; style-src *; report-uri /csp.cgi?type=inline
The "type" parameter in the report-uri is equivalent to the 'reportType' parameter of this method.
The results should also be filtered to ensure that only reports sent by fully compatible browsers
are taken into account. This implementation does not handle URIs in any special way. That is, it does
not add or remove ports, path/query components, or replace them with the 'self' keyword.
The result is a basic Policy containing one whitelisted resource (corresponding to the violated directive).
In practice, it should not be used alone, but be combined with basic policies generated for other violations
on the same web site. It should also be prepended with "default-src 'none'" to ensure that only the
whitelisted resources are allowed. (The standard behaviour of CSP in absence of any default Directive is
to assume "default-src *", which may not be the desired behaviour.)
The result is Policy.INVALID() if (1) the violated directive is missing, Directive.INVALID() or 'default-src',
or a special type incompatible with 'reportType', (2) if 'reportType' is none out of 'regular', 'inline' or 'eval',
(3) the 'blocked-uri' is URI.INVALID() or not a regular URI in the 'reportType'=='regular' case.
"""
if (self == Report.INVALID()
or 'violated-directive' not in self
or 'blocked-uri' not in self):
return Policy.INVALID()
violated = self['violated-directive']
blocked = self['blocked-uri']
generated = violated.generateDirective(reportType, blocked)
if generated == Directive.INVALID():
return Policy.INVALID()
else:
return Policy((generated,))
def exitDirectiveEQU(self, ctx: tl69asmParser.DirectiveEQUContext):
ctx.directive = Directive(DirectiveType.EQU,
ctx.expr().expr,
ctx.label().label_name)
ctx.expr = ctx.expr().expr
def exitDirectiveORG(self, ctx: tl69asmParser.DirectiveORGContext):
ctx.directive = Directive(DirectiveType.ORG, ctx.expr().expr)
def exitDirectiveDW(self, ctx: tl69asmParser.DirectiveDWContext):
ctx.directive = Directive(DirectiveType.DW, ctx.expr().expr)
class Policy(object):
"""
A CSP policy that consists of one or more directives. Immutable
"""
_invalid = None
_defaultSrcDirectiveIfNotSpecified = Directive(
"default-src", (URISourceExpression(None, "*", None, None), ))
def __init__(self, directives):
"""
Initialises this CSP policy from the given 'directives' list/set/tuple.
'directives': An iterable of Directives. May contain at most one Directive of each type.
Any Directive that is not regular will be dropped.
"""
self._hash = None
self._str = None
self._isInvalid = False
# eliminate directives with duplicate type and irregular directives
onlyRegular = filter(lambda x: x.isRegularDirective(), directives)
self._directives = frozenset(
dict(map(lambda x: (x.getType(), x), onlyRegular)).values())
@staticmethod
def INVALID():
"""
Special static singleton Policy representing an invalid policy (could not be parsed).
"""
if Policy._invalid is None:
Policy._invalid = Policy(())
Policy._invalid._isInvalid = True
return Policy._invalid
def combinedPolicy(self, otherPolicy):
"""
Returns a new Policy that is the recursive combination of the Directives contained in this
Policy and 'otherPolicy' (combining Directives of the same type, and adding Directives
of a type that is missing in one of the policies). This extends the whitelists and results
in a more permissive overall Policy. This method can be used when generating policies from
violation reports, but should not be used to enforce multiple policies at the same time
(the semantics are different).
If this Policy or 'otherPolicy' is Policy.INVALID(), or if the combination of any of the
contained Directives is Directive.INVALID(), this method returns Policy.INVALID().
Because of the special meaning of 'default-src', two policies containing 'default-src'
(in one or both policies) can be combined only if 'default-src' is the only directive
type in both policies (else Policy.INVALID() is returned).
"""
if (self == Policy.INVALID() or otherPolicy == Policy.INVALID()):
return Policy.INVALID()
myDirectives = dict(
map(lambda x: (x.getType(), x), self.getDirectives()))
otherDirectives = dict(
map(lambda x: (x.getType(), x), otherPolicy.getDirectives()))
allDirectiveTypes = set(myDirectives.keys()) | set(
otherDirectives.keys())
if ('default-src' in allDirectiveTypes and
((not 'default-src' in myDirectives and len(myDirectives) > 0) or
('default-src' in myDirectives and len(myDirectives) > 1) or
(not 'default-src' in otherDirectives
and len(otherDirectives) > 0) or
('default-src' in otherDirectives and len(otherDirectives) > 1))):
return Policy.INVALID()
combinedDirectives = set([])
for directiveType in allDirectiveTypes:
if directiveType not in myDirectives:
combinedDirectives.add(otherDirectives[directiveType])
elif directiveType not in otherDirectives:
combinedDirectives.add(myDirectives[directiveType])
else:
combined = myDirectives[directiveType].combinedDirective(
otherDirectives[directiveType])
if combined == Directive.INVALID():
return Policy.INVALID()
else:
combinedDirectives.add(combined)
return Policy(combinedDirectives)
def getDirectives(self):
"""
Returns a frozen set of all the directives contained in this policy.
"""
return self._directives # is already immutable
def matches(self,
resourceURI,
resourceType,
protectedDocumentURI,
schemePortMappings=defaults.schemePortMappings,
defaultSrcTypes=defaults.defaultSrcReplacementDirectiveTypes):
"""
Returns whether the given resourceURI is allowed under this Policy. Attempts to match
'resourceURI' is an URI object corresponding to the resource that is attempted to be loaded/executed.
Can be either one of the special URI.UNSAFE_EVAL() / URI.UNSAFE_INLINE() URIs, or a regular URI.
In the latter case, escaped characters in the path of the URI should already have been decoded.
If 'resourceURI' designates a directory (as opposed to a file), its path must end with a '/'.
May not be None.
'resourceType' indicates the type of the 'resourceURI', which is a directive type as
returned by Directive.getType() (such as 'img-src' or 'script-src' for image and script resources,
respectively).
'protectedDocumentURI' is the URI of the document in the context of which 'resourceURI' is being
attempted to be loaded/executed (the host document). May not be None.
'schemePortMappings': A dictionary with mappings from (lowercase) scheme names to the corresponding
default port. Will be used if ports are missing in the 'resourceURI' or 'protectedDocumentURI'.
'defaultSrcTypes': A list of (lowercase) directive types to be used for matching, to determine if the
"default-src" directive can be used if no directive of the needed type is available.
Example: 'resourceType' is 'script-src', but this Policy contains no 'script-src' Directive.
If 'defaultSrcTypes' contains 'script-src', the 'default-src' Directive can be used instead of the
missing 'script-src' Directive, if available.
This implementation requires schemes to be present in both URIs, and either port numbers or a successful
scheme-to-port-number look up in 'schemePortMappings' for both URIs (otherwise, False is returned).
For details about the implementation, see http://www.w3.org/TR/2014/WD-CSP11-20140211/#matching
"""
if self == Policy.INVALID():
return False
# if directive of resource type is available, use it for matching
resourceType = resourceType.lower()
for direct in self._directives:
if direct.getType() == resourceType:
return direct.matches(resourceURI, protectedDocumentURI,
schemePortMappings)
# check if type of resource admits matching with default-src per CSP 1.1 draft
if resourceType not in defaultSrcTypes:
return False
# match with default-src
# if no default directive is available, assume default-src '*' according to # http://www.w3.org/TR/2014/WD-CSP11-20140211/#default-src
for direct in self._directives:
if direct.getType() == "default-src":
return direct.matches(resourceURI, protectedDocumentURI,
schemePortMappings)
return Policy._defaultSrcDirectiveIfNotSpecified.matches(
resourceURI, protectedDocumentURI, schemePortMappings)
def withoutPaths(self, schemeOnly=defaults.schemeOnly):
"""
Returns a copy of this Policy that has the path components removed from all URISourceExpressions
in the contained Directives.
'schemeOnly' is a list of scheme names. If the scheme of any source expression is contained
in this list, not only the path will be removed, but the host and port, too. This is useful
for data or chrome-extension URIs, for example.
"""
if self == Policy.INVALID():
return self
pathsRemoved = []
for direct in self._directives:
pathsRemoved.append(direct.withoutPaths(schemeOnly))
return Policy(pathsRemoved)
def asBasicPolicies(self):
"""
Returns a set of Policies that contain each exactly one Directive with exactly one SourceExpression.
'Decomposes' this Policy into basic Policies and Directives.
Returns the empty set if this policy is invalid.
"""
if self == Policy.INVALID():
return set([])
if len(self._directives) == 0:
return frozenset((self, ))
policies = set([])
for direct in self._directives:
for bDirect in direct.asBasicDirectives():
policies.add(Policy((bDirect, )))
return policies
def compareTo(self, otherPolicy):
"""
Compares this Policy to 'otherPolicy' and returns three sets of basic Policies (that is, Policies with
exactly one Directive and one SourceExpression): (common basic policies, basic policies only in 'self',
basic policies only in 'otherPolicy'). If this Policy or 'otherPolicy' is INVALID(), returns three
empty sets.
"""
if self == Policy.INVALID() or otherPolicy == Policy.INVALID():
return (set([]), set([]), set([]))
selfBasic = self.asBasicPolicies()
otherBasic = otherPolicy.asBasicPolicies()
common = selfBasic & otherBasic
onlySelf = selfBasic - common
onlyOther = otherBasic - common
return (common, onlySelf, onlyOther)
def isBasicPolicy(self):
"""
Returns if this Policy is basic. That is, whether it consists of exactly one Directive, and whether
that Directive is basic.
"""
if self == Policy.INVALID():
return False
if len(self._directives) != 1:
return False
return self._directives.__iter__().next().isBasicDirective()
def isBasicNonePolicy(self):
"""
Returns if this Policy is basic, and has an empty whitelisted resource list in the Directive.
"""
if self == Policy.INVALID():
return False
if len(self._directives) != 1:
return False
direct = self._directives.__iter__().next()
return len(direct.getWhitelistedSourceExpressions()) == 0
def hasDefaultDirective(self):
"""
Returns whether this Policy has a 'default-src' Directive.
"""
if self == Policy.INVALID():
return False
for direct in self._directives:
if direct.getType() == "default-src":
return True
return False
def __repr__(self):
"""
Returns a full representation of this Policy. Equivalent to __str__().
"""
return str(self)
def __str__(self):
"""
Returns a string representation of this policy (directives in sorted order).
"""
if self._str is None:
if self == Policy.INVALID():
self._str = "[invalid]"
else:
stringPolicyList = map(lambda x: str(x), self._directives)
stringPolicyList.sort()
self._str = ("; ".join(stringPolicyList))
return self._str
def __eq__(self, other):
"""
Returns whether the two policies have the elements. This is NOT the same as checking whether
they have the same effect.
"""
if type(other) != Policy:
return False
return (self._isInvalid == other._isInvalid
and self._directives == other._directives)
def __hash__(self):
"""
Returns a hash value for this object that is guaranteed to be the same for two objects
that are equal (the opposite is not necessarily true).
"""
if self._hash is None:
self._hash = hash(self._directives) ^ hash(self._isInvalid)
return self._hash
from pathlib import Path
from directive import Directive
p = Path("../QuestionBank/fopp")
flist = [x for x in p.glob("**/*") if x.is_file()]
activecodes = {}
for f in flist:
with f.open() as df:
try:
d = Directive(df.read())
if d.kind == "activecode":
activecodes[f] = d
except Exception as e:
print(f"Error in {f} is {e}")
for q1 in activecodes.values():
for q2 in activecodes.values():
if q1 is not q2:
if q1.instructions and q1.same_question(q2):
if q1.starter_code == q2.starter_code:
print(f"{q1.identifier} and {q2.identifier} are the same")
if bool(q1.hidden_code) != bool(q2.hidden_code):
if q1.hidden_code:
print(f"{q1.identifier} has unit tests")
else:
print(f"{q2.identifier} has unit tests")
print(q2.hidden_code)
else:
print(q1.starter_code)
"WAND":
Instruction("WAND", 003000, OperandType.EXPRESSION,
AddressType.CHANNEL),
"WOR":
Instruction("WOR", 005000, OperandType.EXPRESSION,
AddressType.CHANNEL),
"WRITE":
Instruction("WRITE", 001000, OperandType.EXPRESSION,
AddressType.CHANNEL),
"ZQ":
Instruction("ZQ", 022007)
},
OpcodeType.DIRECTIVE: {
# Name Method Mnemonic Operand Type Optional? Words
"-1DNADR":
Directive("MinusDNADR", "-1DNADR", OperandType.EXPRESSION, False,
1),
"-2CADR":
Directive("Minus2CADR", "-2CADR", OperandType.EXPRESSION, False,
2),
"-2DNADR":
Directive("MinusDNADR", "-2DNADR", OperandType.EXPRESSION, False,
1),
"-3DNADR":
Directive("MinusDNADR", "-3DNADR", OperandType.EXPRESSION, False,
1),
"-4DNADR":
Directive("MinusDNADR", "-4DNADR", OperandType.EXPRESSION, False,
1),
"-5DNADR":
Directive("MinusDNADR", "-5DNADR", OperandType.EXPRESSION, False,
1),
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from directive import Directive
import sys
p = Directive()
ast = p.parse_file(sys.argv[1])
print(ast.to_yml())
print("> imports:", ast.kimports)
print("> typenames:", ast.ktypenames)