def scan_directory_entry_after_edit(request, page):
from scanner import scanner
if isinstance(page, DirectoryEntry):
try:
scanner.scan(page, commit=True)
messages.success(request, "Scan of '{}' complete.".format(page.title))
except Exception as e:
messages.error(
request,
"Error during scan of '{}': {!r}".format(page.title, e)
)
def test_scan_and_no_commit(self):
"""
When scanner.scan is called without commit=True, it should not save
any results to the database
"""
securedrop = DirectoryEntryFactory.create(
title='Freedom of the Press Foundation',
landing_page_url='https://securedrop.org',
onion_address='notreal.onion')
scanner.scan(securedrop)
self.assertEqual(
0,
DirectoryEntry.objects.get(pk=securedrop.pk).results.count())
def main():
try:
fp = open('./data/LR1_parse_table.dat', 'rb')
except FileNotFoundError:
print(
"Please run \"python3 %s make-parse-table\" before you first use this program."
% sys.argv[0])
return
show_tree_after_iteration = True if 'show-tree-animation' in sys.argv else False
show_simplify_tree = True if 'show-simplify-tree' in sys.argv else False
productions, parse_tab = pickle.load(fp)
try:
tokens, symbol_table, const_string_table = scan(sys.argv[1])
res = lr1.prase(tokens,
parse_tab,
productions,
Terminal,
symbol_table,
const_string_table,
simpify=show_simplify_tree,
show_tree_after_iteration=show_tree_after_iteration)
lr1.show_tree(res, symbol_table, const_string_table)
print('符号表:')
for s in symbol_table:
print('\t', s)
print('字符串表:')
for s in const_string_table:
print("\t'%s'" % s)
except ScanError as e:
e.show_error(sys.argv[1])
except lr1.ParseError as e:
e.show_error(sys.argv[1])
def test_forces_https_should_not_be_none(self):
domain = 'https://sourceanonyme.radio-canada.ca'
entry = DirectoryEntryFactory.create(title='Source Anonyme',
landing_page_url=domain,
onion_address='notreal.onion')
r = scanner.scan(entry, commit=True)
self.assertIsNotNone(r.forces_https)
def test_scan_and_commit(self):
"""
When scanner.scan is called with commit=True, the result of the scan
should be newly saved to the database and associated with the
correct DirectoryEntry
"""
securedrop = DirectoryEntryFactory.create(
title='Freedom of the Press Foundation',
landing_page_url='https://securedrop.org',
onion_address='notreal.onion')
self.assertEqual(
0,
DirectoryEntry.objects.get(pk=securedrop.pk).results.count())
scanner.scan(securedrop, commit=True)
self.assertEqual(
1,
DirectoryEntry.objects.get(pk=securedrop.pk).results.count())
def test_scan_with_permitted_domain(self):
securedrop = DirectoryEntryFactory.create(
title='Freedom of the Press Foundation',
landing_page_url='https://www.nytimes.com/tips',
onion_address='notreal.onion',
permitted_domains_for_assets=['nyt.com'],
)
result = scanner.scan(securedrop)
self.assertEqual(result.no_cross_domain_assets, True)
def test_scan_with_permitted_domains_with_subdomain(self):
securedrop = DirectoryEntryFactory.create(
title='Freedom of the Press Foundation',
landing_page_url='https://securedrop.org',
onion_address='notreal.onion',
permitted_domains_for_assets=['analytics.freedom.press'],
)
result = scanner.scan(securedrop)
self.assertEqual(result.no_cross_domain_assets, True)
def test_scan_detects_absence_of_trackers(self):
"""
If a site contains no known trackers, result.no_analytics should be True
"""
fpf_site = DirectoryEntry(title='FPF',
landing_page_url='https://freedom.press/',
onion_address='notreal.onion')
result = scanner.scan(fpf_site)
self.assertTrue(result.no_analytics)
def test_redirect_target_saved(self):
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url='https://httpbin.org/redirect/3',
onion_address='notreal.onion',
)
result = scanner.scan(entry)
self.assertEqual(result.redirect_target, 'https://httpbin.org/get')
def test_redirect_from_subdomain(self):
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url='http://health.nytimes.com',
onion_address='notreal.onion',
)
r = scanner.scan(entry)
self.assertTrue(r.subdomain)
self.assertTrue(r.no_cross_domain_redirects)
def test_scan_detects_presence_of_trackers(self):
"""
If a site contains common trackers, result.no_analytics should be False
"""
ap_site = DirectoryEntry(title='AP',
landing_page_url='https://www.ap.org/en-us/',
onion_address='notreal.onion')
result = scanner.scan(ap_site)
self.assertFalse(result.no_analytics)
def test_redirect_target_not_saved_if_not_redirect(self):
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url='https://securedrop.org',
onion_address='notreal.onion',
)
result = scanner.scan(entry)
self.assertIsNone(result.redirect_target)
def scan_view(self, request):
if request.method == 'POST':
form = ScannerForm(request.POST)
# if form is invalid, this will skip to returning the form with errors
if form.is_valid():
data = form.cleaned_data
instance = DirectoryEntry(landing_page_url=data['url'], )
result = scanner.scan(instance)
result.save()
context = {
'landing_page_url':
data['url'],
'result':
result,
'submission_form':
DirectoryEntryForm(
directory_page=self,
user=request.user
if request.user.is_authenticated else None,
initial={
'landing_page_url': data['url'],
},
),
'submission_url':
reverse('securedroppage_add'),
'form_title':
self.org_details_form_title
}
if self.org_details_form_text:
context['form_text'] = self.org_details_form_text
return render(
request,
'directory/result.html',
context,
)
elif request.method == 'GET':
form = ScannerForm()
context = {
'form':
form,
'submit_url':
'{base}{scan_url}'.format(
base=self.get_url(request=request),
scan_url=SCAN_URL,
),
'form_title':
self.scanner_form_title,
}
if self.scanner_form_text:
context['text'] = self.scanner_form_text
return render(request, 'directory/scanner_form.html', context)
def test_scan_returns_result_if_site_live(self):
"""
If a site can be connected to, scanner should return a result with
result.live True
"""
securedrop = DirectoryEntry(title='Freedom of the Press Foundation',
landing_page_url='https://securedrop.org',
onion_address='notreal.onion')
result = scanner.scan(securedrop)
self.assertTrue(result.live)
def test_cross_domain_redirect_detected_and_saved(self):
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url=
'https://httpbin.org/redirect-to?url=http%3A%2F%2Fwww.google.com&status_code=302',
onion_address='notreal.onion',
)
r = scanner.scan(entry)
self.assertFalse(r.no_cross_domain_redirects)
def test_permanent_redirect_target_saved(self):
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url=
'https://httpbin.org/redirect-to?status_code=301&url=https%3A%2F%2Fhttpbin.org%2Fget',
onion_address='notreal.onion',
)
result = scanner.scan(entry)
self.assertEqual(result.redirect_target, 'https://httpbin.org/get')
def test_scan_returns_reurns_url_if_site_not_live(self):
"""
If a site cannot be connected to, scanner should return a ScanResult
with the URL attribute set.
"""
securedrop = DirectoryEntry(title='Freedom of the Press Foundation',
landing_page_url=NON_EXISTENT_URL,
onion_address='notreal.onion')
result = scanner.scan(securedrop)
self.assertEqual(result.landing_page_url, NON_EXISTENT_URL)
def test_if_cross_domain_redirect_found_continue_to_scan(self):
"""if a cross-domain redirect is found, then we should make a full scan
of target domain"""
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url=
'https://httpbin.org/redirect-to?url=http%3A%2F%2Fwww.google.com&status_code=302',
onion_address='notreal.onion',
)
r = scanner.scan(entry)
self.assertTrue(r.live)
self.assertTrue(r.no_cross_domain_assets)
def test_redirection_not_200(self):
entry = DirectoryEntryFactory.create(
title='SecureDrop',
landing_page_url=
'https://httpbin.org/redirect-to?url=https%3A%2F%2Fhttpbin.org%2Fstatus%2F404',
onion_address='notreal.onion',
)
result = scanner.scan(entry)
self.assertEqual(result.redirect_target,
'https://httpbin.org/status/404')
self.assertFalse(result.http_status_200_ok)
def test_scan_returns_result_if_site_not_live(self):
"""
If a site cannot be connected to, scanner should return a ScanResult
with result.live False
In addition to vcrpy, this test mocks requests.get to simulate a
ConnectionError for a URL that does not exist without actually sending
an HTTP request to that URL
"""
securedrop = DirectoryEntry(title='Freedom of the Press Foundation',
landing_page_url=NON_EXISTENT_URL,
onion_address='notreal.onion')
result = scanner.scan(securedrop)
self.assertFalse(result.live)
def test_scan_result_attributes(self):
"""
If the site can be connected to, scanner should return a result with
all its attributes set
"""
securedrop = DirectoryEntry(title='Freedom of the Press Foundation',
landing_page_url='https://securedrop.org',
onion_address='notreal.onion')
result = scanner.scan(securedrop)
self.assertTrue(result.forces_https)
self.assertTrue(result.http_status_200_ok)
self.assertTrue(result.hsts)
self.assertTrue(result.hsts_max_age)
self.assertTrue(result.hsts_entire_domain)
self.assertTrue(result.hsts_preloaded)
self.assertIs(result.subdomain, False)
self.assertIs(result.no_cookies, False)
self.assertTrue(result.safe_onion_address)
self.assertIs(result.no_cdn, False)
self.assertTrue(result.no_cross_domain_redirects)
self.assertTrue(result.expected_encoding)
self.assertTrue(result.no_analytics)
self.assertTrue(result.no_server_info)
self.assertTrue(result.no_server_version)
self.assertTrue(result.csp_origin_only)
self.assertTrue(result.mime_sniffing_blocked)
self.assertIs(result.noopen_download, False)
self.assertTrue(result.xss_protection)
self.assertIs(result.clickjacking_protection, False)
self.assertIs(result.good_cross_domain_policy, False)
self.assertIs(result.http_1_0_caching_disabled, False)
self.assertIs(result.expires_set, False)
self.assertTrue(result.cache_control_set)
self.assertIs(result.cache_control_revalidate_set, False)
self.assertIs(result.cache_control_nocache_set, False)
self.assertIs(result.cache_control_notransform_set, False)
self.assertIs(result.cache_control_nostore_set, False)
self.assertIs(result.cache_control_private_set, False)
self.assertIs(result.referrer_policy_set_to_no_referrer, False)
self.assertIs(result.no_cross_domain_assets, False)
self.assertNotEqual(result.cross_domain_asset_summary, '')
def test_scan_detects_presence_of_cross_domain_assets(self):
"""
If a site contains cross-domain assets, result.no_cross_domain_assets should be False
"""
ap_site = DirectoryEntry(title='AP',
landing_page_url='https://www.ap.org/en-us/',
onion_address='notreal.onion')
result = scanner.scan(ap_site)
self.assertIs(result.no_cross_domain_assets, False)
expected_urls = (
'https://www.googletagmanager.com/ns.html?id=GTM-TSGB826',
'//searchg2-assets.crownpeak.net/crownpeak.searchg2-1.0.2.min.js',
'https://cdn.cookielaw.org/langswitch/ead3872f-33b9-4b16-a7f2-4ea8137893d3.js',
)
for url in expected_urls:
self.assertIn(url, result.cross_domain_asset_summary)
ignored_urls = (
'https://www.google-analytics.com/analytics.js',
'pardot.com/pd.js',
'https://www.googletagmanager.com/gtm.js?id=',
'www.crownpeak.com',
'searchg2.crownpeak.net/',
'http://www.w3.org/2000/svg',
'click.bs.carousel.data',
'click.bs.collapse.data',
'element.id',
'click.bs.modal.data',
'hidden.bs.tab',
'shown.bs.tab',
'bs.tab',
'hide.bs.tab',
'show.bs.tab',
'click.bs.tab.data',
)
for url in ignored_urls:
self.assertIn(url, result.ignored_cross_domain_assets)
# encoding=utf-8 """ Импортирование пакета """ from scanner import scanner scanner.scan()
def assemble(lines):
stripped_lines = strip_comments(map(str.strip, lines))
tokenized_lines = [(line_number, scanner.scan(line)) for
line_number, line in stripped_lines]
# nop insertion pass
hot_register = None
for i, line_tuple in enumerate(tokenized_lines):
line_number, line = line_tuple
tokens, rest = line
token_type, value = tokens[0]
if token_type == 'operator':
if value in ('ld', 'ldi', 'ldg'):
hot_register = tokens[1][1]
else:
for token in tokens[1:]:
token_type, value = token
if token_type == 'register' and value == hot_register:
tokenized_lines.insert(
i, (-1, ([('operator', 'nop')], '')))
hot_register = None
# simple label pass
labels = {}
current_address = 1
for line_number, line in tokenized_lines:
tokens, rest = line
token_type, value = tokens[0]
if token_type == 'label':
labels[value] = current_address
elif token_type in ('operator', 'if'):
current_address += 1
#actual assemble pass
assembly = []
for line_number, line in tokenized_lines:
tokens, rest = line
if rest:
print '[%s]: Tokenizer error near' % line_number, rest
return
tokens = iter(tokens)
condition = Cond.ALWAYS
try:
while True:
token = tokens.next()
token_type, value = token
if token_type == 'if':
token = tokens.next()
token_type, value = token
prefix = ''
if token == ('operator', 'not'):
token = tokens.next()
token_type, value = token
prefix = 'not '
if token_type == 'condition':
condition = Cond.fromString(prefix + value)
else:
print 'Your if makes no sense. Line number:',
print line_number
elif token_type == 'operator':
instruction = value.lower()
# Three params
if instruction in ['add', 'addi', 'and', 'andi', 'ld',
'mul', 'muli', 'or', 'ori', 'sll',
'slli', 'sra', 'srai', 'srl', 'srli',
'st', 'sub', 'subi', 'xor', 'xori']:
_, a = tokens.next()
_, b = tokens.next()
_, c = tokens.next()
instruction = instruction_map[
value.capitalize()](a, b, c)
# Two params
elif instruction in ['jmp', 'ldi', 'sti', 'stg',
'cmp', 'mv', 'neg', 'not']:
maybe_label, a = tokens.next()
if maybe_label == 'label':
b = labels[a]
a = 0
else:
_, b = tokens.next()
instruction = instruction_map[value.capitalize()](a, b)
# One param
elif instruction in ['ldg', 'setg']:
_, a = tokens.next()
instruction = instruction_map[value.capitalize()](a)
# No params
elif instruction in ['nop', 'ret']:
instruction = instruction_map[value.capitalize()]()
# special cases
elif instruction == 'call':
maybe_label, a = tokens.next()
if maybe_label == 'label':
b = labels[a]
a = 0
else:
_, b = tokens.next()
instruction = instruction_map[value.capitalize()](a, b)
if instruction.cond == Cond.UNSET:
instruction.cond = condition
assembly.append(instruction)
except StopIteration:
pass
except Exception, e:
print "[%s]:" % line_number, e
def assemble(lines):
stripped_lines = strip_comments(map(str.strip, lines))
tokenized_lines = [(line_number, scanner.scan(line))
for line_number, line in stripped_lines]
# nop insertion pass
hot_register = None
for i, line_tuple in enumerate(tokenized_lines):
line_number, line = line_tuple
tokens, rest = line
token_type, value = tokens[0]
if token_type == 'operator':
if value in ('ld', 'ldi', 'ldg'):
hot_register = tokens[1][1]
else:
for token in tokens[1:]:
token_type, value = token
if token_type == 'register' and value == hot_register:
tokenized_lines.insert(i,
(-1,
([('operator', 'nop')], '')))
hot_register = None
# simple label pass
labels = {}
current_address = 1
for line_number, line in tokenized_lines:
tokens, rest = line
token_type, value = tokens[0]
if token_type == 'label':
labels[value] = current_address
elif token_type in ('operator', 'if'):
current_address += 1
#actual assemble pass
assembly = []
for line_number, line in tokenized_lines:
tokens, rest = line
if rest:
print '[%s]: Tokenizer error near' % line_number, rest
return
tokens = iter(tokens)
condition = Cond.ALWAYS
try:
while True:
token = tokens.next()
token_type, value = token
if token_type == 'if':
token = tokens.next()
token_type, value = token
prefix = ''
if token == ('operator', 'not'):
token = tokens.next()
token_type, value = token
prefix = 'not '
if token_type == 'condition':
condition = Cond.fromString(prefix + value)
else:
print 'Your if makes no sense. Line number:',
print line_number
elif token_type == 'operator':
instruction = value.lower()
# Three params
if instruction in [
'add', 'addi', 'and', 'andi', 'ld', 'mul', 'muli',
'or', 'ori', 'sll', 'slli', 'sra', 'srai', 'srl',
'srli', 'st', 'sub', 'subi', 'xor', 'xori'
]:
_, a = tokens.next()
_, b = tokens.next()
_, c = tokens.next()
instruction = instruction_map[value.capitalize()](a, b,
c)
# Two params
elif instruction in [
'jmp', 'ldi', 'sti', 'stg', 'cmp', 'mv', 'neg',
'not'
]:
maybe_label, a = tokens.next()
if maybe_label == 'label':
b = labels[a]
a = 0
else:
_, b = tokens.next()
instruction = instruction_map[value.capitalize()](a, b)
# One param
elif instruction in ['ldg', 'setg']:
_, a = tokens.next()
instruction = instruction_map[value.capitalize()](a)
# No params
elif instruction in ['nop', 'ret']:
instruction = instruction_map[value.capitalize()]()
# special cases
elif instruction == 'call':
maybe_label, a = tokens.next()
if maybe_label == 'label':
b = labels[a]
a = 0
else:
_, b = tokens.next()
instruction = instruction_map[value.capitalize()](a, b)
if instruction.cond == Cond.UNSET:
instruction.cond = condition
assembly.append(instruction)
except StopIteration:
pass
except Exception, e:
print "[%s]:" % line_number, e
def do_GET(self):
pages = ['','full','info']
url_properties = self.get_fields(self.path)
page_len = self.path.rfind('?')
if page_len == -1: page_len = len(self.path)
page = self.path[1:page_len]
print "sending %s" % page
if page not in pages:
print '%s rejected' % page
self.send_response(404)
return
self.send_response(200)
if page == 'info':
self.send_header('Content-type','text/html; charset=utf-8')
self.end_headers()
self.wfile.write("<html><head><title></title></head><body>")
self.wfile.write("<h1>Foxall Scanner Server</h1>")
self.wfile.write("Attached Scanner: %s %s" % (scanner.device.model, scanner.device.name))
self.wfile.write("<h3>Available Pages</h3>")
self.wfile.write("<p><b>full</b> Get back a live mjpeg stream of the overall scan from the scanner</p><a href=\"/full\">/full</a>")
self.wfile.write("<p><b>info</b> Get this page</p><a href=\"/info\">/info</a>")
self.wfile.write("<h3>Available options</h3>")
self.wfile.write("<p>The following options have been pulled from the scanner driver just now, some properties may not be implemented. Also some options may not be named the same as on other scanners, this is because the driver is written by many developers and they dont all keep to the same naming standards! to select one of theese options you have to put them in on the scan request, simply put in <i>http://thedomain.orip/full?option=value&option=value&option=value</i> like a standard web API </p>")
self.wfile.write("<table border=1><tr><td><b>Property</b></td><td><b>Current setting</b></td><td><b>Constraints/Available Values</b></td></tr>")
for item in scanner.device.options:
try: getVal = scanner.device.options[item].value
except: getVal = 'unknown'
getConst = scanner.device.options[item].constraint
if getConst == None: getConst = ''
self.wfile.write("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" % (item, getVal, getConst))
self.wfile.write("</table>")
self.wfile.write("</body></html>")
elif not scanner.is_scanning and page in ['full', '']:
# Header for mpjpeg style stream
self.send_header('Content-type','image/jpeg')
self.end_headers()
if url_properties != None:
for property in url_properties:
scanner.set_option(property, url_properties[property])
if 'br-x' not in url_properties.keys() and 'br-y' not in url_properties.keys():
scanner.max_end_scan()
else:
scanner.max_end_scan()
print page
# Begin scan, it feeds a frame at a time
scanner.start_scan()
while scanner.is_scanning:
scanner.scan()
scanner.img.save(self.wfile,'JPEG')
else:
print 'Scan in progress, this request is sent single image'
self.send_header('Content-type','image/jpeg')
self.end_headers()
scanner.img.save(self.wfile,'JPEG')
return
def do_GET(self):
pages = ['', 'full', 'info']
url_properties = self.get_fields(self.path)
page_len = self.path.rfind('?')
if page_len == -1: page_len = len(self.path)
page = self.path[1:page_len]
print "sending %s" % page
if page not in pages:
print '%s rejected' % page
self.send_response(404)
return
self.send_response(200)
if page == 'info':
self.send_header('Content-type', 'text/html; charset=utf-8')
self.end_headers()
self.wfile.write("<html><head><title></title></head><body>")
self.wfile.write("<h1>Foxall Scanner Server</h1>")
self.wfile.write("Attached Scanner: %s %s" %
(scanner.device.model, scanner.device.name))
self.wfile.write("<h3>Available Pages</h3>")
self.wfile.write(
"<p><b>full</b> Get back a live mjpeg stream of the overall scan from the scanner</p><a href=\"/full\">/full</a>"
)
self.wfile.write(
"<p><b>info</b> Get this page</p><a href=\"/info\">/info</a>")
self.wfile.write("<h3>Available options</h3>")
self.wfile.write(
"<p>The following options have been pulled from the scanner driver just now, some properties may not be implemented. Also some options may not be named the same as on other scanners, this is because the driver is written by many developers and they dont all keep to the same naming standards! to select one of theese options you have to put them in on the scan request, simply put in <i>http://thedomain.orip/full?option=value&option=value&option=value</i> like a standard web API </p>"
)
self.wfile.write(
"<table border=1><tr><td><b>Property</b></td><td><b>Current setting</b></td><td><b>Constraints/Available Values</b></td></tr>"
)
for item in scanner.device.options:
try:
getVal = scanner.device.options[item].value
except:
getVal = 'unknown'
getConst = scanner.device.options[item].constraint
if getConst == None: getConst = ''
self.wfile.write("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" %
(item, getVal, getConst))
self.wfile.write("</table>")
self.wfile.write("</body></html>")
elif not scanner.is_scanning and page in ['full', '']:
# Header for mpjpeg style stream
self.send_header('Content-type', 'image/jpeg')
self.end_headers()
if url_properties != None:
for property in url_properties:
scanner.set_option(property, url_properties[property])
if 'br-x' not in url_properties.keys(
) and 'br-y' not in url_properties.keys():
scanner.max_end_scan()
else:
scanner.max_end_scan()
print page
# Begin scan, it feeds a frame at a time
scanner.start_scan()
while scanner.is_scanning:
scanner.scan()
scanner.img.save(self.wfile, 'JPEG')
else:
print 'Scan in progress, this request is sent single image'
self.send_header('Content-type', 'image/jpeg')
self.end_headers()
scanner.img.save(self.wfile, 'JPEG')
return
