def scan_directory_entry_after_edit(request, page): from scanner import scanner if isinstance(page, DirectoryEntry): try: scanner.scan(page, commit=True) messages.success(request, "Scan of '{}' complete.".format(page.title)) except Exception as e: messages.error( request, "Error during scan of '{}': {!r}".format(page.title, e) )
def test_scan_and_no_commit(self): """ When scanner.scan is called without commit=True, it should not save any results to the database """ securedrop = DirectoryEntryFactory.create( title='Freedom of the Press Foundation', landing_page_url='https://securedrop.org', onion_address='notreal.onion') scanner.scan(securedrop) self.assertEqual( 0, DirectoryEntry.objects.get(pk=securedrop.pk).results.count())
def main(): try: fp = open('./data/LR1_parse_table.dat', 'rb') except FileNotFoundError: print( "Please run \"python3 %s make-parse-table\" before you first use this program." % sys.argv[0]) return show_tree_after_iteration = True if 'show-tree-animation' in sys.argv else False show_simplify_tree = True if 'show-simplify-tree' in sys.argv else False productions, parse_tab = pickle.load(fp) try: tokens, symbol_table, const_string_table = scan(sys.argv[1]) res = lr1.prase(tokens, parse_tab, productions, Terminal, symbol_table, const_string_table, simpify=show_simplify_tree, show_tree_after_iteration=show_tree_after_iteration) lr1.show_tree(res, symbol_table, const_string_table) print('符号表:') for s in symbol_table: print('\t', s) print('字符串表:') for s in const_string_table: print("\t'%s'" % s) except ScanError as e: e.show_error(sys.argv[1]) except lr1.ParseError as e: e.show_error(sys.argv[1])
def test_forces_https_should_not_be_none(self): domain = 'https://sourceanonyme.radio-canada.ca' entry = DirectoryEntryFactory.create(title='Source Anonyme', landing_page_url=domain, onion_address='notreal.onion') r = scanner.scan(entry, commit=True) self.assertIsNotNone(r.forces_https)
def test_scan_and_commit(self): """ When scanner.scan is called with commit=True, the result of the scan should be newly saved to the database and associated with the correct DirectoryEntry """ securedrop = DirectoryEntryFactory.create( title='Freedom of the Press Foundation', landing_page_url='https://securedrop.org', onion_address='notreal.onion') self.assertEqual( 0, DirectoryEntry.objects.get(pk=securedrop.pk).results.count()) scanner.scan(securedrop, commit=True) self.assertEqual( 1, DirectoryEntry.objects.get(pk=securedrop.pk).results.count())
def test_scan_with_permitted_domain(self): securedrop = DirectoryEntryFactory.create( title='Freedom of the Press Foundation', landing_page_url='https://www.nytimes.com/tips', onion_address='notreal.onion', permitted_domains_for_assets=['nyt.com'], ) result = scanner.scan(securedrop) self.assertEqual(result.no_cross_domain_assets, True)
def test_scan_with_permitted_domains_with_subdomain(self): securedrop = DirectoryEntryFactory.create( title='Freedom of the Press Foundation', landing_page_url='https://securedrop.org', onion_address='notreal.onion', permitted_domains_for_assets=['analytics.freedom.press'], ) result = scanner.scan(securedrop) self.assertEqual(result.no_cross_domain_assets, True)
def test_scan_detects_absence_of_trackers(self): """ If a site contains no known trackers, result.no_analytics should be True """ fpf_site = DirectoryEntry(title='FPF', landing_page_url='https://freedom.press/', onion_address='notreal.onion') result = scanner.scan(fpf_site) self.assertTrue(result.no_analytics)
def test_redirect_target_saved(self): entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url='https://httpbin.org/redirect/3', onion_address='notreal.onion', ) result = scanner.scan(entry) self.assertEqual(result.redirect_target, 'https://httpbin.org/get')
def test_redirect_from_subdomain(self): entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url='http://health.nytimes.com', onion_address='notreal.onion', ) r = scanner.scan(entry) self.assertTrue(r.subdomain) self.assertTrue(r.no_cross_domain_redirects)
def test_scan_detects_presence_of_trackers(self): """ If a site contains common trackers, result.no_analytics should be False """ ap_site = DirectoryEntry(title='AP', landing_page_url='https://www.ap.org/en-us/', onion_address='notreal.onion') result = scanner.scan(ap_site) self.assertFalse(result.no_analytics)
def test_redirect_target_not_saved_if_not_redirect(self): entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url='https://securedrop.org', onion_address='notreal.onion', ) result = scanner.scan(entry) self.assertIsNone(result.redirect_target)
def scan_view(self, request): if request.method == 'POST': form = ScannerForm(request.POST) # if form is invalid, this will skip to returning the form with errors if form.is_valid(): data = form.cleaned_data instance = DirectoryEntry(landing_page_url=data['url'], ) result = scanner.scan(instance) result.save() context = { 'landing_page_url': data['url'], 'result': result, 'submission_form': DirectoryEntryForm( directory_page=self, user=request.user if request.user.is_authenticated else None, initial={ 'landing_page_url': data['url'], }, ), 'submission_url': reverse('securedroppage_add'), 'form_title': self.org_details_form_title } if self.org_details_form_text: context['form_text'] = self.org_details_form_text return render( request, 'directory/result.html', context, ) elif request.method == 'GET': form = ScannerForm() context = { 'form': form, 'submit_url': '{base}{scan_url}'.format( base=self.get_url(request=request), scan_url=SCAN_URL, ), 'form_title': self.scanner_form_title, } if self.scanner_form_text: context['text'] = self.scanner_form_text return render(request, 'directory/scanner_form.html', context)
def test_scan_returns_result_if_site_live(self): """ If a site can be connected to, scanner should return a result with result.live True """ securedrop = DirectoryEntry(title='Freedom of the Press Foundation', landing_page_url='https://securedrop.org', onion_address='notreal.onion') result = scanner.scan(securedrop) self.assertTrue(result.live)
def test_cross_domain_redirect_detected_and_saved(self): entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url= 'https://httpbin.org/redirect-to?url=http%3A%2F%2Fwww.google.com&status_code=302', onion_address='notreal.onion', ) r = scanner.scan(entry) self.assertFalse(r.no_cross_domain_redirects)
def test_permanent_redirect_target_saved(self): entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url= 'https://httpbin.org/redirect-to?status_code=301&url=https%3A%2F%2Fhttpbin.org%2Fget', onion_address='notreal.onion', ) result = scanner.scan(entry) self.assertEqual(result.redirect_target, 'https://httpbin.org/get')
def test_scan_returns_reurns_url_if_site_not_live(self): """ If a site cannot be connected to, scanner should return a ScanResult with the URL attribute set. """ securedrop = DirectoryEntry(title='Freedom of the Press Foundation', landing_page_url=NON_EXISTENT_URL, onion_address='notreal.onion') result = scanner.scan(securedrop) self.assertEqual(result.landing_page_url, NON_EXISTENT_URL)
def test_if_cross_domain_redirect_found_continue_to_scan(self): """if a cross-domain redirect is found, then we should make a full scan of target domain""" entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url= 'https://httpbin.org/redirect-to?url=http%3A%2F%2Fwww.google.com&status_code=302', onion_address='notreal.onion', ) r = scanner.scan(entry) self.assertTrue(r.live) self.assertTrue(r.no_cross_domain_assets)
def test_redirection_not_200(self): entry = DirectoryEntryFactory.create( title='SecureDrop', landing_page_url= 'https://httpbin.org/redirect-to?url=https%3A%2F%2Fhttpbin.org%2Fstatus%2F404', onion_address='notreal.onion', ) result = scanner.scan(entry) self.assertEqual(result.redirect_target, 'https://httpbin.org/status/404') self.assertFalse(result.http_status_200_ok)
def test_scan_returns_result_if_site_not_live(self): """ If a site cannot be connected to, scanner should return a ScanResult with result.live False In addition to vcrpy, this test mocks requests.get to simulate a ConnectionError for a URL that does not exist without actually sending an HTTP request to that URL """ securedrop = DirectoryEntry(title='Freedom of the Press Foundation', landing_page_url=NON_EXISTENT_URL, onion_address='notreal.onion') result = scanner.scan(securedrop) self.assertFalse(result.live)
def test_scan_result_attributes(self): """ If the site can be connected to, scanner should return a result with all its attributes set """ securedrop = DirectoryEntry(title='Freedom of the Press Foundation', landing_page_url='https://securedrop.org', onion_address='notreal.onion') result = scanner.scan(securedrop) self.assertTrue(result.forces_https) self.assertTrue(result.http_status_200_ok) self.assertTrue(result.hsts) self.assertTrue(result.hsts_max_age) self.assertTrue(result.hsts_entire_domain) self.assertTrue(result.hsts_preloaded) self.assertIs(result.subdomain, False) self.assertIs(result.no_cookies, False) self.assertTrue(result.safe_onion_address) self.assertIs(result.no_cdn, False) self.assertTrue(result.no_cross_domain_redirects) self.assertTrue(result.expected_encoding) self.assertTrue(result.no_analytics) self.assertTrue(result.no_server_info) self.assertTrue(result.no_server_version) self.assertTrue(result.csp_origin_only) self.assertTrue(result.mime_sniffing_blocked) self.assertIs(result.noopen_download, False) self.assertTrue(result.xss_protection) self.assertIs(result.clickjacking_protection, False) self.assertIs(result.good_cross_domain_policy, False) self.assertIs(result.http_1_0_caching_disabled, False) self.assertIs(result.expires_set, False) self.assertTrue(result.cache_control_set) self.assertIs(result.cache_control_revalidate_set, False) self.assertIs(result.cache_control_nocache_set, False) self.assertIs(result.cache_control_notransform_set, False) self.assertIs(result.cache_control_nostore_set, False) self.assertIs(result.cache_control_private_set, False) self.assertIs(result.referrer_policy_set_to_no_referrer, False) self.assertIs(result.no_cross_domain_assets, False) self.assertNotEqual(result.cross_domain_asset_summary, '')
def test_scan_detects_presence_of_cross_domain_assets(self): """ If a site contains cross-domain assets, result.no_cross_domain_assets should be False """ ap_site = DirectoryEntry(title='AP', landing_page_url='https://www.ap.org/en-us/', onion_address='notreal.onion') result = scanner.scan(ap_site) self.assertIs(result.no_cross_domain_assets, False) expected_urls = ( 'https://www.googletagmanager.com/ns.html?id=GTM-TSGB826', '//searchg2-assets.crownpeak.net/crownpeak.searchg2-1.0.2.min.js', 'https://cdn.cookielaw.org/langswitch/ead3872f-33b9-4b16-a7f2-4ea8137893d3.js', ) for url in expected_urls: self.assertIn(url, result.cross_domain_asset_summary) ignored_urls = ( 'https://www.google-analytics.com/analytics.js', 'pardot.com/pd.js', 'https://www.googletagmanager.com/gtm.js?id=', 'www.crownpeak.com', 'searchg2.crownpeak.net/', 'http://www.w3.org/2000/svg', 'click.bs.carousel.data', 'click.bs.collapse.data', 'element.id', 'click.bs.modal.data', 'hidden.bs.tab', 'shown.bs.tab', 'bs.tab', 'hide.bs.tab', 'show.bs.tab', 'click.bs.tab.data', ) for url in ignored_urls: self.assertIn(url, result.ignored_cross_domain_assets)
# encoding=utf-8 """ Импортирование пакета """ from scanner import scanner scanner.scan()
def assemble(lines): stripped_lines = strip_comments(map(str.strip, lines)) tokenized_lines = [(line_number, scanner.scan(line)) for line_number, line in stripped_lines] # nop insertion pass hot_register = None for i, line_tuple in enumerate(tokenized_lines): line_number, line = line_tuple tokens, rest = line token_type, value = tokens[0] if token_type == 'operator': if value in ('ld', 'ldi', 'ldg'): hot_register = tokens[1][1] else: for token in tokens[1:]: token_type, value = token if token_type == 'register' and value == hot_register: tokenized_lines.insert( i, (-1, ([('operator', 'nop')], ''))) hot_register = None # simple label pass labels = {} current_address = 1 for line_number, line in tokenized_lines: tokens, rest = line token_type, value = tokens[0] if token_type == 'label': labels[value] = current_address elif token_type in ('operator', 'if'): current_address += 1 #actual assemble pass assembly = [] for line_number, line in tokenized_lines: tokens, rest = line if rest: print '[%s]: Tokenizer error near' % line_number, rest return tokens = iter(tokens) condition = Cond.ALWAYS try: while True: token = tokens.next() token_type, value = token if token_type == 'if': token = tokens.next() token_type, value = token prefix = '' if token == ('operator', 'not'): token = tokens.next() token_type, value = token prefix = 'not ' if token_type == 'condition': condition = Cond.fromString(prefix + value) else: print 'Your if makes no sense. Line number:', print line_number elif token_type == 'operator': instruction = value.lower() # Three params if instruction in ['add', 'addi', 'and', 'andi', 'ld', 'mul', 'muli', 'or', 'ori', 'sll', 'slli', 'sra', 'srai', 'srl', 'srli', 'st', 'sub', 'subi', 'xor', 'xori']: _, a = tokens.next() _, b = tokens.next() _, c = tokens.next() instruction = instruction_map[ value.capitalize()](a, b, c) # Two params elif instruction in ['jmp', 'ldi', 'sti', 'stg', 'cmp', 'mv', 'neg', 'not']: maybe_label, a = tokens.next() if maybe_label == 'label': b = labels[a] a = 0 else: _, b = tokens.next() instruction = instruction_map[value.capitalize()](a, b) # One param elif instruction in ['ldg', 'setg']: _, a = tokens.next() instruction = instruction_map[value.capitalize()](a) # No params elif instruction in ['nop', 'ret']: instruction = instruction_map[value.capitalize()]() # special cases elif instruction == 'call': maybe_label, a = tokens.next() if maybe_label == 'label': b = labels[a] a = 0 else: _, b = tokens.next() instruction = instruction_map[value.capitalize()](a, b) if instruction.cond == Cond.UNSET: instruction.cond = condition assembly.append(instruction) except StopIteration: pass except Exception, e: print "[%s]:" % line_number, e
def assemble(lines): stripped_lines = strip_comments(map(str.strip, lines)) tokenized_lines = [(line_number, scanner.scan(line)) for line_number, line in stripped_lines] # nop insertion pass hot_register = None for i, line_tuple in enumerate(tokenized_lines): line_number, line = line_tuple tokens, rest = line token_type, value = tokens[0] if token_type == 'operator': if value in ('ld', 'ldi', 'ldg'): hot_register = tokens[1][1] else: for token in tokens[1:]: token_type, value = token if token_type == 'register' and value == hot_register: tokenized_lines.insert(i, (-1, ([('operator', 'nop')], ''))) hot_register = None # simple label pass labels = {} current_address = 1 for line_number, line in tokenized_lines: tokens, rest = line token_type, value = tokens[0] if token_type == 'label': labels[value] = current_address elif token_type in ('operator', 'if'): current_address += 1 #actual assemble pass assembly = [] for line_number, line in tokenized_lines: tokens, rest = line if rest: print '[%s]: Tokenizer error near' % line_number, rest return tokens = iter(tokens) condition = Cond.ALWAYS try: while True: token = tokens.next() token_type, value = token if token_type == 'if': token = tokens.next() token_type, value = token prefix = '' if token == ('operator', 'not'): token = tokens.next() token_type, value = token prefix = 'not ' if token_type == 'condition': condition = Cond.fromString(prefix + value) else: print 'Your if makes no sense. Line number:', print line_number elif token_type == 'operator': instruction = value.lower() # Three params if instruction in [ 'add', 'addi', 'and', 'andi', 'ld', 'mul', 'muli', 'or', 'ori', 'sll', 'slli', 'sra', 'srai', 'srl', 'srli', 'st', 'sub', 'subi', 'xor', 'xori' ]: _, a = tokens.next() _, b = tokens.next() _, c = tokens.next() instruction = instruction_map[value.capitalize()](a, b, c) # Two params elif instruction in [ 'jmp', 'ldi', 'sti', 'stg', 'cmp', 'mv', 'neg', 'not' ]: maybe_label, a = tokens.next() if maybe_label == 'label': b = labels[a] a = 0 else: _, b = tokens.next() instruction = instruction_map[value.capitalize()](a, b) # One param elif instruction in ['ldg', 'setg']: _, a = tokens.next() instruction = instruction_map[value.capitalize()](a) # No params elif instruction in ['nop', 'ret']: instruction = instruction_map[value.capitalize()]() # special cases elif instruction == 'call': maybe_label, a = tokens.next() if maybe_label == 'label': b = labels[a] a = 0 else: _, b = tokens.next() instruction = instruction_map[value.capitalize()](a, b) if instruction.cond == Cond.UNSET: instruction.cond = condition assembly.append(instruction) except StopIteration: pass except Exception, e: print "[%s]:" % line_number, e
def do_GET(self): pages = ['','full','info'] url_properties = self.get_fields(self.path) page_len = self.path.rfind('?') if page_len == -1: page_len = len(self.path) page = self.path[1:page_len] print "sending %s" % page if page not in pages: print '%s rejected' % page self.send_response(404) return self.send_response(200) if page == 'info': self.send_header('Content-type','text/html; charset=utf-8') self.end_headers() self.wfile.write("<html><head><title></title></head><body>") self.wfile.write("<h1>Foxall Scanner Server</h1>") self.wfile.write("Attached Scanner: %s %s" % (scanner.device.model, scanner.device.name)) self.wfile.write("<h3>Available Pages</h3>") self.wfile.write("<p><b>full</b> Get back a live mjpeg stream of the overall scan from the scanner</p><a href=\"/full\">/full</a>") self.wfile.write("<p><b>info</b> Get this page</p><a href=\"/info\">/info</a>") self.wfile.write("<h3>Available options</h3>") self.wfile.write("<p>The following options have been pulled from the scanner driver just now, some properties may not be implemented. Also some options may not be named the same as on other scanners, this is because the driver is written by many developers and they dont all keep to the same naming standards! to select one of theese options you have to put them in on the scan request, simply put in <i>http://thedomain.orip/full?option=value&option=value&option=value</i> like a standard web API </p>") self.wfile.write("<table border=1><tr><td><b>Property</b></td><td><b>Current setting</b></td><td><b>Constraints/Available Values</b></td></tr>") for item in scanner.device.options: try: getVal = scanner.device.options[item].value except: getVal = 'unknown' getConst = scanner.device.options[item].constraint if getConst == None: getConst = '' self.wfile.write("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" % (item, getVal, getConst)) self.wfile.write("</table>") self.wfile.write("</body></html>") elif not scanner.is_scanning and page in ['full', '']: # Header for mpjpeg style stream self.send_header('Content-type','image/jpeg') self.end_headers() if url_properties != None: for property in url_properties: scanner.set_option(property, url_properties[property]) if 'br-x' not in url_properties.keys() and 'br-y' not in url_properties.keys(): scanner.max_end_scan() else: scanner.max_end_scan() print page # Begin scan, it feeds a frame at a time scanner.start_scan() while scanner.is_scanning: scanner.scan() scanner.img.save(self.wfile,'JPEG') else: print 'Scan in progress, this request is sent single image' self.send_header('Content-type','image/jpeg') self.end_headers() scanner.img.save(self.wfile,'JPEG') return
def do_GET(self): pages = ['', 'full', 'info'] url_properties = self.get_fields(self.path) page_len = self.path.rfind('?') if page_len == -1: page_len = len(self.path) page = self.path[1:page_len] print "sending %s" % page if page not in pages: print '%s rejected' % page self.send_response(404) return self.send_response(200) if page == 'info': self.send_header('Content-type', 'text/html; charset=utf-8') self.end_headers() self.wfile.write("<html><head><title></title></head><body>") self.wfile.write("<h1>Foxall Scanner Server</h1>") self.wfile.write("Attached Scanner: %s %s" % (scanner.device.model, scanner.device.name)) self.wfile.write("<h3>Available Pages</h3>") self.wfile.write( "<p><b>full</b> Get back a live mjpeg stream of the overall scan from the scanner</p><a href=\"/full\">/full</a>" ) self.wfile.write( "<p><b>info</b> Get this page</p><a href=\"/info\">/info</a>") self.wfile.write("<h3>Available options</h3>") self.wfile.write( "<p>The following options have been pulled from the scanner driver just now, some properties may not be implemented. Also some options may not be named the same as on other scanners, this is because the driver is written by many developers and they dont all keep to the same naming standards! to select one of theese options you have to put them in on the scan request, simply put in <i>http://thedomain.orip/full?option=value&option=value&option=value</i> like a standard web API </p>" ) self.wfile.write( "<table border=1><tr><td><b>Property</b></td><td><b>Current setting</b></td><td><b>Constraints/Available Values</b></td></tr>" ) for item in scanner.device.options: try: getVal = scanner.device.options[item].value except: getVal = 'unknown' getConst = scanner.device.options[item].constraint if getConst == None: getConst = '' self.wfile.write("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" % (item, getVal, getConst)) self.wfile.write("</table>") self.wfile.write("</body></html>") elif not scanner.is_scanning and page in ['full', '']: # Header for mpjpeg style stream self.send_header('Content-type', 'image/jpeg') self.end_headers() if url_properties != None: for property in url_properties: scanner.set_option(property, url_properties[property]) if 'br-x' not in url_properties.keys( ) and 'br-y' not in url_properties.keys(): scanner.max_end_scan() else: scanner.max_end_scan() print page # Begin scan, it feeds a frame at a time scanner.start_scan() while scanner.is_scanning: scanner.scan() scanner.img.save(self.wfile, 'JPEG') else: print 'Scan in progress, this request is sent single image' self.send_header('Content-type', 'image/jpeg') self.end_headers() scanner.img.save(self.wfile, 'JPEG') return