def setUp(self): """ Load the scanner, which helps to set up the test environment. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir) os.chdir(project_dir) conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) self.scanner = Scanner(config) self.scanner.log_dir = os.path.join(self.scanner.logroot_dir, 'testlog') self.scanner.tmp_dir = 'testtmp' self.scanner.creative_db = 'sqlite:///%s/creative.db' % self.scanner.log_dir self.scanner.max_scan = 1 # Turn on the debug mode to temporarily allow the private network. self.scanner.debug = True self.scanner.setup_environment() self.copy_resources(self.scanner.workspace.dirname) self.hostname = socket.gethostname()
def setUp(self): """ Load the scanner, which helps to set up the test environment. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir) os.chdir(project_dir) conf_file = os.path.join("conf", CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) self.scanner = Scanner(config) self.scanner.log_dir = os.path.join(self.scanner.logroot_dir, "testlog") self.scanner.tmp_dir = "testtmp" self.scanner.creative_db = "sqlite:///%s/creative.db" % self.scanner.log_dir self.scanner.max_scan = 1 # Turn on the debug mode to temporarily allow the private network. self.scanner.debug = True self.scanner.setup_environment() self.copy_resources(self.scanner.workspace.dirname) self.hostname = socket.gethostname()
def setUp(self): """ Load the scanner, which helps to set up the test environment. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir) os.chdir(project_dir) conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) self.scanner = Scanner(config) self.scanner.log_dir = os.path.join(self.scanner.logroot_dir, 'testlog') self.scanner.tmp_dir = 'testtmp' self.scanner.creative_db = 'sqlite:///%s/creative.db' % self.scanner.log_dir self.scanner.setup_environment()
def test_setup_environment_and_remove_temp_files(self): """ Test if the temporary files are deleted. """ conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) scanner = Scanner(config) scanner.log_dir = os.path.join(scanner.logroot_dir, 'test_remove_temp_files') scanner.tmp_dir = 'test_remove_temp_files' scanner.creative_db = 'sqlite:///%s/creative.db' % scanner.log_dir scanner.setup_environment() assert os.path.exists(scanner.workspace.dirname) scanner.remove_temp_files() self.assertFalse(os.path.exists(scanner.workspace.dirname)) adscan.fs.rmdirs(scanner.log_dir) adscan.fs.rmdirs(scanner.tmp_dir)
def test_compress_log_file(self): """ Test if the log file is compressed. """ conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) scanner = Scanner(config) scanner.log_dir = os.path.join(scanner.logroot_dir, 'test_compress_log_file') tarname = '%s.tgz' % scanner.log_dir scanner.setup_environment() assert os.path.exists(scanner.log_dir) if os.path.exists(tarname): os.remove(tarname) scanner.compress_log_file() assert os.path.exists(tarname) adscan.fs.rmdirs(scanner.log_dir) os.remove(tarname)
class BrowserTestCase(unittest.TestCase): """ Test the browser controller. """ @classmethod def copy_resources(cls, dest): """ Copy resource files to the """ for fp in os.listdir(TEST_RESOURCE): path = os.path.join(TEST_RESOURCE, fp) if os.path.isfile(path): shutil.copy(path, dest) def setUp(self): """ Load the scanner, which helps to set up the test environment. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir) os.chdir(project_dir) conf_file = os.path.join("conf", CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) self.scanner = Scanner(config) self.scanner.log_dir = os.path.join(self.scanner.logroot_dir, "testlog") self.scanner.tmp_dir = "testtmp" self.scanner.creative_db = "sqlite:///%s/creative.db" % self.scanner.log_dir self.scanner.max_scan = 1 # Turn on the debug mode to temporarily allow the private network. self.scanner.debug = True self.scanner.setup_environment() self.copy_resources(self.scanner.workspace.dirname) self.hostname = socket.gethostname() def tearDown(self): """ Delete the exising processes and clear the directory. """ adscan.fs.rmdirs(self.scanner.log_dir) self.scanner.workspace.delete() def _browse_creative(self, creative_id, snippet): """ Set up a server and a browser to view the snippet. """ creatives = ( self.scanner.db_session.query(Creative) .filter(Creative.created_at == datetime.date.today(), Creative.creative_id == creative_id) .all() ) if creatives and len(creatives) > 0: self.scanner.db_session.delete(creatives[0]) self.scanner.db_session.commit() creative = Creative(creative_id=creative_id, snippet=snippet) self.scanner.db_session.add(creative) self.scanner.db_session.commit() self.scanner.browse_creatives("https") self.scanner.db_session.delete(creative) self.scanner.db_session.commit() netlog = None logfile = "%s/https/netlog/%d.json" % (self.scanner.log_dir, creative_id) with open(logfile, "r") as fp: netlog = json.load(fp) return netlog def test_browser_view_image(self): """ Test if the browser can view an image. """ creative_id = 11111 snippet = '<img src="pixel.png">' netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + ".html") or url.endswith("pixel.png"): assert value["response"]["status"] == 200 self.assertFalse(value["error"]) else: self.fail() def test_browser_with_error_code(self): """ Test if the browser can catch 404 error code. """ creative_id = 11111 snippet = '<img src="notfound.png">' netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + ".html"): assert value["response"]["status"] == 200 self.assertFalse(value["error"]) elif url.endswith("notfound.png"): assert value["response"]["status"] == 404 assert value["error"]["errorCode"] == 203 else: self.fail() def test_browser_with_no_external_request(self): """ Test if the browser does not make unintended requests. """ creative_id = 11111 snippet = "<b>text</b>" netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 1 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + ".html"): assert value["response"]["status"] == 200 self.assertFalse(value["error"]) else: self.fail() def test_browser_with_requests_made_by_flash(self): """ Test if the browser can catch requests made by a flash content. """ creative_id = 11111 flash = "request_pixel.swf" snippet = ( " <object classid='clsid:D27CDB6E-AE6D-11cf-96B8-444553540000' " " codebase='http://download.macromedia.com/pub/shockwave/cabs/flash/swflash.cab#version=6,0,0,0' " " width='300' height='250'> " " <param name='movie' value='%s'> " " <param name='quality' value='high'> " " <param name='bgcolor' value='#ffffff'> " " <embed src='%s' quality='high' bgcolor='#ffffff' width='300' height='250' " " type='application/x-shockwave-flash' pluginpage='http://www.macromedia.com/go/getflashplayer'></embed> " " </object> " % (flash, flash) ) netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 3 for url, value in netlog.iteritems(): if ( url.endswith(str(creative_id) + ".html") or url.endswith("request_pixel.swf") or url.endswith("pixel.png") ): assert value["response"]["status"] == 200 self.assertFalse(value["error"]) else: self.fail() def test_browser_with_url_snippet(self): """ Test if the browser sends a request to the url if the snippet is a url. """ creative_id = 11111 snippet = "https://invalidurl/nocontent" netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 1 for url, value in netlog.iteritems(): if url.endswith("nocontent"): assert value["error"]["errorCode"] == 3 # Host not found. else: self.fail() def test_browser_with_private_network(self): """ Test if the browser blocks a request to a private network. When the request is aborted, PhantomJS will set 1 (Connection Refused Error) for the error code. """ creative_id = 11111 snippet = '<img src="https://localhost/pixel.png">' # Desable debug mode to check the private network access. self.scanner.debug = False netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + ".html"): assert value["response"]["status"] == 200 self.assertFalse(value["error"]) elif url.endswith("pixel.png"): assert value["error"]["errorCode"] == 999 # Private network access error else: self.fail() # Turn on the debug mode again for the next test case self.scanner.debug = True def test_browser_with_private_network_with_ip_url(self): """ Test if the browser blocks a request to a private network. When IP address is used, PhantomJS will set 301 (Protocol Unknown Error) for the error code. """ creative_id = 11111 snippet = '<img src="http://172.21.78.62/pixel.png">' # Desable debug mode to check the private network access. self.scanner.debug = False netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + ".html"): assert value["response"]["status"] == 200 self.assertFalse(value["error"]) elif url.endswith("pixel.png"): assert value["error"]["errorCode"] == 999 # Private network access error else: self.fail() # Turn on the debug mode again for the next test case self.scanner.debug = True
class ScannerTestCase(unittest.TestCase): """ Test the scanner module. """ def setUp(self): """ Load the scanner, which helps to set up the test environment. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir) os.chdir(project_dir) conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) self.scanner = Scanner(config) self.scanner.log_dir = os.path.join(self.scanner.logroot_dir, 'testlog') self.scanner.tmp_dir = 'testtmp' self.scanner.creative_db = 'sqlite:///%s/creative.db' % self.scanner.log_dir self.scanner.setup_environment() def tearDown(self): """ Delete the exising processes and clear the directory. """ adscan.fs.rmdirs(self.scanner.log_dir) self.scanner.workspace.delete() def test_no_certificate(self): """ Test if an exception is throws when no certificate is set at the browse_creatives step. """ original_cert = self.scanner.certificate_file self.scanner.certificate_file = None self.assertRaises(Exception, self.scanner.browse_creatives, 'https') self.scanner.certificate_file = 'cert.pem' self.assertRaises(Exception, self.scanner.browse_creatives, 'https') self.scanner.certificate_file = original_cert def test_no_privatekey(self): """ Test if an exception is throws when no private key is set at the browse_creatives step. """ original_pkey = self.scanner.privatekey_file self.scanner.privatekey_file = None self.assertRaises(Exception, self.scanner.browse_creatives, 'https') self.scanner.privatekey_file = 'pkey.pem' self.assertRaises(Exception, self.scanner.browse_creatives, 'https') self.scanner.privatekey_file = original_pkey def test_setup_environment_and_remove_temp_files(self): """ Test if the temporary files are deleted. """ conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) scanner = Scanner(config) scanner.log_dir = os.path.join(scanner.logroot_dir, 'test_remove_temp_files') scanner.tmp_dir = 'test_remove_temp_files' scanner.creative_db = 'sqlite:///%s/creative.db' % scanner.log_dir scanner.setup_environment() assert os.path.exists(scanner.workspace.dirname) scanner.remove_temp_files() self.assertFalse(os.path.exists(scanner.workspace.dirname)) adscan.fs.rmdirs(scanner.log_dir) adscan.fs.rmdirs(scanner.tmp_dir) def test_compress_log_file(self): """ Test if the log file is compressed. """ conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) scanner = Scanner(config) scanner.log_dir = os.path.join(scanner.logroot_dir, 'test_compress_log_file') tarname = '%s.tgz' % scanner.log_dir scanner.setup_environment() assert os.path.exists(scanner.log_dir) if os.path.exists(tarname): os.remove(tarname) scanner.compress_log_file() assert os.path.exists(tarname) adscan.fs.rmdirs(scanner.log_dir) os.remove(tarname)
def main(): """ The main method to launch the scanner. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) os.chdir(project_dir) conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) scanner = Scanner(config) scanner.setup_environment() if get_boolean(config, 'Steps', 'download_new_creative_ids'): scanner.download_new_creative_ids() if get_boolean(config, 'Steps', 'download_creatives'): scanner.download_creatives() if get_boolean(config, 'Steps', 'browse_creatives'): scanner.browse_creatives('https') if get_boolean(config, 'Steps', 'browse_creatives_over_http'): scanner.browse_creatives('http') if get_boolean(config, 'Steps', 'check_compliance'): scanner.check_compliance() if get_boolean(config, 'Steps', 'upload_creatives'): scanner.upload_creatives() if get_boolean(config, 'Steps', 'compress_log_file'): scanner.compress_log_file() if get_boolean(config, 'Steps', 'remove_temp_files'): scanner.remove_temp_files() sys.exit(0)
class BrowserTestCase(unittest.TestCase): """ Test the browser controller. """ @classmethod def copy_resources(cls, dest): """ Copy resource files to the """ for fp in os.listdir(TEST_RESOURCE): path = os.path.join(TEST_RESOURCE, fp) if (os.path.isfile(path)): shutil.copy(path, dest) def setUp(self): """ Load the scanner, which helps to set up the test environment. """ # Move to the project directory. project_dir = os.path.join(os.path.dirname(__file__), os.pardir) os.chdir(project_dir) conf_file = os.path.join('conf', CONFIG_FILE) config = SafeConfigParser() config.read(conf_file) self.scanner = Scanner(config) self.scanner.log_dir = os.path.join(self.scanner.logroot_dir, 'testlog') self.scanner.tmp_dir = 'testtmp' self.scanner.creative_db = 'sqlite:///%s/creative.db' % self.scanner.log_dir self.scanner.max_scan = 1 # Turn on the debug mode to temporarily allow the private network. self.scanner.debug = True self.scanner.setup_environment() self.copy_resources(self.scanner.workspace.dirname) self.hostname = socket.gethostname() def tearDown(self): """ Delete the exising processes and clear the directory. """ adscan.fs.rmdirs(self.scanner.log_dir) self.scanner.workspace.delete() def _browse_creative(self, creative_id, snippet): """ Set up a server and a browser to view the snippet. """ creatives = self.scanner.db_session.query(Creative).filter( Creative.created_at == datetime.date.today(), Creative.creative_id == creative_id).all() if creatives and len(creatives) > 0: self.scanner.db_session.delete(creatives[0]) self.scanner.db_session.commit() creative = Creative(creative_id=creative_id, snippet=snippet) self.scanner.db_session.add(creative) self.scanner.db_session.commit() self.scanner.browse_creatives('https') self.scanner.db_session.delete(creative) self.scanner.db_session.commit() netlog = None logfile = '%s/https/netlog/%d.json' % (self.scanner.log_dir, creative_id) with open(logfile, 'r') as fp: netlog = json.load(fp) return netlog def test_browser_view_image(self): """ Test if the browser can view an image. """ creative_id = 11111 snippet = '<img src="pixel.png">' netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + '.html') or url.endswith('pixel.png'): assert value['response']['status'] == 200 self.assertFalse(value['error']) else: self.fail() def test_browser_with_error_code(self): """ Test if the browser can catch 404 error code. """ creative_id = 11111 snippet = '<img src="notfound.png">' netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + '.html'): assert value['response']['status'] == 200 self.assertFalse(value['error']) elif url.endswith('notfound.png'): assert value['response']['status'] == 404 assert value['error']['errorCode'] == 203 else: self.fail() def test_browser_with_no_external_request(self): """ Test if the browser does not make unintended requests. """ creative_id = 11111 snippet = '<b>text</b>' netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 1 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + '.html'): assert value['response']['status'] == 200 self.assertFalse(value['error']) else: self.fail() def test_browser_with_requests_made_by_flash(self): """ Test if the browser can catch requests made by a flash content. """ creative_id = 11111 flash = 'request_pixel.swf' snippet = " <object classid='clsid:D27CDB6E-AE6D-11cf-96B8-444553540000' " \ " codebase='http://download.macromedia.com/pub/shockwave/cabs/flash/swflash.cab#version=6,0,0,0' " \ " width='300' height='250'> " \ " <param name='movie' value='%s'> " \ " <param name='quality' value='high'> " \ " <param name='bgcolor' value='#ffffff'> " \ " <embed src='%s' quality='high' bgcolor='#ffffff' width='300' height='250' " \ " type='application/x-shockwave-flash' pluginpage='http://www.macromedia.com/go/getflashplayer'></embed> " \ " </object> " % (flash, flash) netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 3 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + '.html') or url.endswith( 'request_pixel.swf') or url.endswith('pixel.png'): assert value['response']['status'] == 200 self.assertFalse(value['error']) else: self.fail() def test_browser_with_url_snippet(self): """ Test if the browser sends a request to the url if the snippet is a url. """ creative_id = 11111 snippet = 'https://invalidurl/nocontent' netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 1 for url, value in netlog.iteritems(): if url.endswith('nocontent'): assert value['error']['errorCode'] == 3 # Host not found. else: self.fail() def test_browser_with_private_network(self): """ Test if the browser blocks a request to a private network. When the request is aborted, PhantomJS will set 1 (Connection Refused Error) for the error code. """ creative_id = 11111 snippet = '<img src="https://localhost/pixel.png">' # Desable debug mode to check the private network access. self.scanner.debug = False netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + '.html'): assert value['response']['status'] == 200 self.assertFalse(value['error']) elif url.endswith('pixel.png'): assert value['error'][ 'errorCode'] == 999 # Private network access error else: self.fail() # Turn on the debug mode again for the next test case self.scanner.debug = True def test_browser_with_private_network_with_ip_url(self): """ Test if the browser blocks a request to a private network. When IP address is used, PhantomJS will set 301 (Protocol Unknown Error) for the error code. """ creative_id = 11111 snippet = '<img src="http://172.21.78.62/pixel.png">' # Desable debug mode to check the private network access. self.scanner.debug = False netlog = self._browse_creative(creative_id, snippet) assert netlog assert len(netlog) == 2 for url, value in netlog.iteritems(): if url.endswith(str(creative_id) + '.html'): assert value['response']['status'] == 200 self.assertFalse(value['error']) elif url.endswith('pixel.png'): assert value['error'][ 'errorCode'] == 999 # Private network access error else: self.fail() # Turn on the debug mode again for the next test case self.scanner.debug = True