def test_log_output_depth_one(self):
with no_stderrout():
paths = self.make_paths(
output_path='test/raw/log_outputs/dir_depth')
log_files_in_output(paths, depth=1)
self.assertIn('depth_1.txt', read_file(paths['output_statslog']))
self.assertIn('depth_2.txt', read_file(paths['output_statslog']))
def test_log_output_character(self):
with no_stderrout():
paths = self.make_paths()
inputs = link_inputs(paths, ['test/raw/log_sources/move_╬▓.txt'])
write_source_logs(paths, inputs)
self.check_makelog(paths)
self.assertIn('dir_╬▓', read_file(paths['source_maplog']))
self.assertIn('file_╬▓.txt', read_file(paths['source_statslog']))
def test_log_output_depth_one(self):
with no_stderrout():
paths = self.make_paths()
inputs = link_inputs(paths,
['test/raw/log_sources/move_depth.txt'])
write_source_logs(paths, inputs, depth=1)
self.assertIn('depth_1.txt', read_file(paths['source_statslog']))
self.assertIn('depth_2.txt', read_file(paths['source_statslog']))
def setUpTestCase(self, mocker):
# Setup query page
mocker.get('https://yande.re/post?page=1',
text=read_file('test/crawler/yandere/query_list_safe_test'))
# Setup detail page
mocker.get('https://yande.re/post/show/377505',
text=read_file('test/crawler/yandere/detail_page_377505'))
# Setup target file
mocker.get(
'https://files.yande.re/image/64c7aeade1c09807a1417cfe873b81e1/yande.re%20377505%20kimi_no_na_wa%20miyamizu_mitsuha%20seifuku.png',
content=read_as_binary('test/crawler/yandere/picture_target'))
def check_output(self,
paths,
pdf_name = ''):
pdf_name = pdf_name if pdf_name else ('%s_file.pdf' % self.app)
makelog = read_file(paths['makelog'])
self.assertTrue(os.path.isfile('test/output/%s' % pdf_name))
def test_log_output_space(self):
with no_stderrout():
paths = self.make_paths(
output_path='test/raw/log_outputs/dir space')
log_files_in_output(paths)
self.check_makelog(paths)
self.assertIn('file space.txt',
read_file(paths['output_statslog']))
def test_log_space(self):
with no_stderrout():
paths = self.make_paths()
program_name = 'test/raw/run_program/%s_script.%s' % (self.app, self.ext)
run_function(paths, program = program_name, log = 'test/output/log space.log')
self.check_output(paths)
self.assertIn('Test script complete', read_file('test/output/log space.log'))
def test_program_arg(self):
with no_stderrout():
paths = self.make_paths()
program_name = 'test/raw/run_program/%s_script_arg.%s' % (self.app, self.ext)
run_function(paths, program = program_name, args = self.arg)
self.check_output(paths)
output = read_file('test/output/output.csv')
self.assertTrue(re.search('arg', output))
def test_log_output_local_string(self):
with no_stderrout():
paths = self.make_paths(
output_path='test/raw/log_outputs/dir',
output_local_path='test/raw/log_outputs/dir_local')
log_files_in_output(paths)
self.check_logs(paths)
self.assertIn('file_local.txt',
read_file(paths['output_statslog']))
def check_output(self):
# Check output directory exists
self.assertTrue(os.path.exists('test/output/'))
# Check output directory empty
file_list = os.listdir('test/output/')
self.assertEqual(len(file_list), 0)
# Check status message printed
self.assertIn('Cleared:', read_file('test/stdout.txt'))
def test_quiet(self):
self.make_output()
# Redirect stdout
with open('test/stdout.txt', 'w') as f:
with redirect_stdout(f):
remove_dir(['test/output/'], quiet=True)
# Check stdout
self.assertNotIn('Removed:', read_file('test/stdout.txt'))
def test_log_character(self):
with no_stderrout():
paths = self.make_paths()
program_name = 'test/raw/run_program/%s_file.%s' % (self.app,
self.ext)
run_function(paths,
program=program_name,
log='test/output/log_╬▓.log')
self.check_output(paths)
self.assertIn('Executing command', read_file('test/output/log_╬▓.log'))
def test_run_with_default_set(self, mocker):
"""
Enables safe selector
"""
self.setUpTestCase(mocker)
crawler = YandereCrawler(page_limit=1, score_filter=20)
crawler.run()
self.assertTrue(path.isfile('yandere-377505.png'))
self.assertEqual(read_file('yandere-377505.png'),
'If you read this, it is right!\n')
remove('yandere-377505.png')
remove('yandere.log')
def test_parse_query_test_sample(self):
sample = read_file('test/crawler/konachan/query_page_sample')
result = parse_query_page(sample)
self.assertIn({
'id': 234859,
'rating': 'Safe',
'score': 14,
'tags': ['animal ears', 'mazume', 'original'],
'link': 'http://konachan.net/post/show/234859',
}, result)
self.assertIn({
'id': 234858,
'rating': 'Safe',
'score': 55,
'tags': ['cropped', 'ichinose shiki', 'idolmaster',
'idolmaster cinderella girls', 'mossi'],
'link': 'http://konachan.net/post/show/234858'
}, result)
def test_parse_detail_page(self):
sample = read_file('test/crawler/konachan/detail_page_png')
result = parse_detail_page(sample)
self.assertIn({
'type': 'png',
'link': 'http://konachan.net/image/507cc0ad7ced2572b81c0aa5372ec30'
'9/Konachan.com%20-%20230187%20animal%20animal_ears%20biki'
'ni%20blush%20breasts%20cat%20catgirl%20cleavage%20fang%20'
'kantoku%20kurumi_%28kantoku%29%20long_hair%20original%20p'
'hotoshop%20scan%20skirt%20swimsuit%20tail%20white.png'
}, result)
self.assertIn({
'type': 'jpeg',
'link': 'http://konachan.net/jpeg/507cc0ad7ced2572b81c0aa5372ec309'
'/Konachan.com%20-%20230187%20animal%20animal_ears%20bikini%20blus'
'h%20breasts%20cat%20catgirl%20cleavage%20fang%20kantoku%20kurumi_'
'%28kantoku%29%20long_hair%20original%20photoshop%20scan%20skirt'
'%20swimsuit%20tail%20white.jpg'
}, result)
def test_dir_not_exist(self):
"""
Note
----
When removing a directory that does not exist, nothing
is ever printed.
"""
self.assertFalse(os.path.exists('test/output/'))
# Redirect stdout
with open('test/stdout.txt', 'w') as f:
with redirect_stdout(f):
remove_dir(['test/output/'], quiet=False)
# Check stdout
self.assertNotIn('Removed:', read_file('test/stdout.txt'))
self.check_output()
def check_logs(self, paths):
self.check_makelog(paths)
self.assertIn('dir', read_file(paths['source_maplog']))
self.assertIn('file.txt', read_file(paths['source_statslog']))
def check_makelog(self, paths):
self.assertIn('Source logs successfully written!',
read_file(paths['makelog']))
def check_makelog(self, paths):
makelog = read_file(paths['makelog'])
self.assertTrue(re.search('Makelog started: ', makelog))
self.assertTrue(re.search('Hello, World!', makelog))
self.assertTrue(re.search('Makelog ended: ', makelog))
def check_makelog(self, paths):
message = 'Input links successfully created!'
self.assertIn(message, read_file(paths['makelog']))
def check_logs(self, paths):
self.check_makelog(paths)
self.assertIn('file.txt', read_file(paths['output_statslog']))
def check_output(self, paths):
makelog = read_file(paths['makelog'])
self.assertTrue(re.search('Test script complete', makelog))
self.assertTrue(os.path.isfile('test/output/output.csv'))
def check_makelog(self, paths):
message = 'External copies successfully created!'
self.assertIn(message, read_file(paths['makelog']))
def setUp(self):
self.test_html = read_file('test/crawler/yandere/query_list_test_1')
