def test_new_file_and_clobber(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url('/static/my_file.txt')])
with cd_tempdir() as temp_dir:
engine = Builder(args).build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
expected_filename = os.path.join(temp_dir, 'my_file.txt')
self.assertTrue(os.path.exists(expected_filename))
with open(expected_filename, 'rb') as in_file:
self.assertIn(b'END', in_file.read())
engine = Builder(args).build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
expected_filename = os.path.join(temp_dir, 'my_file.txt.1')
self.assertTrue(os.path.exists(expected_filename))
def test_timestamping_hit_orig(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([
self.get_url('/lastmod'),
'--timestamping'
])
with cd_tempdir() as temp_dir:
filename = os.path.join(temp_dir, 'lastmod')
filename_orig = os.path.join(temp_dir, 'lastmod')
with open(filename, 'wb') as out_file:
out_file.write(b'HI')
with open(filename_orig, 'wb') as out_file:
out_file.write(b'HI')
os.utime(filename_orig, (631152000, 631152000))
builder = Builder(args)
engine = builder.build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
with open(filename, 'rb') as in_file:
self.assertEqual(b'HI', in_file.read())
with open(filename_orig, 'rb') as in_file:
self.assertEqual(b'HI', in_file.read())
def test_timestamping_hit_orig(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args(
[self.get_url('/lastmod'), '--timestamping'])
with cd_tempdir() as temp_dir:
filename = os.path.join(temp_dir, 'lastmod')
filename_orig = os.path.join(temp_dir, 'lastmod')
with open(filename, 'wb') as out_file:
out_file.write(b'HI')
with open(filename_orig, 'wb') as out_file:
out_file.write(b'HI')
os.utime(filename_orig, (631152000, 631152000))
builder = Builder(args)
engine = builder.build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
with open(filename, 'rb') as in_file:
self.assertEqual(b'HI', in_file.read())
with open(filename_orig, 'rb') as in_file:
self.assertEqual(b'HI', in_file.read())
def test_new_file_and_clobber(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url('/static/my_file.txt')])
with cd_tempdir() as temp_dir:
engine = Builder(args).build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
expected_filename = os.path.join(temp_dir, 'my_file.txt')
self.assertTrue(os.path.exists(expected_filename))
with open(expected_filename, 'rb') as in_file:
self.assertIn(b'END', in_file.read())
engine = Builder(args).build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
expected_filename = os.path.join(temp_dir, 'my_file.txt.1')
self.assertTrue(os.path.exists(expected_filename))
def test_anti_clobber_dir_path(self):
with cd_tempdir():
self.assertEqual('a', anti_clobber_dir_path('a'))
with cd_tempdir():
self.assertEqual('a/b/c/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g/'))
with cd_tempdir():
self.assertEqual('a/b/c/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g'))
with cd_tempdir():
with open('a', 'w'):
pass
self.assertEqual('a.d/b/c/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g'))
with cd_tempdir():
os.makedirs('a/b')
with open('a/b/c', 'w'):
pass
self.assertEqual('a/b/c.d/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g'))
with cd_tempdir():
os.makedirs('a/b/c/d/e/f')
with open('a/b/c/d/e/f/g', 'w'):
pass
self.assertEqual('a/b/c/d/e/f/g.d',
anti_clobber_dir_path('a/b/c/d/e/f/g'))
def test_dir_or_file(self):
arg_parser = AppArgumentParser()
with cd_tempdir():
args = arg_parser.parse_args([
self.get_url('/dir_or_file'),
'--recursive',
'--no-host-directories',
])
engine = Builder(args).build()
os.mkdir('dir_or_file')
exit_code = yield engine()
self.assertEqual(0, exit_code)
print(list(os.walk('.')))
self.assertTrue(os.path.isdir('dir_or_file'))
self.assertTrue(os.path.isfile('dir_or_file.f'))
with cd_tempdir():
args = arg_parser.parse_args([
self.get_url('/dir_or_file/'),
'--recursive',
'--no-host-directories',
])
engine = Builder(args).build()
with open('dir_or_file', 'wb'):
pass
exit_code = yield engine()
self.assertEqual(0, exit_code)
print(list(os.walk('.')))
self.assertTrue(os.path.isdir('dir_or_file.d'))
self.assertTrue(os.path.isfile('dir_or_file.d/index.html'))
self.assertTrue(os.path.isfile('dir_or_file'))
def test_dir_or_file(self):
arg_parser = AppArgumentParser()
with cd_tempdir():
args = arg_parser.parse_args([
self.get_url('/dir_or_file'),
'--recursive',
'--no-host-directories',
])
engine = Builder(args).build()
os.mkdir('dir_or_file')
exit_code = yield engine()
self.assertEqual(0, exit_code)
print(list(os.walk('.')))
self.assertTrue(os.path.isdir('dir_or_file'))
self.assertTrue(os.path.isfile('dir_or_file.f'))
with cd_tempdir():
args = arg_parser.parse_args([
self.get_url('/dir_or_file/'),
'--recursive',
'--no-host-directories',
])
engine = Builder(args).build()
with open('dir_or_file', 'wb'):
pass
exit_code = yield engine()
self.assertEqual(0, exit_code)
print(list(os.walk('.')))
self.assertTrue(os.path.isdir('dir_or_file.d'))
self.assertTrue(os.path.isfile('dir_or_file.d/index.html'))
self.assertTrue(os.path.isfile('dir_or_file'))
def test_css_converter(self):
with cd_tempdir() as temp_dir:
url_table = URLTable()
css_filename = os.path.join(temp_dir, 'styles.css')
image_filename = os.path.join(temp_dir, 'image.png')
new_css_filename = os.path.join(temp_dir, 'styles.css-new')
url_table.add([
'http://example.com/styles.css',
'http://example.com/image.png',
'http://example.com/cat.jpg',
'http://example.com/cat.jpg',
])
url_table.update(
'http://example.com/styles.css',
status=Status.done,
link_type='css',
filename=os.path.relpath(css_filename, temp_dir)
)
url_table.update(
'http://example.com/image.png',
status=Status.done,
filename=os.path.relpath(image_filename, temp_dir)
)
with open(css_filename, 'w') as out_file:
out_file.write(CSS_TEXT)
with open(image_filename, 'wb'):
pass
converter = CSSConverter(url_table)
converter.convert(
css_filename, new_css_filename,
base_url='http://example.com/styles.css'
)
with open(new_css_filename, 'r') as in_file:
converted_text = in_file.read()
self.assertIn("url('image.png')", converted_text)
self.assertIn("url('http://example.com/cat.jpg')", converted_text)
def test_file_continue(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args(
[self.get_url('/static/my_file.txt'), '--continue', '--debug'])
with cd_tempdir() as temp_dir:
filename = os.path.join(temp_dir, 'my_file.txt')
with open(filename, 'wb') as out_file:
out_file.write(b'START')
engine = Builder(args).build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
with open(filename, 'rb') as in_file:
data = in_file.read()
self.assertEqual('54388a281352fdb2cfa66009ac0e35dd8916af7c',
hashlib.sha1(data).hexdigest())
def test_file_continue(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url('/static/my_file.txt'),
'--continue', '--debug'])
with cd_tempdir() as temp_dir:
filename = os.path.join(temp_dir, 'my_file.txt')
with open(filename, 'wb') as out_file:
out_file.write(b'START')
engine = Builder(args).build()
exit_code = yield engine()
self.assertEqual(0, exit_code)
with open(filename, 'rb') as in_file:
data = in_file.read()
self.assertEqual('54388a281352fdb2cfa66009ac0e35dd8916af7c',
hashlib.sha1(data).hexdigest())
def test_anti_clobber_dir_path(self):
with cd_tempdir():
self.assertEqual(
'a',
anti_clobber_dir_path('a')
)
with cd_tempdir():
self.assertEqual(
'a/b/c/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g/')
)
with cd_tempdir():
self.assertEqual(
'a/b/c/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g')
)
with cd_tempdir():
with open('a', 'w'):
pass
self.assertEqual(
'a.d/b/c/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g')
)
with cd_tempdir():
os.makedirs('a/b')
with open('a/b/c', 'w'):
pass
self.assertEqual(
'a/b/c.d/d/e/f/g',
anti_clobber_dir_path('a/b/c/d/e/f/g')
)
with cd_tempdir():
os.makedirs('a/b/c/d/e/f')
with open('a/b/c/d/e/f/g', 'w'):
pass
self.assertEqual(
'a/b/c/d/e/f/g.d',
anti_clobber_dir_path('a/b/c/d/e/f/g')
)
def test_timestamping_miss(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([
self.get_url('/lastmod'),
'--timestamping'
])
with cd_tempdir() as temp_dir:
filename = os.path.join(temp_dir, 'lastmod')
with open(filename, 'wb') as out_file:
out_file.write(b'HI')
os.utime(filename, (636249600, 636249600))
builder = Builder(args)
app = builder.build()
exit_code = yield app.run()
self.assertEqual(0, exit_code)
with open(filename, 'rb') as in_file:
self.assertEqual(b'HELLO', in_file.read())
def test_xhtml_converter(self):
with cd_tempdir() as temp_dir:
url_table = URLTable()
image_filename = os.path.join(temp_dir, 'image.png')
tubes_filename = os.path.join(temp_dir, 'tubes.html')
ferret_filename = os.path.join(temp_dir, 'ferret.jpg')
url_table.add([
'http://example.com/styles.css',
'http://example.com/image.png',
'http://example.com/cat.jpg',
'http://example.com/fox.jpg',
'http://example.com/ferret.jpg',
'http://example.com/tubes.html',
])
url_table.update(
'http://example.com/styles.css',
status=Status.done,
link_type='css'
)
url_table.update(
'http://example.com/image.png',
status=Status.done,
filename=os.path.relpath(image_filename, temp_dir)
)
url_table.update(
'http://example.com/tubes.html',
status=Status.done,
filename=os.path.relpath(tubes_filename, temp_dir)
)
url_table.update(
'http://example.com/ferret.jpg',
status=Status.done,
filename=os.path.relpath(ferret_filename, temp_dir)
)
html_filename = os.path.join(temp_dir, 'index.html')
new_html_filename = os.path.join(temp_dir, 'index.html-new')
with open(html_filename, 'w') as out_file:
out_file.write(XHTML_TEXT)
for filename in [image_filename, tubes_filename, ferret_filename]:
with open(filename, 'wb'):
pass
converter = HTMLConverter(url_table)
converter.convert(
html_filename, new_html_filename,
base_url='http://example.com/index.html'
)
with open(new_html_filename, 'r') as in_file:
converted_text = in_file.read()
self.assertIn("url('image.png')", converted_text)
self.assertIn("url('http://example.com/cat.jpg')", converted_text)
self.assertIn('"tubes.html"', converted_text)
self.assertIn('"http://example.com/lol.html"', converted_text)
self.assertIn("url('http://example.com/fox.jpg')", converted_text)
self.assertIn("url('ferret.jpg')", converted_text)
self.assertIn("hello world!!", converted_text)
self.assertIn("<hr/>", converted_text)
