def test_delete(self):
with DataGenerator() as test_scenario:
duplicates = ('1/a.data', '2/a.data', '3/a.data', '4/a.data')
test_scenario.create_duplicates(duplicates, size=10)
connection_string = ':memory:'
with connection_factory(connection_string) as conn, \
repository(conn) as repo, patch('os.remove') as mock_path:
DupScanner(repo).scan((test_scenario.root_path,))
for f in duplicates[:-1]:
deleteable = test_scenario.abs_path(f)
repo.delete_file(deleteable)
mock_path.assert_called_with(deleteable)
mock_path.reset_mock()
try:
not_deleteable = test_scenario.abs_path(duplicates[-1])
repo.delete_file(not_deleteable)
assert False, 'Should have raised an exception'
except Exception as e:
expected = Exception('409 Can\'t delete a file without duplicates: {}'.format(not_deleteable))
assert repr(expected) == repr(e)
assert not mock_path.called, 'File should have not been deleted'
mock_path.reset_mock
def test_delete(self):
with DataGenerator() as test_scenario:
duplicates = ('1/a.data', '2/a.data', '3/a.data', '4/a.data')
test_scenario.create_duplicates(duplicates, size=10)
connection_string = ':memory:'
with connection_factory(connection_string) as conn, \
repository(conn) as repo, patch('os.remove') as mock_path:
DupScanner(repo).scan((test_scenario.root_path, ))
for f in duplicates[:-1]:
deleteable = test_scenario.abs_path(f)
repo.delete_file(deleteable)
mock_path.assert_called_with(deleteable)
mock_path.reset_mock()
try:
not_deleteable = test_scenario.abs_path(duplicates[-1])
repo.delete_file(not_deleteable)
assert False, 'Should have raised an exception'
except Exception as e:
expected = Exception(
'409 Can\'t delete a file without duplicates: {}'.
format(not_deleteable))
assert repr(expected) == repr(e)
assert not mock_path.called, 'File should have not been deleted'
mock_path.reset_mock
def test_false_duplicates_in_path(self):
with DataGenerator() as test_scenario:
duplicates_expected = set()
uniques_expected = set()
uniques_expected.add(test_scenario.create_file('1/a.data', size=4097))
uniques_expected.add(test_scenario.create_file('1/b.data', size=4096))
uniques_expected.add(test_scenario.create_file('1/c.data', size=512))
uniques_expected.add(test_scenario.create_file('1/e.data', size=4069))
uniques_expected.add(test_scenario.create_file('1/x.data', size=2048, readable=False))
uniques_expected.add(test_scenario.create_file('1/xx.data', size=2048, readable=False))
# TODO: This behavior is not consistent with don't follow links
test_scenario.symlink('1/', 'links/1')
uniques_expected.add(path.join(test_scenario.root_path, 'links/1/a.data'))
uniques_expected.add(path.join(test_scenario.root_path, 'links/1/b.data'))
uniques_expected.add(path.join(test_scenario.root_path, 'links/1/c.data'))
uniques_expected.add(path.join(test_scenario.root_path, 'links/1/e.data'))
uniques_expected.add(path.join(test_scenario.root_path, 'links/1/x.data'))
uniques_expected.add(path.join(test_scenario.root_path, 'links/1/xx.data'))
connection_string = ':memory:'
with connection_factory(connection_string) as conn, repository(conn) as repo:
# DupScanner(repo).scan((test_scenario.root_path,))
DupScanner(repo).scan((
path.join(test_scenario.root_path, '1'),
path.join(test_scenario.root_path, 'links'),
))
duplicates_found = {fullname for hash, size, fullname, path, abspath in repo.findBy_duplicate_hash()}
duplicates_missing = duplicates_expected - duplicates_found
assert not duplicates_missing, 'Expected duplicate elements were not found: {}'.format(
duplicates_missing)
duplicates_unexpected = duplicates_found - duplicates_expected
assert not duplicates_unexpected, 'Unexpected duplicate elements were found: {}'.format(
duplicates_unexpected)
assert duplicates_expected == duplicates_found, \
'Expected duplicate set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(duplicates_expected, duplicates_found)
uniques_found = {fullname for hash, size, fullname,
path, abspath in repo.findBy_unique_hash()}
uniques_missing = uniques_expected - uniques_found
assert not uniques_missing, 'Expected unique elements were not found: {}'.format(
uniques_missing)
uniques_unexpected = uniques_found - uniques_expected
assert not uniques_unexpected, 'Unexpected unique elements were found: {}'.format(
uniques_unexpected)
assert uniques_expected == uniques_found, \
'Expected unique set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(uniques_expected, uniques_found)
def test_dont_follow_links(self):
with DataGenerator() as test_scenario:
duplicates_expected = set()
uniques_expected = set()
uniques_expected.add(test_scenario.create_file('1/a.data', size=4097))
uniques_expected.add(test_scenario.create_file('2/b.data', size=4096))
uniques_expected.add(test_scenario.create_file('3/c.data', size=512))
uniques_expected.add(test_scenario.create_file('3/e.data', size=4069))
uniques_expected.add(test_scenario.create_file('4/x.data', size=2048, readable=False))
uniques_expected.add(test_scenario.create_file('4/xx.data', size=2048, readable=False))
ignored_links = set()
ignored_links.add(test_scenario.symlink('1/a.data', '2/lnk-a.data'))
ignored_links.add(test_scenario.symlink('2/b.data', '1/lnk-b.data'))
ignored_links.add(test_scenario.symlink('4/x.data', '4/lnk-x.data'))
ignored_links.add(test_scenario.symlink('4/xx.data', '4/lnk-xx.data'))
connection_string = ':memory:'
with connection_factory(connection_string) as conn, repository(conn) as repo:
DupScanner(repo).scan((test_scenario.root_path,))
duplicates_found = {fullname for hash, size, fullname, path, abspath in repo.findBy_duplicate_hash()}
duplicates_missing = duplicates_expected - duplicates_found
assert not duplicates_missing, 'Expected duplicate elements were not found: {}'.format(
duplicates_missing)
duplicates_unexpected = duplicates_found - duplicates_expected
assert not duplicates_unexpected, 'Unexpected duplicate elements were found: {}'.format(
duplicates_unexpected)
assert duplicates_expected == duplicates_found, \
'Expected duplicate set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(duplicates_expected, duplicates_found)
uniques_found = {fullname for hash, size, fullname,
path, abspath in repo.findBy_unique_hash()}
uniques_missing = uniques_expected - uniques_found
assert not uniques_missing, 'Expected unique elements were not found: {}'.format(
uniques_missing)
uniques_unexpected = uniques_found - uniques_expected
assert not uniques_unexpected, 'Unexpected unique elements were found: {}'.format(
uniques_unexpected)
assert uniques_expected == uniques_found, \
'Expected unique set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(uniques_expected, uniques_found)
links_unexpected = (uniques_found | duplicates_found) & ignored_links
assert not links_unexpected, 'Unexpected links found'
def test_happy_path_with_nested_dirs(self):
with DataGenerator() as test_scenario:
duplicates_expected = test_scenario.create_duplicates(
('1/a.data', '2/a.data', '3/a.data', '4/a.data'), size=4097) # 4097 = block size + 1
duplicates_expected |= test_scenario.create_duplicates(
('2/aa.data', '4/aa.data'), size=4096)
duplicates_expected |= test_scenario.create_duplicates(
('1/aaa.data', '4/aaa.data'), size=1024)
duplicates_expected |= test_scenario.create_duplicates(
('1/aaaa.data', '3/aaaa.data'), size=0)
uniques_expected = set()
uniques_expected.add(test_scenario.create_file('1/b.data', size=4097))
uniques_expected.add(test_scenario.create_file('2/c.data', size=4096))
uniques_expected.add(test_scenario.create_file('3/d.data', size=512))
uniques_expected.add(test_scenario.create_file('1/e.data', size=4097))
uniques_expected.add(test_scenario.create_file('4/x.data', size=2048, readable=False))
uniques_expected.add(test_scenario.create_file('4/xx.data', size=2048, readable=False))
connection_string = ':memory:'
with connection_factory(connection_string) as conn, repository(conn) as repo:
DupScanner(repo).scan((test_scenario.root_path, test_scenario.abs_path('1/')))
duplicates_found = {abspath for hash, size, fullname, path, abspath in repo.findBy_duplicate_hash()}
duplicates_missing = duplicates_expected - duplicates_found
assert not duplicates_missing, 'Expected duplicate elements were not found: {}'.format(
duplicates_missing)
duplicates_unexpected = duplicates_found - duplicates_expected
assert not duplicates_unexpected, 'Unexpected duplicate elements were found: {}'.format(
duplicates_unexpected)
assert duplicates_expected == duplicates_found, 'Expected duplicate set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(
duplicates_expected, duplicates_found)
uniques_found = {abspath for hash, size, fullname, path, abspath in repo.findBy_unique_hash()}
uniques_missing = uniques_expected - uniques_found
assert not uniques_missing, 'Expected unique elements were not found: {}'.format(
uniques_missing)
uniques_unexpected = uniques_found - uniques_expected
assert not uniques_unexpected, 'Unexpected unique elements were found: {}'.format(
uniques_unexpected)
assert uniques_expected == uniques_found, 'Expected unique set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(
uniques_expected, uniques_found)
def test_false_duplicates_in_path(self):
with DataGenerator() as test_scenario:
duplicates_expected = set()
uniques_expected = set()
uniques_expected.add(
test_scenario.create_file('1/a.data', size=4097))
uniques_expected.add(
test_scenario.create_file('1/b.data', size=4096))
uniques_expected.add(
test_scenario.create_file('1/c.data', size=512))
uniques_expected.add(
test_scenario.create_file('1/e.data', size=4069))
uniques_expected.add(
test_scenario.create_file('1/x.data',
size=2048,
readable=False))
uniques_expected.add(
test_scenario.create_file('1/xx.data',
size=2048,
readable=False))
# TODO: This behavior is not consistent with don't follow links
test_scenario.symlink('1/', 'links/1')
uniques_expected.add(
path.join(test_scenario.root_path, 'links/1/a.data'))
uniques_expected.add(
path.join(test_scenario.root_path, 'links/1/b.data'))
uniques_expected.add(
path.join(test_scenario.root_path, 'links/1/c.data'))
uniques_expected.add(
path.join(test_scenario.root_path, 'links/1/e.data'))
uniques_expected.add(
path.join(test_scenario.root_path, 'links/1/x.data'))
uniques_expected.add(
path.join(test_scenario.root_path, 'links/1/xx.data'))
connection_string = ':memory:'
with connection_factory(connection_string) as conn, repository(
conn) as repo:
# DupScanner(repo).scan((test_scenario.root_path,))
DupScanner(repo).scan((
path.join(test_scenario.root_path, '1'),
path.join(test_scenario.root_path, 'links'),
))
duplicates_found = {
fullname
for hash, size, fullname, path, abspath in
repo.findBy_duplicate_hash()
}
duplicates_missing = duplicates_expected - duplicates_found
assert not duplicates_missing, 'Expected duplicate elements were not found: {}'.format(
duplicates_missing)
duplicates_unexpected = duplicates_found - duplicates_expected
assert not duplicates_unexpected, 'Unexpected duplicate elements were found: {}'.format(
duplicates_unexpected)
assert duplicates_expected == duplicates_found, \
'Expected duplicate set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(duplicates_expected, duplicates_found)
uniques_found = {
fullname
for hash, size, fullname, path, abspath in
repo.findBy_unique_hash()
}
uniques_missing = uniques_expected - uniques_found
assert not uniques_missing, 'Expected unique elements were not found: {}'.format(
uniques_missing)
uniques_unexpected = uniques_found - uniques_expected
assert not uniques_unexpected, 'Unexpected unique elements were found: {}'.format(
uniques_unexpected)
assert uniques_expected == uniques_found, \
'Expected unique set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(uniques_expected, uniques_found)
def test_dont_follow_links(self):
with DataGenerator() as test_scenario:
duplicates_expected = set()
uniques_expected = set()
uniques_expected.add(
test_scenario.create_file('1/a.data', size=4097))
uniques_expected.add(
test_scenario.create_file('2/b.data', size=4096))
uniques_expected.add(
test_scenario.create_file('3/c.data', size=512))
uniques_expected.add(
test_scenario.create_file('3/e.data', size=4069))
uniques_expected.add(
test_scenario.create_file('4/x.data',
size=2048,
readable=False))
uniques_expected.add(
test_scenario.create_file('4/xx.data',
size=2048,
readable=False))
ignored_links = set()
ignored_links.add(test_scenario.symlink('1/a.data',
'2/lnk-a.data'))
ignored_links.add(test_scenario.symlink('2/b.data',
'1/lnk-b.data'))
ignored_links.add(test_scenario.symlink('4/x.data',
'4/lnk-x.data'))
ignored_links.add(
test_scenario.symlink('4/xx.data', '4/lnk-xx.data'))
connection_string = ':memory:'
with connection_factory(connection_string) as conn, repository(
conn) as repo:
DupScanner(repo).scan((test_scenario.root_path, ))
duplicates_found = {
fullname
for hash, size, fullname, path, abspath in
repo.findBy_duplicate_hash()
}
duplicates_missing = duplicates_expected - duplicates_found
assert not duplicates_missing, 'Expected duplicate elements were not found: {}'.format(
duplicates_missing)
duplicates_unexpected = duplicates_found - duplicates_expected
assert not duplicates_unexpected, 'Unexpected duplicate elements were found: {}'.format(
duplicates_unexpected)
assert duplicates_expected == duplicates_found, \
'Expected duplicate set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(duplicates_expected, duplicates_found)
uniques_found = {
fullname
for hash, size, fullname, path, abspath in
repo.findBy_unique_hash()
}
uniques_missing = uniques_expected - uniques_found
assert not uniques_missing, 'Expected unique elements were not found: {}'.format(
uniques_missing)
uniques_unexpected = uniques_found - uniques_expected
assert not uniques_unexpected, 'Unexpected unique elements were found: {}'.format(
uniques_unexpected)
assert uniques_expected == uniques_found, \
'Expected unique set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(uniques_expected, uniques_found)
links_unexpected = (uniques_found
| duplicates_found) & ignored_links
assert not links_unexpected, 'Unexpected links found'
def test_happy_path_with_nested_dirs(self):
with DataGenerator() as test_scenario:
duplicates_expected = test_scenario.create_duplicates(
('1/a.data', '2/a.data', '3/a.data', '4/a.data'),
size=4097) # 4097 = block size + 1
duplicates_expected |= test_scenario.create_duplicates(
('2/aa.data', '4/aa.data'), size=4096)
duplicates_expected |= test_scenario.create_duplicates(
('1/aaa.data', '4/aaa.data'), size=1024)
duplicates_expected |= test_scenario.create_duplicates(
('1/aaaa.data', '3/aaaa.data'), size=0)
uniques_expected = set()
uniques_expected.add(
test_scenario.create_file('1/b.data', size=4097))
uniques_expected.add(
test_scenario.create_file('2/c.data', size=4096))
uniques_expected.add(
test_scenario.create_file('3/d.data', size=512))
uniques_expected.add(
test_scenario.create_file('1/e.data', size=4097))
uniques_expected.add(
test_scenario.create_file('4/x.data',
size=2048,
readable=False))
uniques_expected.add(
test_scenario.create_file('4/xx.data',
size=2048,
readable=False))
connection_string = ':memory:'
with connection_factory(connection_string) as conn, repository(
conn) as repo:
DupScanner(repo).scan(
(test_scenario.root_path, test_scenario.abs_path('1/')))
duplicates_found = {
abspath
for hash, size, fullname, path, abspath in
repo.findBy_duplicate_hash()
}
duplicates_missing = duplicates_expected - duplicates_found
assert not duplicates_missing, 'Expected duplicate elements were not found: {}'.format(
duplicates_missing)
duplicates_unexpected = duplicates_found - duplicates_expected
assert not duplicates_unexpected, 'Unexpected duplicate elements were found: {}'.format(
duplicates_unexpected)
assert duplicates_expected == duplicates_found, 'Expected duplicate set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(
duplicates_expected, duplicates_found)
uniques_found = {
abspath
for hash, size, fullname, path, abspath in
repo.findBy_unique_hash()
}
uniques_missing = uniques_expected - uniques_found
assert not uniques_missing, 'Expected unique elements were not found: {}'.format(
uniques_missing)
uniques_unexpected = uniques_found - uniques_expected
assert not uniques_unexpected, 'Unexpected unique elements were found: {}'.format(
uniques_unexpected)
assert uniques_expected == uniques_found, 'Expected unique set doesn\'t match found set. \n Expected: {}\n Found: {}'.format(
uniques_expected, uniques_found)
def setUp(self):
self.conn = sqlite3.connect(":memory:")
self.repo = dupscanner.repository(self.conn)
self.repo.create_schema()
def setUp(self):
self.conn = sqlite3.connect(":memory:")
self.repo = dupscanner.repository(self.conn)
self.repo.create_schema()
def main():
args_parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser()
parser.set_defaults(action='duplicates')
parser.add_argument("path", help="Path where to look for duplicates", nargs='+')
parser.add_argument("-d", "--database", help="Stores a temporary SQLite database in a file", default=":memory:")
parser.add_argument("-lf", "--log-format", help="Logging format", default='%(message)s')
parser.add_argument("-l", "--log", help="File to output the log messages")
# parser.add_argument("-u", "--unique", help="Find unique files", action="store_const", const='unique', dest='action')
parser.add_argument("-u", "--unique", help="Find unique files", action="store_true")
parser.add_argument("-o", "--output-file", help="Output file (default: stdout)", default='-', type=file('w', encoding='UTF-8'))
g = parser.add_mutually_exclusive_group()
g.add_argument(
"-t",
"--template",
help="""Output template. Variables ${hash}, ${size}, ${fullname}, ${path}, ${abspath}, ${realpath}
will be replaced with the actual values""",
default="${hash}\t${size}\t${fullname}"
)
g.add_argument(
"-e",
"--evaluate",
help="""For each result, evaluate the given python code to process the output.
Variables hash, size, filename and output_file will be bounded to the appropiate values"""
)
g.add_argument(
"-x",
"--execute-script",
help="""Executes the given python script to process the results.
Variables results and output_file will be bounded to an iterator and the appropiate output stream"""
)
g.add_argument(
"-p",
"--pretty-print",
action="store_true",
help="Groups the results by hash and file size and displays a pretty output"
)
g.add_argument(
"-i",
"--interactive",
action="store_true",
help="Interactive mode"
)
parser.add_argument(
"-v",
"--verbosity",
help="Verbosity level (default: WARN)",
default='WARN',
choices=['DEBUG','INFO','WARN','ERROR','CRITICAL'],
type=lambda level: level.upper()
)
args = parser.parse_args()
logging.basicConfig(
level=args.verbosity,
format=args.log_format,
filename=args.log
)
connection_string = args.database
path = args.path
action = args.action
template = args.template
with connection_factory(connection_string) as conn, repository(conn) as repo, args.output_file as output_file:
dupscanner = DupScanner(repo)
# command = { 'unique': dupscanner.find_unique, 'duplicates': dupscanner.find_duplicates }
# results = command[action](path)
results = dupscanner.find_unique(path) if args.unique else dupscanner.find_duplicates(path)
if args.execute_script: exec_script(args.execute_script, output_file, results, conn, repo)
elif args.interactive: run_server(repo)
elif args.evaluate: func = exec_command(args.evaluate, output_file, results)
elif args.pretty_print: func = pretty_print(results, output_file)
else: plain_print(template, results, output_file)