def test_find(self):
path = "/home/test/downloads"
size_filter = core.algorithm.SizeFilter()
char_filter = core.algorithm.CharacterFilter()
filters = [
size_filter,
char_filter
]
dup_finder = core.dup_finder.DupFinder([path], filters)
file_a = File("/home/test/downloads/a")
file_b = File("/home/test/downloads/b")
file_insts = [file_a, file_b]
when(core.dup_finder.Walker).walk([path]).thenReturn(file_insts)
when(size_filter).find().thenReturn(None)
size_filter.filtered_files = file_insts
when(size_filter).set_files().thenReturn(None)
when(char_filter).set_files().thenReturn(None)
when(char_filter).find().thenReturn(None)
char_filter.filtered_files = []
dup_finder.find()
verify(core.dup_finder.Walker).walk([path])
verify(size_filter).set_files(file_insts)
verify(size_filter).find()
verify(char_filter).set_files(file_insts)
verify(char_filter).find()
self.assertEqual([], dup_finder.dup_files)
def put(self, path):
try:
context = g.context
connection = context.get_connection()
result = []
for f in request.files.getlist('file'):
file = File()
file.update_file(context, f, path)
result.append({
"status": "OK",
"file_id": file.get_file_id(),
"remote_path": path
})
return make_response(dict({
"results": result,
"status": "ok"
}), 200)
except FileNotFoundInDatabase as err:
logger.exception(f"FileNotFoundInDatabase: {err}")
return make_response({"status": "Err", "error": str(err)}, 404)
except Exception as err:
logger.exception(f"Exception: {err}")
return make_response({"status": "Err", "error": str(err)}, 500)
def get(self, path):
try:
context = g.context
connection = context.get_connection()
file = File()
result = file.read_file(context, path)
if result == None:
return make_response(
{
"status": "Err",
"error": f"File not found: {path}"
}, 404)
response = make_response(result['file'])
response.headers.set('Content-Type', result['mime_type'])
#response.headers.set('Content-Disposition', 'attachment', filename='test.jpg')
response.headers.set('Content-Disposition',
'inline',
filename=result['name'])
return response
except Exception as err:
logger.exception(f"Exception: {err}")
return make_response({"status": "Err", "error": str(err)}, 500)
def __path_walk(self, path):
file_instances = list()
for (root, dirs, files) in os.walk(path):
# LOG.debug("{0} {1} {2}".format(root, dirs, files))
for _file in files:
filepath = os.path.join(root, _file)
if os.path.exists(filepath):
file_instance = File(filepath)
file_instances.append(file_instance)
self.progress = self.progress + 1
return file_instances
def __init__(self, opts):
""" constructor"""
self.opts = opts # all nullscan options
self.logger = Logger()
self.log = self.logger.log
self.file = File()
self.misc = Misc()
# tmp stupid fix to strip bad chars out of options
self.bad_chars = '<>()[]{}:;.,="\'*&^%$#@!+-_/?°²³´`'
return
def main():
path = sys.argv[1]
LOG.info("Start to find duplicated files on {0}".format(path))
if os.path.isfile(path):
start_time = time.time()
print (File(path).md5sum)
print (File(path).size)
end_time = time.time()
print (end_time - start_time)
else:
start_time = time.time()
filters = [
core.algorithm.SizeFilter(),
core.algorithm.CharacterFilter()
]
dup_finder = core.dup_finder.DupFinder([path], filters)
dup_finder.find()
end_time = time.time()
#dup_finder.dump2file("output.txt")
dup_finder.dump2csv("output.csv")
print (end_time - start_time)
print (utils.size_renderer(dup_finder.dup_size))
def __init__(self, target, opts):
""" init """
Helper.__init__(self, target, opts)
ToolsHelper.__init__(self)
Tools.__init__(self, target, opts)
Parser.__init__(self)
self.logger = Logger()
self.log = self.logger.log
self.file = File()
self.logfile = None # logfile for command
return
def __init__(self):
""" constructor """
# logger
self.logger = Logger()
self.log = self.logger.log
# file i/o
self.file = File()
# original terminal state
self.term_fd = sys.stdin.fileno()
self.term_state = termios.tcgetattr(self.term_fd)
return
def __init__(self, date, opts, template_dir, report_dir, logs_dir):
""" constructor """
self.date = date
self.opts = opts
self.res = {} # holds the results out of log dirs
self.template_dir = template_dir
self.report_dir = report_dir
self.logs_dir = logs_dir
self.file = File()
self.file.copy_dirs(self.template_dir, self.report_dir)
self.index_html = f'{self.report_dir}/index.html'
self.res_html = f'{self.report_dir}/results.html'
self.index_html_data = self.file.read_file(self.index_html)
self.res_html_data = self.file.read_file(self.res_html)
return
def __init__(self):
""" constructor """
# options and modules
self.opt = None
self.mod = Module(MOD_PATH)
# logger
self.logger = Logger()
self.log = self.logger.log
# rest we need
self.file = File()
self.check = Check()
self.misc = Misc()
self.parser = None
# nullscan working dir
self.nullscan_dir = None
return
def __init__(self, mod_path):
""" constructor """
# logger
self.logger = Logger()
self.log = self.logger.log
# file i/o
self.file = File()
# get all modules
self.mod_path = mod_path
self.mods = []
self.get_modules()
# all loaded module
self.lmod = {}
# docstrings for all tools
self.docstrings = {}
return
def get(self, **kwargs):
args = self.parser.parse_args()
path = args['path']
if os.path.isfile(path):
return File(path)
raise UnprocessableEntity
