def collect_screenshots(cat, folder, file_tag='FileUsage'):
"""
add events based on screenshot creation
"""
hdr = ['date', 'category', 'size', 'filename', 'path']
ev = core_data.CoreTable(op_folder, tpe='Events', user=usr, header=hdr)
op_file = ev.get_filename(file_tag + '-' + cat)
if os.path.exists(op_file):
os.remove(op_file)
# get collection of screenshots
fl_filename = os.path.join(op_folder, 'filelist_screenshots.csv')
fles = fl.FileList([folder], ['*.jpg', '*.png', '*.gif'],
['\\thumbnails\\'], fl_filename)
fles.save_filelist(fl_filename, ["name", "path", "size", "date"])
files = fles.get_metadata()
for file_dict in files:
ev.add(
core_data.CoreDataWhen(cat, [
file_dict["date"], file_tag, file_dict["size"],
file_dict["name"], file_dict["path"]
]))
lg.record_process(
'event_aggregator.py',
'created ' + str(len(files)) + ' events for ' + cat + '-' + file_tag)
ev.save(file_tag, add_header='Y')
def index():
"""
main function - outputs in following format BEFORE consolidation (which is TODO)
# filename, word, linenumbers
# refAction.csv, ActionTypeName, 1
# refAction.csv, PhysicalType, 1
# goals.csv, Cleanliness, 11
"""
lg = mod_log.Log(mod_cfg.fldrs['localPath'])
lg.record_command('Starting indexing', 'index.py') # sys.modules[self.__module__].__file__)
if silent == 'N':
print('------------------')
print('Rebuilding Indexes')
print('------------------')
ndxFile = ndxPath + '\\ndxFull.txt'
opIndex = ndxPath + '\\ndxWordsToFiles.txt'
with open(ndxFile, "w") as ndx:
ndx.write('filename, word, linenumbers\n')
files_to_index = mod_fl.FileList([mod_cfg.fldrs['public_data_path'] + os.sep + 'core'], ['*.csv'], ["__pycache__", ".git"], "files_to_index_filelist.csv")
for f in files_to_index.get_list():
buildIndex(f, ndxFile, silent)
# now build the one big index file
consolidate(ndxFile, opIndex )
lg.record_command('Finished indexing', 'index.py') #, fle.GetModuleName())
if silent == 'N':
print('Done')
def convert_acute_diary_files_to_text(fldr, opfolder):
"""
Takes a folder of Acute Software Diary files and
parses them to a single file of DATE, TIME, DETAILS
"""
txt = ''
cols = []
try:
os.remove(opfile)
except Exception:
pass
fl = mod_fl.FileList([fldr], ['D2013*.DAT'], ["__pycache__", ".git"],
"temp.csv")
with open(opfile, 'w') as fop:
for f in fl.get_list():
print(f)
with open(f, 'r') as fip:
for line in fip:
if line != '':
try:
cols = line.split(chr(31))
fop.write(cols[1] + ', ' + cols[2] + ', ' +
cols[14] + '\n')
except Exception as ex:
print("Error - ", str(ex))
def add_diary_files_to_list(lst, fldr):
""" adds all Diary files from folder to the lst """
fl = mod_fl.FileList([fldr], ['D2014*.DAT'], ["__pycache__", ".git"],
"temp.csv")
for f in fl.get_list():
lst.append(f)
return lst
def test_11_extract(self):
z = zip_tools.ZipFile(test_file2)
z.extract(op_folder)
lst1 = fl.FileList([op_folder], ['*.*'], [], '')
#print(lst1.get_list())
self.assertEqual(len(lst1.get_list()) > 1, True)
extract_fldr = op_folder + os.sep
def get_list_of_applications():
"""
Get list of applications
"""
apps = mod_prg.Programs('Applications', 'C:\\apps')
fl = mod_fl.FileList(['C:\\apps'], ['*.exe'], ["\\bk\\"])
for f in fl.get_list():
apps.add(f, 'autogenerated list')
apps.list()
apps.save()
def test_05_check_metadata(self):
""" make sure metadata is correct for this file """
lst5 = fl.FileList([os.path.dirname(os.path.abspath(__file__))],
['test_cls_filelist.py'], [], self.fname)
files = lst5.get_metadata()
for file_dict in files:
self.assertEqual(file_dict["name"], 'test_cls_filelist.py')
self.assertTrue(file_dict["size"] > 2500)
self.assertTrue(file_dict["date"] > '2014-08-12 21:32:57')
self.assertEqual(file_dict["path"],
os.path.dirname(os.path.abspath(__file__)))
def create_diary(fldr):
"""
find all created event files in folder and generate
diary files (and display)
"""
print('generating diary from files in ' + fldr)
fles = fl.FileList([op_folder], ['*.' + usr], [], '')
files = fles.get_metadata()
for file_dict in files:
print(' reading ' + str(file_dict['fullfilename']))
with open(str(file_dict['fullfilename']), 'r') as f:
hdr = f.readline().strip('"').split(',')
print('hdr = ', hdr)
def test_04_save_filelist(self):
""" test saving filelist """
if os.path.isfile(self.fname):
os.remove(self.fname)
aikif_fl = fl.FileList([root_folder + os.sep + 'tests'], ['*.py'], [],
self.fname)
#print('FL = ', aikif_fl.get_list())
aikif_fl.save_filelist(self.fname, ["name", "path", "size", "date"])
if os.path.isfile(self.fname):
self.assertEqual("File Exists", "File Exists")
else:
self.assertEqual("File Exists", "Whoops - nope")
def list_all_python_programs(self):
"""
collects a filelist of all .py programs
"""
self.tot_lines = 0
self.tot_bytes = 0
self.tot_files = 0
self.tot_loc = 0
fl = mod_fl.FileList([self.fldr], ['*.py'], ["__pycache__", ".git"])
for file in fl.get_list():
if '__init__.py' not in file:
self.add(file, 'TODO - add comment')
f = mod_file.TextFile(file)
self.tot_lines += f.count_lines_in_file()
self.tot_loc += f.count_lines_of_code()
self.tot_bytes += f.size
self.tot_files += 1
print('All Python Program Statistics')
print('Files = ', self.tot_files, ' Bytes = ', self.tot_bytes,
' Lines = ', self.tot_lines, ' Lines of Code = ', self.tot_loc)
def get_import_list(root_folder):
"""
scans a root folder for all python files and gets a
list of all imports. Wait, stop - this wont work due
to various local imports of aikif local py files.
Shelve for now, use hard coded list
"""
import_statements = []
print("ROOT FOLDER = " + root_folder)
# py_files = cl.clsCollectFiles(os.path.join(os.path.dirname(root_folder), os.pardir), '*.py') # test_cls_collect
py_files = cl.FileList([root_folder], ['*.py'], [], "sample_filelist.csv")
for fname in py_files.fl_metadata:
#print(fl.print_file_details_in_line(f, ["fullfilename"]))
print(fname["fullfilename"])
with open(fname["fullfilename"], "r") as f:
for line in f:
if line.strip()[0:6] == 'import':
import_statements.append(line.strip()[7:])
#print (line)
return list(set(import_statements))
def search_aikif(txt, formatHTML=True):
"""
search for text - currently this looks in all folders in the
root of AIKIF but that also contains binaries so will need to
use the agent_filelist.py to specify the list of folders.
NOTE - this needs to use indexes rather than full search each time
"""
results = []
num_found = 0
import aikif.lib.cls_filelist as mod_fl
my_files = mod_fl.FileList([root_folder + os.sep + 'data', root_folder + os.sep + 'aikif'], ['*.*'], ['*.pyc'])
files = my_files.get_list()
for f in files:
try:
num_found = 0
with open(f, 'r') as cur:
line_num = 0
for line in cur:
line_num += 1
if txt in line:
num_found += 1
if formatHTML == True:
results.append(format_result(f, line, line_num, txt))
else:
results.append([f, line, line_num, txt])
if num_found > 0:
if formatHTML == True:
results.append('<h3>' + f + ' = ' + str(num_found) + ' results</h3>')
else:
print(f + ' = ' + str(num_found) + '')
except:
results.append('problem with file ' + f)
if len(results) == 0:
results.append("No results")
return results
def test_03_exclude_files(self):
"""print("test2 - Collecting multiple file metadata")"""
lst3 = fl.FileList(
[root_folder + os.sep + 'tests' + os.sep + 'test_results'],
['*.*'], ['*.sql'], self.fname)
self.assertEqual(len(lst3.get_list()) > 10, True)
def test_02_create_zip_from_folder(self):
zip_tools.create_zip_from_folder(nested_zipfile, folder_to_zip)
lst1 = fl.FileList([os.getcwd()], ['*.zip'], [], '')
#print(lst1.get_list())
self.assertEqual(len(lst1.get_list()) > 1, True)
def test_01_file_result(self):
"""print("test1 - filelist with one file")"""
lst1 = fl.FileList([root_folder + os.sep + 'tests'], ['test_*.py'], [],
self.fname)
self.assertEqual(len(lst1.get_list()) > 18, True)
def test_02_multiple_file_result(self):
"""print("test2 - Collecting multiple file metadata")"""
lst2 = fl.FileList([root_folder + os.sep + 'tests'], ['*.*'], [],
self.fname)
self.assertEqual(len(lst2.get_list()) > 30, True)
def do_your_job(self, *arg):
"""
the goal of the filelist agent is to collect metadata on files
"""
self.lst = fl.FileList([self.root_folder], ['*'], [], False)
self.lst.save_filelist( self.fl_opname, self.col_list)
def test_01_create_zip_from_file(self):
zip_tools.create_zip_from_file(test_file2, src_file)
lst1 = fl.FileList([os.getcwd()], ['test2.zip'], [], '')
self.assertEqual('test2.zip' in lst1.get_list()[0], True)
