def collect_screenshots(cat, folder, file_tag='FileUsage'): """ add events based on screenshot creation """ hdr = ['date', 'category', 'size', 'filename', 'path'] ev = core_data.CoreTable(op_folder, tpe='Events', user=usr, header=hdr) op_file = ev.get_filename(file_tag + '-' + cat) if os.path.exists(op_file): os.remove(op_file) # get collection of screenshots fl_filename = os.path.join(op_folder, 'filelist_screenshots.csv') fles = fl.FileList([folder], ['*.jpg', '*.png', '*.gif'], ['\\thumbnails\\'], fl_filename) fles.save_filelist(fl_filename, ["name", "path", "size", "date"]) files = fles.get_metadata() for file_dict in files: ev.add( core_data.CoreDataWhen(cat, [ file_dict["date"], file_tag, file_dict["size"], file_dict["name"], file_dict["path"] ])) lg.record_process( 'event_aggregator.py', 'created ' + str(len(files)) + ' events for ' + cat + '-' + file_tag) ev.save(file_tag, add_header='Y')
def index(): """ main function - outputs in following format BEFORE consolidation (which is TODO) # filename, word, linenumbers # refAction.csv, ActionTypeName, 1 # refAction.csv, PhysicalType, 1 # goals.csv, Cleanliness, 11 """ lg = mod_log.Log(mod_cfg.fldrs['localPath']) lg.record_command('Starting indexing', 'index.py') # sys.modules[self.__module__].__file__) if silent == 'N': print('------------------') print('Rebuilding Indexes') print('------------------') ndxFile = ndxPath + '\\ndxFull.txt' opIndex = ndxPath + '\\ndxWordsToFiles.txt' with open(ndxFile, "w") as ndx: ndx.write('filename, word, linenumbers\n') files_to_index = mod_fl.FileList([mod_cfg.fldrs['public_data_path'] + os.sep + 'core'], ['*.csv'], ["__pycache__", ".git"], "files_to_index_filelist.csv") for f in files_to_index.get_list(): buildIndex(f, ndxFile, silent) # now build the one big index file consolidate(ndxFile, opIndex ) lg.record_command('Finished indexing', 'index.py') #, fle.GetModuleName()) if silent == 'N': print('Done')
def convert_acute_diary_files_to_text(fldr, opfolder): """ Takes a folder of Acute Software Diary files and parses them to a single file of DATE, TIME, DETAILS """ txt = '' cols = [] try: os.remove(opfile) except Exception: pass fl = mod_fl.FileList([fldr], ['D2013*.DAT'], ["__pycache__", ".git"], "temp.csv") with open(opfile, 'w') as fop: for f in fl.get_list(): print(f) with open(f, 'r') as fip: for line in fip: if line != '': try: cols = line.split(chr(31)) fop.write(cols[1] + ', ' + cols[2] + ', ' + cols[14] + '\n') except Exception as ex: print("Error - ", str(ex))
def add_diary_files_to_list(lst, fldr): """ adds all Diary files from folder to the lst """ fl = mod_fl.FileList([fldr], ['D2014*.DAT'], ["__pycache__", ".git"], "temp.csv") for f in fl.get_list(): lst.append(f) return lst
def test_11_extract(self): z = zip_tools.ZipFile(test_file2) z.extract(op_folder) lst1 = fl.FileList([op_folder], ['*.*'], [], '') #print(lst1.get_list()) self.assertEqual(len(lst1.get_list()) > 1, True) extract_fldr = op_folder + os.sep
def get_list_of_applications(): """ Get list of applications """ apps = mod_prg.Programs('Applications', 'C:\\apps') fl = mod_fl.FileList(['C:\\apps'], ['*.exe'], ["\\bk\\"]) for f in fl.get_list(): apps.add(f, 'autogenerated list') apps.list() apps.save()
def test_05_check_metadata(self): """ make sure metadata is correct for this file """ lst5 = fl.FileList([os.path.dirname(os.path.abspath(__file__))], ['test_cls_filelist.py'], [], self.fname) files = lst5.get_metadata() for file_dict in files: self.assertEqual(file_dict["name"], 'test_cls_filelist.py') self.assertTrue(file_dict["size"] > 2500) self.assertTrue(file_dict["date"] > '2014-08-12 21:32:57') self.assertEqual(file_dict["path"], os.path.dirname(os.path.abspath(__file__)))
def create_diary(fldr): """ find all created event files in folder and generate diary files (and display) """ print('generating diary from files in ' + fldr) fles = fl.FileList([op_folder], ['*.' + usr], [], '') files = fles.get_metadata() for file_dict in files: print(' reading ' + str(file_dict['fullfilename'])) with open(str(file_dict['fullfilename']), 'r') as f: hdr = f.readline().strip('"').split(',') print('hdr = ', hdr)
def test_04_save_filelist(self): """ test saving filelist """ if os.path.isfile(self.fname): os.remove(self.fname) aikif_fl = fl.FileList([root_folder + os.sep + 'tests'], ['*.py'], [], self.fname) #print('FL = ', aikif_fl.get_list()) aikif_fl.save_filelist(self.fname, ["name", "path", "size", "date"]) if os.path.isfile(self.fname): self.assertEqual("File Exists", "File Exists") else: self.assertEqual("File Exists", "Whoops - nope")
def list_all_python_programs(self): """ collects a filelist of all .py programs """ self.tot_lines = 0 self.tot_bytes = 0 self.tot_files = 0 self.tot_loc = 0 fl = mod_fl.FileList([self.fldr], ['*.py'], ["__pycache__", ".git"]) for file in fl.get_list(): if '__init__.py' not in file: self.add(file, 'TODO - add comment') f = mod_file.TextFile(file) self.tot_lines += f.count_lines_in_file() self.tot_loc += f.count_lines_of_code() self.tot_bytes += f.size self.tot_files += 1 print('All Python Program Statistics') print('Files = ', self.tot_files, ' Bytes = ', self.tot_bytes, ' Lines = ', self.tot_lines, ' Lines of Code = ', self.tot_loc)
def get_import_list(root_folder): """ scans a root folder for all python files and gets a list of all imports. Wait, stop - this wont work due to various local imports of aikif local py files. Shelve for now, use hard coded list """ import_statements = [] print("ROOT FOLDER = " + root_folder) # py_files = cl.clsCollectFiles(os.path.join(os.path.dirname(root_folder), os.pardir), '*.py') # test_cls_collect py_files = cl.FileList([root_folder], ['*.py'], [], "sample_filelist.csv") for fname in py_files.fl_metadata: #print(fl.print_file_details_in_line(f, ["fullfilename"])) print(fname["fullfilename"]) with open(fname["fullfilename"], "r") as f: for line in f: if line.strip()[0:6] == 'import': import_statements.append(line.strip()[7:]) #print (line) return list(set(import_statements))
def search_aikif(txt, formatHTML=True): """ search for text - currently this looks in all folders in the root of AIKIF but that also contains binaries so will need to use the agent_filelist.py to specify the list of folders. NOTE - this needs to use indexes rather than full search each time """ results = [] num_found = 0 import aikif.lib.cls_filelist as mod_fl my_files = mod_fl.FileList([root_folder + os.sep + 'data', root_folder + os.sep + 'aikif'], ['*.*'], ['*.pyc']) files = my_files.get_list() for f in files: try: num_found = 0 with open(f, 'r') as cur: line_num = 0 for line in cur: line_num += 1 if txt in line: num_found += 1 if formatHTML == True: results.append(format_result(f, line, line_num, txt)) else: results.append([f, line, line_num, txt]) if num_found > 0: if formatHTML == True: results.append('<h3>' + f + ' = ' + str(num_found) + ' results</h3>') else: print(f + ' = ' + str(num_found) + '') except: results.append('problem with file ' + f) if len(results) == 0: results.append("No results") return results
def test_03_exclude_files(self): """print("test2 - Collecting multiple file metadata")""" lst3 = fl.FileList( [root_folder + os.sep + 'tests' + os.sep + 'test_results'], ['*.*'], ['*.sql'], self.fname) self.assertEqual(len(lst3.get_list()) > 10, True)
def test_02_create_zip_from_folder(self): zip_tools.create_zip_from_folder(nested_zipfile, folder_to_zip) lst1 = fl.FileList([os.getcwd()], ['*.zip'], [], '') #print(lst1.get_list()) self.assertEqual(len(lst1.get_list()) > 1, True)
def test_01_file_result(self): """print("test1 - filelist with one file")""" lst1 = fl.FileList([root_folder + os.sep + 'tests'], ['test_*.py'], [], self.fname) self.assertEqual(len(lst1.get_list()) > 18, True)
def test_02_multiple_file_result(self): """print("test2 - Collecting multiple file metadata")""" lst2 = fl.FileList([root_folder + os.sep + 'tests'], ['*.*'], [], self.fname) self.assertEqual(len(lst2.get_list()) > 30, True)
def do_your_job(self, *arg): """ the goal of the filelist agent is to collect metadata on files """ self.lst = fl.FileList([self.root_folder], ['*'], [], False) self.lst.save_filelist( self.fl_opname, self.col_list)
def test_01_create_zip_from_file(self): zip_tools.create_zip_from_file(test_file2, src_file) lst1 = fl.FileList([os.getcwd()], ['test2.zip'], [], '') self.assertEqual('test2.zip' in lst1.get_list()[0], True)