def getattr(self, path, fileh=None): auth, realbase, restpath, validops, deniedops = self._getvars(path) if auth == None: stats = os.lstat(settings.MSERVE_DATA) return _stat_to_dict(stats, {"st_uid":self.uid, "st_gid":self.gid}) else: if len(path) == 0: stats = os.lstat(settings.MSERVE_DATA) return _stat_to_dict(stats, {"st_uid":self.uid, "st_gid":self.gid}) else: try: mfile = realbase.get_file_for_paths(restpath) if mfile: stats = os.stat(mfile.file.path) return _stat_to_dict(stats) except Exception as exception: logging.debug(exception) try: realbase.get_folder_for_paths(restpath) # TODO - Check this is secure to return stat for ./ stats = os.lstat(settings.MSERVE_DATA) return _stat_to_dict(stats, {"st_uid":self.uid, "st_gid":self.gid, "st_size":5000}) except Exception as exception: logging.debug(exception) raise FuseOSError(ENOENT)
def _assert_dirs_equal_cmp(self, diff): self.assert_equal(diff.left_only, []) self.assert_equal(diff.right_only, []) self.assert_equal(diff.diff_files, []) self.assert_equal(diff.funny_files, []) for filename in diff.common: path1 = os.path.join(diff.left, filename) path2 = os.path.join(diff.right, filename) s1 = os.lstat(path1) s2 = os.lstat(path2) # Assume path2 is on FUSE if st_dev is different fuse = s1.st_dev != s2.st_dev attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev'] if not fuse or not os.path.isdir(path1): # dir nlink is always 1 on our fuse fileystem attrs.append('st_nlink') d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] if not os.path.islink(path1) or utime_supports_fd: # Older versions of llfuse does not support ns precision properly if fuse and not have_fuse_mtime_ns: d1.append(round(st_mtime_ns(s1), -4)) d2.append(round(st_mtime_ns(s2), -4)) d1.append(round(st_mtime_ns(s1), st_mtime_ns_round)) d2.append(round(st_mtime_ns(s2), st_mtime_ns_round)) d1.append(get_all(path1, follow_symlinks=False)) d2.append(get_all(path2, follow_symlinks=False)) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): self._assert_dirs_equal_cmp(sub_diff)
def glob(pathname): """Return a list of paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la fnmatch. """ if not has_magic(pathname): return [pathname] dirname, basename = os.path.split(pathname) if not dirname: return glob1(os.curdir, basename) elif has_magic(dirname): list = glob(dirname) else: list = [dirname] if not has_magic(basename): result = [] for dirname in list: if basename or os.path.isdir(dirname): name = os.path.join(dirname, basename) try: os.lstat(name) except OSError, err: # if the file does not exist, or if an element of the # path to the file is not a directory, ignore the error if err.errno != errno.ENOENT and err.errno != errno.ENOTDIR: raise else: result.append(name)
def check_dir(inst, type, mode): if type == 'xlogs': limit_ut = int(inst['xlogs_mon_max_lag']) * 3600 elif type == 'snaps': limit_ut = int(inst['snaps_mon_max_lag']) * 3600 elif type == 'other': limit_ut = int(inst['other_mon_max_lag']) * 3600 else: limit_ut = 86400 if mode == 'tarantool': root = '/backup/%s/%s/' % (inst['rsync_modulepath'], type) elif mode == 'silver': name = inst['base_dir'].rsplit('/')[-1] root = '/backup/%s/%s/%s/' % (inst['rsync_modulepath'], name, type) if not os.path.isdir(root): logger.critical("Directory '%s' does not exist" % root) return if os.listdir(root): ### "Other" backups tend to have same mtime for a long period of time, so we will check their ctime if type == 'other': oldest_file = max([os.path.join(root, f) for f in os.listdir(root) if not f.startswith('.')], key=os.path.getctime) if os.lstat(oldest_file).st_ctime < now - limit_ut: hours_ago = int((now - os.lstat(oldest_file).st_ctime) / 60 // 60) logger.critical("Last backup in '%s' was made more than %s hours ago" % (root, hours_ago)) else: oldest_file = max([os.path.join(root, f) for f in os.listdir(root) if not f.startswith('.')], key=os.path.getmtime) if os.lstat(oldest_file).st_mtime < now - limit_ut: hours_ago = int((now - os.lstat(oldest_file).st_mtime) / 60 // 60) logger.critical("Last backup in '%s' was made more than %s hours ago" % (root, hours_ago)) else: logger.critical("Directory '%s' is empty" % root)
def test_lchflags_symlink(self): testfn_st = os.stat(test_support.TESTFN) self.assertTrue(hasattr(testfn_st, 'st_flags')) os.symlink(test_support.TESTFN, _DUMMY_SYMLINK) self.teardown_files.append(_DUMMY_SYMLINK) dummy_symlink_st = os.lstat(_DUMMY_SYMLINK) # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. try: posix.lchflags(_DUMMY_SYMLINK, dummy_symlink_st.st_flags | stat.UF_IMMUTABLE) except OSError as err: if err.errno != errno.EOPNOTSUPP: raise msg = 'chflag UF_IMMUTABLE not supported by underlying fs' self.skipTest(msg) try: new_testfn_st = os.stat(test_support.TESTFN) new_dummy_symlink_st = os.lstat(_DUMMY_SYMLINK) self.assertEqual(testfn_st.st_flags, new_testfn_st.st_flags) self.assertEqual(dummy_symlink_st.st_flags | stat.UF_IMMUTABLE, new_dummy_symlink_st.st_flags) finally: posix.lchflags(_DUMMY_SYMLINK, dummy_symlink_st.st_flags)
def lexists(path: AnyStr) -> bool: """Test whether a path exists. Returns True for broken symbolic links""" try: os.lstat(path) except os.error: return False return True
def tst_link(mnt_dir): name1 = pjoin(mnt_dir, name_generator()) name2 = pjoin(mnt_dir, name_generator()) shutil.copyfile(TEST_FILE, name1) assert filecmp.cmp(name1, TEST_FILE, False) fstat1 = os.lstat(name1) assert fstat1.st_nlink == 1 os.link(name1, name2) fstat1 = os.lstat(name1) fstat2 = os.lstat(name2) for attr in ('st_mode', 'st_dev', 'st_uid', 'st_gid', 'st_size', 'st_atime', 'st_mtime', 'st_ctime'): assert getattr(fstat1, attr) == getattr(fstat2, attr) assert os.path.basename(name2) in os.listdir(mnt_dir) assert filecmp.cmp(name1, name2, False) os.unlink(name2) assert os.path.basename(name2) not in os.listdir(mnt_dir) with pytest.raises(FileNotFoundError): os.lstat(name2) os.unlink(name1)
def lexists(path): """Test whether a path exists. Returns True for broken symbolic links""" try: os.lstat(path) except OSError: return False return True
def ismount(path): """Test whether a path is a mount point""" try: s1 = os.lstat(path) except OSError: # It doesn't exist -- so not a mount point. :-) return False else: # A symlink can never be a mount point if stat.S_ISLNK(s1.st_mode): return False if isinstance(path, bytes): parent = join(path, b'..') else: parent = join(path, '..') try: s2 = os.lstat(parent) except OSError: return False dev1 = s1.st_dev dev2 = s2.st_dev if dev1 != dev2: return True # path/.. on a different device as path ino1 = s1.st_ino ino2 = s2.st_ino if ino1 == ino2: return True # path/.. is the same i-node as path return False
def ignore_tree (root_dir): for dir_path, sd, files in os.walk(root_dir): perm = oct(os.lstat(dir_path).st_mode & 0o777).rjust(4, '0')[1:] yield ('ign', 'd', perm, dir_path) for f in files: perm = oct(os.lstat(dir_path+'/'+f).st_mode & 0o777).rjust(4, '0')[1:] yield ('ign', 'f', perm, dir_path + '/' + f)
def gen_items (rootdir, verbose=False): global locked_dirs if os.path.isfile(rootdir): file_path = rootdir dir_path = os.path.dirname(rootdir) file_name = os.path.basename(rootdir) perm = oct(os.lstat(file_path).st_mode & 0o777).rjust(4, '0')[1:] action = get_file_action(perm, dir_path, file_name) if (action != 'ign' and action != perm) or verbose: yield (action, 'f', perm, dir_path + '/' + file_name) return for dir_path, sub_dirs, files in os.walk(rootdir): perm = oct(os.lstat(dir_path).st_mode & 0o777).rjust(4, '0')[1:] action = get_dir_action(perm, dir_path, sub_dirs, files) if action == 'ign' or action == 'non': if verbose: for i in ignore_tree(dir_path): yield i del sub_dirs[:] else: if action != perm or (action == perm and verbose): yield (action, 'd', perm, dir_path) ign_sub_dir_list = get_ignore_sub_dirs_list(sub_dirs) if verbose: for i in ign_sub_dir_list: for j in ignore_tree(dir_path + '/' + i): yield j for i in ign_sub_dir_list: sub_dirs.remove(i) # may have problem to walk for i in sub_dirs: perm = oct(os.lstat(dir_path + '/' + i).st_mode & 0o777).rjust(4, '0')[1:] if re.match(r'^.*[0123].*$', perm): locked_dirs.append( '[{perm}] {itemname}'.format( perm=perm, itemname=dir_path+'/'+i) ) for file_name in files: perm = oct(os.lstat(dir_path + '/' + file_name).st_mode & 0o777).rjust(4, '0')[1:] action = get_file_action(perm, dir_path, file_name) output = False if (action != 'ign' and action != perm) or verbose: yield (action, 'f', perm, dir_path + '/' + file_name)
def convertGroups( self, dir, repFile, ignorefilelist): for f in os.listdir(dir): pathname = '%s/%s' % (dir, f) mode = os.lstat(pathname)[ST_MODE] if S_ISDIR(mode): # It's a directory therefore it must be a COVISE group for ff in os.listdir(pathname): execName = '%s/%s' % (pathname, ff) if not execName.endswith(".pdb") and not execName.endswith(".ilk") and not execName.endswith(".exp") and not execName.endswith(".lib") and not execName.endswith(".suo"): skipfile = False # check to see, if filename is to be ignored if (ignorefilelist != None): for fignore in ignorefilelist: actualfile = '%s/%s' % (f, ff) #repFile.write("convertGroups - comparing ignorefile=%s w/ actualfile=%s\n" % (fignore, actualfile)) if (fignore == actualfile): #repFile.write("convertGroups - will ignore file=%s\n" % fignore) skipfile = True if not skipfile: repFile.write(bytes("convertGroups - processing file=",'UTF-8')) repFile.write(bytes(execName,'UTF-8')) repFile.write(bytes("\n",'UTF-8')) mmode = os.lstat(execName)[ST_MODE] if S_ISREG(mmode) and (ff[0] != "."): # It's a file, call the callback function self.convertFromFile( execName, repFile) else: repFile.write("convertGroups - ignoring file=%s\n" % execName)
def lexists(filename): "test whether a file with this name exists. does not follow symlinks" try: os.lstat(filename) except: return False return True
def test__write_inventory(self): # The private interface _write_inventory is currently used by transform. tree = self.make_branch_and_tree('.') # if we write write an inventory then do a walkdirs we should get back # missing entries, and actual, and unknowns as appropriate. self.build_tree(['present', 'unknown']) inventory = Inventory(tree.get_root_id()) inventory.add_path('missing', 'file', 'missing-id') inventory.add_path('present', 'file', 'present-id') # there is no point in being able to write an inventory to an unlocked # tree object - its a low level api not a convenience api. tree.lock_write() tree._write_inventory(inventory) tree.unlock() tree.lock_read() try: present_stat = os.lstat('present') unknown_stat = os.lstat('unknown') expected_results = [ (('', tree.get_root_id()), [('missing', 'missing', 'unknown', None, 'missing-id', 'file'), ('present', 'present', 'file', present_stat, 'present-id', 'file'), ('unknown', 'unknown', 'file', unknown_stat, None, None), ] )] self.assertEqual(expected_results, list(tree.walkdirs())) finally: tree.unlock()
def getSubdirSize(directory): # returns size in bytes try: mydev = os.lstat(directory)[stat.ST_DEV] except OSError as e: log.debug("failed to stat %s: %s", directory, e) return 0 try: dirlist = os.listdir(directory) except OSError as e: log.debug("failed to listdir %s: %s", directory, e) return 0 dsize = 0 for f in dirlist: curpath = '%s/%s' % (directory, f) try: sinfo = os.lstat(curpath) except OSError as e: log.debug("failed to stat %s/%s: %s", directory, f, e) continue if stat.S_ISDIR(sinfo[stat.ST_MODE]): if os.path.ismount(curpath): continue if mydev == sinfo[stat.ST_DEV]: dsize += getSubdirSize(curpath) elif stat.S_ISREG(sinfo[stat.ST_MODE]): dsize += sinfo[stat.ST_SIZE] return dsize
def VerEXIST(): # xpath = argv[1] if(argv[1][-1] == '/'): xpath = argv[1][:-1] else: xpath = argv[1] if os.path.exists(xpath): if os.path.isfile(xpath): right = oct(stat.S_IMODE(os.lstat(argv[1]).st_mode)) if right == '0': print("Vous n'avez pas les droits necessaire.") elif right[1] == '4' or right[1] == '5' or right[1] == '6' or right[1] == '7': FileEXIST() else: print("Vous n'avez pas les droits necessaire.") if os.path.isdir(xpath): right = oct(stat.S_IMODE(os.lstat(argv[1]).st_mode)) if right == '0': print("Vous n'avez pas les droits necessaire.") if right[1] == '4' or right[1] == '5' or right[1] == '6' or right[1] == '7': DirectoryEXIST() else: print("Vous n'avez pas les droits necessaire.") else: print(xpath + " --> N'existe pas.")
def rm_pkgs(args): # TODO: This doesn't handle packages that have hard links to files within # themselves, like bin/python3.3 and bin/python3.3m in the Python package from os.path import join, isdir from os import lstat, walk, listdir from conda.install import rm_rf pkgs_dir = config.pkgs_dirs[0] print('Cache location: %s' % pkgs_dir) rmlist = [] pkgs = [i for i in listdir(pkgs_dir) if isdir(join(pkgs_dir, i)) and # Only include actual packages isdir(join(pkgs_dir, i, 'info'))] for pkg in pkgs: breakit = False for root, dir, files in walk(join(pkgs_dir, pkg)): if breakit: break for fn in files: try: stat = lstat(join(root, fn)) except OSError as e: print(e) continue if stat.st_nlink > 1: # print('%s is installed: %s' % (pkg, join(root, fn))) breakit = True break else: rmlist.append(pkg) if not rmlist: print("There are no unused packages to remove") sys.exit(0) print("Will remove the following packages:") print() totalsize = 0 maxlen = len(max(rmlist, key=lambda x: len(str(x)))) fmt = "%-40s %10s" for pkg in rmlist: pkgsize = 0 for root, dir, files in walk(join(pkgs_dir, pkg)): for fn in files: # We don't have to worry about counting things twice: by # definition these files all have a link count of 1! size = lstat(join(root, fn)).st_size totalsize += size pkgsize += size print(fmt % (pkg, human_bytes(pkgsize))) print('-' * (maxlen + 2 + 10)) print(fmt % ('Total:', human_bytes(totalsize))) print() common.confirm_yn(args) for pkg in rmlist: print("removing %s" % pkg) rm_rf(join(pkgs_dir, pkg))
def getattr(self, path, fh=None): if os.path.basename(args.src) in path: st = os.lstat(args.src) else: st = os.lstat(path) return dict((key, getattr(st, key)) for key in ('st_atime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid'))
def walk(path): os.lstat(path) for dir_path, dirs, filenames in os.walk(path): for filename in filenames: full_path = os.path.join(dir_path, filename) rel_path = full_path[len(path) + 1:] yield rel_path.replace(os.path.sep, '/'), str(full_path)
def test_write_renewal_config(self): # Mostly tested by the process of creating and updating lineages, # but we can test that this successfully creates files, removes # unneeded items, and preserves comments. temp = os.path.join(self.config.config_dir, "sample-file") temp2 = os.path.join(self.config.config_dir, "sample-file.new") with open(temp, "w") as f: f.write("[renewalparams]\nuseful = value # A useful value\n" "useless = value # Not needed\n") os.chmod(temp, 0o640) target = {} for x in ALL_FOUR: target[x] = "somewhere" archive_dir = "the_archive" relevant_data = {"useful": "new_value"} from certbot import storage storage.write_renewal_config(temp, temp2, archive_dir, target, relevant_data) with open(temp2, "r") as f: content = f.read() # useful value was updated self.assertTrue("useful = new_value" in content) # associated comment was preserved self.assertTrue("A useful value" in content) # useless value was deleted self.assertTrue("useless" not in content) # check version was stored self.assertTrue("version = {0}".format(certbot.__version__) in content) # ensure permissions are copied self.assertEqual(stat.S_IMODE(os.lstat(temp).st_mode), stat.S_IMODE(os.lstat(temp2).st_mode))
def test_lchflags_symlink(self): testfn_st = os.stat(support.TESTFN) self.assertTrue(hasattr(testfn_st, "st_flags")) os.symlink(support.TESTFN, _DUMMY_SYMLINK) self.teardown_files.append(_DUMMY_SYMLINK) dummy_symlink_st = os.lstat(_DUMMY_SYMLINK) def chflags_nofollow(path, flags): return posix.chflags(path, flags, follow_symlinks=False) for fn in (posix.lchflags, chflags_nofollow): # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. flags = dummy_symlink_st.st_flags | stat.UF_IMMUTABLE try: fn(_DUMMY_SYMLINK, flags) except OSError as err: if err.errno != errno.EOPNOTSUPP: raise msg = "chflag UF_IMMUTABLE not supported by underlying fs" self.skipTest(msg) try: new_testfn_st = os.stat(support.TESTFN) new_dummy_symlink_st = os.lstat(_DUMMY_SYMLINK) self.assertEqual(testfn_st.st_flags, new_testfn_st.st_flags) self.assertEqual(dummy_symlink_st.st_flags | stat.UF_IMMUTABLE, new_dummy_symlink_st.st_flags) finally: fn(_DUMMY_SYMLINK, dummy_symlink_st.st_flags)
def TestOwnerGroupMode(DestinationPath, SourcePath, fc): stat_info = os.lstat(DestinationPath) if SourcePath: stat_info_src = os.lstat(SourcePath) if fc.Owner: Specified_Owner_ID = pwd.getpwnam(fc.Owner)[2] if Specified_Owner_ID != pwd.getpwuid(stat_info.st_uid)[2]: return False elif SourcePath: # Owner wasn't specified, if SourcePath is specified then check that the Owners match if pwd.getpwuid(stat_info.st_uid)[2] != pwd.getpwuid(stat_info_src.st_uid)[2]: return False if fc.Group: Specified_Group_ID = grp.getgrnam(fc.Group)[2] if Specified_Group_ID != grp.getgrgid(stat_info.st_gid)[2]: return False elif SourcePath: # Group wasn't specified, if SourcePath is specified then check that the Groups match if grp.getgrgid(stat_info.st_gid)[2] != grp.getgrgid(stat_info_src.st_gid)[2]: return False # Mode is irrelevant to symlinks if not os.path.islink(DestinationPath): if fc.Mode: if str(oct(stat_info.st_mode))[-3:] != fc.Mode: return False elif SourcePath: # Mode wasn't specified, if SourcePath is specified then check that the Modes match if str(oct(stat_info.st_mode))[-3:] != str(oct(stat_info_src.st_mode))[-3:]: return False return True
def GetURLValues(url, inProps): ns_dict, error = url.resourceValuesForKeys_error_( inProps+[NSURLIsVolumeKey, u'NSURLParentDirectoryURLKey', NSURLIsDirectoryKey, NSURLIsPackageKey] , None ) if error is not None: raise MyError(error.code() , error.localizedDescription()) ns_dict = dict( ns_dict ) #shallow copy, no per-item conversion. # ns_dict = ns_dict.mutableCopy() p = url.path() file_id = os.lstat(p).st_ino # [anotherDict setObject: dict forKey: "sub-dictionary-key"]; ns_dict[NSFileSystemFileNumber] = file_id ns_dict[NSURLPathKey] = p ns_dict['url'] = url if ns_dict[NSURLIsDirectoryKey]: ns_dict.update( { "NSURLTotalFileSizeKey": 0 }) # file size is zero for directories if ns_dict[NSURLIsVolumeKey]: ns_dict[NSFileSystemFolderNumber] = 1L else: folder_url = ns_dict[NSURLParentDirectoryURLKey] fp = folder_url.path() folder_id = os.lstat(fp).st_ino ns_dict[NSFileSystemFolderNumber] = int(folder_id) return ns_dict
def daten(): """ Anzeige der /daten Seite mit Auflistung der Download-Dateien """ body_list = db.get_body() body_dict = {} for body in body_list: body_dict[str(body['_id'])] = body['name'] data_list = [] for file in os.listdir(app.config['data_dump_folder']): if file.endswith(".tar.bz2"): stat = os.lstat(app.config['data_dump_folder'] + os.sep + file) data_list.append({ 'id': file.split('.')[0], 'name': body_dict[file.split('.')[0]], 'size': "%d" % (stat.st_size / 1024.0 / 1024.0) }) file_list = [] for file in os.listdir(app.config['files_dump_folder']): if file.endswith(".tar.bz2"): stat = os.lstat(app.config['files_dump_folder'] + os.sep + file) file_list.append({ 'id': file.split('.')[0], 'name': body_dict[file.split('.')[0]], 'size': "%d" % (stat.st_size / 1024.0 / 1024.0 / 1024.0) }) return render_template('daten.html', data_list=data_list, file_list=file_list)
def _get_all_modules(self): ''' Scans the overlay modules dir for loadable modules @rtype dict of module_plugins ''' module_dir = self._module_path importables = [] names = os.listdir(module_dir) for entry in names: if entry.startswith('__'): continue try: os.lstat(os.path.join(module_dir, entry, '__init__.py')) importables.append(entry) except EnvironmentError: pass kids = {} for entry in importables: new_module = Module(entry, self._namepath, self.output) for module_name in new_module.kids: kid = new_module.kids[module_name] kid['parent'] = new_module kids[kid['name']] = kid return kids
def lexists(path): try: os.lstat(path) except os.error: return False return True
def test_rename_away_tree_entry(self) -> None: """ Rename a tree entry away and back again """ # We should be able to rename files that are in the Tree hello = os.path.join(self.mount, "hello") targetname = os.path.join(self.mount, "a-new-target") os.rename(hello, targetname) with self.assertRaises(OSError) as context: os.lstat(hello) self.assertEqual( errno.ENOENT, context.exception.errno, msg="no longer visible as old name" ) self.assert_checkout_root_entries({".eden", "a-new-target", "adir", "slink"}) with open(targetname, "r") as f: self.assertEqual("hola\n", f.read(), msg="materialized correct data") # Now, while we hold this file open, check that a rename # leaves the handle connected to the file contents when # we rename it back to its old name. os.rename(targetname, hello) self.assert_checkout_root_entries({".eden", "adir", "hello", "slink"}) with open(hello, "r+") as write_f: write_f.seek(0, os.SEEK_END) write_f.write("woot") f.seek(0) self.assertEqual("hola\nwoot", f.read())
def GetURLResourceValuesForKeys(url, inProps): """raises custom exception MyError when, eg, the file does not exist""" # add keys needed by this routine values, error = url.resourceValuesForKeys_error_( inProps+[NSURLIsVolumeKey, u'NSURLParentDirectoryURLKey', NSURLIsDirectoryKey] , None ) if error is not None: raise MyError(error.code() , error.localizedDescription()) # convert unicode key strings to string # convert objc types to python types (for mysql converter) item_dict = dict( zip( [str(z) for z in values.allKeys() ] , [df2fk(v) for v in values.allValues()] ) ) # add fields that are filesystem related, but not directly gotten as keys in the URL values p = url.path() file_id = os.lstat(p).st_ino item_dict[NSFileSystemFileNumber] = file_id item_dict[NSURLPathKey] = p if item_dict[NSURLIsDirectoryKey]: item_dict.update( { "NSURLTotalFileSizeKey": 0 }) # file size is zero for directories if item_dict[NSURLIsVolumeKey]: item_dict[NSFileSystemFolderNumber] = 1L else: folder_url = values[NSURLParentDirectoryURLKey] fp = folder_url.path() folder_id = os.lstat(fp).st_ino item_dict[NSFileSystemFolderNumber] = int(folder_id) return item_dict
def load(self): """ Load data from most recent file. Update the cache if needed. If the file has not changed, use the cache instead (typically named something like filename.ftstpk) in a pickle format. If file is newer, uses inherited class's loadFromLog method. Then, save in pickle cache. """ if not self.has_changed(): # cache is newer, just load the cache return self.loadCache() while 1: #could need more than one loop if the log file is changing fstat = os.lstat(self.logname) start_logtime = fstat[stat.ST_MTIME] del fstat self.loadFromLog() try: self.saveCache() except IOError: return # silently ignore, this was a load in the end # the log may have changed -> check fstat = os.lstat(self.logname) logtime = fstat[stat.ST_MTIME] del fstat if logtime <= start_logtime: return # OK, not changed, can exit return # should never reach this point
def getattr(self, path): for backup in self.backups: fpath = backup + path if os.path.exists(fpath): f_stat = os.lstat(fpath) mode = f_stat.st_mode if S_ISDIR(mode): return f_stat else: # it is a file, show as dir f_stat = list(f_stat) f_stat[0] = mode & 0b111111111111 | S_IFDIR f_stat = stat_result(f_stat) assert S_ISDIR(f_stat.st_mode) parent = os.lstat(os.path.dirname(fpath)) # print 'done getattr - fake dir' # print 'fstat: ', f_stat # print 'parent: ', parent return parent # XXX workaround return f_stat # either the path does not exist or it actually is a file real_path = self._get_real_path(path) # print 'done getattr ', real_path if real_path: return os.lstat(real_path)
def _check_path(self, src, path_type, dest=None, force=False): """Check a new destination path in the archive. Since it is possible for multiple plugins to collect the same paths, and since plugins can now run concurrently, it is possible for two threads to race in archive methods: historically the archive class only needed to test for the actual presence of a path, since it was impossible for another `Archive` client to enter the class while another method invocation was being dispatched. Deal with this by implementing a locking scheme for operations that modify the path structure of the archive, and by testing explicitly for conflicts with any existing content at the specified destination path. It is not an error to attempt to create a path that already exists in the archive so long as the type of the object to be added matches the type of object already found at the path. It is an error to attempt to re-create an existing path with a different path type (for example, creating a symbolic link at a path already occupied by a regular file). :param src: the source path to be copied to the archive :param path_type: the type of object to be copied :param dest: an optional destination path :param force: force file creation even if the path exists :returns: An absolute destination path if the path should be copied now or `None` otherwise """ dest = dest or self.dest_path(src) if path_type == P_DIR: dest_dir = dest else: dest_dir = os.path.split(dest)[0] if not dest_dir: return dest # Check containing directory presence and path type if os.path.exists(dest_dir) and not os.path.isdir(dest_dir): raise ValueError("path '%s' exists and is not a directory" % dest_dir) elif not os.path.exists(dest_dir): src_dir = src if path_type == P_DIR else os.path.split(src)[0] self._make_leading_paths(src_dir) def is_special(mode): return any([ stat.S_ISBLK(mode), stat.S_ISCHR(mode), stat.S_ISFIFO(mode), stat.S_ISSOCK(mode) ]) if force: return dest # Check destination path presence and type if os.path.exists(dest): # Use lstat: we care about the current object, not the referent. st = os.lstat(dest) ve_msg = "path '%s' exists and is not a %s" if path_type == P_FILE and not stat.S_ISREG(st.st_mode): raise ValueError(ve_msg % (dest, "regular file")) if path_type == P_LINK and not stat.S_ISLNK(st.st_mode): raise ValueError(ve_msg % (dest, "symbolic link")) if path_type == P_NODE and not is_special(st.st_mode): raise ValueError(ve_msg % (dest, "special file")) if path_type == P_DIR and not stat.S_ISDIR(st.st_mode): raise ValueError(ve_msg % (dest, "directory")) # Path has already been copied: skip return None return dest
def makepkginfo(installeritem, options): '''Return a pkginfo dictionary for item''' if isinstance(options, dict): options = AttributeDict(options) pkginfo = {} installs = [] if installeritem and os.path.exists(installeritem): # Check if the item is a mount point for a disk image if dmgutils.pathIsVolumeMountPoint(installeritem): # Get the disk image path for the mount point # and use that instead of the original item installeritem = dmgutils.diskImageForMountPoint(installeritem) # get size of installer item itemsize = 0 itemhash = "N/A" if os.path.isfile(installeritem): itemsize = int(os.path.getsize(installeritem)) itemhash = munkihash.getsha256hash(installeritem) if pkgutils.hasValidDiskImageExt(installeritem): if dmgutils.DMGisWritable( installeritem) and options.print_warnings: print >> sys.stderr, ( "WARNING: %s is a writable disk image. " "Checksum verification is not supported." % installeritem) print >> sys.stderr, ( "WARNING: Consider converting %s to a read-only disk" "image." % installeritem) itemhash = "N/A" pkginfo = get_catalog_info_from_dmg(installeritem, options) if (pkginfo and pkginfo.get('installer_type') == "AdobeCS5Installer"): raise PkgInfoGenerationError( "This disk image appears to contain an Adobe CS5/CS6 " "product install.\n" "Please use Adobe Application Manager, Enterprise " "Edition (AAMEE) to create an installation package " "for this product.") if not pkginfo: raise PkgInfoGenerationError( "Could not find a supported installer item in %s!" % installeritem) elif pkgutils.hasValidPackageExt(installeritem): pkginfo = get_catalog_info_from_path(installeritem, options) if not pkginfo: raise PkgInfoGenerationError( "%s doesn't appear to be a valid installer item!" % installeritem) if os.path.isdir(installeritem) and options.print_warnings: print >> sys.stderr, ( "WARNING: %s is a bundle-style package!\n" "To use it with Munki, you should encapsulate it " "in a disk image.\n") % installeritem # need to walk the dir and add it all up for (path, dummy_dirs, files) in os.walk(installeritem): for name in files: filename = os.path.join(path, name) # use os.lstat so we don't follow symlinks itemsize += int(os.lstat(filename).st_size) # convert to kbytes itemsize = int(itemsize / 1024) elif pkgutils.hasValidConfigProfileExt(installeritem): try: pkginfo = get_catalog_info_for_profile(installeritem) except ProfileMetadataGenerationError, err: print >> sys.stderr, err raise PkgInfoGenerationError( "%s doesn't appear to be a supported configuration " "profile!" % installeritem) else: raise PkgInfoGenerationError("%s is not a valid installer item!" % installeritem) pkginfo['installer_item_size'] = int(itemsize / 1024) if itemhash != "N/A": pkginfo['installer_item_hash'] = itemhash # try to generate the correct item location temppath = installeritem location = "" while len(temppath) > 4: if temppath.endswith('/pkgs'): location = installeritem[len(temppath) + 1:] break else: temppath = os.path.dirname(temppath) if not location: #just the filename location = os.path.split(installeritem)[1] pkginfo['installer_item_location'] = location # ADOBE STUFF - though maybe generalizable in the future? if (pkginfo.get('installer_type') == "AdobeCCPInstaller" and not options.uninstalleritem) and options.print_warnings: print >> sys.stderr, ( "WARNING: This item appears to be an Adobe Creative " "Cloud product install.\n" "No uninstaller package was specified so product " "removal will not be possible.") pkginfo['uninstallable'] = False if 'uninstall_method' in pkginfo: del pkginfo['uninstall_method'] if options.uninstalleritem: uninstallerpath = options.uninstalleritem if os.path.exists(uninstallerpath): # try to generate the correct item location temppath = uninstallerpath location = "" while len(temppath) > 4: if temppath.endswith('/pkgs'): location = uninstallerpath[len(temppath) + 1:] break else: temppath = os.path.dirname(temppath) if not location: #just the filename location = os.path.split(uninstallerpath)[1] pkginfo['uninstaller_item_location'] = location itemsize = int(os.path.getsize(uninstallerpath)) itemhash = munkihash.getsha256hash(uninstallerpath) pkginfo['uninstaller_item_size'] = int(itemsize / 1024) pkginfo['uninstaller_item_hash'] = itemhash else: raise PkgInfoGenerationError("No uninstaller item at %s" % uninstallerpath) # if we have receipts, assume we can uninstall using them if pkginfo.get('receipts', None): pkginfo['uninstallable'] = True pkginfo['uninstall_method'] = "removepackages"
consider_gcc("", "-%(version)s" % vars()) consider_gcc("%(gnu_host)s-" % vars(), "-%(version)s" % vars()) for gnu_host in os.listdir(gcccross_dir): consider_gcc("%(gnu_host)s-" % vars(), "") for version in os.listdir(gcccross_dir + "/" + gnu_host): consider_gcc("", "-%(version)s" % vars()) consider_gcc("%(gnu_host)s-" % vars(), "-%(version)s" % vars()) consider_clang("") for ent in os.listdir("/usr/lib"): if ent.startswith("llvm-"): version = ent.split("-")[1] consider_clang("-%(version)s" % vars()) for name in os.listdir(distcc_dir): mode = os.lstat(distcc_dir + "/" + name).st_mode if stat.S_ISLNK(mode): if os.access(distcc_dir + "/" + name, os.X_OK): old_symlinks.add(name) else: os.unlink(distcc_dir + "/" + name) for link in old_symlinks: if link not in new_symlinks: os.unlink(distcc_dir + "/" + link) for link in new_symlinks: if link not in old_symlinks: if os.access("/usr/bin/distcc", os.X_OK): os.symlink("../../bin/distcc", distcc_dir + "/" + link) else:
def TestStats(self): """Collect inodes and blocks usage.""" # Find the loopback device that was mounted to ROOT_A. loop_device = None root_path = os.path.abspath(os.readlink(image_test_lib.ROOT_A)) for mtab in osutils.IterateMountPoints(): if mtab.destination == root_path: loop_device = mtab.source break self.assertTrue(loop_device, 'Cannot find loopback device for ROOT_A.') # Gather file system stats with tune2fs. cmd = ['tune2fs', '-l', loop_device] # tune2fs produces output like this: # # tune2fs 1.42 (29-Nov-2011) # Filesystem volume name: ROOT-A # Last mounted on: <not available> # Filesystem UUID: <none> # Filesystem magic number: 0xEF53 # Filesystem revision #: 1 (dynamic) # ... # # So we need to ignore the first line. ret = cros_build_lib.SudoRunCommand(cmd, capture_output=True, extra_env={'LC_ALL': 'C'}) fs_stat = dict( line.split(':', 1) for line in ret.output.splitlines() if ':' in line) free_inodes = int(fs_stat['Free inodes']) free_blocks = int(fs_stat['Free blocks']) inode_count = int(fs_stat['Inode count']) block_count = int(fs_stat['Block count']) block_size = int(fs_stat['Block size']) sum_file_size = 0 for root, _, filenames in os.walk(image_test_lib.ROOT_A): for file_name in filenames: full_name = os.path.join(root, file_name) file_stat = os.lstat(full_name) sum_file_size += file_stat.st_size metadata_size = (block_count - free_blocks) * block_size - sum_file_size self.OutputPerfValue('free_inodes_over_inode_count', free_inodes * 100.0 / inode_count, 'percent', graph='free_over_used_ratio') self.OutputPerfValue('free_blocks_over_block_count', free_blocks * 100.0 / block_count, 'percent', graph='free_over_used_ratio') self.OutputPerfValue('apparent_size', sum_file_size, 'bytes', higher_is_better=False, graph='filesystem_stats') self.OutputPerfValue('metadata_size', metadata_size, 'bytes', higher_is_better=False, graph='filesystem_stats')
def is_socket(path): # type: (str) -> bool return stat.S_ISSOCK(os.lstat(path).st_mode)
def expand(self): import os import stat from BlockDevice import BlockDevice from CharacterDevice import CharacterDevice from File import File from Link import Link from NamedPipe import NamedPipe from Socket import Socket import journal debug = journal.debug("pyre.filesystem") files = [] subdirectories = [] root = self.path children = os.listdir(root) debug.log("directory '%s' has %d files" % (self.name, len(children))) count = 0 for name in children: count += 1 if name in self._children: continue pathname = os.path.join(root, name) # PORTABILITY: lstat is unix only mode = os.lstat(pathname)[stat.ST_MODE] if stat.S_ISDIR(mode): node = Directory(name, self) subdirectories.append(node) elif stat.S_ISREG(mode): node = File(name, self) files.append(node) elif stat.S_ISLNK(mode): node = Link(name, self) elif stat.S_ISSOCK(mode): node = Socket(name, self) elif stat.S_ISFIFO(mode): node = NamedPipe(name, self) elif stat.S_ISCHR(mode): node = CharacterDevice(name, self) elif stat.S_ISBLK(mode): node = BlockDevice(name, self) else: Firewall.hit("unknown file type: mode=%x" % mode) self._children[node.name] = node if not count % 1000: debug.log("processed %d files" % count) debug.log("total files processed: %d" % count) self._files = files self._subdirectories = subdirectories return subdirectories
def __getCurrentLocation(self, homePath): statTuple = os.lstat(homePath) if not stat.S_ISLNK(statTuple[stat.ST_MODE]): raise SystemExit("PDAQ_HOME '%s' is not a symlink" % homePath) return os.readlink(homePath)
def lstat(self, path=None): return os.lstat(self.join(path))
def get_perm(path): return stat.S_IMODE(os.lstat(path)[stat.ST_MODE])
def lstat(path): if isinstance(path, str): return os.lstat(path) else: return os.lstat(path.as_bytes())
def test_index_mutation(self, rw_repo): index = rw_repo.index num_entries = len(index.entries) cur_head = rw_repo.head uname = "Some Developer" umail = "*****@*****.**" rw_repo.config_writer().set_value("user", "name", uname) rw_repo.config_writer().set_value("user", "email", umail) # remove all of the files, provide a wild mix of paths, BaseIndexEntries, # IndexEntries def mixed_iterator(): count = 0 for entry in index.entries.itervalues(): type_id = count % 4 if type_id == 0: # path yield entry.path elif type_id == 1: # blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) elif type_id == 2: # BaseIndexEntry yield BaseIndexEntry(entry[:4]) elif type_id == 3: # IndexEntry yield entry else: raise AssertionError("Invalid Type") count += 1 # END for each entry # END mixed iterator deleted_files = index.remove(mixed_iterator(), working_tree=False) assert deleted_files assert self._count_existing(rw_repo, deleted_files) == len(deleted_files) assert len(index.entries) == 0 # reset the index to undo our changes index.reset() assert len(index.entries) == num_entries # remove with working copy deleted_files = index.remove(mixed_iterator(), working_tree=True) assert deleted_files assert self._count_existing(rw_repo, deleted_files) == 0 # reset everything index.reset(working_tree=True) assert self._count_existing(rw_repo, deleted_files) == len(deleted_files) # invalid type self.failUnlessRaises(TypeError, index.remove, [1]) # absolute path deleted_files = index.remove([os.path.join(rw_repo.working_tree_dir, "lib")], r=True) assert len(deleted_files) > 1 self.failUnlessRaises(ValueError, index.remove, ["/doesnt/exists"]) # TEST COMMITTING # commit changed index cur_commit = cur_head.commit commit_message = "commit default head" new_commit = index.commit(commit_message, head=False) assert cur_commit != new_commit assert new_commit.author.name == uname assert new_commit.author.email == umail assert new_commit.committer.name == uname assert new_commit.committer.email == umail assert new_commit.message == commit_message assert new_commit.parents[0] == cur_commit assert len(new_commit.parents) == 1 assert cur_head.commit == cur_commit # same index, no parents commit_message = "index without parents" commit_no_parents = index.commit(commit_message, parent_commits=list(), head=True) assert commit_no_parents.message == commit_message assert len(commit_no_parents.parents) == 0 assert cur_head.commit == commit_no_parents # same index, multiple parents commit_message = "Index with multiple parents\n commit with another line" commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) assert commit_multi_parent.message == commit_message assert len(commit_multi_parent.parents) == 2 assert commit_multi_parent.parents[0] == commit_no_parents assert commit_multi_parent.parents[1] == new_commit assert cur_head.commit == commit_multi_parent # re-add all files in lib # get the lib folder back on disk, but get an index without it index.reset(new_commit.parents[0], working_tree=True).reset(new_commit, working_tree=False) lib_file_path = os.path.join("lib", "git", "__init__.py") assert (lib_file_path, 0) not in index.entries assert os.path.isfile(os.path.join(rw_repo.working_tree_dir, lib_file_path)) # directory entries = index.add(['lib'], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert len(entries) > 1 # glob entries = index.reset(new_commit).add([os.path.join('lib', 'git', '*.py')], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert len(entries) == 14 # same file entries = index.reset(new_commit).add( [os.path.abspath(os.path.join('lib', 'git', 'head.py'))] * 2, fprogress=self._fprogress_add) self._assert_entries(entries) assert entries[0].mode & 0644 == 0644 # would fail, test is too primitive to handle this case # self._assert_fprogress(entries) self._reset_progress() assert len(entries) == 2 # missing path self.failUnlessRaises(OSError, index.reset(new_commit).add, ['doesnt/exist/must/raise']) # blob from older revision overrides current index revision old_blob = new_commit.parents[0].tree.blobs[0] entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert index.entries[(old_blob.path, 0)].hexsha == old_blob.hexsha and len(entries) == 1 # mode 0 not allowed null_hex_sha = Diff.NULL_HEX_SHA null_bin_sha = "\0" * 20 self.failUnlessRaises(ValueError, index.reset( new_commit).add, [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))]) # add new file new_file_relapath = "my_new_file" new_file_path = self._make_file(new_file_relapath, "hello world", rw_repo) entries = index.reset(new_commit).add( [BaseIndexEntry((010644, null_bin_sha, 0, new_file_relapath))], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert len(entries) == 1 and entries[0].hexsha != null_hex_sha # add symlink if sys.platform != "win32": basename = "my_real_symlink" target = "/etc/that" link_file = os.path.join(rw_repo.working_tree_dir, basename) os.symlink(target, link_file) entries = index.reset(new_commit).add([link_file], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert len(entries) == 1 and S_ISLNK(entries[0].mode) assert S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode) # we expect only the target to be written assert index.repo.odb.stream(entries[0].binsha).read() == target # END real symlink test # add fake symlink and assure it checks-our as symlink fake_symlink_relapath = "my_fake_symlink" link_target = "/etc/that" fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) fake_entry = BaseIndexEntry((0120000, null_bin_sha, 0, fake_symlink_relapath)) entries = index.reset(new_commit).add([fake_entry], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert entries[0].hexsha != null_hex_sha assert len(entries) == 1 and S_ISLNK(entries[0].mode) # assure this also works with an alternate method full_index_entry = IndexEntry.from_base(BaseIndexEntry((0120000, entries[0].binsha, 0, entries[0].path))) entry_key = index.entry_key(full_index_entry) index.reset(new_commit) assert entry_key not in index.entries index.entries[entry_key] = full_index_entry index.write() index.update() # force reread of entries new_entry = index.entries[entry_key] assert S_ISLNK(new_entry.mode) # a tree created from this should contain the symlink tree = index.write_tree() assert fake_symlink_relapath in tree index.write() # flush our changes for the checkout # checkout the fakelink, should be a link then assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE]) os.remove(fake_symlink_path) index.checkout(fake_symlink_path) # on windows we will never get symlinks if os.name == 'nt': # simlinks should contain the link as text ( which is what a # symlink actually is ) open(fake_symlink_path, 'rb').read() == link_target else: assert S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE]) # TEST RENAMING def assert_mv_rval(rval): for source, dest in rval: assert not os.path.exists(source) and os.path.exists(dest) # END for each renamed item # END move assertion utility self.failUnlessRaises(ValueError, index.move, ['just_one_path']) # file onto existing file files = ['AUTHORS', 'LICENSE'] self.failUnlessRaises(GitCommandError, index.move, files) # again, with force assert_mv_rval(index.move(files, f=True)) # files into directory - dry run paths = ['LICENSE', 'VERSION', 'doc'] rval = index.move(paths, dry_run=True) assert len(rval) == 2 assert os.path.exists(paths[0]) # again, no dry run rval = index.move(paths) assert_mv_rval(rval) # dir into dir rval = index.move(['doc', 'test']) assert_mv_rval(rval) # TEST PATH REWRITING ###################### count = [0] def rewriter(entry): rval = str(count[0]) count[0] += 1 return rval # END rewriter def make_paths(): # two existing ones, one new one yield 'CHANGES' yield 'ez_setup.py' yield index.entries[index.entry_key('README', 0)] yield index.entries[index.entry_key('.gitignore', 0)] for fid in range(3): fname = 'newfile%i' % fid open(fname, 'wb').write("abcd") yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0100644, fname) # END for each new file # END path producer paths = list(make_paths()) self._assert_entries(index.add(paths, path_rewriter=rewriter)) for filenum in range(len(paths)): assert index.entry_key(str(filenum), 0) in index.entries # TEST RESET ON PATHS ###################### arela = "aa" brela = "bb" afile = self._make_file(arela, "adata", rw_repo) bfile = self._make_file(brela, "bdata", rw_repo) akey = index.entry_key(arela, 0) bkey = index.entry_key(brela, 0) keys = (akey, bkey) absfiles = (afile, bfile) files = (arela, brela) for fkey in keys: assert not fkey in index.entries index.add(files, write=True) nc = index.commit("2 files committed", head=False) for fkey in keys: assert fkey in index.entries # just the index index.reset(paths=(arela, afile)) assert not akey in index.entries assert bkey in index.entries # now with working tree - files on disk as well as entries must be recreated rw_repo.head.commit = nc for absfile in absfiles: os.remove(absfile) index.reset(working_tree=True, paths=files) for fkey in keys: assert fkey in index.entries for absfile in absfiles: assert os.path.isfile(absfile)
def clean_leftovers(): stat_cache = {} files_by_name = {} new_paths = [] for f in self.files.list: files_by_name.setdefault(os.path.basename(f.path), []).append(f) new_paths.append(f.path) if not self.old_files: return for old_file in self.old_files.list: old_file_path = os.path.join(ctx.config.dest_dir(), old_file.path) if old_file.path in new_paths: continue if old_file_path not in new_paths: if os.path.islink(old_file_path): os.unlink(old_file_path) continue try: old_file_stat = os.lstat(old_file_path) except OSError: continue old_filename = os.path.basename(old_file.path) # If one of the parent directories is a symlink, it is possible # that the new and old file paths refer to the same file. # In this case, we must not remove the old file. # # e.g. /lib/libdl.so and /lib64/libdl.so when /lib64 is # a symlink to /lib. for new_file in files_by_name.get(old_filename, []): new_file_stat = stat_cache.get(new_file.path) if new_file_stat is None: path = os.path.join(ctx.config.dest_dir(), new_file.path) try: new_file_stat = os.lstat(path) except OSError: continue stat_cache[new_file.path] = new_file_stat if os.path.samestat(new_file_stat, old_file_stat): break else: remove_permanent = not ctx.config.get_option( "preserve_permanent") Remove.remove_file(old_file, self.pkginfo.name, remove_permanent, store_old_paths=self.store_old_paths)
def delete(path, shred=False, ignore_missing=False, allow_shred=True): """Delete path that is either file, directory, link or FIFO. If shred is enabled as a function parameter or the BleachBit global parameter, the path will be shredded unless allow_shred = False. """ from bleachbit.Options import options is_special = False path = extended_path(path) if not os.path.lexists(path): if ignore_missing: return raise OSError(2, 'No such file or directory', path) if 'posix' == os.name: # With certain (relatively rare) files on Windows os.lstat() # may return Access Denied mode = os.lstat(path)[stat.ST_MODE] is_special = stat.S_ISFIFO(mode) or stat.S_ISLNK(mode) if is_special: os.remove(path) elif os.path.isdir(path): delpath = path if allow_shred and (shred or options.get('shred')): delpath = wipe_name(path) try: os.rmdir(delpath) except OSError as e: # [Errno 39] Directory not empty # https://bugs.launchpad.net/bleachbit/+bug/1012930 if errno.ENOTEMPTY == e.errno: logger.info("directory is not empty: %s", path) else: raise except WindowsError as e: # WindowsError: [Error 145] The directory is not empty: # 'C:\\Documents and Settings\\username\\Local Settings\\Temp\\NAILogs' # Error 145 may happen if the files are scheduled for deletion # during reboot. if 145 == e.winerror: logger.info("directory is not empty: %s", path) else: raise elif os.path.isfile(path): # wipe contents if allow_shred and (shred or options.get('shred')): try: wipe_contents(path) except pywinerror as e: # 2 = The system cannot find the file specified. # This can happen with a broken symlink # https://github.com/bleachbit/bleachbit/issues/195 if 2 != e.winerror: raise # If a broken symlink, try os.remove() below. except IOError as e: # permission denied (13) happens shredding MSIE 8 on Windows 7 logger.debug("IOError #%s shredding '%s'", e.errno, path, exc_info=True) # wipe name os.remove(wipe_name(path)) else: # unlink os.remove(path) else: logger.info("special file type cannot be deleted: %s", path)
def wallet_permissions(self): return oct(os.lstat(self.wallet_path).st_mode)[-3:]
def subtest_repair_file_permissions(self): try: # for this test we need to make sure the warning is NOT suppressed, but set it back to current setting at the end original_suppress_warning_value = os.environ.get( 'OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING') os.environ['OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING'] = 'False' # capture stdout / stderr so we can validate warnings if oci_cli.cli_util.is_windows(): with util.capture() as out: # create a temporary file and set some unnecessary permissions tmp = tempfile.NamedTemporaryFile() subprocess.check_output( 'icacls "{path}" /grant Everyone:F'.format( path=tmp.name), stderr=subprocess.STDOUT) # warning should be emitted because permissions are too loose oci_cli.cli_util.FilePermissionChecker.warn_on_invalid_file_permissions( tmp.name) assert 'WARNING' in out[1].getvalue() # reset captured stderr out[1] = StringIO() result = self.invoke([ 'setup', 'repair-file-permissions', '--file', tmp.name ]) assert result.exit_code == 0 # no warning should be emitted because we repaired the permissions oci_cli.cli_util.FilePermissionChecker.warn_on_invalid_file_permissions( tmp.name) assert 'WARNING' not in out[1].getvalue() else: with util.capture() as out: # create a temporary file and set some unnecessary permissions tmp = tempfile.NamedTemporaryFile() os.chmod(tmp.name, 509) # octal 775 # warning should be emitted because permissions are too loose oci_cli.cli_util.FilePermissionChecker.warn_on_invalid_file_permissions( tmp.name) assert 'WARNING' in out[1].getvalue() # reset captured stderr out[1] = StringIO() result = self.invoke([ 'setup', 'repair-file-permissions', '--file', tmp.name ]) assert result.exit_code == 0 assert oct(stat.S_IMODE(os.lstat( tmp.name).st_mode)) == oct(384) # 600 # no warning should be emitted because we repaired the permissions oci_cli.cli_util.FilePermissionChecker.warn_on_invalid_file_permissions( tmp.name) assert 'WARNING' not in out[1].getvalue() with util.capture() as out: # validate that 400 file permissions are accepted as well os.chmod(tmp.name, 256) # octal 400 oci_cli.cli_util.FilePermissionChecker.warn_on_invalid_file_permissions( tmp.name) assert 'WARNING' not in out[1].getvalue() finally: if original_suppress_warning_value is None: del os.environ['OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING'] else: os.environ[ 'OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING'] = original_suppress_warning_value
def ego_owner(filename): """Return whether current user owns the file""" return os.lstat(filename).st_uid == os.getuid()
if os.system('../fusecompress -o dedup,detach test') != 0: os.rmdir('test') sys.exit(2) text1 = 'foo' text2 = 'bar' open('test/1', 'w').write(text1) os.chmod('test/1', 0647) open('test/2', 'w').write(text1) os.chmod('test/2', 0600) os.system('fusermount -u test') time.sleep(1) assert (os.lstat('test/1').st_nlink == 2) os.system('../fusecompress -o dedup,detach test') time.sleep(.1) stat1 = os.lstat('test/1') stat2 = os.lstat('test/2') assert (stat1.st_nlink == 1) assert (stat2.st_nlink == 1) assert (stat.S_IMODE(stat1.st_mode) == 0647) assert (stat.S_IMODE(stat2.st_mode) == 0600) os.system('fusermount -u test') time.sleep(1) shutil.rmtree('test')
def getsize(name): try: return os.lstat( name).st_size # same as above; use instead of os.path.... except: return 0
def check(self, test): """ """ success = 1 [testname, args] = self.rule.split("=") if testname[0] == "!": self.false = 1 testname = testname[1:] [kind, case] = testname.split("_") if "|" in args: [key, value] = args.split("|", 1) else: [key, value] = [args, None] if kind == "PACMAN": if case == "RETCODE": if test.retcode != int(key): success = 0 elif case == "OUTPUT": logfile = os.path.join(test.root, util.LOGFILE) if not os.access(logfile, os.F_OK): tap.diag("LOGFILE not found, cannot validate 'OUTPUT' rule") success = 0 elif not util.grep(logfile, key): success = 0 else: tap.diag("PACMAN rule '%s' not found" % case) success = -1 elif kind == "PKG": localdb = test.db["local"] newpkg = localdb.db_read(key) if not newpkg: success = 0 else: if case == "EXIST": success = 1 elif case == "VERSION": if value != newpkg.version: success = 0 elif case == "DESC": if value != newpkg.desc: success = 0 elif case == "GROUPS": if not value in newpkg.groups: success = 0 elif case == "PROVIDES": if not value in newpkg.provides: success = 0 elif case == "DEPENDS": if not value in newpkg.depends: success = 0 elif case == "OPTDEPENDS": success = 0 for optdep in newpkg.optdepends: if value == optdep.split(':', 1)[0]: success = 1 break elif case == "REASON": if newpkg.reason != int(value): success = 0 elif case == "FILES": if not value in newpkg.files: success = 0 elif case == "BACKUP": success = 0 for f in newpkg.backup: if f.startswith(value + "\t"): success = 1 break; else: tap.diag("PKG rule '%s' not found" % case) success = -1 elif kind == "FILE": filename = os.path.join(test.root, key) if case == "EXIST": if not os.path.isfile(filename): success = 0 elif case == "EMPTY": if not (os.path.isfile(filename) and os.path.getsize(filename) == 0): success = 0 elif case == "CONTENTS": try: with open(filename, 'r') as f: success = f.read() == value except: success = 0 elif case == "MODIFIED": for f in test.files: if f.name == key: if not f.ismodified(): success = 0 break elif case == "MODE": if not os.path.isfile(filename): success = 0 else: mode = os.lstat(filename)[stat.ST_MODE] if int(value, 8) != stat.S_IMODE(mode): success = 0 elif case == "TYPE": if value == "dir": if not os.path.isdir(filename): success = 0 elif value == "file": if not os.path.isfile(filename): success = 0 elif value == "link": if not os.path.islink(filename): success = 0 elif case == "PACNEW": if not os.path.isfile("%s.pacnew" % filename): success = 0 elif case == "PACSAVE": if not os.path.isfile("%s.pacsave" % filename): success = 0 else: tap.diag("FILE rule '%s' not found" % case) success = -1 elif kind == "DIR": filename = os.path.join(test.root, key) if case == "EXIST": if not os.path.isdir(filename): success = 0 else: tap.diag("DIR rule '%s' not found" % case) success = -1 elif kind == "LINK": filename = os.path.join(test.root, key) if case == "EXIST": if not os.path.islink(filename): success = 0 else: tap.diag("LINK rule '%s' not found" % case) success = -1 elif kind == "CACHE": cachedir = os.path.join(test.root, util.PM_CACHEDIR) if case == "EXISTS": pkg = test.findpkg(key, value, allow_local=True) if not pkg or not os.path.isfile( os.path.join(cachedir, pkg.filename())): success = 0 else: tap.diag("Rule kind '%s' not found" % kind) success = -1 if self.false and success != -1: success = not success self.result = success return success
def runall(self): print "server listening..." self.s.listen(5) self.conn, self.addr = self.s.accept() self.data = self.conn.recv(1024) self.data=self.data.split() #print len(self.data) if len(self.data)==0: return elif self.data[0]=='download': if len(self.data)!=3: print "INVALID COMMAND" elif self.data[1]=='UDP': if os.path.exists(self.data[2])==False: print "File not exists" return self.conn.send(str(self.data[2])+' '+str(os.stat(self.data[2]).st_size)+' '+str(time.ctime(os.stat(self.data[2]).st_mtime))+' '+str(self.md5(self.data[2]))+' '+str(oct(stat.S_IMODE(os.lstat(self.data[2]).st_mode)))+' ') serverSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) serverSocket.bind((socket.gethostname(),40000)) message, clientAddress = serverSocket.recvfrom(1024) modifiedMessage = message.upper() self.filename=self.data[2] f = open(self.filename,'rb') l = f.read(1024) while (l): serverSocket.sendto(l,clientAddress) l = f.read(1024) serverSocket.sendto(l,clientAddress) f.close() serverSocket.close() self.conn.close() elif self.data[1]=='TCP': if os.path.exists(self.data[2])==False: print "File not exists" return self.filename=self.data[2] self.conn.send(str(self.data[2])+' '+str(os.stat(self.data[2]).st_size)+' '+str(time.ctime(os.stat(self.data[2]).st_mtime))+' '+str(self.md5(self.data[2]))+' '+str(oct(stat.S_IMODE(os.lstat(self.data[2]).st_mode)))+' ') f=open(self.filename,'rb') l = f.read(1024) while (l): self.conn.send(l) l = f.read(1024) f.close() self.conn.close() elif self.data[0]=='index': if len(self.data)<2: print "INVALID COMMAND" self.conn.send("INVALID COMMAND") elif self.data[1]=='longlist': for file in os.listdir(os.getcwd()): a=os.stat(file) '''self.conn.send(file) self.conn.send("\t") self.conn.send(str(a.st_size)) self.conn.send("\t") if os.path.isfile(file): self.conn.send("file") else: self.conn.send("Directory") self.conn.send("\t") self.conn.send(str(time.ctime(a.st_mtime))) self.conn.send("\n")''' filename=str(file) size=str(a.st_size) if os.path.isfile(file): type="file" else: type="Directory" mtime=str(time.ctime(a.st_mtime)) self.conn.send(filename+" "+size+" "+type+" "+mtime+"\n") self.conn.close() elif self.data[1]=='shortlist' and len(self.data)==4: for file in os.listdir(os.getcwd()): a=os.stat(file) mint=time.mktime(time.strptime(' '.join(self.data[2:6]), '%d %m %Y %H:%M:%S')) maxt=time.mktime(time.strptime(' '.join(self.data[6:10]), '%d %m %Y %H:%M:%S')) if mint<os.path.getmtime(file)<maxt: '''self.conn.send(file) self.conn.send("\t") self.conn.send(str(a.st_size)) self.conn.send("\t") if os.path.isfile(file): self.conn.send("file") else: self.conn.send("Directory") self.conn.send("\t") self.conn.send(str(time.ctime(a.st_mtime))) self.conn.send("\n")''' filename=str(file) size=str(a.st_size) if os.path.isfile(file): type="file" else: type="Directory" mtime=str(time.ctime(a.st_mtime)) self.conn.send(filename+" "+size+" "+type+" "+mtime+"\n") self.conn.close() elif self.data[1]=='regex' and len(self.data)==3: reprog=re.compile(' '.join(self.data[2:]).strip()) for file in os.listdir(os.getcwd()): if reprog.match(file): a=os.stat(file) '''self.conn.send(file) self.conn.send("\t") self.conn.send(str(a.st_size)) self.conn.send("\t") if os.path.isfile(file): self.conn.send("file") else: self.conn.send("Directory") self.conn.send("\t") self.conn.send(str(time.ctime(a.st_mtime))) self.conn.send("\n")''' filename=str(file) size=str(a.st_size) if os.path.isfile(file): type="file" else: type="Directory" mtime=str(time.ctime(a.st_mtime)) self.conn.send(filename+" "+size+" "+type+" "+mtime+"\n") self.conn.close() else: print "INVALID COMMAND" self.conn.send("INVALID COMMAND") elif self.data[0]=='hash': if len(self.data)<2: print "INVALID COMMAND" self.conn.send("INVALID COMMAND") elif self.data[1]=='checkall': for file in os.listdir(os.getcwd()): '''self.conn.send(file) self.conn.send("\t") self.conn.send(self.md5(file)) self.conn.send("\t") self.conn.send(str(time.ctime(os.stat(file).st_mtime))) self.conn.send("\n")''' self.conn.send(str(file)+' '+str(self.md5(file))+' '+str(time.ctime(os.stat(file).st_mtime))+' ') self.conn.close() elif self.data[1]=='verify' and len(self.data)==3: '''self.conn.send(self.data[2]) self.conn.send("\t") self.conn.send(self.md5(self.data[2])) self.conn.send("\t") self.conn.send(str(time.ctime(os.stat(self.data[2]).st_mtime))) self.conn.send("\n")''' self.conn.send(str(self.data[2])+' '+str(self.md5(self.data[2]))+' '+str(time.ctime(os.stat(self.data[2]).st_mtime))+' ') self.conn.close() else: print "INVALID COMMAND" self.conn.send("INVALID COMMAND") elif self.data[0]=='getfileslist': ata='' for file in os.listdir(os.getcwd()): '''self.conn.send(str(file)) self.conn.send("\t") self.conn.send(str(self.md5(file)))''' self.conn.send(str(file)+' '+str(self.md5(file))+' ') self.conn.send(ata) self.conn.close() elif self.data[0]=='exit': global exit exit=True else: print "command not found"
def _make_writable(fname): """Make a file writable.""" os.chmod(fname, stat.S_IMODE(os.lstat(fname)[stat.ST_MODE]) | 128) # write
def copy(self, dest, skip_if_older=True): assert isinstance(dest, basestring) # The logic in this function is complicated by the fact that symlinks # aren't universally supported. So, where symlinks aren't supported, we # fall back to file copying. Keep in mind that symlink support is # per-filesystem, not per-OS. # Handle the simple case where symlinks are definitely not supported by # falling back to file copy. if not hasattr(os, 'symlink'): return File.copy(self, dest, skip_if_older=skip_if_older) # Always verify the symlink target path exists. if not os.path.exists(self.path): raise ErrorMessage('Symlink target path does not exist: %s' % self.path) st = None try: st = os.lstat(dest) except OSError as ose: if ose.errno != errno.ENOENT: raise # If the dest is a symlink pointing to us, we have nothing to do. # If it's the wrong symlink, the filesystem must support symlinks, # so we replace with a proper symlink. if st and stat.S_ISLNK(st.st_mode): link = os.readlink(dest) if link == self.path: return False os.remove(dest) os.symlink(self.path, dest) return True # If the destination doesn't exist, we try to create a symlink. If that # fails, we fall back to copy code. if not st: try: os.symlink(self.path, dest) return True except OSError: return File.copy(self, dest, skip_if_older=skip_if_older) # Now the complicated part. If the destination exists, we could be # replacing a file with a symlink. Or, the filesystem may not support # symlinks. We want to minimize I/O overhead for performance reasons, # so we keep the existing destination file around as long as possible. # A lot of the system calls would be eliminated if we cached whether # symlinks are supported. However, even if we performed a single # up-front test of whether the root of the destination directory # supports symlinks, there's no guarantee that all operations for that # dest (or source) would be on the same filesystem and would support # symlinks. # # Our strategy is to attempt to create a new symlink with a random # name. If that fails, we fall back to copy mode. If that works, we # remove the old destination and move the newly-created symlink into # its place. temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4())) try: os.symlink(self.path, temp_dest) # TODO Figure out exactly how symlink creation fails and only trap # that. except EnvironmentError: return File.copy(self, dest, skip_if_older=skip_if_older) # If removing the original file fails, don't forget to clean up the # temporary symlink. try: os.remove(dest) except EnvironmentError: os.remove(temp_dest) raise os.rename(temp_dest, dest) return True
def action(self, path): rootdir = self.rootdir pubpath = pkg.misc.relpath(path, rootdir) pstat = os.lstat(path) mode = oct(stat.S_IMODE(pstat.st_mode)) timestamp = pkg.misc.time_to_timestamp(pstat.st_mtime) # Set default root and group. owner = "root" group = "bin" # Check whether need to change owner. if not self.use_default_owner: try: owner = pwd.getpwuid(pstat.st_uid).pw_name except KeyError as e: owner = None try: group = grp.getgrgid(pstat.st_gid).gr_name except KeyError as e: group = None if not owner and not group: raise pkg.bundle.InvalidOwnershipException(path, uid=pstat.st_uid, gid=pstat.st_gid) elif not owner: raise pkg.bundle.InvalidOwnershipException(path, uid=pstat.st_uid) elif not group: raise pkg.bundle.InvalidOwnershipException(path, gid=pstat.st_gid) if stat.S_ISREG(pstat.st_mode): inode = pstat.st_ino # Any inode in self.inodes will either have been visited # before or will have been pre-populated from the list # of known targets. Create file actions for known # targets and unvisited inodes. if pubpath in self.targetpaths or \ inode not in self.inodes: if pstat.st_nlink > 1: self.inodes.setdefault(inode, path) return pkg.actions.file.FileAction(open(path, "rb"), mode=mode, owner=owner, group=group, path=pubpath, timestamp=timestamp) else: # Find the relative path to the link target. target = pkg.misc.relpath(self.inodes[inode], os.path.dirname(path)) return pkg.actions.hardlink.HardLinkAction(path=pubpath, target=target) elif stat.S_ISLNK(pstat.st_mode): return pkg.actions.link.LinkAction(target=os.readlink(path), path=pubpath) elif stat.S_ISDIR(pstat.st_mode): return pkg.actions.directory.DirectoryAction(timestamp=timestamp, mode=mode, owner=owner, group=group, path=pubpath)
def _dir_model(self, path, content=True): """Build a model for a directory if content is requested, will include a listing of the directory """ os_path = self._get_os_path(path) four_o_four = u'directory does not exist: %r' % path if not os.path.isdir(os_path): raise web.HTTPError(404, four_o_four) elif is_hidden(os_path, self.root_dir) and not self.allow_hidden: self.log.info( "Refusing to serve hidden directory %r, via 404 Error", os_path) raise web.HTTPError(404, four_o_four) model = self._base_model(path) model['type'] = 'directory' model['size'] = None if content: model['content'] = contents = [] os_dir = self._get_os_path(path) for name in os.listdir(os_dir): try: os_path = os.path.join(os_dir, name) except UnicodeDecodeError as e: self.log.warning("failed to decode filename '%s': %s", name, e) continue try: st = os.lstat(os_path) except OSError as e: # skip over broken symlinks in listing if e.errno == errno.ENOENT: self.log.warning("%s doesn't exist", os_path) elif e.errno != errno.EACCES: # Don't provide clues about protected files self.log.warning("Error stat-ing %s: %s", os_path, e) continue if (not stat.S_ISLNK(st.st_mode) and not stat.S_ISREG(st.st_mode) and not stat.S_ISDIR(st.st_mode)): self.log.debug("%s not a regular file", os_path) continue try: if self.should_list(name): if self.allow_hidden or not is_file_hidden( os_path, stat_res=st): contents.append( self.get(path='%s/%s' % (path, name), content=False)) except OSError as e: # ELOOP: recursive symlink, also don't show failure due to permissions if e.errno not in [errno.ELOOP, errno.EACCES]: self.log.warning( "Unknown error checking if file %r is hidden", os_path, exc_info=True, ) model['format'] = 'json' return model
def get_dir_meta(worker_name, path, cliargs, reindex_dict): """This is the get directory meta data function. It gets directory metadata and returns dir meta dict. It checks if meta data is in Redis and compares times mtime and ctime on disk compared to Redis and if same returns sametimes string. """ try: lstat_path = os.lstat(path) mtime_unix = lstat_path.st_mtime mtime_utc = datetime.utcfromtimestamp(mtime_unix) \ .strftime('%Y-%m-%dT%H:%M:%S') atime_unix = lstat_path.st_atime atime_utc = datetime.utcfromtimestamp(atime_unix) \ .strftime('%Y-%m-%dT%H:%M:%S') ctime_unix = lstat_path.st_ctime ctime_utc = datetime.utcfromtimestamp(ctime_unix) \ .strftime('%Y-%m-%dT%H:%M:%S') if cliargs['index2']: # check if directory times cached in Redis redis_dirtime = redis_conn.get( base64.encodestring(path.encode('utf-8', errors='ignore'))) if redis_dirtime: cached_times = float(redis_dirtime.decode('utf-8')) # check if cached times are the same as on disk current_times = float(mtime_unix + ctime_unix) if cached_times == current_times: return "sametimes" # get time now in utc indextime_utc = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") # get user id of owner uid = lstat_path.st_uid # try to get owner user name # first check cache if uid in uids: owner = owners[uid] # not in cache else: try: owner = pwd.getpwuid(uid).pw_name.split('\\') # remove domain before owner if len(owner) == 2: owner = owner[1] else: owner = owner[0] # if we can't find the owner's user name, use the uid number except KeyError: owner = uid # store it in cache if not uid in uids: uids.append(uid) owners[uid] = owner # get group id gid = lstat_path.st_gid # try to get group name # first check cache if gid in gids: group = groups[gid] # not in cache else: try: group = grp.getgrgid(gid).gr_name.split('\\') # remove domain before group if len(group) == 2: group = group[1] else: group = group[0] # if we can't find the group name, use the gid number except KeyError: group = gid # store in cache if not gid in gids: gids.append(gid) groups[gid] = group inode = lstat_path.st_ino hardlinks = lstat_path.st_nlink filename = os.path.basename(path) parentdir = os.path.abspath(os.path.join(path, os.pardir)) fullpath = os.path.abspath(os.path.join(parentdir, filename)) dirmeta_dict = { "filename": filename, "path_parent": parentdir, "filesize": 0, "items": 1, # 1 for itself "items_files": 0, "items_subdirs": 0, "last_modified": mtime_utc, "last_access": atime_utc, "last_change": ctime_utc, "hardlinks": hardlinks, "inode": inode, "owner": owner, "group": group, "tag": "", "tag_custom": "", "crawl_time": 0, "change_percent_filesize": "", "change_percent_items": "", "change_percent_items_files": "", "change_percent_items_subdirs": "", "worker_name": worker_name, "indexing_date": indextime_utc, "_type": "directory" } # check plugins for adding extra meta data to dirmeta_dict for plugin in diskover.plugins: try: # check if plugin is for directory doc mappings = {'mappings': {'directory': {'properties': {}}}} plugin.add_mappings(mappings) dirmeta_dict.update(plugin.add_meta(fullpath)) except KeyError: pass # add any autotags to dirmeta_dict if cliargs['autotag'] and len(diskover.config['autotag_dirs']) > 0: auto_tag(dirmeta_dict, 'directory', mtime_unix, atime_unix, ctime_unix) # search for and copy over any existing tags from reindex_dict for sublist in reindex_dict['directory']: if sublist[0] == fullpath: dirmeta_dict['tag'] = sublist[1] dirmeta_dict['tag_custom'] = sublist[2] break except (IOError, OSError): return None # cache directory times in Redis, encode path (key) using base64 if diskover.config['redis_cachedirtimes'] == 'True' or diskover.config[ 'redis_cachedirtimes'] == 'true': redis_conn.set(base64.encodestring( path.encode('utf-8', errors='ignore')), mtime_unix + ctime_unix, ex=diskover.config['redis_dirtimesttl']) return dirmeta_dict
def VerifyFileOwner(filename): stat_info = os.lstat(filename) return os.getuid() == stat_info.st_uid
def main(): module = AssibleModule( argument_spec=dict( path=dict(type='path', required=True, aliases=['dest', 'name']), follow=dict(type='bool', default=False), get_md5=dict(type='bool', default=False), get_checksum=dict(type='bool', default=True), get_mime=dict(type='bool', default=True, aliases=['mime', 'mime_type', 'mime-type']), get_attributes=dict(type='bool', default=True, aliases=['attr', 'attributes']), checksum_algorithm=dict(type='str', default='sha1', choices=[ 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512' ], aliases=['checksum', 'checksum_algo']), ), supports_check_mode=True, ) path = module.params.get('path') b_path = to_bytes(path, errors='surrogate_or_strict') follow = module.params.get('follow') get_mime = module.params.get('get_mime') get_attr = module.params.get('get_attributes') get_checksum = module.params.get('get_checksum') checksum_algorithm = module.params.get('checksum_algorithm') # NOTE: undocumented option since 2.9 to be removed at a later date if possible (3.0+) # no real reason for keeping other than fear we may break older content. get_md5 = module.params.get('get_md5') # main stat data try: if follow: st = os.stat(b_path) else: st = os.lstat(b_path) except OSError as e: if e.errno == errno.ENOENT: output = {'exists': False} module.exit_json(changed=False, stat=output) module.fail_json(msg=e.strerror) # process base results output = format_output(module, path, st) # resolved permissions for perm in [('readable', os.R_OK), ('writeable', os.W_OK), ('executable', os.X_OK)]: output[perm[0]] = os.access(b_path, perm[1]) # symlink info if output.get('islnk'): output['lnk_source'] = os.path.realpath(b_path) output['lnk_target'] = os.readlink(b_path) try: # user data pw = pwd.getpwuid(st.st_uid) output['pw_name'] = pw.pw_name except (TypeError, KeyError): pass try: # group data grp_info = grp.getgrgid(st.st_gid) output['gr_name'] = grp_info.gr_name except (KeyError, ValueError, OverflowError): pass # checksums if output.get('isreg') and output.get('readable'): # NOTE: see above about get_md5 if get_md5: # Will fail on FIPS-140 compliant systems try: output['md5'] = module.md5(b_path) except ValueError: output['md5'] = None if get_checksum: output['checksum'] = module.digest_from_file( b_path, checksum_algorithm) # try to get mime data if requested if get_mime: output['mimetype'] = output['charset'] = 'unknown' mimecmd = module.get_bin_path('file') if mimecmd: mimecmd = [mimecmd, '--mime-type', '--mime-encoding', b_path] try: rc, out, err = module.run_command(mimecmd) if rc == 0: mimetype, charset = out.rsplit(':', 1)[1].split(';') output['mimetype'] = mimetype.strip() output['charset'] = charset.split('=')[1].strip() except Exception: pass # try to get attr data if get_attr: output['version'] = None output['attributes'] = [] output['attr_flags'] = '' out = module.get_file_attributes(b_path) for x in ('version', 'attributes', 'attr_flags'): if x in out: output[x] = out[x] module.exit_json(changed=False, stat=output)
def get_file_meta(worker_name, path, cliargs, reindex_dict): """This is the get file meta data function. It scrapes file meta and ignores files smaller than minsize Bytes, newer than mtime and in excluded_files. Returns file meta dict. """ try: filename = os.path.basename(path) # check if file is in exluded_files list extension = os.path.splitext(filename)[1][1:].strip().lower() if file_excluded(filename, extension, path, cliargs['verbose']): return None # use lstat to get meta and not follow sym links stat = os.lstat(path) # get file size (bytes) size = stat.st_size # Skip files smaller than minsize cli flag if size < cliargs['minsize']: return None # check file modified time mtime_unix = stat.st_mtime mtime_utc = \ datetime.utcfromtimestamp(mtime_unix).strftime('%Y-%m-%dT%H:%M:%S') # Convert time in days (mtime cli arg) to seconds time_sec = cliargs['mtime'] * 86400 file_mtime_sec = time.time() - mtime_unix # Only process files modified at least x days ago if file_mtime_sec < time_sec: return None # get access time atime_unix = stat.st_atime atime_utc = \ datetime.utcfromtimestamp(atime_unix).strftime('%Y-%m-%dT%H:%M:%S') # get change time ctime_unix = stat.st_ctime ctime_utc = \ datetime.utcfromtimestamp(ctime_unix).strftime('%Y-%m-%dT%H:%M:%S') # get user id of owner uid = stat.st_uid # try to get owner user name # first check cache if uid in uids: owner = owners[uid] # not in cache else: try: owner = pwd.getpwuid(uid).pw_name.split('\\') # remove domain before owner if len(owner) == 2: owner = owner[1] else: owner = owner[0] # if we can't find the owner's user name, use the uid number except KeyError: owner = uid # store it in cache if not uid in uids: uids.append(uid) owners[uid] = owner # get group id gid = stat.st_gid # try to get group name # first check cache if gid in gids: group = groups[gid] # not in cache else: try: group = grp.getgrgid(gid).gr_name.split('\\') # remove domain before group if len(group) == 2: group = group[1] else: group = group[0] # if we can't find the group name, use the gid number except KeyError: group = gid # store in cache if not gid in gids: gids.append(gid) groups[gid] = group # get inode number inode = stat.st_ino # get number of hardlinks hardlinks = stat.st_nlink # create md5 hash of file using metadata filesize and mtime filestring = str(size) + str(mtime_unix) filehash = hashlib.md5(filestring.encode('utf-8')).hexdigest() # get time indextime_utc = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") # get absolute path of parent directory parentdir = os.path.abspath(os.path.join(path, os.pardir)) # create file metadata dictionary filemeta_dict = { "filename": filename, "extension": extension, "path_parent": parentdir, "filesize": size, "owner": owner, "group": group, "last_modified": mtime_utc, "last_access": atime_utc, "last_change": ctime_utc, "hardlinks": hardlinks, "inode": inode, "filehash": filehash, "tag": "", "tag_custom": "", "dupe_md5": "", "worker_name": worker_name, "indexing_date": indextime_utc, "_type": "file" } # check plugins for adding extra meta data to filemeta_dict for plugin in diskover.plugins: try: # check if plugin is for file doc mappings = {'mappings': {'file': {'properties': {}}}} plugin.add_mappings(mappings) filemeta_dict.update(plugin.add_meta(path)) except KeyError: pass # add any autotags to filemeta_dict if cliargs['autotag'] and len(diskover.config['autotag_files']) > 0: auto_tag(filemeta_dict, 'file', mtime_unix, atime_unix, ctime_unix) # search for and copy over any existing tags from reindex_dict for sublist in reindex_dict['file']: if sublist[0] == path: filemeta_dict['tag'] = sublist[1] filemeta_dict['tag_custom'] = sublist[2] break except (IOError, OSError): return None return filemeta_dict
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return m = self.recipe.magic[path] if not m: return # FIXME: should be: #if (m.name == "ELF" or m.name == "ar") and \ # m.contents['hasDebug']): # but this has to wait until ewt writes debug detection # for archives as well as elf files if (m.name == "ELF" and m.contents['hasDebug']) or \ (m.name == "ar"): oldmode = None fullpath = self.dm.destdir+path mode = os.lstat(fullpath)[stat.ST_MODE] if mode & 0600 != 0600: # need to be able to read and write the file to strip it oldmode = mode os.chmod(fullpath, mode|0600) if self.debuginfo and m.name == 'ELF' and not path.endswith('.o'): dir=os.path.dirname(path) b=os.path.basename(path) if not b.endswith('.debug'): b += '.debug' debuglibdir = '%(destdir)s%(debuglibdir)s' %self.dm +dir debuglibpath = util.joinPaths(debuglibdir, b) if os.path.exists(debuglibpath): return self._openDb() if (_findProgPath(self.macros.debugedit, self.db, self.recipe, error=False) and _findProgPath(self.macros.strip, self.db, self.recipe, error=False)): # null-separated AND terminated list, so we need to throw # away the last (empty) item before updating self.debugfiles self.debugfiles |= set(util.popen( '%(debugedit)s -b %(topbuilddir)s -d %(debugsrcdir)s' ' -l /dev/stdout '%self.dm +fullpath).read().split('\x00')[:-1]) util.mkdirChain(debuglibdir) util.execute('%s -f %s %s' %( self.dm.strip, debuglibpath, fullpath)) else: self._openDb() if m.name == 'ar' or path.endswith('.o'): # just in case strip is eu-strip, which segfaults # whenever it touches an ar archive, and seems to # break some .o files if _findProgPath(self.macros.strip_archive, self.db, self.recipe, error=False): util.execute('%(strip_archive)s ' %self.dm +fullpath) else: if _findProgPath(self.macros.strip, self.db, self.recipe, error=False): util.execute('%(strip)s ' %self.dm +fullpath) del self.recipe.magic[path] if oldmode is not None: os.chmod(fullpath, oldmode)
def lstat(self): return os.lstat(self)