def test_symlink_latest_log_directory(self): handler = FileProcessorHandler(base_log_folder=self.base_log_folder, filename_template=self.filename) handler.dag_dir = self.dag_dir date1 = (timezone.utcnow() + timedelta(days=1)).strftime("%Y-%m-%d") date2 = (timezone.utcnow() + timedelta(days=2)).strftime("%Y-%m-%d") p1 = os.path.join(self.base_log_folder, date1, "log1") p2 = os.path.join(self.base_log_folder, date1, "log2") if os.path.exists(p1): os.remove(p1) if os.path.exists(p2): os.remove(p2) link = os.path.join(self.base_log_folder, "latest") with freeze_time(date1): handler.set_context(filename=os.path.join(self.dag_dir, "log1")) self.assertTrue(os.path.islink(link)) self.assertEqual(os.path.basename(os.readlink(link)), date1) self.assertTrue(os.path.exists(os.path.join(link, "log1"))) with freeze_time(date2): handler.set_context(filename=os.path.join(self.dag_dir, "log2")) self.assertTrue(os.path.islink(link)) self.assertEqual(os.path.basename(os.readlink(link)), date2) self.assertTrue(os.path.exists(os.path.join(link, "log2")))
def init(self): if os.path.isfile("/tmp/extip"): os.system("rm -f /tmp/extip") self.softcam = CamControl('softcam') print "# selected ", self.softcam.current() self.link_softcam = os.readlink('/etc/init.d/softcam') self.link_cardserver = os.readlink('/etc/init.d/cardserver') if self.link_softcam.find('mgcamd') != -1: self.readNewcsPort() if self.link_cardserver.find('newcs') != -1: self.type = "NEWCAMD" self.setTitle("SETUP LOADER - NEWCAMD"); else: self.type = "None" self.setTitle("SETUP LOADER - None"); elif self.link_softcam.find('CCcam') != -1: self.readCCcamPort() self.setTitle("SETUP LOADER - CCCAM"); if self.link_cardserver.find('oscam') != -1: self.type = "CCCAM+OSCAM" elif self.link_cardserver.find('newcs') != -1: self.type = "CCCAM+NEWCS" else: self.type = "CCCAM" else: self.type = "None" self.setTitle("SETUP LOADER - None");
def _persist_symlink(self, abspath): """Persist symbolic link and bind mount it back to its current location """ persisted_path = self._config_path(abspath) current_target = os.readlink(abspath) if os.path.exists(persisted_path): stored_target = os.readlink(persisted_path) if stored_target == current_target: self._logger.warn('Symlink "%s" had already been persisted', abspath) return else: # Write the new symlink to an alternate location and atomically # rename self._prepare_dir(abspath, persisted_path) tmp_path = persisted_path + '.ovirtnode.atom' try: os.symlink(current_target, tmp_path) except Exception: raise else: os.rename(tmp_path, persisted_path) else: self._prepare_dir(abspath, persisted_path) os.symlink(current_target, persisted_path) self.copy_attributes(abspath, persisted_path) self._logger.info('Symbolic link "%s" successfully persisted', abspath) self._add_path_entry(abspath)
def testNewEntry(self): # test the default case d = dumpStorage.DumpStorage(self.testDir) dateLeafSet = set() expectedLeafs = set(["55", "00", "20"]) for k, v in self.testData.items(): nd, dd = d.newEntry(k, v[1]) dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd, k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k)))) assert e == g, "Expected %s, got %s" % (e, g) assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet) # test the for JsonDumpStorage default d = dumpStorage.DumpStorage(self.testDir, subSlotCount=1) dateLeafSet = set() expectedLeafs = set(["55_0", "00_0", "20_0"]) for k, v in self.testData.items(): nd, dd = d.newEntry(k, v[1]) dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd, k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k)))) assert e == g, "Expected %s, got %s" % (e, g) assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet) # test the trailing _n case at same level d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3) dateLeafSet = set() expectedLeafs = set(["00_0", "20_0", "55_0"]) for k, v in self.testData.items(): nd, dd = d.newEntry(k, v[1]) dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd, k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k)))) assert e == g, "Expected %s, got %s" % (e, g) assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet) # test with subdirectory further down d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3) dateLeafSet = set() expectedLeafs = set(["wh_0", "wh_1", "wh_2"]) for k, v in self.testData.items(): nd, dd = d.newEntry(k, v[1], webheadName="wh") dateLeafSet.add(os.path.split(dd)[1]) assert os.path.isdir(nd) assert os.path.isdir(dd) assert os.path.islink(os.path.join(dd, k)) e = os.path.abspath(nd) g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k)))) assert e == g, "Expected %s, got %s" % (e, g) assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet)
def linkBestCurrentSolution(dpath): alldsz = glob.glob(dpath + '/*dsz') currentLink = None bestDsz = None bestN = None stu = None for x in alldsz: if x.endswith('current.dsz'): assert os.path.islink(x) currentLink = os.readlink(x) continue fname = os.path.basename(x) m = CURSOL_RE_.match(fname) assert m is not None, ('failed to parse %s' % fname) nstu = m.group(1).upper() if stu is None: stu = nstu assert nstu == stu nth = int(m.group(2)) if (bestDsz is None) or (nth > bestN): bestN = nth bestDsz = fname if bestDsz is None: return None currentDsz = os.path.join(dpath, stu + 'current.dsz') if os.path.islink(currentDsz): odsz = os.readlink(currentDsz) if odsz == bestDsz: return bestDsz os.unlink(currentDsz) os.symlink(bestDsz, currentDsz) return bestDsz
def AddDirectory(self, dir_path): """Adds all files under the given directory to the checksum. This adds both the contents of the files as well as their names and locations to the checksum. If the checksums of two directories are equal this means they have exactly the same files, and contents. Args: dir_path: str, The directory path to add all files from. Returns: self, For method chaining. """ for root, dirs, files in os.walk(dir_path): dirs.sort(key=os.path.normcase) files.sort(key=os.path.normcase) for d in dirs: path = os.path.join(root, d) # We don't traverse directory links, but add the fact that it was found # in the tree. if os.path.islink(path): relpath = os.path.relpath(path, dir_path) self.__files.add(relpath) self.AddContents(relpath) self.AddContents(os.readlink(path)) for f in files: path = os.path.join(root, f) relpath = os.path.relpath(path, dir_path) self.__files.add(relpath) self.AddContents(relpath) if os.path.islink(path): self.AddContents(os.readlink(path)) else: self.AddFileContents(path) return self
def copytree(src, dst): """Similar to shutil.copytree, but always copies symlinks and doesn't error out if the destination path already exists. """ # If the source tree is a symlink, duplicate the link and we're done. if os.path.islink(src): os.symlink(os.readlink(src), dst) return try: os.mkdir(dst) except OSError as e: if e.errno != errno.EEXIST: raise names = os.listdir(src) for name in names: srcname = os.path.join(src, name) dstname = os.path.join(dst, name) if os.path.islink(srcname): os.symlink(os.readlink(srcname), dstname) elif os.path.isdir(srcname): copytree(srcname, dstname) else: copy2(srcname, dstname) try: shutil.copystat(src, dst) except OSError as e: if e.errno != errno.EPERM: raise
def copy(src, dst, hardlink=False, keep_symlink=True): assert not P.isdir(src), 'Source path must not be a dir' assert not P.isdir(dst), 'Destination path must not be a dir' if keep_symlink and P.islink(src): assert not P.isabs(readlink(src)), 'Cannot copy symlink that points to an absolute path (%s)' % src logger.debug('%8s %s -> %s' % ('symlink', src, dst)) if P.exists(dst): assert readlink(dst) == readlink(src), 'Refusing to retarget already-exported symlink %s' % dst else: symlink(readlink(src), dst) return if P.exists(dst): assert hash_file(src) == hash_file(dst), 'Refusing to overwrite already exported dst %s' % dst else: if hardlink: try: link(src, dst) logger.debug('%8s %s -> %s' % ('hardlink', src, dst)) return except OSError, o: if o.errno != errno.EXDEV: # Invalid cross-device link, not an error, fall back to copy raise logger.debug('%8s %s -> %s' % ('copy', src, dst)) shutil.copy2(src, dst)
def create_link(opts, src, dst): if not os.path.lexists(dst): return _create_link(opts, src, dst) if not os.path.islink(dst): return "[%s] is not a symbolic link as we expected, " \ "please adjust if this is not what you intended." % dst if not os.path.exists(os.readlink(dst)): environment.warning('BROKEN LINK: [%s] attempting to delete and fix it to point to %s.' % (dst, src)) try: os.unlink(dst) return _create_link(opts, src, dst) except: msg = "[%s] was a broken symlink, failed to delete " \ "and relink to [%s], please fix this manually" % (dst, src) return msg environment.debug(opts, 'verifying link: %s points to %s' % (dst, src)) dst_stat = os.stat(dst) src_stat = os.stat(src) if dst_stat.st_ino != src_stat.st_ino: msg = "[%s] is pointing to [%s] which is different than " \ "the intended target [%s]" % (dst, os.readlink(dst), src) return msg
def TestLink(DestinationPath, SourcePath, fc): if SourcePath: if os.path.islink(SourcePath): if fc.Links == "follow": if os.path.isdir(SourcePath): if TestDirectory(DestinationPath, os.path.realpath(SourcePath), fc) is False: return False elif os.path.isfile(SourcePath): if TestFile(DestinationPath, os.path.realpath(SourcePath), fc) is False: return False elif fc.Links == "manage": if not os.path.islink(DestinationPath): return False if os.readlink(DestinationPath) != os.readlink(SourcePath): return False elif fc.Links == "ignore": return True else: if not os.path.exists(DestinationPath) or not os.path.exists(SourcePath) or not os.path.islink(DestinationPath) : return False if os.readlink(DestinationPath) != SourcePath: return False if os.path.exists(DestinationPath) != True: return False if TestOwnerGroupMode(DestinationPath, SourcePath, fc) is False: return False return True
def diff(self, file_struct): self._validate_struct(file_struct) temp_file, temp_dirs = self.process(file_struct) path = file_struct['path'] sectx_result = '' result = '' cur_sectx = lgetfilecon(path)[1] if cur_sectx == None: cur_sectx = '' if file_struct.has_key('selinux_ctx') and file_struct['selinux_ctx']: if cur_sectx != file_struct['selinux_ctx']: sectx_result = "SELinux contexts differ: actual: [%s], expected: [%s]\n" % (cur_sectx, file_struct['selinux_ctx']) if file_struct['filetype'] == 'symlink': try: curlink = os.path.abspath(os.readlink(path)) newlink = os.path.abspath(os.readlink(temp_file)) if curlink == newlink: result = '' else: result = "Link targets differ for [%s]: actual: [%s], expected: [%s]\n" % (path, curlink, newlink) except OSError, e: if e.errno == 22: result = "Deployed symlink is no longer a symlink!" else: raise e
def copy(source, destination, sym = True): '''recursively copy a "source" file or directory to "destination"''' sourceGlob = glob.glob(source) if len(sourceGlob) == 0: error(_("ActionsAPI [copy]: No file matched pattern \"%s\".") % source) for filePath in sourceGlob: if isFile(filePath) and not isLink(filePath): try: shutil.copy(filePath, destination) except IOError: error(_('ActionsAPI [copy]: Permission denied: %s to %s') % (filePath, destination)) elif isLink(filePath) and sym: if isDirectory(destination): os.symlink(os.readlink(filePath), join_path(destination, os.path.basename(filePath))) else: if isFile(destination): os.remove(destination) os.symlink(os.readlink(filePath), destination) elif isLink(filePath) and not sym: if isDirectory(filePath): copytree(filePath, destination) else: shutil.copy(filePath, destination) elif isDirectory(filePath): copytree(filePath, destination, sym) else: error(_('ActionsAPI [copy]: File %s does not exist.') % filePath)
def TestFile(DestinationPath, SourcePath, fc): if '://' in SourcePath and fc.LocalPath == '': # we cannot verify the remote has not changed until the Set return False if not os.path.exists(DestinationPath) or not os.path.isfile(DestinationPath) or os.path.islink(DestinationPath): return False if TestOwnerGroupMode(DestinationPath, SourcePath, fc) is False: return False if SourcePath and len(SourcePath) > 0: if not os.path.isfile(SourcePath): return False if os.path.islink(SourcePath): if fc.Links == "follow": if os.path.isdir(os.path.realpath(SourcePath)): print("Error: Expecting a file, but source link points to directory") LG().Log("ERROR", "Expecting a file, but source link points to directory") return False else: if not os.path.islink(DestinationPath): return False if os.readlink(DestinationPath) != os.readlink(SourcePath): return False elif CompareFiles(DestinationPath, SourcePath, fc.Checksum) == -1: return False elif fc.Contents: dest_file, error = ReadFile(DestinationPath) if fc.Contents.encode('utf8') != dest_file: return False return True
def lndir(src, dst, overwrite=True, summary_print_func=None): """ Create a recursive symlink shadow copy of src in dst. Directories in src will be created in dst. All other files will be symlinked from dst to src. If the src is a symlink then the destination symlink will be re-linked to the origin target. :param src: src path, if relative, relative to dst not cwd :param dst: dst mpath, must exit. :param overwrite: defaul True, overwrite existing symlinks in dst :param summary_print_func: Function to print a summary after each interation :return: count of directores created and files symlinked :rtype: tuple """ if summary_print_func is None: summary_print_func = lambda *args: None dst = os.path.abspath(dst) if os.path.isabs(src): calculate_src = lambda ss, sd: ss else: # following the X11 lndir convention if src is relative, # it is relative to dst not to cwd. src = os.path.abspath(os.path.join(dst, src)) calculate_src = os.path.relpath # start walking the source target dir_c = 0 # counter for dires file_c = 0 # counter for files for root, dirs, files in os.walk(src): for f in files: real_src = os.path.join(root, f) sym_dst_dir, sym_dst = calculate_src_dst(dst, f, root, src) delete_if_exists_and_overwrite(overwrite, sym_dst) if os.path.islink(real_src): # assume relative symlink does not reach above src # and just use the original link sym_src = os.readlink(real_src) improved_error_symlink(sym_src, sym_dst) else: sym_src = calculate_src(os.path.join(root, f), sym_dst_dir) improved_error_symlink(sym_src, sym_dst) file_c += 1 for d in dirs: real_src = os.path.join(root, d) sym_dst_dir, sym_dst = calculate_src_dst(dst, d, root, src) delete_if_exists_and_overwrite(overwrite, sym_dst) if os.path.islink(real_src): # assume relative symlink does not reach above src # and just use the original link sym_src = os.readlink(real_src) improved_error_symlink(sym_src, sym_dst) else: try: os.mkdir(sym_dst) except OSError: logging.exception("dst = %s", sym_dst) dir_c += 1 summary_print_func(dir_c, file_c, root) return dir_c, file_c
def _fix_symlinks(debdir): ''' Sometimes debs will contain absolute symlinks (e.g. if the relative path would go all the way to root, they just do absolute). We can't have that, so instead clean those absolute symlinks. Some unpacked items will also contain suid binaries which we do not want in the resulting snap. ''' for root, dirs, files in os.walk(debdir): # Symlinks to directories will be in dirs, while symlinks to # non-directories will be in files. for entry in itertools.chain(files, dirs): path = os.path.join(root, entry) if os.path.islink(path) and os.path.isabs(os.readlink(path)): target = os.path.join(debdir, os.readlink(path)[1:]) if _skip_link(os.readlink(path)): logger.debug('Skipping {}'.format(target)) continue if not os.path.exists(target): if not _try_copy_local(path, target): continue os.remove(path) os.symlink(os.path.relpath(target, root), path) elif os.path.exists(path): _fix_filemode(path)
def get_name_of_init(run=process.run): """ Internal function to determine what executable is PID 1 It does that by checking /proc/1/exe. Fall back to checking /proc/1/cmdline (local execution). :return: executable name for PID 1, aka init :rtype: str """ if run == process.run: # On a local run, there are better ways to check # our PID 1 executable name. try: return os.path.basename(os.readlink('/proc/1/exe')) except OSError: with open('/proc/1/cmdline', 'r') as cmdline: init = cmdline.read().split(chr(0))[0] try: init = os.readlink(init) except OSError: pass return os.path.basename(init) else: output = run("readlink /proc/1/exe").stdout.strip() return os.path.basename(output)
def install(self): path = self.options['path'] logger = logging.getLogger(self.name) logger.info( 'Creating directory %s' % os.path.basename(path)) if not os.path.exists(path): os.makedirs(path) files = (file for file in self.options['files'].split('\n') if file) for file in files: file = file.split(None, 1) if len(file) == 2: file, as_ = file else: file = file[0] as_ = os.path.basename(file) if '://' in file: file = self._get_resource_filename(file) to = os.path.join(path, as_) if os.path.islink(to) and os.readlink(to) != file: logger.info('Removing symlink from "%s" to "%s"' % (os.readlink(to), to)) os.remove(to) if not os.path.exists(to): logger.info('Making symlink from "%s" to "%s"' % (file, to)) os.symlink(file, to) # Since other processes may create resources in path it # should be excluded during uninstall, so return empty list. # We can't return list of created symlinks as buildout does not # currently support symlink removal, see uninstall_symlinks below. return []
def run(self): while True: try: p_result = self.polling.poll() print p_result if p_result[0][1] <= select.POLLPRI: self.ev = self.conn.recv_event() else: continue except Exception, e: print e if self.ev.what == connector.PROC_EVENT_FORK: linkpath = os.path.join('/proc/', str(self.ev.child_pid), 'exe') if os.path.exists(linkpath): name = os.readlink(linkpath) else: name = 'non ecsiste' print "Parent: ", self.ev.parent_pid, "Forked: ", self.ev.child_pid, name elif self.ev.what == connector.PROC_EVENT_EXEC: linkpath = os.path.join('/proc/', str(self.ev.process_pid), 'exe') name = os.readlink(linkpath) print "Exec: ", self.ev.process_pid, name elif self.ev.what == connector.PROC_EVENT_EXIT: linkpath = os.path.join('/proc', str(self.ev.process_pid), 'exe') if os.path.exists(linkpath): name = os.readlink(linkpath) else: name = None print "Exit: ", self.ev.process_pid, name
def test_save(self): image = VImage.new_rgba(width=1, height=1, ink=rgba(r=0, g=0, b=0, a=0)) self.storage.save(x=0, y=1, z=2, image=image) self.storage.save(x=1, y=0, z=2, image=image) self.storage.save(x=1, y=0, z=3, image=image) self.storage.waitall() self.assertEqual(set(recursive_listdir(self.outputdir)), set(['2/', '2/0/', '2/0/1.png', '2/1/', '2/1/0.png', '3/', '3/1/', '3/1/0.png'])) # Is this a real file? self.assertFalse( os.path.islink(os.path.join(self.outputdir, '2', '0', '1.png')) ) # Does the symlinking work? self.assertEqual( os.readlink(os.path.join(self.outputdir, '2', '1', '0.png')), os.path.join(os.path.pardir, '0', '1.png') ) self.assertEqual( os.readlink(os.path.join(self.outputdir, '3', '1', '0.png')), os.path.join(os.path.pardir, os.path.pardir, '2', '0', '1.png') )
def con2prog(src_ip, src_port, dst_ip, dst_port, proto): proc_tcp = readFile("/proc/net/%s" % proto)[1:] inode = None for p in proc_tcp: con = p.split() src = con[1].split(":") dst = con[2].split(":") if hex2ip(src[0]) == src_ip and int(src[1], 16) == src_port and \ hex2ip(dst[0]) == dst_ip and int(dst[1], 16) == dst_port: inode = con[9] if not inode: return None pids = glob.glob("/proc/[0-9]*") for pid in pids: fds = glob.glob("%s/fd/*" % pid) for fd in fds: try: fdlink = os.readlink(fd) if re.match("socket:\[%s\]" % inode, fdlink): prog = os.readlink("%s/exe" % pid) cmd = readFile("%s/cmdline" % pid, lines=False).\ replace("\x00", " ").strip() return pid, prog, cmd except OSError: pass return None
def test_readlink_nonexistent(self): with test_support.temp_cwd() as new_cwd: nonexistent_file = os.path.join(new_cwd, "nonexistent-file") with self.assertRaises(OSError) as cm: os.readlink(nonexistent_file) self.assertEqual(cm.exception.errno, errno.ENOENT) self.assertEqual(cm.exception.filename, nonexistent_file)
def test_resolve_simple_dep_graph(self): dep = self._create_dep(provide_bin=True) app = self._create_app(dep) _, node_paths = self._resolve_target(app) dep_node_path = node_paths.node_path(dep) app_node_path = node_paths.node_path(app) self.assertIsNotNone(dep_node_path) self.assertIsNotNone(app_node_path) # Verify that 'app/node_modules' has a correct symlink to 'dep' app_node_modules_path = os.path.join(app_node_path, 'node_modules') link_dep_path = os.path.join(app_node_modules_path, 'dep') self.assertTrue(os.path.exists(app_node_modules_path)) self.assertTrue(os.path.islink(link_dep_path)) expected = os.path.relpath(dep_node_path, app_node_modules_path) self.assertEqual(os.readlink(link_dep_path), expected) # Verify that 'app/node_modules/.bin' has a correct symlink to 'dep' app_bin_dep_path = os.path.join(dep_node_path, 'cli.js') dep_bin_path = os.path.join(app_node_path, 'node_modules', '.bin') link_dep_bin_path = os.path.join(dep_bin_path, 'dep') self.assertTrue(os.path.islink(link_dep_bin_path)) relative_path = os.readlink(link_dep_bin_path) expected = os.path.relpath(app_bin_dep_path, dep_bin_path) self.assertEqual(relative_path, expected)
def testTarFileWithSymlink(self): outfd = StringIO.StringIO() infd1 = StringIO.StringIO("this is a test string") st1 = os.stat_result((0644, 0, 0, 0, 0, 0, len(infd1.getvalue()), 0, 0, 0)) infd2 = StringIO.StringIO("this is another test string") st2 = os.stat_result((0644, 0, 0, 0, 0, 0, len(infd2.getvalue()), 0, 0, 0)) # Write the zip into a file like object. with utils.StreamingTarWriter(outfd, mode="w:gz") as writer: writer.WriteFromFD(infd1, "test1.txt", st=st1) writer.WriteFromFD(infd2, "subdir/test2.txt", st=st2) writer.WriteSymlink("test1.txt", "test1.txt.link") writer.WriteSymlink("subdir/test2.txt", "test2.txt.link") with tarfile.open( fileobj=StringIO.StringIO(outfd.getvalue()), mode="r") as test_fd: test_fd.extractall(self.temp_dir) link_path = os.path.join(self.temp_dir, "test1.txt.link") self.assertTrue(os.path.islink(link_path)) self.assertEqual(os.readlink(link_path), "test1.txt") link_path = os.path.join(self.temp_dir, "test2.txt.link") self.assertTrue(os.path.islink(link_path)) self.assertEqual(os.readlink(link_path), "subdir/test2.txt")
def defaults(self): self.id_format = u"%(type)s-run%(run)06d-host%(hostname)s-pid%(pid)06d" self.doc["type"] = "dqm-source-state" self.doc["run"] = 0 # figure out the tag c = self.doc["cmdline"] for l in c: if l.endswith(".py"): t = os.path.basename(l) t = t.replace(".py", "") t = t.replace("_cfg", "") self.doc["tag"] = t pr = l.split("=") if len(pr) > 1 and pr[0] == "runNumber" and pr[1].isdigit(): run = long(pr[1]) self.doc["run"] = run self.make_id() #if os.environ.has_key("GZIP_LOG"): # self.doc["stdlog_gzip"] = os.environ["GZIP_LOG"] try: self.doc["stdout_fn"] = os.readlink("/proc/self/fd/1") self.doc["stderr_fn"] = os.readlink("/proc/self/fd/2") except: pass self.update_doc({ "extra": { "environ": dict(os.environ) }})
def test_publish(self): repo = mock.Mock(spec=Repository) repo.working_dir = self.repo_working_dir repo.id = "test_publish" num_units = 10 relative_url = "rel_a/rel_b/rel_c/" existing_units = self.get_units(count=num_units) publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir) config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, relative_url=relative_url, http=False, https=True) distributor = YumDistributor() distributor.process_repo_auth_certificate_bundle = mock.Mock() status, msg = distributor.validate_config(repo, config, None) self.assertTrue(status) report = distributor.publish_repo(repo, publish_conduit, config) self.assertTrue(report.success_flag) summary = report.summary self.assertEqual(summary["num_package_units_attempted"], num_units) self.assertEqual(summary["num_package_units_published"], num_units) self.assertEqual(summary["num_package_units_errors"], 0) # Verify we did not attempt to publish to http expected_repo_http_publish_dir = os.path.join(self.http_publish_dir, relative_url) self.assertFalse(os.path.exists(expected_repo_http_publish_dir)) expected_repo_https_publish_dir = os.path.join(self.https_publish_dir, relative_url).rstrip('/') self.assertEqual(summary["https_publish_dir"], expected_repo_https_publish_dir) self.assertTrue(os.path.exists(expected_repo_https_publish_dir)) details = report.details self.assertEqual(len(details["errors"]), 0) # # Add a verification of the publish directory # self.assertTrue(os.path.exists(summary["https_publish_dir"])) self.assertTrue(os.path.islink(summary["https_publish_dir"].rstrip("/"))) source_of_link = os.readlink(expected_repo_https_publish_dir.rstrip("/")) self.assertEquals(source_of_link, repo.working_dir) # # Verify the expected units # for u in existing_units: expected_link = os.path.join(expected_repo_https_publish_dir, u.metadata["relativepath"]) self.assertTrue(os.path.exists(expected_link)) actual_target = os.readlink(expected_link) expected_target = u.storage_path self.assertEqual(actual_target, expected_target) # # Now test flipping so https is disabled and http is enabled # config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, http_publish_dir=self.http_publish_dir, relative_url=relative_url, http=True, https=False) report = distributor.publish_repo(repo, publish_conduit, config) self.assertTrue(report.success_flag) # Verify we did publish to http self.assertTrue(os.path.exists(expected_repo_http_publish_dir)) # Verify we did not publish to https self.assertFalse(os.path.exists(expected_repo_https_publish_dir)) # Verify we cleaned up the misc dirs under the https dir self.assertEquals(len(os.listdir(self.https_publish_dir)), 0)
def TestFile(DestinationPath, SourcePath, fc): if not os.path.exists(DestinationPath) or not os.path.isfile(DestinationPath) or os.path.islink(DestinationPath): return False if TestOwnerGroupMode(DestinationPath, SourcePath, fc) == False: return False if SourcePath: if not os.path.isfile(SourcePath): return False if os.path.islink(SourcePath): if fc.Links == "follow": if os.path.isdir(os.path.realpath(SourcePath)): print("Error: Expecting a file, but source link points to directory") return False else: if not os.path.islink(DestinationPath): return False if os.readlink(DestinationPath) != os.readlink(SourcePath): return False elif CompareFiles(DestinationPath, SourcePath, fc) == False: return False elif fc.Contents: dest_file = open(DestinationPath, "rb").read() if fc.Contents != dest_file: return False return True
def same_file(left_stat, left_dir, right_stat, right_dir, filename): """Are two filesystem objects the same?""" if S_IFMT(left_stat.st_mode) != S_IFMT(right_stat.st_mode): # Different fundamental types return False elif S_ISREG(left_stat.st_mode): # Files with the same size and MD5sum are the same if left_stat.st_size != right_stat.st_size: return False elif md5sum("%s/%s" % (left_dir, filename)) \ != md5sum("%s/%s" % (right_dir, filename)): return False else: return True elif S_ISDIR(left_stat.st_mode) or S_ISFIFO(left_stat.st_mode) \ or S_ISSOCK(left_stat.st_mode): # Directories, fifos and sockets are always the same return True elif S_ISCHR(left_stat.st_mode) or S_ISBLK(left_stat.st_mode): # Char/block devices are the same if they have the same rdev if left_stat.st_rdev != right_stat.st_rdev: return False else: return True elif S_ISLNK(left_stat.st_mode): # Symbolic links are the same if they have the same target if os.readlink("%s/%s" % (left_dir, filename)) \ != os.readlink("%s/%s" % (right_dir, filename)): return False else: return True else: return True
def device_is_zfcp(info): """ Return True if the device is a zfcp device. """ if info.get("DEVTYPE") != "disk": return False subsystem = device_get_sysfs_path(info) while True: topdir = os.path.realpath(os.path.dirname(subsystem)) driver = "%s/driver" % (topdir,) if os.path.islink(driver): subsystemname = os.path.basename(os.readlink(subsystem)) drivername = os.path.basename(os.readlink(driver)) if subsystemname == 'ccw' and drivername == 'zfcp': return True newsubsystem = os.path.dirname(topdir) if newsubsystem == topdir: break subsystem = newsubsystem + "/subsystem" return False
def does_file_need_update(src_name, src_st, dest_name): try: target_st = os.lstat(dest_name) except OSError as exc: if exc.errno != errno.ENOENT: raise return True if src_st.st_size != target_st.st_size: return True if stat.S_IFMT(src_st.st_mode) != stat.S_IFMT(target_st.st_mode): return True if stat.S_ISLNK(src_st.st_mode): return os.readlink(src_name) != os.readlink(dest_name) if not stat.S_ISREG(src_st.st_mode): return True # They might have the same content; compare. with open(src_name, "rb") as sf, open(dest_name, "rb") as df: chunk_size = 8192 while True: src_data = sf.read(chunk_size) dest_data = df.read(chunk_size) if src_data != dest_data: return True if len(src_data) < chunk_size: # EOF break return False
def __hasNoLinkOrFail(self,jsonStorage,uuid): linkPath = jsonStorage.getJson(uuid)[:-len(jsonStorage.jsonSuffix)] try: os.readlink(linkPath) assert False, 'Expected to find no link: %s '%linkPath except OSError,x: assert 2 == x.errno, "Expected errno=2, got %d for linkpath %s"%(x.errno,linkPath)
def test_save_successor(self, mock_rv): # Mock relevant_values() to claim that all values are relevant here # (to avoid instantiating parser) mock_rv.side_effect = lambda x: x for ver in six.moves.range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) self.assertEqual( 6, self.test_rc.save_successor(3, b'new cert', None, b'new chain', self.config)) with open(self.test_rc.version("cert", 6)) as f: self.assertEqual(f.read(), "new cert") with open(self.test_rc.version("chain", 6)) as f: self.assertEqual(f.read(), "new chain") with open(self.test_rc.version("fullchain", 6)) as f: self.assertEqual(f.read(), "new cert" + "new chain") # version 6 of the key should be a link back to version 3 self.assertFalse(os.path.islink(self.test_rc.version("privkey", 3))) self.assertTrue(os.path.islink(self.test_rc.version("privkey", 6))) # Let's try two more updates self.assertEqual( 7, self.test_rc.save_successor(6, b'again', None, b'newer chain', self.config)) self.assertEqual( 8, self.test_rc.save_successor(7, b'hello', None, b'other chain', self.config)) # All of the subsequent versions should link directly to the original # privkey. for i in (6, 7, 8): self.assertTrue(os.path.islink(self.test_rc.version("privkey", i))) self.assertEqual("privkey3.pem", os.path.basename(os.readlink( self.test_rc.version("privkey", i)))) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 9))) self.assertEqual(self.test_rc.current_version(kind), 3) # Test updating from latest version rather than old version self.test_rc.update_all_links_to(8) self.assertEqual( 9, self.test_rc.save_successor(8, b'last', None, b'attempt', self.config)) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 10))) self.assertEqual(self.test_rc.current_version(kind), 8) with open(self.test_rc.version("fullchain", 9)) as f: self.assertEqual(f.read(), "last" + "attempt") temp_config_file = os.path.join(self.config.renewal_configs_dir, self.test_rc.lineagename) + ".conf.new" with open(temp_config_file, "w") as f: f.write("We previously crashed while writing me :(") # Test updating when providing a new privkey. The key should # be saved in a new file rather than creating a new symlink. self.assertEqual( 10, self.test_rc.save_successor(9, b'with', b'a', b'key', self.config)) self.assertTrue(os.path.exists(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.exists(temp_config_file))
def package(self, package: TargetPackage, version: FullVersion) -> str: package.build() package.initialize_buildroot({'libarchive'}) log.info('PKG %s', self._get_filename(package, version)) # The package needs to be installed in /usr/arch> on the Red Hat # system. installdir = os.path.join(config.DIR_BUILDROOT, 'install') arch = package.get_arch() prefix = os.path.join('/usr', arch) package.extract(installdir, prefix) files = sorted(util.walk_files(installdir)) # Create an xz compressed cpio payload containing all files. listing = os.path.join(config.DIR_BUILDROOT, 'listing') with open(listing, 'w') as f: f.write('#mtree\n') for path in files: relpath = os.path.join(prefix, os.path.relpath(path, installdir)) if os.path.islink(path): f.write( '%s type=link mode=0777 uname=root gname=root time=0 link=%s\n' % (relpath, os.readlink(path))) else: f.write( '%s type=file mode=0%o uname=root gname=root time=0 contents=%s\n' % (relpath, self._get_suggested_mode(path), path)) data = os.path.join(config.DIR_BUILDROOT, 'data.cpio.xz') self._run_tar([ '-cJf', data, '--format=newc', '-C', installdir, '@' + listing, ]) # The header, based on the following documentation: # http://www.rpm.org/max-rpm/s1-rpm-file-format-rpm-file-format.html # http://refspecs.linux-foundation.org/LSB_5.0.0/LSB-Core-generic/LSB-Core-generic/pkgformat.html name = package.get_redhat_name() lib_depends = sorted(dep.get_redhat_name() for dep in package.get_lib_depends()) dirs = sorted({os.path.dirname(f) for f in files}) header = bytes( rpm.Header({ 100: rpm.StringArray(['C']), 1000: rpm.String(name), 1001: rpm.String(str(version.get_version())), 1002: rpm.String(str(version.get_revision())), 1003: rpm.Int32([version.get_epoch()]), 1004: rpm.I18NString('%s for %s' % (name, arch)), 1005: rpm.I18NString('%s for %s' % (name, arch)), 1009: rpm.Int32([sum(self._file_size(f) for f in files)]), 1014: rpm.String('Unknown'), 1016: rpm.I18NString('Development/Libraries'), 1020: rpm.String(package.get_homepage()), 1021: rpm.String('linux'), 1022: rpm.String('noarch'), 1028: rpm.Int32(os.lstat(f).st_size for f in files), 1030: rpm.Int16(self._file_mode(f) for f in files), 1033: rpm.Int16(0 for f in files), 1034: rpm.Int32(0 for f in files), 1035: rpm.StringArray(self._file_md5(f) for f in files), 1036: rpm.StringArray(self._file_linkto(f) for f in files), 1037: rpm.Int32(0 for f in files), 1039: rpm.StringArray('root' for f in files), 1040: rpm.StringArray('root' for f in files), 1047: rpm.StringArray([name]), 1048: rpm.Int32(0 for dep in lib_depends), 1049: rpm.StringArray(lib_depends), 1050: rpm.StringArray('' for dep in lib_depends), 1095: rpm.Int32(1 for f in files), 1096: rpm.Int32(range(1, len(files) + 1)), 1097: rpm.StringArray('' for f in files), 1112: rpm.Int32(8 for dep in lib_depends), 1113: rpm.StringArray([version.get_redhat_version()]), 1116: rpm.Int32(dirs.index(os.path.dirname(f)) for f in files), 1117: rpm.StringArray(os.path.basename(f) for f in files), 1118: rpm.StringArray( os.path.join(prefix, os.path.relpath(d, installdir)) + '/' for d in dirs), 1124: rpm.String('cpio'), 1125: rpm.String('xz'), 1126: rpm.String('9'), })) # The signature. checksum = hashlib.md5() checksum.update(header) util.hash_file(data, checksum) signature = bytes( rpm.Header({ 1000: rpm.Int32([len(header) + os.stat(data).st_size]), 1004: rpm.Bin(checksum.digest()), })) # Create the RPM file. output = os.path.join(config.DIR_BUILDROOT, 'output.rpm') with open(output, 'wb') as fb: # The lead. fb.write(b'\xed\xab\xee\xdb\x03\x00\x00\x00\x00\x00') fullname = '%s-%s' % (name, version.get_redhat_version()) fb.write(bytes(fullname, encoding='ASCII')[:66].ljust(66, b'\0')) fb.write(b'\x00\x01\x00\x05') fb.write(b'\x00' * 16) # The signature, aligned up to eight bytes in size. fb.write(signature) fb.write(b'\0' * ((8 - len(signature) % 8) % 8)) # The header. fb.write(header) # The payload. with open(data, 'rb') as fin: shutil.copyfileobj(fin, fb) return output
def assemble(self): """ This method is the MAIN method for finding all necessary files to be bundled. """ from ..config import CONF for m in self.excludes: logger.debug("Excluding module '%s'" % m) self.graph = initialize_modgraph(excludes=self.excludes, user_hook_dirs=self.hookspath) # TODO Find a better place where to put 'base_library.zip' and when to created it. # For Python 3 it is necessary to create file 'base_library.zip' # containing core Python modules. In Python 3 some built-in modules # are written in pure Python. base_library.zip is a way how to have # those modules as "built-in". libzip_filename = os.path.join(CONF['workpath'], 'base_library.zip') create_py3_base_library(libzip_filename, graph=self.graph) # Bundle base_library.zip as data file. # Data format of TOC item: ('relative_path_in_dist_dir', 'absolute_path_on_disk', 'DATA') self.datas.append( (os.path.basename(libzip_filename), libzip_filename, 'DATA')) # Expand sys.path of module graph. # The attribute is the set of paths to use for imports: sys.path, # plus our loader, plus other paths from e.g. --path option). self.graph.path = self.pathex + self.graph.path self.graph.set_setuptools_nspackages() logger.info("running Analysis %s", self.tocbasename) # Get paths to Python and, in Windows, the manifest. python = sys.executable if not is_win: # Linux/MacOS: get a real, non-link path to the running Python executable. while os.path.islink(python): python = os.path.join(os.path.dirname(python), os.readlink(python)) depmanifest = None else: # Windows: Create a manifest to embed into built .exe, containing the same # dependencies as python.exe. depmanifest = winmanifest.Manifest( type_="win32", name=CONF['specnm'], processorArchitecture=winmanifest.processor_architecture(), version=(1, 0, 0, 0)) depmanifest.filename = os.path.join( CONF['workpath'], CONF['specnm'] + ".exe.manifest") # We record "binaries" separately from the modulegraph, as there # is no way to record those dependencies in the graph. These include # the python executable and any binaries added by hooks later. # "binaries" are not the same as "extensions" which are .so or .dylib # that are found and recorded as extension nodes in the graph. # Reset seen variable before running bindepend. We use bindepend only for # the python executable. bindepend.seen.clear() # Add binary and assembly dependencies of Python.exe. # This also ensures that its assembly depencies under Windows get added to the # built .exe's manifest. Python 2.7 extension modules have no assembly # dependencies, and rely on the app-global dependencies set by the .exe. self.binaries.extend( bindepend.Dependencies([('', python, '')], manifest=depmanifest, redirects=self.binding_redirects)[1:]) if is_win: depmanifest.writeprettyxml() ### Module graph. # # Construct the module graph of import relationships between modules # required by this user's application. For each entry point (top-level # user-defined Python script), all imports originating from this entry # point are recursively parsed into a subgraph of the module graph. This # subgraph is then connected to this graph's root node, ensuring # imported module nodes will be reachable from the root node -- which is # is (arbitrarily) chosen to be the first entry point's node. # List to hold graph nodes of scripts and runtime hooks in use order. priority_scripts = [] # Assume that if the script does not exist, Modulegraph will raise error. # Save the graph nodes of each in sequence. for script in self.inputs: logger.info("Analyzing %s", script) priority_scripts.append(self.graph.run_script(script)) # Analyze the script's hidden imports (named on the command line) self.graph.add_hiddenimports(self.hiddenimports) ### Post-graph hooks. self.graph.process_post_graph_hooks() # Update 'binaries' TOC and 'datas' TOC. deps_proc = DependencyProcessor(self.graph, self.graph._additional_files_cache) self.binaries.extend(deps_proc.make_binaries_toc()) self.datas.extend(deps_proc.make_datas_toc()) self.zipped_data.extend(deps_proc.make_zipped_data_toc()) # Note: zipped eggs are collected below ### Look for dlls that are imported by Python 'ctypes' module. # First get code objects of all modules that import 'ctypes'. logger.info('Looking for ctypes DLLs') ctypes_code_objs = self.graph.get_co_using_ctypes( ) # dict like: {'module1': code_obj, 'module2': code_obj} for name, co in ctypes_code_objs.items(): # Get dlls that might be needed by ctypes. logger.debug('Scanning %s for shared libraries or dlls', name) ctypes_binaries = scan_code_for_ctypes(co) self.binaries.extend(set(ctypes_binaries)) # Analyze run-time hooks. # Run-time hooks has to be executed before user scripts. Add them # to the beginning of 'priority_scripts'. priority_scripts = self.graph.analyze_runtime_hooks( self.custom_runtime_hooks) + priority_scripts # 'priority_scripts' is now a list of the graph nodes of custom runtime # hooks, then regular runtime hooks, then the PyI loader scripts. # Further on, we will make sure they end up at the front of self.scripts ### Extract the nodes of the graph as TOCs for further processing. # Initialize the scripts list with priority scripts in the proper order. self.scripts = self.graph.nodes_to_toc(priority_scripts) # Extend the binaries list with all the Extensions modulegraph has found. self.binaries = self.graph.make_binaries_toc(self.binaries) # Fill the "pure" list with pure Python modules. assert len(self.pure) == 0 self.pure = self.graph.make_pure_toc() # And get references to module code objects constructed by ModuleGraph # to avoid writing .pyc/pyo files to hdd. self.pure._code_cache = self.graph.get_code_objects() # Add remaining binary dependencies - analyze Python C-extensions and what # DLLs they depend on. logger.info('Looking for dynamic libraries') self.binaries.extend( bindepend.Dependencies(self.binaries, redirects=self.binding_redirects)) ### Include zipped Python eggs. logger.info('Looking for eggs') self.zipfiles.extend(deps_proc.make_zipfiles_toc()) # Verify that Python dynamic library can be found. # Without dynamic Python library PyInstaller cannot continue. self._check_python_library(self.binaries) if is_win: # Remove duplicate redirects self.binding_redirects[:] = list(set(self.binding_redirects)) logger.info("Found binding redirects: \n%s", self.binding_redirects) # Place Python source in data files for the noarchive case. if self.noarchive: # Create a new TOC of ``(dest path for .pyc, source for .py, type)``. new_toc = TOC() for name, path, typecode in self.pure: assert typecode == 'PYMODULE' # Transform a python module name into a file name. name = name.replace('.', os.sep) # Special case: modules have an implied filename to add. if os.path.splitext(os.path.basename(path))[0] == '__init__': name += os.sep + '__init__' # Append the extension for the compiled result. # In python 3.5 (PEP-488) .pyo files were replaced by # .opt-1.pyc and .opt-2.pyc. However, it seems that for # bytecode-only module distribution, we always need to # use the .pyc extension. name += '.pyc' new_toc.append((name, path, typecode)) # Put the result of byte-compiling this TOC in datas. Mark all entries as data. for name, path, typecode in compile_py_files( new_toc, CONF['workpath']): self.datas.append((name, path, 'DATA')) # Store no source in the archive. self.pure = TOC() # Write warnings about missing modules. self._write_warnings() # Write debug information about hte graph self._write_graph_debug()
def up(self): self.sftp.lstat(self.remote) local_files = {} spinner.waitfor('Testing') for root, dirs, files in os.walk(self.local.encode('utf-8')): root = to_unicode(root) if self._exclude(root): continue for d in dirs: d = to_unicode(d) path = os.path.relpath(os.path.join(root, d), self.local) if self._exclude(path): continue self.check_dir(path) if self.subdir: root = os.path.join(self.subdir, root) for f in files: f = to_unicode(f) if f in ('.files',): continue lfile = os.path.join(root, f) filename = os.path.relpath(lfile, self.local_root) if filename.split(os.path.sep)[0] == os.path.curdir: filename = filename[2:] if self._exclude(lfile): continue rfile = os.path.join(self.remote_root, filename) s = os.lstat(lfile) spinner.status(string_shortener(filename)) local_files[filename] = File(int(s.st_mtime), int(s.st_size), int(s.st_mode)) if filename not in self.revision_file: print_file_info(filename, s) upload = True else: lf = local_files[filename] rf = self.revision_file[filename] upload = different(self.sftp, filename, rf, lf, self.local_root, self.remote_root) if upload: spinner.succeeded() info(" Uploading: %s\n" % filename) try: try: rstat = self.sftp.lstat(rfile) except KeyboardInterrupt: raise except: pass else: if rstat.st_mtime > s.st_mtime: raise ValueError("Conflict with file %s (remote file is newer)" % filename) start = time.time() def status(total, size): if size == 0: return speed = total / (time.time() - start) * 1.0 if speed > 1024 ** 2: speeds = "%.2f MiByte/s" % (speed / 1024 ** 2,) elif speed > 1024: speeds = "%.2f KiByte/s" % (speed / 1024,) else: speeds = "%f Byte/s" % speed remaining = timedelta(seconds=int((size - total) / speed)) sys.stdout.write("\r%02d%% %d/%d %s %s" % ( total * 100 / size, total, size, speeds, remaining)) sys.stdout.flush() if not self.dry_run: if stat.S_ISLNK(s.st_mode): target = os.readlink(lfile) print() info("Creating remote symlink %s -> %s\n" % (rfile, target)) try: self.sftp.symlink(target, rfile) except paramiko.SSHException as e: error("Failed: %s\n" % (e,)) except IOError: pass else: self.sftp.put(lfile, rfile, status) f = self.sftp.file(rfile) f.utime((s.st_atime, s.st_mtime)) f.close() spinner.waitfor('Testing') except ValueError: del local_files[filename] self.revision_file.update(local_files) if not self.dry_run: self.revision_file.save() raise except KeyboardInterrupt: del local_files[filename] if not self.dry_run: self.sftp.unlink(rfile) self.revision_file.update(local_files) if not self.dry_run: self.revision_file.save() exit(1) except Exception as e: del local_files[filename] if not self.dry_run and filename not in self.revision_file: self.sftp.unlink(rfile) if not self.skip_on_error: self.revision_file.update(local_files) if not self.dry_run: self.revision_file.save() raise else: if filename in self.revision_file: # prevent deletion local_files[filename] = self.revision_file[filename] error("Error during upload of %s: %s\n" % (filename, str(e))) spinner.succeeded() for filename in self.revision_file.keys(): if self.subdir: if not filename.startswith(self.subdir): continue if filename not in local_files and not self._exclude(filename): warning("Deleted file locally: %s\n" % filename) print(" Last mod time: %d\n Size: %d\n Mode: %o" % self.revision_file[filename]) if not self.dry_run: try: self.sftp.lstat(os.path.join(self.remote_root, filename)) except KeyboardInterrupt: raise except: warning("Can't stat remote file. Maybe already deleted?\n") del self.revision_file[filename] continue answer = prompt("Delete it on remote?[y/n]") if answer == 'y': self.sftp.unlink(os.path.join(self.remote_root, filename)) del self.revision_file[filename] self.revision_file.update(local_files) if not self.dry_run: self.revision_file.save()
def _getExeName(self, pid): return os.readlink('/proc/%d/file' % pid)
def walk(top, topdown=True, onerror=None, followlinks=False, blacklist=()): """Directory tree generator. Custom implementation of os.walk that doesn't block if the destination of a symlink is on an unreachable blacklisted path (typically a nfs mount). All the general os.walk documentation applies. """ # We may not have read permission for top, in which case we can't # get a list of the files the directory contains. os.path.walk # always suppressed the exception then, rather than blow up for a # minor reason when (say) a thousand readable directories are still # left to visit. That logic is copied here. try: names = os.listdir(top) except OSError as err: if onerror is not None: onerror(err) return # Use absolute and normalized blacklist paths normblacklist = [os.path.abspath(x) for x in blacklist] dirs, nondirs = [], [] for name in names: path = os.path.join(top, name) # Begin of the part where we handle the unreachable symlinks if os.path.abspath(path) in normblacklist: continue if not followlinks: # Don't use os.path.islink because it relies on the syscall # lstat which is getting stuck if the destination is unreachable try: os.readlink(path) except OSError as err: # EINVAL is thrown when "path" is not a symlink, in such # case continue normally if err.errno != errno.EINVAL: raise # There is an hidden code path here, if we fail to read the # link and the errno is EINVAL then skip the following else # code block: else: nondirs.append(name) continue # End of the part where we handle the unreachable symlinks if os.path.isdir(path): dirs.append(name) else: nondirs.append(name) if topdown: yield top, dirs, nondirs for name in dirs: path = os.path.join(top, name) if followlinks or not os.path.islink(path): for x in walk(path, topdown, onerror, followlinks, blacklist): yield x if not topdown: yield top, dirs, nondirs
def get_interfaces(): """ Collect the information on all network interfaces. Returns ------- dict The information on the interfaces. keys: physical : boolean, true if physical interface mac : mac address pci : PCI device virtfns : dict of virtual function """ ret = {} pci_id_to_iface = {} for n in os.listdir(_CLASS_NET_DIR): physical = True iface = "{}/{}".format(_CLASS_NET_DIR, n) try: link = os.readlink(iface) if link.startswith('../../devices/virtual'): physical = False except OSError: continue mac = open('{}/address'.format(iface)).read().strip().lower() iface_info = {'physical': physical, 'mac': mac} if physical: # Check to see if this is a physical or virtual # function dev = '{}/device'.format(iface) pci_id = os.readlink(dev) pci_id = pci_id[pci_id.rfind('/') + 1:] pci_id_to_iface[pci_id] = n iface_info['pci'] = pci_id try: phys_id = os.readlink('{}/physfn'.format(dev))[3:] iface_info['physfn'] = phys_id except OSError: # If there is no physical function backing this # interface, then it must itself be one virt_ifaces = {} dirs = os.listdir(dev) for d in dirs: if not d.startswith('virtfn'): continue virt_pci_id = os.readlink('{}/{}'.format(dev, d))[3:] virt_ifaces[int(d[6:])] = {'pci_id': virt_pci_id} # TODO: find a better way to get mac addresses for virtual functions for line in subprocess.check_output( ['/usr/sbin/ip', 'link', 'show', n]).splitlines(): line = line.strip().decode() if not line.startswith('vf '): continue ents = line.split(' ') vf_num = int(ents[1]) vf_mac = ents[3][:-1] virt_ifaces[vf_num]['mac'] = vf_mac iface_info['virtfns'] = virt_ifaces ret[n] = iface_info # Populate any potentially invalid mac addresses with # the correct data for n, info in ret.items(): if not info['physical']: continue virt_fns = info.get('virtfns') if virt_fns is None: continue for _, v in virt_fns.items(): try: v['mac'] = ret[pci_id_to_iface[v['pci_id']]]['mac'] except Exception: pass return ret
def _assess_event_links(event_spec): self.assertTrue(self.hooks_dir / event_spec.event_name in self.hooks_dir.iterdir()) for event_hook in all_event_hooks: self.assertTrue((self.JUJU_CHARM_DIR / event_hook).exists(), f'Missing hook: {event_hook}') self.assertEqual(os.readlink(self.JUJU_CHARM_DIR / event_hook), self.charm_exec_path)
def _get_left_part(self, bar): # pylint: disable=too-many-branches,too-many-statements left = bar.left if self.column is not None and self.column.target is not None\ and self.column.target.is_directory: target = self.column.target.pointed_obj else: directory = self.fm.thistab.at_level(0) if directory: target = directory.pointed_obj else: return try: stat = target.stat except AttributeError: return if stat is None: return if self.fm.mode != 'normal': perms = '--%s--' % self.fm.mode.upper() else: perms = target.get_permission_string() how = 'good' if getuid() == stat.st_uid else 'bad' left.add(perms, 'permissions', how) left.add_space() left.add(str(stat.st_nlink), 'nlink') left.add_space() left.add(self._get_owner(target), 'owner') left.add_space() left.add(self._get_group(target), 'group') if target.is_link: how = 'good' if target.exists else 'bad' try: dest = readlink(target.path) except OSError: dest = '?' left.add(' -> ' + dest, 'link', how) else: left.add_space() if self.settings.display_size_in_status_bar and target.infostring: left.add(target.infostring.replace(" ", "")) left.add_space() left.add(strftime(self.timeformat, localtime(stat.st_mtime)), 'mtime') directory = target if target.is_directory else \ target.fm.get_directory(os.path.dirname(target.path)) if directory.vcs and directory.vcs.track: if directory.vcs.rootvcs.branch: vcsinfo = '({0:s}: {1:s})'.format( directory.vcs.rootvcs.repotype, directory.vcs.rootvcs.branch) else: vcsinfo = '({0:s})'.format(directory.vcs.rootvcs.repotype) left.add_space() left.add(vcsinfo, 'vcsinfo') left.add_space() if directory.vcs.rootvcs.obj.vcsremotestatus: vcsstr, vcscol = self.vcsremotestatus_symb[ directory.vcs.rootvcs.obj.vcsremotestatus] left.add(vcsstr.strip(), 'vcsremote', *vcscol) if target.vcsstatus: vcsstr, vcscol = self.vcsstatus_symb[target.vcsstatus] left.add(vcsstr.strip(), 'vcsfile', *vcscol) if directory.vcs.rootvcs.head: left.add_space() left.add(directory.vcs.rootvcs.head['date'].strftime(self.timeformat), 'vcsdate') left.add_space() left.add(directory.vcs.rootvcs.head['summary'], 'vcscommit')
def get_interfaces_info(self, ip_path, default_ipv4, default_ipv6): interfaces = {} ips = dict( all_ipv4_addresses=[], all_ipv6_addresses=[], ) # FIXME: maybe split into smaller methods? # FIXME: this is pretty much a constructor for path in glob.glob('/sys/class/net/*'): if not os.path.isdir(path): continue device = os.path.basename(path) interfaces[device] = {'device': device} if os.path.exists(os.path.join(path, 'address')): macaddress = get_file_content(os.path.join(path, 'address'), default='') if macaddress and macaddress != '00:00:00:00:00:00': interfaces[device]['macaddress'] = macaddress if os.path.exists(os.path.join(path, 'mtu')): interfaces[device]['mtu'] = int(get_file_content(os.path.join(path, 'mtu'))) if os.path.exists(os.path.join(path, 'operstate')): interfaces[device]['active'] = get_file_content(os.path.join(path, 'operstate')) != 'down' if os.path.exists(os.path.join(path, 'device', 'driver', 'module')): interfaces[device]['module'] = os.path.basename(os.path.realpath(os.path.join(path, 'device', 'driver', 'module'))) if os.path.exists(os.path.join(path, 'type')): _type = get_file_content(os.path.join(path, 'type')) interfaces[device]['type'] = self.INTERFACE_TYPE.get(_type, 'unknown') if os.path.exists(os.path.join(path, 'bridge')): interfaces[device]['type'] = 'bridge' interfaces[device]['interfaces'] = [os.path.basename(b) for b in glob.glob(os.path.join(path, 'brif', '*'))] if os.path.exists(os.path.join(path, 'bridge', 'bridge_id')): interfaces[device]['id'] = get_file_content(os.path.join(path, 'bridge', 'bridge_id'), default='') if os.path.exists(os.path.join(path, 'bridge', 'stp_state')): interfaces[device]['stp'] = get_file_content(os.path.join(path, 'bridge', 'stp_state')) == '1' if os.path.exists(os.path.join(path, 'bonding')): interfaces[device]['type'] = 'bonding' interfaces[device]['slaves'] = get_file_content(os.path.join(path, 'bonding', 'slaves'), default='').split() interfaces[device]['mode'] = get_file_content(os.path.join(path, 'bonding', 'mode'), default='').split()[0] interfaces[device]['miimon'] = get_file_content(os.path.join(path, 'bonding', 'miimon'), default='').split()[0] interfaces[device]['lacp_rate'] = get_file_content(os.path.join(path, 'bonding', 'lacp_rate'), default='').split()[0] primary = get_file_content(os.path.join(path, 'bonding', 'primary')) if primary: interfaces[device]['primary'] = primary path = os.path.join(path, 'bonding', 'all_slaves_active') if os.path.exists(path): interfaces[device]['all_slaves_active'] = get_file_content(path) == '1' if os.path.exists(os.path.join(path, 'bonding_slave')): interfaces[device]['perm_macaddress'] = get_file_content(os.path.join(path, 'bonding_slave', 'perm_hwaddr'), default='') if os.path.exists(os.path.join(path, 'device')): interfaces[device]['pciid'] = os.path.basename(os.readlink(os.path.join(path, 'device'))) if os.path.exists(os.path.join(path, 'speed')): speed = get_file_content(os.path.join(path, 'speed')) if speed is not None: interfaces[device]['speed'] = int(speed) # Check whether an interface is in promiscuous mode if os.path.exists(os.path.join(path, 'flags')): promisc_mode = False # The second byte indicates whether the interface is in promiscuous mode. # 1 = promisc # 0 = no promisc data = int(get_file_content(os.path.join(path, 'flags')), 16) promisc_mode = (data & 0x0100 > 0) interfaces[device]['promisc'] = promisc_mode # TODO: determine if this needs to be in a nested scope/closure def parse_ip_output(output, secondary=False): for line in output.splitlines(): if not line: continue words = line.split() broadcast = '' if words[0] == 'inet': if '/' in words[1]: address, netmask_length = words[1].split('/') if len(words) > 3: broadcast = words[3] else: # pointopoint interfaces do not have a prefix address = words[1] netmask_length = "32" address_bin = struct.unpack('!L', socket.inet_aton(address))[0] netmask_bin = (1 << 32) - (1 << 32 >> int(netmask_length)) netmask = socket.inet_ntoa(struct.pack('!L', netmask_bin)) network = socket.inet_ntoa(struct.pack('!L', address_bin & netmask_bin)) iface = words[-1] # NOTE: device is ref to outside scope # NOTE: interfaces is also ref to outside scope if iface != device: interfaces[iface] = {} if not secondary and "ipv4" not in interfaces[iface]: interfaces[iface]['ipv4'] = {'address': address, 'broadcast': broadcast, 'netmask': netmask, 'network': network} else: if "ipv4_secondaries" not in interfaces[iface]: interfaces[iface]["ipv4_secondaries"] = [] interfaces[iface]["ipv4_secondaries"].append({ 'address': address, 'broadcast': broadcast, 'netmask': netmask, 'network': network, }) # add this secondary IP to the main device if secondary: if "ipv4_secondaries" not in interfaces[device]: interfaces[device]["ipv4_secondaries"] = [] if device != iface: interfaces[device]["ipv4_secondaries"].append({ 'address': address, 'broadcast': broadcast, 'netmask': netmask, 'network': network, }) # NOTE: default_ipv4 is ref to outside scope # If this is the default address, update default_ipv4 if 'address' in default_ipv4 and default_ipv4['address'] == address: default_ipv4['broadcast'] = broadcast default_ipv4['netmask'] = netmask default_ipv4['network'] = network # NOTE: macadress is ref from outside scope default_ipv4['macaddress'] = macaddress default_ipv4['mtu'] = interfaces[device]['mtu'] default_ipv4['type'] = interfaces[device].get("type", "unknown") default_ipv4['alias'] = words[-1] if not address.startswith('127.'): ips['all_ipv4_addresses'].append(address) elif words[0] == 'inet6': if 'peer' == words[2]: address = words[1] _, prefix = words[3].split('/') scope = words[5] else: address, prefix = words[1].split('/') scope = words[3] if 'ipv6' not in interfaces[device]: interfaces[device]['ipv6'] = [] interfaces[device]['ipv6'].append({ 'address': address, 'prefix': prefix, 'scope': scope }) # If this is the default address, update default_ipv6 if 'address' in default_ipv6 and default_ipv6['address'] == address: default_ipv6['prefix'] = prefix default_ipv6['scope'] = scope default_ipv6['macaddress'] = macaddress default_ipv6['mtu'] = interfaces[device]['mtu'] default_ipv6['type'] = interfaces[device].get("type", "unknown") if not address == '::1': ips['all_ipv6_addresses'].append(address) ip_path = self.module.get_bin_path("ip") args = [ip_path, 'addr', 'show', 'primary', device] rc, primary_data, stderr = self.module.run_command(args, errors='surrogate_then_replace') if rc == 0: parse_ip_output(primary_data) else: # possibly busybox, fallback to running without the "primary" arg # https://github.com/ansible/ansible/issues/50871 args = [ip_path, 'addr', 'show', device] rc, data, stderr = self.module.run_command(args, errors='surrogate_then_replace') if rc == 0: parse_ip_output(data) args = [ip_path, 'addr', 'show', 'secondary', device] rc, secondary_data, stderr = self.module.run_command(args, errors='surrogate_then_replace') if rc == 0: parse_ip_output(secondary_data, secondary=True) interfaces[device].update(self.get_ethtool_data(device)) # replace : by _ in interface name since they are hard to use in template new_interfaces = {} # i is a dict key (string) not an index int for i in interfaces: if ':' in i: new_interfaces[i.replace(':', '_')] = interfaces[i] else: new_interfaces[i] = interfaces[i] return new_interfaces, ips
def dircheck(environ, dir_tree): """Checks if dir_tree matches the actual directory tree in the filesystem""" # Helper functions def check_owner(path, props, is_link=False): """For owner permissions we only look up if its a normal user or the root user because we can't create other users just for the sake of these tests""" stat = os.lstat if is_link else os.stat if "rootuser" not in props: props["rootuser"] = False if "rootgroup" not in props: props["rootgroup"] = False if bool(stat(path).st_uid) == props["rootuser"]: user = "******" if props["rootuser"] else "user" raise ValueError((False, path + " is a not owned by " + user)) if bool(stat(path).st_gid) == props["rootgroup"]: group = "root" if props["rootgroup"] else "group" raise ValueError((False, path + " is a not owned by " + group)) def check_permission(path, permission): perm_real = str(oct(os.stat(path).st_mode))[-3:] if perm_real != str(permission): raise ValueError((False, path + " has permission " + perm_real)) # Collect all files/links in environment file_list = [] for root, _, files in os.walk(environ): for file in files: file_list.append(os.path.abspath(os.path.join(root, file))) # Compare dir_tree against actual directory tree in environment for dir_name, dir_props in dir_tree.items(): # Add environment to directory dir_name = os.path.normpath(os.path.join(environ, dir_name)) # Directory existance if not os.path.isdir(dir_name): raise ValueError((False, dir_name + " is a not a directory")) # Directory permission if "permission" in dir_props: check_permission(dir_name, dir_props["permission"]) # Directory owner check_owner(dir_name, dir_props) # Files if "files" in dir_props: for file_props in dir_props["files"]: file_path = os.path.join(dir_name, file_props["name"]) # File existance if os.path.islink(file_path) or not os.path.isfile(file_path): raise ValueError((False, file_path + " is a not a file")) # File permission if "permission" in file_props: check_permission(file_path, file_props["permission"]) # File owner check_owner(file_path, file_props) # File content md5 = hashlib.md5(open(file_path, "rb").read()).hexdigest() if "content" in file_props and md5 != file_props["content"]: msg = file_path + " has wrong content:\n" msg += open(file_path, "rb").read().decode() raise ValueError((False, msg)) file_list.remove(file_path) # Links if "links" in dir_props: for link_props in dir_props["links"]: link_path = os.path.join(dir_name, link_props["name"]) # Link existance if not os.path.islink(link_path): raise ValueError((False, link_path + " is a not a link")) # Link permission if "permission" in link_props: check_permission(link_path, link_props["permission"]) # Link owner check_owner(link_path, link_props, True) # Link target target_path = os.path.normpath( os.path.join(dir_name, os.readlink(link_path))) link_props["target"] = os.path.abspath(link_props["target"]) if target_path != link_props["target"]: msg = link_path + " should point to " + link_props['target'] msg += ", but points to " + target_path raise ValueError((False, msg)) # Link target content md5 = hashlib.md5(open(target_path, "rb").read()).hexdigest() if "content" in link_props and md5 != link_props["content"]: msg = link_path + " has wrong content:\n" msg += open(target_path, "rb").read().decode() raise ValueError((False, msg)) file_list.remove(link_path) # Check if there are files left, that wasn't described by the dir_tree if file_list: msg = "Test created unexpected files:\n" for file in file_list: msg += " " + file + "\n" raise ValueError((False, msg))
def copytree(start_site, src, end_site, dst, current_depth, symlinks=False, ignore=None, arguments=argparse.Namespace()): """Recursively copy a directory (src) to another location (dst)""" if arguments.debug: print("copytree:") print("\tsrc:", src) print("\tdst:", dst) print("\tdepth:", str(arguments.depth)) print("\tcurrent_depth:", str(current_depth)) print("\tsymlinks:", str(symlinks)) print("\tdry_run:", str(arguments.dry_run)) if current_depth >= arguments.depth: return if not arguments.dry_run: made_dir = make_directory(end_site, dst, arguments.protocol, arguments.debug) if not made_dir: raise Exception( "ERROR::copytree::Unable to make the destination directory.") files = get_list_of_files(arguments.protocol, start_site, arguments.sample, src, arguments.debug) if arguments.debug: print("copytree:") print("\tList of files:", files) if arguments.diff: files = do_diff(files, arguments.protocol, end_site, arguments.sample, dst, arguments.debug) if ignore is not None: ignored_names = ignore(src, files) else: ignored_names = set() errors = [] for file in files: if file in ignored_names: continue srcname = os.path.join(src, file) \ if not (os.path.isfile(src) or (start_site.alias != 'local' \ and not remote_is_dir(start_site, src))) else src dstname = os.path.join(dst, file) if arguments.debug: print("copytree:") print("\tsrcname:", srcname) print("\tdstname:", dstname) try: if symlinks and os.path.islink(srcname): linkto = os.readlink(srcname) os.symlink(linkto, dstname) elif os.path.isdir(srcname) or \ (arguments.protocol == "xrootd" and start_site.alias != 'local' and remote_is_dir(start_site, srcname)): copytree(start_site, srcname, end_site, dstname, current_depth + 1, symlinks, ignore, arguments) else: srel = os.path.relpath( src, src[:src.find(start_site.path) + len(start_site.path)]) + "/" if srel == "./": srel = "" copy_command, start_location, end_location = init_commands( start_site, end_site, arguments) print("Copying file " + file + " from " + srel) if start_site.alias != 'local' and start_site.alias != 'local' and not remote_is_dir( start_site, src): command = copy_command + " " + start_location + " " + end_location elif start_site.alias != 'local': command = copy_command + " " + start_location + srel + file + " " + end_location + srel + file elif os.path.isfile(start_location): command = copy_command + " " + start_location + " " + end_location + srel + file else: command = copy_command + " " + start_location + srel + file + " " + end_location + srel + file print("\tcopy command:", command) if not arguments.dry_run: os.system(command) print("") # catch the Error from the recursive copytree so that we can # continue with other files except Error as err: errors.extend(err.args[0]) except EnvironmentError as why: errors.append((srcname, dstname, str(why))) if errors: raise Error(errors)
def main(): module = AnsibleModule( argument_spec=dict( path=dict(type='path', required=True, aliases=['dest', 'name']), follow=dict(type='bool', default=False), get_md5=dict(type='bool', default=False), get_checksum=dict(type='bool', default=True), get_mime=dict(type='bool', default=True, aliases=['mime', 'mime_type', 'mime-type']), get_attributes=dict(type='bool', default=True, aliases=['attr', 'attributes']), checksum_algorithm=dict(type='str', default='sha1', choices=['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'], aliases=['checksum', 'checksum_algo']), ), supports_check_mode=True, ) path = module.params.get('path') b_path = to_bytes(path, errors='surrogate_or_strict') follow = module.params.get('follow') get_mime = module.params.get('get_mime') get_attr = module.params.get('get_attributes') get_checksum = module.params.get('get_checksum') checksum_algorithm = module.params.get('checksum_algorithm') # NOTE: undocumented option since 2.9 to be removed at a later date if possible (3.0+) # no real reason for keeping other than fear we may break older content. get_md5 = module.params.get('get_md5') # main stat data try: if follow: st = os.stat(b_path) else: st = os.lstat(b_path) except OSError as e: if e.errno == errno.ENOENT: output = {'exists': False} module.exit_json(changed=False, stat=output) module.fail_json(msg=e.strerror) # process base results output = format_output(module, path, st) # resolved permissions for perm in [('readable', os.R_OK), ('writeable', os.W_OK), ('executable', os.X_OK)]: output[perm[0]] = os.access(b_path, perm[1]) # symlink info if output.get('islnk'): output['lnk_source'] = os.path.realpath(b_path) output['lnk_target'] = os.readlink(b_path) try: # user data pw = pwd.getpwuid(st.st_uid) output['pw_name'] = pw.pw_name except (TypeError, KeyError): pass try: # group data grp_info = grp.getgrgid(st.st_gid) output['gr_name'] = grp_info.gr_name except (KeyError, ValueError, OverflowError): pass # checksums if output.get('isreg') and output.get('readable'): # NOTE: see above about get_md5 if get_md5: # Will fail on FIPS-140 compliant systems try: output['md5'] = module.md5(b_path) except ValueError: output['md5'] = None if get_checksum: output['checksum'] = module.digest_from_file(b_path, checksum_algorithm) # try to get mime data if requested if get_mime: output['mimetype'] = output['charset'] = 'unknown' mimecmd = module.get_bin_path('file') if mimecmd: mimecmd = [mimecmd, '--mime-type', '--mime-encoding', b_path] try: rc, out, err = module.run_command(mimecmd) if rc == 0: mimetype, charset = out.rsplit(':', 1)[1].split(';') output['mimetype'] = mimetype.strip() output['charset'] = charset.split('=')[1].strip() except Exception: pass # try to get attr data if get_attr: output['version'] = None output['attributes'] = [] output['attr_flags'] = '' out = module.get_file_attributes(b_path) for x in ('version', 'attributes', 'attr_flags'): if x in out: output[x] = out[x] module.exit_json(changed=False, stat=output)
# # License: see LICENSE file for details # from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import random import sys # point to absolute path of peda.py PEDAFILE = os.path.abspath(os.path.expanduser(__file__)) while os.path.islink(PEDAFILE): PEDAFILE = os.path.abspath(os.path.join(os.path.dirname(PEDAFILE), os.path.expanduser(os.readlink(PEDAFILE)))) sys.path.insert(0, os.path.dirname(PEDAFILE)) from peda import * try: import zlib except ImportError: zlib = None __version__ = 'alpha-1.0' class Asm(AsmBase): """ Wrapper class for assemble/disassemble using as
def apport_excepthook(exc_type, exc_obj, exc_tb): '''Catch an uncaught exception and make a traceback.''' # create and save a problem report. Note that exceptions in this code # are bad, and we probably need a per-thread reentrancy guard to # prevent that happening. However, on Ubuntu there should never be # a reason for an exception here, other than [say] a read only var # or some such. So what we do is use a try - finally to ensure that # the original excepthook is invoked, and until we get bug reports # ignore the other issues. # import locally here so that there is no routine overhead on python # startup time - only when a traceback occurs will this trigger. try: # ignore 'safe' exit types. if exc_type in (KeyboardInterrupt, ): return # do not do anything if apport was disabled if not enabled(): return try: from cStringIO import StringIO StringIO # pyflakes except ImportError: from io import StringIO import re, traceback from apport.fileutils import likely_packaged, get_recent_crashes # apport will look up the package from the executable path. try: binary = os.path.realpath(os.path.join(os.getcwd(), sys.argv[0])) except (TypeError, AttributeError, IndexError): # the module has mutated sys.argv, plan B try: binary = os.readlink('/proc/%i/exe' % os.getpid()) except OSError: return # for interactive python sessions, sys.argv[0] == ''; catch that and # other irregularities if not os.access(binary, os.X_OK) or not os.path.isfile(binary): return # filter out binaries in user accessible paths if not likely_packaged(binary): return import apport.report pr = apport.report.Report() # special handling of dbus-python exceptions if hasattr(exc_obj, 'get_dbus_name'): if exc_obj.get_dbus_name() == 'org.freedesktop.DBus.Error.NoReply': # NoReply is an useless crash, we do not even get the method it # was trying to call; needs actual crash from D-BUS backend (LP #914220) return if exc_obj.get_dbus_name( ) == 'org.freedesktop.DBus.Error.ServiceUnknown': dbus_service_unknown_analysis(exc_obj, pr) # append a basic traceback. In future we may want to include # additional data such as the local variables, loaded modules etc. tb_file = StringIO() traceback.print_exception(exc_type, exc_obj, exc_tb, file=tb_file) pr['Traceback'] = tb_file.getvalue().strip() pr.add_proc_info(extraenv=['PYTHONPATH', 'PYTHONHOME']) pr.add_user_info() # override the ExecutablePath with the script that was actually running pr['ExecutablePath'] = binary if 'ExecutableTimestamp' in pr: pr['ExecutableTimestamp'] = str(int(os.stat(binary).st_mtime)) try: pr['PythonArgs'] = '%r' % sys.argv except AttributeError: pass if pr.check_ignored(): return mangled_program = re.sub('/', '_', binary) # get the uid for now, user name later user = os.getuid() pr_filename = '%s/%s.%i.crash' % (os.environ.get( 'APPORT_REPORT_DIR', '/var/crash'), mangled_program, user) crash_counter = 0 if os.path.exists(pr_filename): if apport.fileutils.seen_report(pr_filename): # flood protection with open(pr_filename, 'rb') as f: crash_counter = get_recent_crashes(f) + 1 if crash_counter > 1: return # remove the old file, so that we can create the new one with # os.O_CREAT|os.O_EXCL os.unlink(pr_filename) else: # don't clobber existing report return if crash_counter: pr['CrashCounter'] = str(crash_counter) with os.fdopen( os.open(pr_filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640), 'wb') as f: pr.write(f) finally: # resume original processing to get the default behaviour, # but do not trigger an AttributeError on interpreter shutdown. if sys: sys.__excepthook__(exc_type, exc_obj, exc_tb)
def zvol_device_name(self): return os.path.split(os.readlink(self.zvol_dev))[-1]
def _create_and_check_link(self, source, link): absolute_symlink(source, link) self.assertTrue(os.path.islink(link)) self.assertEqual(source, os.readlink(link))
def copy(src, dst, hardlink=False, keep_symlink=True): '''Copy a file to another location with a bunch of link handling''' assert not P.isdir(src), 'Source path must not be a dir' assert not P.isdir(dst), 'Destination path must not be a dir' # There is nothing we can do about absolute sym links in system dirs. We just # trace those to the source and copy. if keep_symlink and P.islink(src): if P.isabs(readlink(src)): m = re.match('^/usr', readlink(src)) if m: print("Resolving absolute link: ", src) while P.islink(src) and \ P.isabs(readlink(src)) \ and os.path.basename(src) == os.path.basename(readlink(src)): src = readlink(src) print("Resolved to: ", src) if keep_symlink and P.islink(src): assert not P.isabs(readlink(src)), \ 'Cannot copy symlink that points to an absolute path (%s)' % src logger.debug('%8s %s -> %s' % ('symlink', src, dst)) # Some of the libraries are both in our install dir and in USGS conda's package. # That is because they do not provide headers for cspice for example. # So below we will run into trouble. Just overwrite any library we built # with the one from conda. if P.exists(dst): is_soft_link = True try: link_val = readlink(dst) except: is_soft_link = False if (not is_soft_link) or (readlink(dst) != readlink(src)): print("Will overwrite " + dst + " with " + src) os.remove(dst) if not P.exists(dst): try: symlink(readlink(src), dst) except: pass return if P.exists(dst): if hash_file(src) != hash_file(dst): print("Will overwrite " + dst + " with " + src + " having a different hash.") else: if hardlink: try: link(src, dst) logger.debug('%8s %s -> %s' % ('hardlink', src, dst)) return except OSError as o: # Invalid cross-device link, not an error, fall back to copy if o.errno != errno.EXDEV: raise logger.debug('%8s %s -> %s' % ('copy', src, dst)) shutil.copyfile(src, dst) # Bugfix, make it writeable mode = os.stat(dst)[stat.ST_MODE] os.chmod(dst, mode | stat.S_IWUSR)
def formatted_filename(path, filename, mode): if not stat.S_ISLINK(mode): return filename symlink_target = os.readlink(filename) return filename + ' -> ' + symlink_target
def main(): module = AnsibleModule( argument_spec=dict( state=dict(choices=[ 'file', 'directory', 'link', 'hard', 'touch', 'absent' ], default=None), path=dict(aliases=['dest', 'name'], required=True, type='path'), original_basename=dict( required=False), # Internal use only, for recursive ops recurse=dict(default=False, type='bool'), force=dict(required=False, default=False, type='bool'), follow=dict(required=False, default=False, type='bool'), diff_peek=dict( default=None ), # Internal use only, for internal checks in the action plugins validate=dict( required=False, default=None), # Internal use only, for template and copy src=dict(required=False, default=None, type='path'), ), add_file_common_args=True, supports_check_mode=True) params = module.params state = params['state'] recurse = params['recurse'] force = params['force'] diff_peek = params['diff_peek'] src = params['src'] b_src = to_bytes(src, errors='surrogate_or_strict') follow = params['follow'] # modify source as we later reload and pass, specially relevant when used by other modules. path = params['path'] b_path = to_bytes(path, errors='surrogate_or_strict') # short-circuit for diff_peek if diff_peek is not None: appears_binary = False try: f = open(b_path, 'rb') head = f.read(8192) f.close() if b("\x00") in head: appears_binary = True except: pass module.exit_json(path=path, changed=False, appears_binary=appears_binary) prev_state = get_state(b_path) # state should default to file, but since that creates many conflicts, # default to 'current' when it exists. if state is None: if prev_state != 'absent': state = prev_state elif recurse: state = 'directory' else: state = 'file' # source is both the source of a symlink or an informational passing of the src for a template module # or copy module, even if this module never uses it, it is needed to key off some things if src is None: if state in ('link', 'hard'): if follow and state == 'link': # use the current target of the link as the source src = to_native(os.path.realpath(b_path), errors='strict') b_src = to_bytes(os.path.realpath(b_path), errors='strict') else: module.fail_json( msg='src and dest are required for creating links') # original_basename is used by other modules that depend on file. if state not in ("link", "absent") and os.path.isdir(b_path): basename = None if params['original_basename']: basename = params['original_basename'] elif src is not None: basename = os.path.basename(src) if basename: params['path'] = path = os.path.join(path, basename) b_path = to_bytes(path, errors='surrogate_or_strict') prev_state = get_state(b_path) # make sure the target path is a directory when we're doing a recursive operation if recurse and state != 'directory': module.fail_json(path=path, msg="recurse option requires state to be 'directory'") file_args = module.load_file_common_arguments(params) changed = False diff = { 'before': { 'path': path }, 'after': { 'path': path }, } state_change = False if prev_state != state: diff['before']['state'] = prev_state diff['after']['state'] = state state_change = True if state == 'absent': if state_change: if not module.check_mode: if prev_state == 'directory': try: shutil.rmtree(b_path, ignore_errors=False) except Exception as e: module.fail_json(msg="rmtree failed: %s" % to_native(e)) else: try: os.unlink(b_path) except Exception as e: module.fail_json(path=path, msg="unlinking failed: %s " % to_native(e)) module.exit_json(path=path, changed=True, diff=diff) else: module.exit_json(path=path, changed=False) elif state == 'file': if state_change: if follow and prev_state == 'link': # follow symlink and operate on original b_path = os.path.realpath(b_path) path = to_native(b_path, errors='strict') prev_state = get_state(b_path) file_args['path'] = path if prev_state not in ('file', 'hard'): # file is not absent and any other state is a conflict module.fail_json(path=path, msg='file (%s) is %s, cannot continue' % (path, prev_state)) changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) module.exit_json(path=path, changed=changed, diff=diff) elif state == 'directory': if follow and prev_state == 'link': b_path = os.path.realpath(b_path) path = to_native(b_path, errors='strict') prev_state = get_state(b_path) if prev_state == 'absent': if module.check_mode: module.exit_json(changed=True, diff=diff) changed = True curpath = '' try: # Split the path so we can apply filesystem attributes recursively # from the root (/) directory for absolute paths or the base path # of a relative path. We can then walk the appropriate directory # path to apply attributes. for dirname in path.strip('/').split('/'): curpath = '/'.join([curpath, dirname]) # Remove leading slash if we're creating a relative path if not os.path.isabs(path): curpath = curpath.lstrip('/') b_curpath = to_bytes(curpath, errors='surrogate_or_strict') if not os.path.exists(b_curpath): try: os.mkdir(b_curpath) except OSError as ex: # Possibly something else created the dir since the os.path.exists # check above. As long as it's a dir, we don't need to error out. if not (ex.errno == errno.EEXIST and os.path.isdir(b_curpath)): raise tmp_file_args = file_args.copy() tmp_file_args['path'] = curpath changed = module.set_fs_attributes_if_different( tmp_file_args, changed, diff, expand=False) except Exception as e: module.fail_json( path=path, msg='There was an issue creating %s as requested: %s' % (curpath, to_native(e))) # We already know prev_state is not 'absent', therefore it exists in some form. elif prev_state != 'directory': module.fail_json(path=path, msg='%s already exists as a %s' % (path, prev_state)) changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) if recurse: changed |= recursive_set_attributes( module, to_bytes(file_args['path'], errors='surrogate_or_strict'), follow, file_args) module.exit_json(path=path, changed=changed, diff=diff) elif state in ('link', 'hard'): if not os.path.islink(b_path) and os.path.isdir(b_path): relpath = path else: b_relpath = os.path.dirname(b_path) relpath = to_native(b_relpath, errors='strict') absrc = os.path.join(relpath, src) b_absrc = to_bytes(absrc, errors='surrogate_or_strict') if not force and not os.path.exists(b_absrc): module.fail_json( path=path, src=src, msg= 'src file does not exist, use "force=yes" if you really want to create the link: %s' % absrc) if state == 'hard': if not os.path.isabs(b_src): module.fail_json(msg="absolute paths are required") elif prev_state == 'directory': if not force: module.fail_json( path=path, msg='refusing to convert between %s and %s for %s' % (prev_state, state, path)) elif os.listdir(b_path): # refuse to replace a directory that has files in it module.fail_json( path=path, msg='the directory %s is not empty, refusing to convert it' % path) elif prev_state in ('file', 'hard') and not force: module.fail_json( path=path, msg='refusing to convert between %s and %s for %s' % (prev_state, state, path)) if prev_state == 'absent': changed = True elif prev_state == 'link': b_old_src = os.readlink(b_path) if b_old_src != b_src: diff['before']['src'] = to_native(b_old_src, errors='strict') diff['after']['src'] = src changed = True elif prev_state == 'hard': if not (state == 'hard' and os.stat(b_path).st_ino == os.stat(b_src).st_ino): changed = True if not force: module.fail_json( dest=path, src=src, msg= 'Cannot link, different hard link exists at destination' ) elif prev_state == 'file': changed = True if not force: module.fail_json(dest=path, src=src, msg='Cannot link, %s exists at destination' % prev_state) elif prev_state == 'directory': changed = True if os.path.exists(b_path): if state == 'hard' and os.stat(b_path).st_ino == os.stat( b_src).st_ino: module.exit_json(path=path, changed=False) elif not force: module.fail_json( dest=path, src=src, msg= 'Cannot link, different hard link exists at destination' ) else: module.fail_json(dest=path, src=src, msg='unexpected position reached') if changed and not module.check_mode: if prev_state != 'absent': # try to replace atomically b_tmppath = to_bytes(os.path.sep).join([ os.path.dirname(b_path), to_bytes(".%s.%s.tmp" % (os.getpid(), time.time())) ]) try: if prev_state == 'directory' and state == 'link': os.rmdir(b_path) elif prev_state == 'directory' and state == 'hard': if os.path.exists(b_path): os.remove(b_path) if state == 'hard': os.link(b_src, b_tmppath) else: os.symlink(b_src, b_tmppath) os.rename(b_tmppath, b_path) except OSError as e: if os.path.exists(b_tmppath): os.unlink(b_tmppath) module.fail_json(path=path, msg='Error while replacing: %s' % to_native(e, nonstring='simplerepr')) else: try: if state == 'hard': os.link(b_src, b_path) else: os.symlink(b_src, b_path) except OSError as e: module.fail_json(path=path, msg='Error while linking: %s' % to_native(e, nonstring='simplerepr')) if module.check_mode and not os.path.exists(b_path): module.exit_json(dest=path, src=src, changed=changed, diff=diff) changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) module.exit_json(dest=path, src=src, changed=changed, diff=diff) elif state == 'touch': if not module.check_mode: if prev_state == 'absent': try: open(b_path, 'wb').close() except OSError as e: module.fail_json(path=path, msg='Error, could not touch target: %s' % to_native(e, nonstring='simplerepr')) elif prev_state in ('file', 'directory', 'hard'): try: os.utime(b_path, None) except OSError as e: module.fail_json( path=path, msg='Error while touching existing target: %s' % to_native(e, nonstring='simplerepr')) else: module.fail_json( msg= 'Cannot touch other than files, directories, and hardlinks (%s is %s)' % (path, prev_state)) try: module.set_fs_attributes_if_different(file_args, True, diff, expand=False) except SystemExit as e: if e.code: # We take this to mean that fail_json() was called from # somewhere in basic.py if prev_state == 'absent': # If we just created the file we can safely remove it os.remove(b_path) raise e module.exit_json(dest=path, changed=True, diff=diff) module.fail_json(path=path, msg='unexpected position reached')
def _deleted_files(): ''' Iterates over /proc/PID/maps and /proc/PID/fd links and returns list of desired deleted files. Returns: List of deleted files to analyze, False on failure. ''' deleted_files = [] for proc in psutil.process_iter(): # pylint: disable=too-many-nested-blocks try: pinfo = proc.as_dict(attrs=['pid', 'name']) try: with salt.utils.files.fopen('/proc/{0}/maps'.format( pinfo['pid'])) as maps: # pylint: disable=resource-leakage dirpath = '/proc/' + six.text_type(pinfo['pid']) + '/fd/' listdir = os.listdir(dirpath) maplines = maps.readlines() except (OSError, IOError): yield False # /proc/PID/maps mapline = re.compile( r'^[\da-f]+-[\da-f]+ [r-][w-][x-][sp-] ' r'[\da-f]+ [\da-f]{2}:[\da-f]{2} (\d+) *(.+)( \(deleted\))?\n$' ) for line in maplines: line = salt.utils.stringutils.to_unicode(line) matched = mapline.match(line) if not matched: continue path = matched.group(2) if not path: continue valid = _valid_deleted_file(path) if not valid: continue val = (pinfo['name'], pinfo['pid'], path[0:-10]) if val not in deleted_files: deleted_files.append(val) yield val # /proc/PID/fd try: for link in listdir: path = dirpath + link readlink = os.readlink(path) filenames = [] if os.path.isfile(readlink): filenames.append(readlink) elif os.path.isdir(readlink) and readlink != '/': for root, dummy_dirs, files in salt.utils.path.os_walk( readlink, followlinks=True): for name in files: filenames.append(os.path.join(root, name)) for filename in filenames: valid = _valid_deleted_file(filename) if not valid: continue val = (pinfo['name'], pinfo['pid'], filename) if val not in deleted_files: deleted_files.append(val) yield val except OSError: pass except psutil.NoSuchProcess: pass
def copy_tree(self, src, dst, preserve_mode=1, preserve_times=1, preserve_symlinks=0, update=0, verbose=1, dry_run=0): """Same as dir_util.copy_tree, but clobber existing symlinks.""" from distutils.file_util import copy_file from distutils.dir_util import mkpath if not dry_run and not os.path.isdir(src): raise DistutilsFileError("cannot copy tree '%s': not a directory" % src) try: names = os.listdir(src) except OSError as e: if dry_run: names = [] else: raise DistutilsFileError("error listing files in '%s': %s" % (src, e.strerror)) if not dry_run: mkpath(dst, verbose=verbose) outputs = [] for n in names: src_name = os.path.join(src, n) dst_name = os.path.join(dst, n) if preserve_symlinks and os.path.islink(src_name): link_dest = os.readlink(src_name) if verbose >= 1: log.info("linking %s -> %s", dst_name, link_dest) if not dry_run: if os.path.islink(dst_name): os.unlink(dst_name) os.symlink(link_dest, dst_name) outputs.append(dst_name) elif os.path.isdir(src_name): outputs.extend( self.copy_tree(src_name, dst_name, preserve_mode, preserve_times, preserve_symlinks, update, verbose=verbose, dry_run=dry_run)) else: copy_file(src_name, dst_name, preserve_mode, preserve_times, update, verbose=verbose, dry_run=dry_run) outputs.append(dst_name) return outputs
def ensure_hardlink(path, src, follow, force, timestamps): b_path = to_bytes(path, errors='surrogate_or_strict') b_src = to_bytes(src, errors='surrogate_or_strict') prev_state = get_state(b_path) file_args = module.load_file_common_arguments(module.params) mtime = get_timestamp_for_time(timestamps['modification_time'], timestamps['modification_time_format']) atime = get_timestamp_for_time(timestamps['access_time'], timestamps['access_time_format']) # src is the source of a hardlink. We require it if we are creating a new hardlink. # We require path in the argument_spec so we know it is present at this point. if src is None: raise AssibleModuleError(results={'msg': 'src is required for creating new hardlinks'}) if not os.path.exists(b_src): raise AssibleModuleError(results={'msg': 'src does not exist', 'dest': path, 'src': src}) diff = initial_diff(path, 'hard', prev_state) changed = False if prev_state == 'absent': changed = True elif prev_state == 'link': b_old_src = os.readlink(b_path) if b_old_src != b_src: diff['before']['src'] = to_native(b_old_src, errors='strict') diff['after']['src'] = src changed = True elif prev_state == 'hard': if not os.stat(b_path).st_ino == os.stat(b_src).st_ino: changed = True if not force: raise AssibleModuleError(results={'msg': 'Cannot link, different hard link exists at destination', 'dest': path, 'src': src}) elif prev_state == 'file': changed = True if not force: raise AssibleModuleError(results={'msg': 'Cannot link, %s exists at destination' % prev_state, 'dest': path, 'src': src}) elif prev_state == 'directory': changed = True if os.path.exists(b_path): if os.stat(b_path).st_ino == os.stat(b_src).st_ino: return {'path': path, 'changed': False} elif not force: raise AssibleModuleError(results={'msg': 'Cannot link: different hard link exists at destination', 'dest': path, 'src': src}) else: raise AssibleModuleError(results={'msg': 'unexpected position reached', 'dest': path, 'src': src}) if changed and not module.check_mode: if prev_state != 'absent': # try to replace atomically b_tmppath = to_bytes(os.path.sep).join( [os.path.dirname(b_path), to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))] ) try: if prev_state == 'directory': if os.path.exists(b_path): try: os.unlink(b_path) except OSError as e: if e.errno != errno.ENOENT: # It may already have been removed raise os.link(b_src, b_tmppath) os.rename(b_tmppath, b_path) except OSError as e: if os.path.exists(b_tmppath): os.unlink(b_tmppath) raise AssibleModuleError(results={'msg': 'Error while replacing: %s' % to_native(e, nonstring='simplerepr'), 'path': path}) else: try: os.link(b_src, b_path) except OSError as e: raise AssibleModuleError(results={'msg': 'Error while linking: %s' % to_native(e, nonstring='simplerepr'), 'path': path}) if module.check_mode and not os.path.exists(b_path): return {'dest': path, 'src': src, 'changed': changed, 'diff': diff} changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) changed |= update_timestamp_for_file(file_args['path'], mtime, atime, diff) return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
def restartcheck(ignorelist=None, blacklist=None, excludepid=None, **kwargs): ''' Analyzes files openeded by running processes and seeks for packages which need to be restarted. Args: ignorelist: string or list of packages to be ignored blacklist: string or list of file paths to be ignored excludepid: string or list of process IDs to be ignored verbose: boolean, enables extensive output timeout: int, timeout in minute Returns: Dict on error: { 'result': False, 'comment': '<reason>' } String with checkrestart output if some package seems to need to be restarted or if no packages need restarting. .. versionadded:: 2015.8.3 CLI Example: .. code-block:: bash salt '*' restartcheck.restartcheck ''' kwargs = salt.utils.args.clean_kwargs(**kwargs) start_time = int(round(time.time() * 1000)) kernel_restart = True verbose = kwargs.pop('verbose', True) timeout = kwargs.pop('timeout', 5) if __grains__.get('os_family') == 'Debian': cmd_pkg_query = 'dpkg-query --listfiles ' systemd_folder = '/lib/systemd/system/' systemd = '/bin/systemd' kernel_versions = _kernel_versions_debian() elif __grains__.get('os_family') == 'RedHat': cmd_pkg_query = 'repoquery -l ' systemd_folder = '/usr/lib/systemd/system/' systemd = '/usr/bin/systemctl' kernel_versions = _kernel_versions_redhat() elif __grains__.get('os_family') == NILRT_FAMILY_NAME: cmd_pkg_query = 'opkg files ' systemd = '' kernel_versions = _kernel_versions_nilrt() else: return { 'result': False, 'comment': 'Only available on Debian, Red Hat and NI Linux Real-Time based systems.' } # Check kernel versions kernel_current = __salt__['cmd.run']('uname -a') for kernel in kernel_versions: _check_timeout(start_time, timeout) if kernel in kernel_current: if __grains__.get('os_family') == 'NILinuxRT': # Check kernel modules and hardware API's for version changes if not _kernel_modules_changed_nilrt( kernel) and not _sysapi_changed_nilrt(): kernel_restart = False break else: kernel_restart = False break packages = {} running_services = {} restart_services = [] if ignorelist: if not isinstance(ignorelist, list): ignorelist = [ignorelist] else: ignorelist = ['screen', 'systemd'] if blacklist: if not isinstance(blacklist, list): blacklist = [blacklist] else: blacklist = [] if excludepid: if not isinstance(excludepid, list): excludepid = [excludepid] else: excludepid = [] for service in __salt__['service.get_running'](): _check_timeout(start_time, timeout) service_show = __salt__['service.show'](service) if 'ExecMainPID' in service_show: running_services[service] = int(service_show['ExecMainPID']) owners_cache = {} for deleted_file in _deleted_files(): if deleted_file is False: return { 'result': False, 'comment': 'Could not get list of processes.' ' (Do you have root access?)' } _check_timeout(start_time, timeout) name, pid, path = deleted_file[0], deleted_file[1], deleted_file[2] if path in blacklist or pid in excludepid: continue try: readlink = os.readlink('/proc/{0}/exe'.format(pid)) except OSError: excludepid.append(pid) continue try: packagename = owners_cache[readlink] except KeyError: packagename = __salt__['pkg.owner'](readlink) if not packagename: packagename = name owners_cache[readlink] = packagename for running_service in running_services: _check_timeout(start_time, timeout) if running_service not in restart_services and pid == running_services[ running_service]: if packagename and packagename not in ignorelist: restart_services.append(running_service) name = running_service if packagename and packagename not in ignorelist: program = '\t' + six.text_type( pid) + ' ' + readlink + ' (file: ' + six.text_type(path) + ')' if packagename not in packages: packages[packagename] = { 'initscripts': [], 'systemdservice': [], 'processes': [program], 'process_name': name } else: if program not in packages[packagename]['processes']: packages[packagename]['processes'].append(program) if not packages and not kernel_restart: return 'No packages seem to need to be restarted.' for package in packages: _check_timeout(start_time, timeout) cmd = cmd_pkg_query + package paths = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) while True: _check_timeout(start_time, timeout) line = salt.utils.stringutils.to_unicode(paths.stdout.readline()) if not line: break pth = line[:-1] if pth.startswith('/etc/init.d/') and not pth.endswith('.sh'): packages[package]['initscripts'].append(pth[12:]) if os.path.exists(systemd) and pth.startswith(systemd_folder) and pth.endswith('.service') and \ pth.find('.wants') == -1: is_oneshot = False try: servicefile = salt.utils.files.fopen(pth) # pylint: disable=resource-leakage except IOError: continue sysfold_len = len(systemd_folder) for line in servicefile.readlines(): line = salt.utils.stringutils.to_unicode(line) if line.find('Type=oneshot') > 0: # scripts that does a single job and then exit is_oneshot = True continue servicefile.close() if not is_oneshot: packages[package]['systemdservice'].append( pth[sysfold_len:]) sys.stdout.flush() paths.stdout.close() # Alternatively, find init.d script or service that match the process name for package in packages: _check_timeout(start_time, timeout) if not packages[package]['systemdservice'] and not packages[package][ 'initscripts']: service = __salt__['service.available']( packages[package]['process_name']) if service: if os.path.exists('/etc/init.d/' + packages[package]['process_name']): packages[package]['initscripts'].append( packages[package]['process_name']) else: packages[package]['systemdservice'].append( packages[package]['process_name']) restartable = [] nonrestartable = [] restartinitcommands = [] restartservicecommands = [] for package in packages: _check_timeout(start_time, timeout) if packages[package]['initscripts']: restartable.append(package) restartinitcommands.extend([ 'service ' + s + ' restart' for s in packages[package]['initscripts'] ]) elif packages[package]['systemdservice']: restartable.append(package) restartservicecommands.extend([ 'systemctl restart ' + s for s in packages[package]['systemdservice'] ]) else: nonrestartable.append(package) if packages[package]['process_name'] in restart_services: restart_services.remove(packages[package]['process_name']) for restart_service in restart_services: _check_timeout(start_time, timeout) restartservicecommands.extend(['systemctl restart ' + restart_service]) ret = _format_output(kernel_restart, packages, verbose, restartable, nonrestartable, restartservicecommands, restartinitcommands) return ret
def compare_link(link_path, dst_path): return os.path.abspath(os.readlink(link_path)) == os.path.abspath(dst_path)
def bind_or_link(src, target): if path.islink(src): symlink(readlink(src), target) else: bind_mount(src, target)
def visit( self, obj: CWLObjectType, stagedir: str, basedir: str, copy: bool = False, staged: bool = False, ) -> None: stagedir = cast(Optional[str], obj.get("dirname")) or stagedir tgt = os.path.join( stagedir, cast(str, obj["basename"]), ) if obj["location"] in self._pathmap: return if obj["class"] == "Directory": location = cast(str, obj["location"]) if location.startswith("file://"): resolved = uri_file_path(location) else: resolved = location self._pathmap[location] = MapperEnt( resolved, tgt, "WritableDirectory" if copy else "Directory", staged ) if location.startswith("file://"): staged = False self.visitlisting( cast(List[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, staged=staged, ) elif obj["class"] == "File": path = cast(str, obj["location"]) ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( obj["contents"], tgt, "CreateWritableFile" if copy else "CreateFile", staged, ) else: with SourceLine( obj, "location", ValidationException, _logger.isEnabledFor(logging.DEBUG), ): deref = ab if urllib.parse.urlsplit(deref).scheme in ["http", "https"]: deref = downloadHttpFile(path) else: # Dereference symbolic links st = os.lstat(deref) while stat.S_ISLNK(st.st_mode): rl = os.readlink(deref) deref = ( rl if os.path.isabs(rl) else os.path.join(os.path.dirname(deref), rl) ) st = os.lstat(deref) self._pathmap[path] = MapperEnt( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( cast(List[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, )
def ensure_symlink(path, src, follow, force, timestamps): b_path = to_bytes(path, errors='surrogate_or_strict') b_src = to_bytes(src, errors='surrogate_or_strict') prev_state = get_state(b_path) mtime = get_timestamp_for_time(timestamps['modification_time'], timestamps['modification_time_format']) atime = get_timestamp_for_time(timestamps['access_time'], timestamps['access_time_format']) # source is both the source of a symlink or an informational passing of the src for a template module # or copy module, even if this module never uses it, it is needed to key off some things if src is None: if follow: # use the current target of the link as the source src = to_native(os.path.realpath(b_path), errors='strict') b_src = to_bytes(src, errors='surrogate_or_strict') if not os.path.islink(b_path) and os.path.isdir(b_path): relpath = path else: b_relpath = os.path.dirname(b_path) relpath = to_native(b_relpath, errors='strict') absrc = os.path.join(relpath, src) b_absrc = to_bytes(absrc, errors='surrogate_or_strict') if not force and not os.path.exists(b_absrc): raise AssibleModuleError(results={'msg': 'src file does not exist, use "force=yes" if you' ' really want to create the link: %s' % absrc, 'path': path, 'src': src}) if prev_state == 'directory': if not force: raise AssibleModuleError(results={'msg': 'refusing to convert from %s to symlink for %s' % (prev_state, path), 'path': path}) elif os.listdir(b_path): # refuse to replace a directory that has files in it raise AssibleModuleError(results={'msg': 'the directory %s is not empty, refusing to' ' convert it' % path, 'path': path}) elif prev_state in ('file', 'hard') and not force: raise AssibleModuleError(results={'msg': 'refusing to convert from %s to symlink for %s' % (prev_state, path), 'path': path}) diff = initial_diff(path, 'link', prev_state) changed = False if prev_state in ('hard', 'file', 'directory', 'absent'): changed = True elif prev_state == 'link': b_old_src = os.readlink(b_path) if b_old_src != b_src: diff['before']['src'] = to_native(b_old_src, errors='strict') diff['after']['src'] = src changed = True else: raise AssibleModuleError(results={'msg': 'unexpected position reached', 'dest': path, 'src': src}) if changed and not module.check_mode: if prev_state != 'absent': # try to replace atomically b_tmppath = to_bytes(os.path.sep).join( [os.path.dirname(b_path), to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))] ) try: if prev_state == 'directory': os.rmdir(b_path) os.symlink(b_src, b_tmppath) os.rename(b_tmppath, b_path) except OSError as e: if os.path.exists(b_tmppath): os.unlink(b_tmppath) raise AssibleModuleError(results={'msg': 'Error while replacing: %s' % to_native(e, nonstring='simplerepr'), 'path': path}) else: try: os.symlink(b_src, b_path) except OSError as e: raise AssibleModuleError(results={'msg': 'Error while linking: %s' % to_native(e, nonstring='simplerepr'), 'path': path}) if module.check_mode and not os.path.exists(b_path): return {'dest': path, 'src': src, 'changed': changed, 'diff': diff} # Now that we might have created the symlink, get the arguments. # We need to do it now so we can properly follow the symlink if needed # because load_file_common_arguments sets 'path' according # the value of follow and the symlink existence. file_args = module.load_file_common_arguments(module.params) # Whenever we create a link to a nonexistent target we know that the nonexistent target # cannot have any permissions set on it. Skip setting those and emit a warning (the user # can set follow=False to remove the warning) if follow and os.path.islink(b_path) and not os.path.exists(file_args['path']): module.warn('Cannot set fs attributes on a non-existent symlink target. follow should be' ' set to False to avoid this.') else: changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) changed |= update_timestamp_for_file(file_args['path'], mtime, atime, diff) return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
def is_broken_link(path): """Returns True if the path given as is a broken symlink""" path = os.readlink(path) return not os.path.exists(path)
def move_repos_into_place(src, dst): # Find all the stuff in src/*, move it to a freshly-created # directory beside dst, then play some games with symlinks so that # dst is a name the new stuff and dst+".old" names the previous # one. This feels like a lot of hooey for something so trivial. # First, make a crispy fresh new directory to put the stuff in. i = 0 while True: date_suffix = time.strftime("%Y-%m-%d") dname = dst + ".%s.%d" % (date_suffix, i) try: os.mkdir(dname) break except OSError: exc = sys.exc_value if exc.errno == errno.EEXIST: pass else: raise exc i = i + 1 # Put the stuff in our new directory. for r in os.listdir(src): sysassert(["cp", "-rv", src + "/" + r, dname]) # Make a symlink to the new directory; the symlink will be renamed # to dst shortly. i = 0 while True: tmpnam = dst + ".TMP.%d" % i try: os.symlink(dname, tmpnam) break except OSError: # as exc: # Python >2.5 exc = sys.exc_value if exc.errno == errno.EEXIST: pass else: raise exc i = i + 1 # Make a symlink to the old directory; this symlink will be # renamed shortly, too. oldnam = None if os.path.exists(dst): i = 0 while True: oldnam = dst + ".old.%d" % i try: os.symlink(os.readlink(dst), oldnam) break except OSError: # as exc: # Python >2.5 exc = sys.exc_value if exc.errno == errno.EEXIST: pass else: raise exc os.rename(tmpnam, dst) if oldnam: os.rename(oldnam, dst + ".old")