def test_symlink_latest_log_directory(self):
handler = FileProcessorHandler(base_log_folder=self.base_log_folder,
filename_template=self.filename)
handler.dag_dir = self.dag_dir
date1 = (timezone.utcnow() + timedelta(days=1)).strftime("%Y-%m-%d")
date2 = (timezone.utcnow() + timedelta(days=2)).strftime("%Y-%m-%d")
p1 = os.path.join(self.base_log_folder, date1, "log1")
p2 = os.path.join(self.base_log_folder, date1, "log2")
if os.path.exists(p1):
os.remove(p1)
if os.path.exists(p2):
os.remove(p2)
link = os.path.join(self.base_log_folder, "latest")
with freeze_time(date1):
handler.set_context(filename=os.path.join(self.dag_dir, "log1"))
self.assertTrue(os.path.islink(link))
self.assertEqual(os.path.basename(os.readlink(link)), date1)
self.assertTrue(os.path.exists(os.path.join(link, "log1")))
with freeze_time(date2):
handler.set_context(filename=os.path.join(self.dag_dir, "log2"))
self.assertTrue(os.path.islink(link))
self.assertEqual(os.path.basename(os.readlink(link)), date2)
self.assertTrue(os.path.exists(os.path.join(link, "log2")))
def init(self):
if os.path.isfile("/tmp/extip"):
os.system("rm -f /tmp/extip")
self.softcam = CamControl('softcam')
print "# selected ", self.softcam.current()
self.link_softcam = os.readlink('/etc/init.d/softcam')
self.link_cardserver = os.readlink('/etc/init.d/cardserver')
if self.link_softcam.find('mgcamd') != -1:
self.readNewcsPort()
if self.link_cardserver.find('newcs') != -1:
self.type = "NEWCAMD"
self.setTitle("SETUP LOADER - NEWCAMD");
else:
self.type = "None"
self.setTitle("SETUP LOADER - None");
elif self.link_softcam.find('CCcam') != -1:
self.readCCcamPort()
self.setTitle("SETUP LOADER - CCCAM");
if self.link_cardserver.find('oscam') != -1:
self.type = "CCCAM+OSCAM"
elif self.link_cardserver.find('newcs') != -1:
self.type = "CCCAM+NEWCS"
else:
self.type = "CCCAM"
else:
self.type = "None"
self.setTitle("SETUP LOADER - None");
def _persist_symlink(self, abspath):
"""Persist symbolic link and bind mount it back to its current location
"""
persisted_path = self._config_path(abspath)
current_target = os.readlink(abspath)
if os.path.exists(persisted_path):
stored_target = os.readlink(persisted_path)
if stored_target == current_target:
self._logger.warn('Symlink "%s" had already been persisted',
abspath)
return
else:
# Write the new symlink to an alternate location and atomically
# rename
self._prepare_dir(abspath, persisted_path)
tmp_path = persisted_path + '.ovirtnode.atom'
try:
os.symlink(current_target, tmp_path)
except Exception:
raise
else:
os.rename(tmp_path, persisted_path)
else:
self._prepare_dir(abspath, persisted_path)
os.symlink(current_target, persisted_path)
self.copy_attributes(abspath, persisted_path)
self._logger.info('Symbolic link "%s" successfully persisted', abspath)
self._add_path_entry(abspath)
def testNewEntry(self):
# test the default case
d = dumpStorage.DumpStorage(self.testDir)
dateLeafSet = set()
expectedLeafs = set(["55", "00", "20"])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, "Expected %s, got %s" % (e, g)
assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet)
# test the for JsonDumpStorage default
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=1)
dateLeafSet = set()
expectedLeafs = set(["55_0", "00_0", "20_0"])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, "Expected %s, got %s" % (e, g)
assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet)
# test the trailing _n case at same level
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
dateLeafSet = set()
expectedLeafs = set(["00_0", "20_0", "55_0"])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1])
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, "Expected %s, got %s" % (e, g)
assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet)
# test with subdirectory further down
d = dumpStorage.DumpStorage(self.testDir, subSlotCount=3)
dateLeafSet = set()
expectedLeafs = set(["wh_0", "wh_1", "wh_2"])
for k, v in self.testData.items():
nd, dd = d.newEntry(k, v[1], webheadName="wh")
dateLeafSet.add(os.path.split(dd)[1])
assert os.path.isdir(nd)
assert os.path.isdir(dd)
assert os.path.islink(os.path.join(dd, k))
e = os.path.abspath(nd)
g = os.path.abspath(os.path.join(dd, os.readlink(os.path.join(dd, k))))
assert e == g, "Expected %s, got %s" % (e, g)
assert expectedLeafs == dateLeafSet, "Expected %s, got %s" % (expectedLeafs, dateLeafSet)
def linkBestCurrentSolution(dpath):
alldsz = glob.glob(dpath + '/*dsz')
currentLink = None
bestDsz = None
bestN = None
stu = None
for x in alldsz:
if x.endswith('current.dsz'):
assert os.path.islink(x)
currentLink = os.readlink(x)
continue
fname = os.path.basename(x)
m = CURSOL_RE_.match(fname)
assert m is not None, ('failed to parse %s' % fname)
nstu = m.group(1).upper()
if stu is None:
stu = nstu
assert nstu == stu
nth = int(m.group(2))
if (bestDsz is None) or (nth > bestN):
bestN = nth
bestDsz = fname
if bestDsz is None:
return None
currentDsz = os.path.join(dpath, stu + 'current.dsz')
if os.path.islink(currentDsz):
odsz = os.readlink(currentDsz)
if odsz == bestDsz:
return bestDsz
os.unlink(currentDsz)
os.symlink(bestDsz, currentDsz)
return bestDsz
def AddDirectory(self, dir_path):
"""Adds all files under the given directory to the checksum.
This adds both the contents of the files as well as their names and
locations to the checksum. If the checksums of two directories are equal
this means they have exactly the same files, and contents.
Args:
dir_path: str, The directory path to add all files from.
Returns:
self, For method chaining.
"""
for root, dirs, files in os.walk(dir_path):
dirs.sort(key=os.path.normcase)
files.sort(key=os.path.normcase)
for d in dirs:
path = os.path.join(root, d)
# We don't traverse directory links, but add the fact that it was found
# in the tree.
if os.path.islink(path):
relpath = os.path.relpath(path, dir_path)
self.__files.add(relpath)
self.AddContents(relpath)
self.AddContents(os.readlink(path))
for f in files:
path = os.path.join(root, f)
relpath = os.path.relpath(path, dir_path)
self.__files.add(relpath)
self.AddContents(relpath)
if os.path.islink(path):
self.AddContents(os.readlink(path))
else:
self.AddFileContents(path)
return self
def copytree(src, dst):
"""Similar to shutil.copytree, but always copies symlinks and doesn't
error out if the destination path already exists.
"""
# If the source tree is a symlink, duplicate the link and we're done.
if os.path.islink(src):
os.symlink(os.readlink(src), dst)
return
try:
os.mkdir(dst)
except OSError as e:
if e.errno != errno.EEXIST:
raise
names = os.listdir(src)
for name in names:
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if os.path.islink(srcname):
os.symlink(os.readlink(srcname), dstname)
elif os.path.isdir(srcname):
copytree(srcname, dstname)
else:
copy2(srcname, dstname)
try:
shutil.copystat(src, dst)
except OSError as e:
if e.errno != errno.EPERM:
raise
def copy(src, dst, hardlink=False, keep_symlink=True):
assert not P.isdir(src), 'Source path must not be a dir'
assert not P.isdir(dst), 'Destination path must not be a dir'
if keep_symlink and P.islink(src):
assert not P.isabs(readlink(src)), 'Cannot copy symlink that points to an absolute path (%s)' % src
logger.debug('%8s %s -> %s' % ('symlink', src, dst))
if P.exists(dst):
assert readlink(dst) == readlink(src), 'Refusing to retarget already-exported symlink %s' % dst
else:
symlink(readlink(src), dst)
return
if P.exists(dst):
assert hash_file(src) == hash_file(dst), 'Refusing to overwrite already exported dst %s' % dst
else:
if hardlink:
try:
link(src, dst)
logger.debug('%8s %s -> %s' % ('hardlink', src, dst))
return
except OSError, o:
if o.errno != errno.EXDEV: # Invalid cross-device link, not an error, fall back to copy
raise
logger.debug('%8s %s -> %s' % ('copy', src, dst))
shutil.copy2(src, dst)
def create_link(opts, src, dst):
if not os.path.lexists(dst):
return _create_link(opts, src, dst)
if not os.path.islink(dst):
return "[%s] is not a symbolic link as we expected, " \
"please adjust if this is not what you intended." % dst
if not os.path.exists(os.readlink(dst)):
environment.warning('BROKEN LINK: [%s] attempting to delete and fix it to point to %s.' % (dst, src))
try:
os.unlink(dst)
return _create_link(opts, src, dst)
except:
msg = "[%s] was a broken symlink, failed to delete " \
"and relink to [%s], please fix this manually" % (dst, src)
return msg
environment.debug(opts, 'verifying link: %s points to %s' % (dst, src))
dst_stat = os.stat(dst)
src_stat = os.stat(src)
if dst_stat.st_ino != src_stat.st_ino:
msg = "[%s] is pointing to [%s] which is different than " \
"the intended target [%s]" % (dst, os.readlink(dst), src)
return msg
def TestLink(DestinationPath, SourcePath, fc):
if SourcePath:
if os.path.islink(SourcePath):
if fc.Links == "follow":
if os.path.isdir(SourcePath):
if TestDirectory(DestinationPath, os.path.realpath(SourcePath), fc) is False:
return False
elif os.path.isfile(SourcePath):
if TestFile(DestinationPath, os.path.realpath(SourcePath), fc) is False:
return False
elif fc.Links == "manage":
if not os.path.islink(DestinationPath):
return False
if os.readlink(DestinationPath) != os.readlink(SourcePath):
return False
elif fc.Links == "ignore":
return True
else:
if not os.path.exists(DestinationPath) or not os.path.exists(SourcePath) or not os.path.islink(DestinationPath) :
return False
if os.readlink(DestinationPath) != SourcePath:
return False
if os.path.exists(DestinationPath) != True:
return False
if TestOwnerGroupMode(DestinationPath, SourcePath, fc) is False:
return False
return True
def diff(self, file_struct):
self._validate_struct(file_struct)
temp_file, temp_dirs = self.process(file_struct)
path = file_struct['path']
sectx_result = ''
result = ''
cur_sectx = lgetfilecon(path)[1]
if cur_sectx == None:
cur_sectx = ''
if file_struct.has_key('selinux_ctx') and file_struct['selinux_ctx']:
if cur_sectx != file_struct['selinux_ctx']:
sectx_result = "SELinux contexts differ: actual: [%s], expected: [%s]\n" % (cur_sectx, file_struct['selinux_ctx'])
if file_struct['filetype'] == 'symlink':
try:
curlink = os.path.abspath(os.readlink(path))
newlink = os.path.abspath(os.readlink(temp_file))
if curlink == newlink:
result = ''
else:
result = "Link targets differ for [%s]: actual: [%s], expected: [%s]\n" % (path, curlink, newlink)
except OSError, e:
if e.errno == 22:
result = "Deployed symlink is no longer a symlink!"
else:
raise e
def copy(source, destination, sym = True):
'''recursively copy a "source" file or directory to "destination"'''
sourceGlob = glob.glob(source)
if len(sourceGlob) == 0:
error(_("ActionsAPI [copy]: No file matched pattern \"%s\".") % source)
for filePath in sourceGlob:
if isFile(filePath) and not isLink(filePath):
try:
shutil.copy(filePath, destination)
except IOError:
error(_('ActionsAPI [copy]: Permission denied: %s to %s') % (filePath, destination))
elif isLink(filePath) and sym:
if isDirectory(destination):
os.symlink(os.readlink(filePath), join_path(destination, os.path.basename(filePath)))
else:
if isFile(destination):
os.remove(destination)
os.symlink(os.readlink(filePath), destination)
elif isLink(filePath) and not sym:
if isDirectory(filePath):
copytree(filePath, destination)
else:
shutil.copy(filePath, destination)
elif isDirectory(filePath):
copytree(filePath, destination, sym)
else:
error(_('ActionsAPI [copy]: File %s does not exist.') % filePath)
def TestFile(DestinationPath, SourcePath, fc):
if '://' in SourcePath and fc.LocalPath == '': # we cannot verify the remote has not changed until the Set
return False
if not os.path.exists(DestinationPath) or not os.path.isfile(DestinationPath) or os.path.islink(DestinationPath):
return False
if TestOwnerGroupMode(DestinationPath, SourcePath, fc) is False:
return False
if SourcePath and len(SourcePath) > 0:
if not os.path.isfile(SourcePath):
return False
if os.path.islink(SourcePath):
if fc.Links == "follow":
if os.path.isdir(os.path.realpath(SourcePath)):
print("Error: Expecting a file, but source link points to directory")
LG().Log("ERROR", "Expecting a file, but source link points to directory")
return False
else:
if not os.path.islink(DestinationPath):
return False
if os.readlink(DestinationPath) != os.readlink(SourcePath):
return False
elif CompareFiles(DestinationPath, SourcePath, fc.Checksum) == -1:
return False
elif fc.Contents:
dest_file, error = ReadFile(DestinationPath)
if fc.Contents.encode('utf8') != dest_file:
return False
return True
def lndir(src, dst, overwrite=True, summary_print_func=None):
"""
Create a recursive symlink shadow copy of src in dst.
Directories in src will be created in dst. All other files will be symlinked from dst to src.
If the src is a symlink then the destination symlink will be re-linked to the origin target.
:param src: src path, if relative, relative to dst not cwd
:param dst: dst mpath, must exit.
:param overwrite: defaul True, overwrite existing symlinks in dst
:param summary_print_func: Function to print a summary after each interation
:return: count of directores created and files symlinked
:rtype: tuple
"""
if summary_print_func is None:
summary_print_func = lambda *args: None
dst = os.path.abspath(dst)
if os.path.isabs(src):
calculate_src = lambda ss, sd: ss
else:
# following the X11 lndir convention if src is relative,
# it is relative to dst not to cwd.
src = os.path.abspath(os.path.join(dst, src))
calculate_src = os.path.relpath
# start walking the source target
dir_c = 0 # counter for dires
file_c = 0 # counter for files
for root, dirs, files in os.walk(src):
for f in files:
real_src = os.path.join(root, f)
sym_dst_dir, sym_dst = calculate_src_dst(dst, f, root, src)
delete_if_exists_and_overwrite(overwrite, sym_dst)
if os.path.islink(real_src):
# assume relative symlink does not reach above src
# and just use the original link
sym_src = os.readlink(real_src)
improved_error_symlink(sym_src, sym_dst)
else:
sym_src = calculate_src(os.path.join(root, f), sym_dst_dir)
improved_error_symlink(sym_src, sym_dst)
file_c += 1
for d in dirs:
real_src = os.path.join(root, d)
sym_dst_dir, sym_dst = calculate_src_dst(dst, d, root, src)
delete_if_exists_and_overwrite(overwrite, sym_dst)
if os.path.islink(real_src):
# assume relative symlink does not reach above src
# and just use the original link
sym_src = os.readlink(real_src)
improved_error_symlink(sym_src, sym_dst)
else:
try:
os.mkdir(sym_dst)
except OSError:
logging.exception("dst = %s", sym_dst)
dir_c += 1
summary_print_func(dir_c, file_c, root)
return dir_c, file_c
def _fix_symlinks(debdir):
'''
Sometimes debs will contain absolute symlinks (e.g. if the relative
path would go all the way to root, they just do absolute). We can't
have that, so instead clean those absolute symlinks.
Some unpacked items will also contain suid binaries which we do not want in
the resulting snap.
'''
for root, dirs, files in os.walk(debdir):
# Symlinks to directories will be in dirs, while symlinks to
# non-directories will be in files.
for entry in itertools.chain(files, dirs):
path = os.path.join(root, entry)
if os.path.islink(path) and os.path.isabs(os.readlink(path)):
target = os.path.join(debdir, os.readlink(path)[1:])
if _skip_link(os.readlink(path)):
logger.debug('Skipping {}'.format(target))
continue
if not os.path.exists(target):
if not _try_copy_local(path, target):
continue
os.remove(path)
os.symlink(os.path.relpath(target, root), path)
elif os.path.exists(path):
_fix_filemode(path)
def get_name_of_init(run=process.run):
"""
Internal function to determine what executable is PID 1
It does that by checking /proc/1/exe.
Fall back to checking /proc/1/cmdline (local execution).
:return: executable name for PID 1, aka init
:rtype: str
"""
if run == process.run:
# On a local run, there are better ways to check
# our PID 1 executable name.
try:
return os.path.basename(os.readlink('/proc/1/exe'))
except OSError:
with open('/proc/1/cmdline', 'r') as cmdline:
init = cmdline.read().split(chr(0))[0]
try:
init = os.readlink(init)
except OSError:
pass
return os.path.basename(init)
else:
output = run("readlink /proc/1/exe").stdout.strip()
return os.path.basename(output)
def install(self):
path = self.options['path']
logger = logging.getLogger(self.name)
logger.info(
'Creating directory %s' % os.path.basename(path))
if not os.path.exists(path):
os.makedirs(path)
files = (file for file in self.options['files'].split('\n') if file)
for file in files:
file = file.split(None, 1)
if len(file) == 2:
file, as_ = file
else:
file = file[0]
as_ = os.path.basename(file)
if '://' in file:
file = self._get_resource_filename(file)
to = os.path.join(path, as_)
if os.path.islink(to) and os.readlink(to) != file:
logger.info('Removing symlink from "%s" to "%s"' % (os.readlink(to), to))
os.remove(to)
if not os.path.exists(to):
logger.info('Making symlink from "%s" to "%s"' % (file, to))
os.symlink(file, to)
# Since other processes may create resources in path it
# should be excluded during uninstall, so return empty list.
# We can't return list of created symlinks as buildout does not
# currently support symlink removal, see uninstall_symlinks below.
return []
def run(self):
while True:
try:
p_result = self.polling.poll()
print p_result
if p_result[0][1] <= select.POLLPRI:
self.ev = self.conn.recv_event()
else:
continue
except Exception, e:
print e
if self.ev.what == connector.PROC_EVENT_FORK:
linkpath = os.path.join('/proc/', str(self.ev.child_pid), 'exe')
if os.path.exists(linkpath):
name = os.readlink(linkpath)
else:
name = 'non ecsiste'
print "Parent: ", self.ev.parent_pid, "Forked: ", self.ev.child_pid, name
elif self.ev.what == connector.PROC_EVENT_EXEC:
linkpath = os.path.join('/proc/', str(self.ev.process_pid), 'exe')
name = os.readlink(linkpath)
print "Exec: ", self.ev.process_pid, name
elif self.ev.what == connector.PROC_EVENT_EXIT:
linkpath = os.path.join('/proc', str(self.ev.process_pid), 'exe')
if os.path.exists(linkpath):
name = os.readlink(linkpath)
else:
name = None
print "Exit: ", self.ev.process_pid, name
def test_save(self):
image = VImage.new_rgba(width=1, height=1,
ink=rgba(r=0, g=0, b=0, a=0))
self.storage.save(x=0, y=1, z=2, image=image)
self.storage.save(x=1, y=0, z=2, image=image)
self.storage.save(x=1, y=0, z=3, image=image)
self.storage.waitall()
self.assertEqual(set(recursive_listdir(self.outputdir)),
set(['2/',
'2/0/',
'2/0/1.png',
'2/1/',
'2/1/0.png',
'3/',
'3/1/',
'3/1/0.png']))
# Is this a real file?
self.assertFalse(
os.path.islink(os.path.join(self.outputdir, '2', '0', '1.png'))
)
# Does the symlinking work?
self.assertEqual(
os.readlink(os.path.join(self.outputdir, '2', '1', '0.png')),
os.path.join(os.path.pardir, '0', '1.png')
)
self.assertEqual(
os.readlink(os.path.join(self.outputdir, '3', '1', '0.png')),
os.path.join(os.path.pardir, os.path.pardir, '2', '0', '1.png')
)
def con2prog(src_ip, src_port, dst_ip, dst_port, proto):
proc_tcp = readFile("/proc/net/%s" % proto)[1:]
inode = None
for p in proc_tcp:
con = p.split()
src = con[1].split(":")
dst = con[2].split(":")
if hex2ip(src[0]) == src_ip and int(src[1], 16) == src_port and \
hex2ip(dst[0]) == dst_ip and int(dst[1], 16) == dst_port:
inode = con[9]
if not inode:
return None
pids = glob.glob("/proc/[0-9]*")
for pid in pids:
fds = glob.glob("%s/fd/*" % pid)
for fd in fds:
try:
fdlink = os.readlink(fd)
if re.match("socket:\[%s\]" % inode, fdlink):
prog = os.readlink("%s/exe" % pid)
cmd = readFile("%s/cmdline" % pid, lines=False).\
replace("\x00", " ").strip()
return pid, prog, cmd
except OSError:
pass
return None
def test_readlink_nonexistent(self):
with test_support.temp_cwd() as new_cwd:
nonexistent_file = os.path.join(new_cwd, "nonexistent-file")
with self.assertRaises(OSError) as cm:
os.readlink(nonexistent_file)
self.assertEqual(cm.exception.errno, errno.ENOENT)
self.assertEqual(cm.exception.filename, nonexistent_file)
def test_resolve_simple_dep_graph(self):
dep = self._create_dep(provide_bin=True)
app = self._create_app(dep)
_, node_paths = self._resolve_target(app)
dep_node_path = node_paths.node_path(dep)
app_node_path = node_paths.node_path(app)
self.assertIsNotNone(dep_node_path)
self.assertIsNotNone(app_node_path)
# Verify that 'app/node_modules' has a correct symlink to 'dep'
app_node_modules_path = os.path.join(app_node_path, 'node_modules')
link_dep_path = os.path.join(app_node_modules_path, 'dep')
self.assertTrue(os.path.exists(app_node_modules_path))
self.assertTrue(os.path.islink(link_dep_path))
expected = os.path.relpath(dep_node_path, app_node_modules_path)
self.assertEqual(os.readlink(link_dep_path), expected)
# Verify that 'app/node_modules/.bin' has a correct symlink to 'dep'
app_bin_dep_path = os.path.join(dep_node_path, 'cli.js')
dep_bin_path = os.path.join(app_node_path, 'node_modules', '.bin')
link_dep_bin_path = os.path.join(dep_bin_path, 'dep')
self.assertTrue(os.path.islink(link_dep_bin_path))
relative_path = os.readlink(link_dep_bin_path)
expected = os.path.relpath(app_bin_dep_path, dep_bin_path)
self.assertEqual(relative_path, expected)
def testTarFileWithSymlink(self):
outfd = StringIO.StringIO()
infd1 = StringIO.StringIO("this is a test string")
st1 = os.stat_result((0644, 0, 0, 0, 0, 0, len(infd1.getvalue()), 0, 0, 0))
infd2 = StringIO.StringIO("this is another test string")
st2 = os.stat_result((0644, 0, 0, 0, 0, 0, len(infd2.getvalue()), 0, 0, 0))
# Write the zip into a file like object.
with utils.StreamingTarWriter(outfd, mode="w:gz") as writer:
writer.WriteFromFD(infd1, "test1.txt", st=st1)
writer.WriteFromFD(infd2, "subdir/test2.txt", st=st2)
writer.WriteSymlink("test1.txt", "test1.txt.link")
writer.WriteSymlink("subdir/test2.txt", "test2.txt.link")
with tarfile.open(
fileobj=StringIO.StringIO(outfd.getvalue()), mode="r") as test_fd:
test_fd.extractall(self.temp_dir)
link_path = os.path.join(self.temp_dir, "test1.txt.link")
self.assertTrue(os.path.islink(link_path))
self.assertEqual(os.readlink(link_path), "test1.txt")
link_path = os.path.join(self.temp_dir, "test2.txt.link")
self.assertTrue(os.path.islink(link_path))
self.assertEqual(os.readlink(link_path), "subdir/test2.txt")
def defaults(self):
self.id_format = u"%(type)s-run%(run)06d-host%(hostname)s-pid%(pid)06d"
self.doc["type"] = "dqm-source-state"
self.doc["run"] = 0
# figure out the tag
c = self.doc["cmdline"]
for l in c:
if l.endswith(".py"):
t = os.path.basename(l)
t = t.replace(".py", "")
t = t.replace("_cfg", "")
self.doc["tag"] = t
pr = l.split("=")
if len(pr) > 1 and pr[0] == "runNumber" and pr[1].isdigit():
run = long(pr[1])
self.doc["run"] = run
self.make_id()
#if os.environ.has_key("GZIP_LOG"):
# self.doc["stdlog_gzip"] = os.environ["GZIP_LOG"]
try:
self.doc["stdout_fn"] = os.readlink("/proc/self/fd/1")
self.doc["stderr_fn"] = os.readlink("/proc/self/fd/2")
except:
pass
self.update_doc({ "extra": {
"environ": dict(os.environ)
}})
def test_publish(self):
repo = mock.Mock(spec=Repository)
repo.working_dir = self.repo_working_dir
repo.id = "test_publish"
num_units = 10
relative_url = "rel_a/rel_b/rel_c/"
existing_units = self.get_units(count=num_units)
publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, relative_url=relative_url,
http=False, https=True)
distributor = YumDistributor()
distributor.process_repo_auth_certificate_bundle = mock.Mock()
status, msg = distributor.validate_config(repo, config, None)
self.assertTrue(status)
report = distributor.publish_repo(repo, publish_conduit, config)
self.assertTrue(report.success_flag)
summary = report.summary
self.assertEqual(summary["num_package_units_attempted"], num_units)
self.assertEqual(summary["num_package_units_published"], num_units)
self.assertEqual(summary["num_package_units_errors"], 0)
# Verify we did not attempt to publish to http
expected_repo_http_publish_dir = os.path.join(self.http_publish_dir, relative_url)
self.assertFalse(os.path.exists(expected_repo_http_publish_dir))
expected_repo_https_publish_dir = os.path.join(self.https_publish_dir, relative_url).rstrip('/')
self.assertEqual(summary["https_publish_dir"], expected_repo_https_publish_dir)
self.assertTrue(os.path.exists(expected_repo_https_publish_dir))
details = report.details
self.assertEqual(len(details["errors"]), 0)
#
# Add a verification of the publish directory
#
self.assertTrue(os.path.exists(summary["https_publish_dir"]))
self.assertTrue(os.path.islink(summary["https_publish_dir"].rstrip("/")))
source_of_link = os.readlink(expected_repo_https_publish_dir.rstrip("/"))
self.assertEquals(source_of_link, repo.working_dir)
#
# Verify the expected units
#
for u in existing_units:
expected_link = os.path.join(expected_repo_https_publish_dir, u.metadata["relativepath"])
self.assertTrue(os.path.exists(expected_link))
actual_target = os.readlink(expected_link)
expected_target = u.storage_path
self.assertEqual(actual_target, expected_target)
#
# Now test flipping so https is disabled and http is enabled
#
config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir,
http_publish_dir=self.http_publish_dir, relative_url=relative_url, http=True, https=False)
report = distributor.publish_repo(repo, publish_conduit, config)
self.assertTrue(report.success_flag)
# Verify we did publish to http
self.assertTrue(os.path.exists(expected_repo_http_publish_dir))
# Verify we did not publish to https
self.assertFalse(os.path.exists(expected_repo_https_publish_dir))
# Verify we cleaned up the misc dirs under the https dir
self.assertEquals(len(os.listdir(self.https_publish_dir)), 0)
def TestFile(DestinationPath, SourcePath, fc):
if not os.path.exists(DestinationPath) or not os.path.isfile(DestinationPath) or os.path.islink(DestinationPath):
return False
if TestOwnerGroupMode(DestinationPath, SourcePath, fc) == False:
return False
if SourcePath:
if not os.path.isfile(SourcePath):
return False
if os.path.islink(SourcePath):
if fc.Links == "follow":
if os.path.isdir(os.path.realpath(SourcePath)):
print("Error: Expecting a file, but source link points to directory")
return False
else:
if not os.path.islink(DestinationPath):
return False
if os.readlink(DestinationPath) != os.readlink(SourcePath):
return False
elif CompareFiles(DestinationPath, SourcePath, fc) == False:
return False
elif fc.Contents:
dest_file = open(DestinationPath, "rb").read()
if fc.Contents != dest_file:
return False
return True
def same_file(left_stat, left_dir, right_stat, right_dir, filename):
"""Are two filesystem objects the same?"""
if S_IFMT(left_stat.st_mode) != S_IFMT(right_stat.st_mode):
# Different fundamental types
return False
elif S_ISREG(left_stat.st_mode):
# Files with the same size and MD5sum are the same
if left_stat.st_size != right_stat.st_size:
return False
elif md5sum("%s/%s" % (left_dir, filename)) \
!= md5sum("%s/%s" % (right_dir, filename)):
return False
else:
return True
elif S_ISDIR(left_stat.st_mode) or S_ISFIFO(left_stat.st_mode) \
or S_ISSOCK(left_stat.st_mode):
# Directories, fifos and sockets are always the same
return True
elif S_ISCHR(left_stat.st_mode) or S_ISBLK(left_stat.st_mode):
# Char/block devices are the same if they have the same rdev
if left_stat.st_rdev != right_stat.st_rdev:
return False
else:
return True
elif S_ISLNK(left_stat.st_mode):
# Symbolic links are the same if they have the same target
if os.readlink("%s/%s" % (left_dir, filename)) \
!= os.readlink("%s/%s" % (right_dir, filename)):
return False
else:
return True
else:
return True
def device_is_zfcp(info):
""" Return True if the device is a zfcp device. """
if info.get("DEVTYPE") != "disk":
return False
subsystem = device_get_sysfs_path(info)
while True:
topdir = os.path.realpath(os.path.dirname(subsystem))
driver = "%s/driver" % (topdir,)
if os.path.islink(driver):
subsystemname = os.path.basename(os.readlink(subsystem))
drivername = os.path.basename(os.readlink(driver))
if subsystemname == 'ccw' and drivername == 'zfcp':
return True
newsubsystem = os.path.dirname(topdir)
if newsubsystem == topdir:
break
subsystem = newsubsystem + "/subsystem"
return False
def does_file_need_update(src_name, src_st, dest_name):
try:
target_st = os.lstat(dest_name)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
return True
if src_st.st_size != target_st.st_size:
return True
if stat.S_IFMT(src_st.st_mode) != stat.S_IFMT(target_st.st_mode):
return True
if stat.S_ISLNK(src_st.st_mode):
return os.readlink(src_name) != os.readlink(dest_name)
if not stat.S_ISREG(src_st.st_mode):
return True
# They might have the same content; compare.
with open(src_name, "rb") as sf, open(dest_name, "rb") as df:
chunk_size = 8192
while True:
src_data = sf.read(chunk_size)
dest_data = df.read(chunk_size)
if src_data != dest_data:
return True
if len(src_data) < chunk_size:
# EOF
break
return False
def __hasNoLinkOrFail(self,jsonStorage,uuid):
linkPath = jsonStorage.getJson(uuid)[:-len(jsonStorage.jsonSuffix)]
try:
os.readlink(linkPath)
assert False, 'Expected to find no link: %s '%linkPath
except OSError,x:
assert 2 == x.errno, "Expected errno=2, got %d for linkpath %s"%(x.errno,linkPath)
def test_save_successor(self, mock_rv):
# Mock relevant_values() to claim that all values are relevant here
# (to avoid instantiating parser)
mock_rv.side_effect = lambda x: x
for ver in six.moves.range(1, 6):
for kind in ALL_FOUR:
self._write_out_kind(kind, ver)
self.test_rc.update_all_links_to(3)
self.assertEqual(
6, self.test_rc.save_successor(3, b'new cert', None,
b'new chain', self.config))
with open(self.test_rc.version("cert", 6)) as f:
self.assertEqual(f.read(), "new cert")
with open(self.test_rc.version("chain", 6)) as f:
self.assertEqual(f.read(), "new chain")
with open(self.test_rc.version("fullchain", 6)) as f:
self.assertEqual(f.read(), "new cert" + "new chain")
# version 6 of the key should be a link back to version 3
self.assertFalse(os.path.islink(self.test_rc.version("privkey", 3)))
self.assertTrue(os.path.islink(self.test_rc.version("privkey", 6)))
# Let's try two more updates
self.assertEqual(
7, self.test_rc.save_successor(6, b'again', None,
b'newer chain', self.config))
self.assertEqual(
8, self.test_rc.save_successor(7, b'hello', None,
b'other chain', self.config))
# All of the subsequent versions should link directly to the original
# privkey.
for i in (6, 7, 8):
self.assertTrue(os.path.islink(self.test_rc.version("privkey", i)))
self.assertEqual("privkey3.pem", os.path.basename(os.readlink(
self.test_rc.version("privkey", i))))
for kind in ALL_FOUR:
self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 9)))
self.assertEqual(self.test_rc.current_version(kind), 3)
# Test updating from latest version rather than old version
self.test_rc.update_all_links_to(8)
self.assertEqual(
9, self.test_rc.save_successor(8, b'last', None,
b'attempt', self.config))
for kind in ALL_FOUR:
self.assertEqual(self.test_rc.available_versions(kind),
list(six.moves.range(1, 10)))
self.assertEqual(self.test_rc.current_version(kind), 8)
with open(self.test_rc.version("fullchain", 9)) as f:
self.assertEqual(f.read(), "last" + "attempt")
temp_config_file = os.path.join(self.config.renewal_configs_dir,
self.test_rc.lineagename) + ".conf.new"
with open(temp_config_file, "w") as f:
f.write("We previously crashed while writing me :(")
# Test updating when providing a new privkey. The key should
# be saved in a new file rather than creating a new symlink.
self.assertEqual(
10, self.test_rc.save_successor(9, b'with', b'a',
b'key', self.config))
self.assertTrue(os.path.exists(self.test_rc.version("privkey", 10)))
self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10)))
self.assertFalse(os.path.exists(temp_config_file))
def package(self, package: TargetPackage, version: FullVersion) -> str:
package.build()
package.initialize_buildroot({'libarchive'})
log.info('PKG %s', self._get_filename(package, version))
# The package needs to be installed in /usr/arch> on the Red Hat
# system.
installdir = os.path.join(config.DIR_BUILDROOT, 'install')
arch = package.get_arch()
prefix = os.path.join('/usr', arch)
package.extract(installdir, prefix)
files = sorted(util.walk_files(installdir))
# Create an xz compressed cpio payload containing all files.
listing = os.path.join(config.DIR_BUILDROOT, 'listing')
with open(listing, 'w') as f:
f.write('#mtree\n')
for path in files:
relpath = os.path.join(prefix,
os.path.relpath(path, installdir))
if os.path.islink(path):
f.write(
'%s type=link mode=0777 uname=root gname=root time=0 link=%s\n'
% (relpath, os.readlink(path)))
else:
f.write(
'%s type=file mode=0%o uname=root gname=root time=0 contents=%s\n'
% (relpath, self._get_suggested_mode(path), path))
data = os.path.join(config.DIR_BUILDROOT, 'data.cpio.xz')
self._run_tar([
'-cJf',
data,
'--format=newc',
'-C',
installdir,
'@' + listing,
])
# The header, based on the following documentation:
# http://www.rpm.org/max-rpm/s1-rpm-file-format-rpm-file-format.html
# http://refspecs.linux-foundation.org/LSB_5.0.0/LSB-Core-generic/LSB-Core-generic/pkgformat.html
name = package.get_redhat_name()
lib_depends = sorted(dep.get_redhat_name()
for dep in package.get_lib_depends())
dirs = sorted({os.path.dirname(f) for f in files})
header = bytes(
rpm.Header({
100:
rpm.StringArray(['C']),
1000:
rpm.String(name),
1001:
rpm.String(str(version.get_version())),
1002:
rpm.String(str(version.get_revision())),
1003:
rpm.Int32([version.get_epoch()]),
1004:
rpm.I18NString('%s for %s' % (name, arch)),
1005:
rpm.I18NString('%s for %s' % (name, arch)),
1009:
rpm.Int32([sum(self._file_size(f) for f in files)]),
1014:
rpm.String('Unknown'),
1016:
rpm.I18NString('Development/Libraries'),
1020:
rpm.String(package.get_homepage()),
1021:
rpm.String('linux'),
1022:
rpm.String('noarch'),
1028:
rpm.Int32(os.lstat(f).st_size for f in files),
1030:
rpm.Int16(self._file_mode(f) for f in files),
1033:
rpm.Int16(0 for f in files),
1034:
rpm.Int32(0 for f in files),
1035:
rpm.StringArray(self._file_md5(f) for f in files),
1036:
rpm.StringArray(self._file_linkto(f) for f in files),
1037:
rpm.Int32(0 for f in files),
1039:
rpm.StringArray('root' for f in files),
1040:
rpm.StringArray('root' for f in files),
1047:
rpm.StringArray([name]),
1048:
rpm.Int32(0 for dep in lib_depends),
1049:
rpm.StringArray(lib_depends),
1050:
rpm.StringArray('' for dep in lib_depends),
1095:
rpm.Int32(1 for f in files),
1096:
rpm.Int32(range(1,
len(files) + 1)),
1097:
rpm.StringArray('' for f in files),
1112:
rpm.Int32(8 for dep in lib_depends),
1113:
rpm.StringArray([version.get_redhat_version()]),
1116:
rpm.Int32(dirs.index(os.path.dirname(f)) for f in files),
1117:
rpm.StringArray(os.path.basename(f) for f in files),
1118:
rpm.StringArray(
os.path.join(prefix, os.path.relpath(d, installdir)) + '/'
for d in dirs),
1124:
rpm.String('cpio'),
1125:
rpm.String('xz'),
1126:
rpm.String('9'),
}))
# The signature.
checksum = hashlib.md5()
checksum.update(header)
util.hash_file(data, checksum)
signature = bytes(
rpm.Header({
1000: rpm.Int32([len(header) + os.stat(data).st_size]),
1004: rpm.Bin(checksum.digest()),
}))
# Create the RPM file.
output = os.path.join(config.DIR_BUILDROOT, 'output.rpm')
with open(output, 'wb') as fb:
# The lead.
fb.write(b'\xed\xab\xee\xdb\x03\x00\x00\x00\x00\x00')
fullname = '%s-%s' % (name, version.get_redhat_version())
fb.write(bytes(fullname, encoding='ASCII')[:66].ljust(66, b'\0'))
fb.write(b'\x00\x01\x00\x05')
fb.write(b'\x00' * 16)
# The signature, aligned up to eight bytes in size.
fb.write(signature)
fb.write(b'\0' * ((8 - len(signature) % 8) % 8))
# The header.
fb.write(header)
# The payload.
with open(data, 'rb') as fin:
shutil.copyfileobj(fin, fb)
return output
def assemble(self):
"""
This method is the MAIN method for finding all necessary files to be bundled.
"""
from ..config import CONF
for m in self.excludes:
logger.debug("Excluding module '%s'" % m)
self.graph = initialize_modgraph(excludes=self.excludes,
user_hook_dirs=self.hookspath)
# TODO Find a better place where to put 'base_library.zip' and when to created it.
# For Python 3 it is necessary to create file 'base_library.zip'
# containing core Python modules. In Python 3 some built-in modules
# are written in pure Python. base_library.zip is a way how to have
# those modules as "built-in".
libzip_filename = os.path.join(CONF['workpath'], 'base_library.zip')
create_py3_base_library(libzip_filename, graph=self.graph)
# Bundle base_library.zip as data file.
# Data format of TOC item: ('relative_path_in_dist_dir', 'absolute_path_on_disk', 'DATA')
self.datas.append(
(os.path.basename(libzip_filename), libzip_filename, 'DATA'))
# Expand sys.path of module graph.
# The attribute is the set of paths to use for imports: sys.path,
# plus our loader, plus other paths from e.g. --path option).
self.graph.path = self.pathex + self.graph.path
self.graph.set_setuptools_nspackages()
logger.info("running Analysis %s", self.tocbasename)
# Get paths to Python and, in Windows, the manifest.
python = sys.executable
if not is_win:
# Linux/MacOS: get a real, non-link path to the running Python executable.
while os.path.islink(python):
python = os.path.join(os.path.dirname(python),
os.readlink(python))
depmanifest = None
else:
# Windows: Create a manifest to embed into built .exe, containing the same
# dependencies as python.exe.
depmanifest = winmanifest.Manifest(
type_="win32",
name=CONF['specnm'],
processorArchitecture=winmanifest.processor_architecture(),
version=(1, 0, 0, 0))
depmanifest.filename = os.path.join(
CONF['workpath'], CONF['specnm'] + ".exe.manifest")
# We record "binaries" separately from the modulegraph, as there
# is no way to record those dependencies in the graph. These include
# the python executable and any binaries added by hooks later.
# "binaries" are not the same as "extensions" which are .so or .dylib
# that are found and recorded as extension nodes in the graph.
# Reset seen variable before running bindepend. We use bindepend only for
# the python executable.
bindepend.seen.clear()
# Add binary and assembly dependencies of Python.exe.
# This also ensures that its assembly depencies under Windows get added to the
# built .exe's manifest. Python 2.7 extension modules have no assembly
# dependencies, and rely on the app-global dependencies set by the .exe.
self.binaries.extend(
bindepend.Dependencies([('', python, '')],
manifest=depmanifest,
redirects=self.binding_redirects)[1:])
if is_win:
depmanifest.writeprettyxml()
### Module graph.
#
# Construct the module graph of import relationships between modules
# required by this user's application. For each entry point (top-level
# user-defined Python script), all imports originating from this entry
# point are recursively parsed into a subgraph of the module graph. This
# subgraph is then connected to this graph's root node, ensuring
# imported module nodes will be reachable from the root node -- which is
# is (arbitrarily) chosen to be the first entry point's node.
# List to hold graph nodes of scripts and runtime hooks in use order.
priority_scripts = []
# Assume that if the script does not exist, Modulegraph will raise error.
# Save the graph nodes of each in sequence.
for script in self.inputs:
logger.info("Analyzing %s", script)
priority_scripts.append(self.graph.run_script(script))
# Analyze the script's hidden imports (named on the command line)
self.graph.add_hiddenimports(self.hiddenimports)
### Post-graph hooks.
self.graph.process_post_graph_hooks()
# Update 'binaries' TOC and 'datas' TOC.
deps_proc = DependencyProcessor(self.graph,
self.graph._additional_files_cache)
self.binaries.extend(deps_proc.make_binaries_toc())
self.datas.extend(deps_proc.make_datas_toc())
self.zipped_data.extend(deps_proc.make_zipped_data_toc())
# Note: zipped eggs are collected below
### Look for dlls that are imported by Python 'ctypes' module.
# First get code objects of all modules that import 'ctypes'.
logger.info('Looking for ctypes DLLs')
ctypes_code_objs = self.graph.get_co_using_ctypes(
) # dict like: {'module1': code_obj, 'module2': code_obj}
for name, co in ctypes_code_objs.items():
# Get dlls that might be needed by ctypes.
logger.debug('Scanning %s for shared libraries or dlls', name)
ctypes_binaries = scan_code_for_ctypes(co)
self.binaries.extend(set(ctypes_binaries))
# Analyze run-time hooks.
# Run-time hooks has to be executed before user scripts. Add them
# to the beginning of 'priority_scripts'.
priority_scripts = self.graph.analyze_runtime_hooks(
self.custom_runtime_hooks) + priority_scripts
# 'priority_scripts' is now a list of the graph nodes of custom runtime
# hooks, then regular runtime hooks, then the PyI loader scripts.
# Further on, we will make sure they end up at the front of self.scripts
### Extract the nodes of the graph as TOCs for further processing.
# Initialize the scripts list with priority scripts in the proper order.
self.scripts = self.graph.nodes_to_toc(priority_scripts)
# Extend the binaries list with all the Extensions modulegraph has found.
self.binaries = self.graph.make_binaries_toc(self.binaries)
# Fill the "pure" list with pure Python modules.
assert len(self.pure) == 0
self.pure = self.graph.make_pure_toc()
# And get references to module code objects constructed by ModuleGraph
# to avoid writing .pyc/pyo files to hdd.
self.pure._code_cache = self.graph.get_code_objects()
# Add remaining binary dependencies - analyze Python C-extensions and what
# DLLs they depend on.
logger.info('Looking for dynamic libraries')
self.binaries.extend(
bindepend.Dependencies(self.binaries,
redirects=self.binding_redirects))
### Include zipped Python eggs.
logger.info('Looking for eggs')
self.zipfiles.extend(deps_proc.make_zipfiles_toc())
# Verify that Python dynamic library can be found.
# Without dynamic Python library PyInstaller cannot continue.
self._check_python_library(self.binaries)
if is_win:
# Remove duplicate redirects
self.binding_redirects[:] = list(set(self.binding_redirects))
logger.info("Found binding redirects: \n%s",
self.binding_redirects)
# Place Python source in data files for the noarchive case.
if self.noarchive:
# Create a new TOC of ``(dest path for .pyc, source for .py, type)``.
new_toc = TOC()
for name, path, typecode in self.pure:
assert typecode == 'PYMODULE'
# Transform a python module name into a file name.
name = name.replace('.', os.sep)
# Special case: modules have an implied filename to add.
if os.path.splitext(os.path.basename(path))[0] == '__init__':
name += os.sep + '__init__'
# Append the extension for the compiled result.
# In python 3.5 (PEP-488) .pyo files were replaced by
# .opt-1.pyc and .opt-2.pyc. However, it seems that for
# bytecode-only module distribution, we always need to
# use the .pyc extension.
name += '.pyc'
new_toc.append((name, path, typecode))
# Put the result of byte-compiling this TOC in datas. Mark all entries as data.
for name, path, typecode in compile_py_files(
new_toc, CONF['workpath']):
self.datas.append((name, path, 'DATA'))
# Store no source in the archive.
self.pure = TOC()
# Write warnings about missing modules.
self._write_warnings()
# Write debug information about hte graph
self._write_graph_debug()
def up(self):
self.sftp.lstat(self.remote)
local_files = {}
spinner.waitfor('Testing')
for root, dirs, files in os.walk(self.local.encode('utf-8')):
root = to_unicode(root)
if self._exclude(root):
continue
for d in dirs:
d = to_unicode(d)
path = os.path.relpath(os.path.join(root, d), self.local)
if self._exclude(path):
continue
self.check_dir(path)
if self.subdir:
root = os.path.join(self.subdir, root)
for f in files:
f = to_unicode(f)
if f in ('.files',):
continue
lfile = os.path.join(root, f)
filename = os.path.relpath(lfile, self.local_root)
if filename.split(os.path.sep)[0] == os.path.curdir:
filename = filename[2:]
if self._exclude(lfile):
continue
rfile = os.path.join(self.remote_root, filename)
s = os.lstat(lfile)
spinner.status(string_shortener(filename))
local_files[filename] = File(int(s.st_mtime), int(s.st_size), int(s.st_mode))
if filename not in self.revision_file:
print_file_info(filename, s)
upload = True
else:
lf = local_files[filename]
rf = self.revision_file[filename]
upload = different(self.sftp, filename, rf, lf,
self.local_root, self.remote_root)
if upload:
spinner.succeeded()
info(" Uploading: %s\n" % filename)
try:
try:
rstat = self.sftp.lstat(rfile)
except KeyboardInterrupt:
raise
except:
pass
else:
if rstat.st_mtime > s.st_mtime:
raise ValueError("Conflict with file %s (remote file is newer)" % filename)
start = time.time()
def status(total, size):
if size == 0:
return
speed = total / (time.time() - start) * 1.0
if speed > 1024 ** 2:
speeds = "%.2f MiByte/s" % (speed / 1024 ** 2,)
elif speed > 1024:
speeds = "%.2f KiByte/s" % (speed / 1024,)
else:
speeds = "%f Byte/s" % speed
remaining = timedelta(seconds=int((size - total) / speed))
sys.stdout.write("\r%02d%% %d/%d %s %s" % (
total * 100 / size, total, size, speeds, remaining))
sys.stdout.flush()
if not self.dry_run:
if stat.S_ISLNK(s.st_mode):
target = os.readlink(lfile)
print()
info("Creating remote symlink %s -> %s\n" % (rfile, target))
try:
self.sftp.symlink(target, rfile)
except paramiko.SSHException as e:
error("Failed: %s\n" % (e,))
except IOError:
pass
else:
self.sftp.put(lfile, rfile, status)
f = self.sftp.file(rfile)
f.utime((s.st_atime, s.st_mtime))
f.close()
spinner.waitfor('Testing')
except ValueError:
del local_files[filename]
self.revision_file.update(local_files)
if not self.dry_run:
self.revision_file.save()
raise
except KeyboardInterrupt:
del local_files[filename]
if not self.dry_run:
self.sftp.unlink(rfile)
self.revision_file.update(local_files)
if not self.dry_run:
self.revision_file.save()
exit(1)
except Exception as e:
del local_files[filename]
if not self.dry_run and filename not in self.revision_file:
self.sftp.unlink(rfile)
if not self.skip_on_error:
self.revision_file.update(local_files)
if not self.dry_run:
self.revision_file.save()
raise
else:
if filename in self.revision_file: # prevent deletion
local_files[filename] = self.revision_file[filename]
error("Error during upload of %s: %s\n" % (filename, str(e)))
spinner.succeeded()
for filename in self.revision_file.keys():
if self.subdir:
if not filename.startswith(self.subdir):
continue
if filename not in local_files and not self._exclude(filename):
warning("Deleted file locally: %s\n" % filename)
print(" Last mod time: %d\n Size: %d\n Mode: %o" % self.revision_file[filename])
if not self.dry_run:
try:
self.sftp.lstat(os.path.join(self.remote_root, filename))
except KeyboardInterrupt:
raise
except:
warning("Can't stat remote file. Maybe already deleted?\n")
del self.revision_file[filename]
continue
answer = prompt("Delete it on remote?[y/n]")
if answer == 'y':
self.sftp.unlink(os.path.join(self.remote_root, filename))
del self.revision_file[filename]
self.revision_file.update(local_files)
if not self.dry_run:
self.revision_file.save()
def _getExeName(self, pid):
return os.readlink('/proc/%d/file' % pid)
def walk(top, topdown=True, onerror=None, followlinks=False, blacklist=()):
"""Directory tree generator.
Custom implementation of os.walk that doesn't block if the destination of
a symlink is on an unreachable blacklisted path (typically a nfs mount).
All the general os.walk documentation applies.
"""
# We may not have read permission for top, in which case we can't
# get a list of the files the directory contains. os.path.walk
# always suppressed the exception then, rather than blow up for a
# minor reason when (say) a thousand readable directories are still
# left to visit. That logic is copied here.
try:
names = os.listdir(top)
except OSError as err:
if onerror is not None:
onerror(err)
return
# Use absolute and normalized blacklist paths
normblacklist = [os.path.abspath(x) for x in blacklist]
dirs, nondirs = [], []
for name in names:
path = os.path.join(top, name)
# Begin of the part where we handle the unreachable symlinks
if os.path.abspath(path) in normblacklist:
continue
if not followlinks:
# Don't use os.path.islink because it relies on the syscall
# lstat which is getting stuck if the destination is unreachable
try:
os.readlink(path)
except OSError as err:
# EINVAL is thrown when "path" is not a symlink, in such
# case continue normally
if err.errno != errno.EINVAL:
raise
# There is an hidden code path here, if we fail to read the
# link and the errno is EINVAL then skip the following else
# code block:
else:
nondirs.append(name)
continue
# End of the part where we handle the unreachable symlinks
if os.path.isdir(path):
dirs.append(name)
else:
nondirs.append(name)
if topdown:
yield top, dirs, nondirs
for name in dirs:
path = os.path.join(top, name)
if followlinks or not os.path.islink(path):
for x in walk(path, topdown, onerror, followlinks, blacklist):
yield x
if not topdown:
yield top, dirs, nondirs
def get_interfaces():
"""
Collect the information on all network interfaces.
Returns
-------
dict
The information on the interfaces.
keys:
physical : boolean, true if physical interface
mac : mac address
pci : PCI device
virtfns : dict of virtual function
"""
ret = {}
pci_id_to_iface = {}
for n in os.listdir(_CLASS_NET_DIR):
physical = True
iface = "{}/{}".format(_CLASS_NET_DIR, n)
try:
link = os.readlink(iface)
if link.startswith('../../devices/virtual'):
physical = False
except OSError:
continue
mac = open('{}/address'.format(iface)).read().strip().lower()
iface_info = {'physical': physical, 'mac': mac}
if physical:
# Check to see if this is a physical or virtual
# function
dev = '{}/device'.format(iface)
pci_id = os.readlink(dev)
pci_id = pci_id[pci_id.rfind('/') + 1:]
pci_id_to_iface[pci_id] = n
iface_info['pci'] = pci_id
try:
phys_id = os.readlink('{}/physfn'.format(dev))[3:]
iface_info['physfn'] = phys_id
except OSError:
# If there is no physical function backing this
# interface, then it must itself be one
virt_ifaces = {}
dirs = os.listdir(dev)
for d in dirs:
if not d.startswith('virtfn'):
continue
virt_pci_id = os.readlink('{}/{}'.format(dev, d))[3:]
virt_ifaces[int(d[6:])] = {'pci_id': virt_pci_id}
# TODO: find a better way to get mac addresses for virtual functions
for line in subprocess.check_output(
['/usr/sbin/ip', 'link', 'show', n]).splitlines():
line = line.strip().decode()
if not line.startswith('vf '):
continue
ents = line.split(' ')
vf_num = int(ents[1])
vf_mac = ents[3][:-1]
virt_ifaces[vf_num]['mac'] = vf_mac
iface_info['virtfns'] = virt_ifaces
ret[n] = iface_info
# Populate any potentially invalid mac addresses with
# the correct data
for n, info in ret.items():
if not info['physical']:
continue
virt_fns = info.get('virtfns')
if virt_fns is None:
continue
for _, v in virt_fns.items():
try:
v['mac'] = ret[pci_id_to_iface[v['pci_id']]]['mac']
except Exception:
pass
return ret
def _assess_event_links(event_spec):
self.assertTrue(self.hooks_dir / event_spec.event_name in self.hooks_dir.iterdir())
for event_hook in all_event_hooks:
self.assertTrue((self.JUJU_CHARM_DIR / event_hook).exists(), f'Missing hook: {event_hook}')
self.assertEqual(os.readlink(self.JUJU_CHARM_DIR / event_hook), self.charm_exec_path)
def _get_left_part(self, bar): # pylint: disable=too-many-branches,too-many-statements
left = bar.left
if self.column is not None and self.column.target is not None\
and self.column.target.is_directory:
target = self.column.target.pointed_obj
else:
directory = self.fm.thistab.at_level(0)
if directory:
target = directory.pointed_obj
else:
return
try:
stat = target.stat
except AttributeError:
return
if stat is None:
return
if self.fm.mode != 'normal':
perms = '--%s--' % self.fm.mode.upper()
else:
perms = target.get_permission_string()
how = 'good' if getuid() == stat.st_uid else 'bad'
left.add(perms, 'permissions', how)
left.add_space()
left.add(str(stat.st_nlink), 'nlink')
left.add_space()
left.add(self._get_owner(target), 'owner')
left.add_space()
left.add(self._get_group(target), 'group')
if target.is_link:
how = 'good' if target.exists else 'bad'
try:
dest = readlink(target.path)
except OSError:
dest = '?'
left.add(' -> ' + dest, 'link', how)
else:
left.add_space()
if self.settings.display_size_in_status_bar and target.infostring:
left.add(target.infostring.replace(" ", ""))
left.add_space()
left.add(strftime(self.timeformat, localtime(stat.st_mtime)), 'mtime')
directory = target if target.is_directory else \
target.fm.get_directory(os.path.dirname(target.path))
if directory.vcs and directory.vcs.track:
if directory.vcs.rootvcs.branch:
vcsinfo = '({0:s}: {1:s})'.format(
directory.vcs.rootvcs.repotype, directory.vcs.rootvcs.branch)
else:
vcsinfo = '({0:s})'.format(directory.vcs.rootvcs.repotype)
left.add_space()
left.add(vcsinfo, 'vcsinfo')
left.add_space()
if directory.vcs.rootvcs.obj.vcsremotestatus:
vcsstr, vcscol = self.vcsremotestatus_symb[
directory.vcs.rootvcs.obj.vcsremotestatus]
left.add(vcsstr.strip(), 'vcsremote', *vcscol)
if target.vcsstatus:
vcsstr, vcscol = self.vcsstatus_symb[target.vcsstatus]
left.add(vcsstr.strip(), 'vcsfile', *vcscol)
if directory.vcs.rootvcs.head:
left.add_space()
left.add(directory.vcs.rootvcs.head['date'].strftime(self.timeformat), 'vcsdate')
left.add_space()
left.add(directory.vcs.rootvcs.head['summary'], 'vcscommit')
def get_interfaces_info(self, ip_path, default_ipv4, default_ipv6):
interfaces = {}
ips = dict(
all_ipv4_addresses=[],
all_ipv6_addresses=[],
)
# FIXME: maybe split into smaller methods?
# FIXME: this is pretty much a constructor
for path in glob.glob('/sys/class/net/*'):
if not os.path.isdir(path):
continue
device = os.path.basename(path)
interfaces[device] = {'device': device}
if os.path.exists(os.path.join(path, 'address')):
macaddress = get_file_content(os.path.join(path, 'address'), default='')
if macaddress and macaddress != '00:00:00:00:00:00':
interfaces[device]['macaddress'] = macaddress
if os.path.exists(os.path.join(path, 'mtu')):
interfaces[device]['mtu'] = int(get_file_content(os.path.join(path, 'mtu')))
if os.path.exists(os.path.join(path, 'operstate')):
interfaces[device]['active'] = get_file_content(os.path.join(path, 'operstate')) != 'down'
if os.path.exists(os.path.join(path, 'device', 'driver', 'module')):
interfaces[device]['module'] = os.path.basename(os.path.realpath(os.path.join(path, 'device', 'driver', 'module')))
if os.path.exists(os.path.join(path, 'type')):
_type = get_file_content(os.path.join(path, 'type'))
interfaces[device]['type'] = self.INTERFACE_TYPE.get(_type, 'unknown')
if os.path.exists(os.path.join(path, 'bridge')):
interfaces[device]['type'] = 'bridge'
interfaces[device]['interfaces'] = [os.path.basename(b) for b in glob.glob(os.path.join(path, 'brif', '*'))]
if os.path.exists(os.path.join(path, 'bridge', 'bridge_id')):
interfaces[device]['id'] = get_file_content(os.path.join(path, 'bridge', 'bridge_id'), default='')
if os.path.exists(os.path.join(path, 'bridge', 'stp_state')):
interfaces[device]['stp'] = get_file_content(os.path.join(path, 'bridge', 'stp_state')) == '1'
if os.path.exists(os.path.join(path, 'bonding')):
interfaces[device]['type'] = 'bonding'
interfaces[device]['slaves'] = get_file_content(os.path.join(path, 'bonding', 'slaves'), default='').split()
interfaces[device]['mode'] = get_file_content(os.path.join(path, 'bonding', 'mode'), default='').split()[0]
interfaces[device]['miimon'] = get_file_content(os.path.join(path, 'bonding', 'miimon'), default='').split()[0]
interfaces[device]['lacp_rate'] = get_file_content(os.path.join(path, 'bonding', 'lacp_rate'), default='').split()[0]
primary = get_file_content(os.path.join(path, 'bonding', 'primary'))
if primary:
interfaces[device]['primary'] = primary
path = os.path.join(path, 'bonding', 'all_slaves_active')
if os.path.exists(path):
interfaces[device]['all_slaves_active'] = get_file_content(path) == '1'
if os.path.exists(os.path.join(path, 'bonding_slave')):
interfaces[device]['perm_macaddress'] = get_file_content(os.path.join(path, 'bonding_slave', 'perm_hwaddr'), default='')
if os.path.exists(os.path.join(path, 'device')):
interfaces[device]['pciid'] = os.path.basename(os.readlink(os.path.join(path, 'device')))
if os.path.exists(os.path.join(path, 'speed')):
speed = get_file_content(os.path.join(path, 'speed'))
if speed is not None:
interfaces[device]['speed'] = int(speed)
# Check whether an interface is in promiscuous mode
if os.path.exists(os.path.join(path, 'flags')):
promisc_mode = False
# The second byte indicates whether the interface is in promiscuous mode.
# 1 = promisc
# 0 = no promisc
data = int(get_file_content(os.path.join(path, 'flags')), 16)
promisc_mode = (data & 0x0100 > 0)
interfaces[device]['promisc'] = promisc_mode
# TODO: determine if this needs to be in a nested scope/closure
def parse_ip_output(output, secondary=False):
for line in output.splitlines():
if not line:
continue
words = line.split()
broadcast = ''
if words[0] == 'inet':
if '/' in words[1]:
address, netmask_length = words[1].split('/')
if len(words) > 3:
broadcast = words[3]
else:
# pointopoint interfaces do not have a prefix
address = words[1]
netmask_length = "32"
address_bin = struct.unpack('!L', socket.inet_aton(address))[0]
netmask_bin = (1 << 32) - (1 << 32 >> int(netmask_length))
netmask = socket.inet_ntoa(struct.pack('!L', netmask_bin))
network = socket.inet_ntoa(struct.pack('!L', address_bin & netmask_bin))
iface = words[-1]
# NOTE: device is ref to outside scope
# NOTE: interfaces is also ref to outside scope
if iface != device:
interfaces[iface] = {}
if not secondary and "ipv4" not in interfaces[iface]:
interfaces[iface]['ipv4'] = {'address': address,
'broadcast': broadcast,
'netmask': netmask,
'network': network}
else:
if "ipv4_secondaries" not in interfaces[iface]:
interfaces[iface]["ipv4_secondaries"] = []
interfaces[iface]["ipv4_secondaries"].append({
'address': address,
'broadcast': broadcast,
'netmask': netmask,
'network': network,
})
# add this secondary IP to the main device
if secondary:
if "ipv4_secondaries" not in interfaces[device]:
interfaces[device]["ipv4_secondaries"] = []
if device != iface:
interfaces[device]["ipv4_secondaries"].append({
'address': address,
'broadcast': broadcast,
'netmask': netmask,
'network': network,
})
# NOTE: default_ipv4 is ref to outside scope
# If this is the default address, update default_ipv4
if 'address' in default_ipv4 and default_ipv4['address'] == address:
default_ipv4['broadcast'] = broadcast
default_ipv4['netmask'] = netmask
default_ipv4['network'] = network
# NOTE: macadress is ref from outside scope
default_ipv4['macaddress'] = macaddress
default_ipv4['mtu'] = interfaces[device]['mtu']
default_ipv4['type'] = interfaces[device].get("type", "unknown")
default_ipv4['alias'] = words[-1]
if not address.startswith('127.'):
ips['all_ipv4_addresses'].append(address)
elif words[0] == 'inet6':
if 'peer' == words[2]:
address = words[1]
_, prefix = words[3].split('/')
scope = words[5]
else:
address, prefix = words[1].split('/')
scope = words[3]
if 'ipv6' not in interfaces[device]:
interfaces[device]['ipv6'] = []
interfaces[device]['ipv6'].append({
'address': address,
'prefix': prefix,
'scope': scope
})
# If this is the default address, update default_ipv6
if 'address' in default_ipv6 and default_ipv6['address'] == address:
default_ipv6['prefix'] = prefix
default_ipv6['scope'] = scope
default_ipv6['macaddress'] = macaddress
default_ipv6['mtu'] = interfaces[device]['mtu']
default_ipv6['type'] = interfaces[device].get("type", "unknown")
if not address == '::1':
ips['all_ipv6_addresses'].append(address)
ip_path = self.module.get_bin_path("ip")
args = [ip_path, 'addr', 'show', 'primary', device]
rc, primary_data, stderr = self.module.run_command(args, errors='surrogate_then_replace')
if rc == 0:
parse_ip_output(primary_data)
else:
# possibly busybox, fallback to running without the "primary" arg
# https://github.com/ansible/ansible/issues/50871
args = [ip_path, 'addr', 'show', device]
rc, data, stderr = self.module.run_command(args, errors='surrogate_then_replace')
if rc == 0:
parse_ip_output(data)
args = [ip_path, 'addr', 'show', 'secondary', device]
rc, secondary_data, stderr = self.module.run_command(args, errors='surrogate_then_replace')
if rc == 0:
parse_ip_output(secondary_data, secondary=True)
interfaces[device].update(self.get_ethtool_data(device))
# replace : by _ in interface name since they are hard to use in template
new_interfaces = {}
# i is a dict key (string) not an index int
for i in interfaces:
if ':' in i:
new_interfaces[i.replace(':', '_')] = interfaces[i]
else:
new_interfaces[i] = interfaces[i]
return new_interfaces, ips
def dircheck(environ, dir_tree):
"""Checks if dir_tree matches the actual directory
tree in the filesystem"""
# Helper functions
def check_owner(path, props, is_link=False):
"""For owner permissions we only look up if its a normal
user or the root user because we can't create other
users just for the sake of these tests"""
stat = os.lstat if is_link else os.stat
if "rootuser" not in props:
props["rootuser"] = False
if "rootgroup" not in props:
props["rootgroup"] = False
if bool(stat(path).st_uid) == props["rootuser"]:
user = "******" if props["rootuser"] else "user"
raise ValueError((False, path + " is a not owned by " + user))
if bool(stat(path).st_gid) == props["rootgroup"]:
group = "root" if props["rootgroup"] else "group"
raise ValueError((False, path + " is a not owned by " + group))
def check_permission(path, permission):
perm_real = str(oct(os.stat(path).st_mode))[-3:]
if perm_real != str(permission):
raise ValueError((False, path + " has permission " + perm_real))
# Collect all files/links in environment
file_list = []
for root, _, files in os.walk(environ):
for file in files:
file_list.append(os.path.abspath(os.path.join(root, file)))
# Compare dir_tree against actual directory tree in environment
for dir_name, dir_props in dir_tree.items():
# Add environment to directory
dir_name = os.path.normpath(os.path.join(environ, dir_name))
# Directory existance
if not os.path.isdir(dir_name):
raise ValueError((False, dir_name + " is a not a directory"))
# Directory permission
if "permission" in dir_props:
check_permission(dir_name, dir_props["permission"])
# Directory owner
check_owner(dir_name, dir_props)
# Files
if "files" in dir_props:
for file_props in dir_props["files"]:
file_path = os.path.join(dir_name, file_props["name"])
# File existance
if os.path.islink(file_path) or not os.path.isfile(file_path):
raise ValueError((False, file_path + " is a not a file"))
# File permission
if "permission" in file_props:
check_permission(file_path, file_props["permission"])
# File owner
check_owner(file_path, file_props)
# File content
md5 = hashlib.md5(open(file_path, "rb").read()).hexdigest()
if "content" in file_props and md5 != file_props["content"]:
msg = file_path + " has wrong content:\n"
msg += open(file_path, "rb").read().decode()
raise ValueError((False, msg))
file_list.remove(file_path)
# Links
if "links" in dir_props:
for link_props in dir_props["links"]:
link_path = os.path.join(dir_name, link_props["name"])
# Link existance
if not os.path.islink(link_path):
raise ValueError((False, link_path + " is a not a link"))
# Link permission
if "permission" in link_props:
check_permission(link_path, link_props["permission"])
# Link owner
check_owner(link_path, link_props, True)
# Link target
target_path = os.path.normpath(
os.path.join(dir_name, os.readlink(link_path)))
link_props["target"] = os.path.abspath(link_props["target"])
if target_path != link_props["target"]:
msg = link_path + " should point to " + link_props['target']
msg += ", but points to " + target_path
raise ValueError((False, msg))
# Link target content
md5 = hashlib.md5(open(target_path, "rb").read()).hexdigest()
if "content" in link_props and md5 != link_props["content"]:
msg = link_path + " has wrong content:\n"
msg += open(target_path, "rb").read().decode()
raise ValueError((False, msg))
file_list.remove(link_path)
# Check if there are files left, that wasn't described by the dir_tree
if file_list:
msg = "Test created unexpected files:\n"
for file in file_list:
msg += " " + file + "\n"
raise ValueError((False, msg))
def copytree(start_site,
src,
end_site,
dst,
current_depth,
symlinks=False,
ignore=None,
arguments=argparse.Namespace()):
"""Recursively copy a directory (src) to another location (dst)"""
if arguments.debug:
print("copytree:")
print("\tsrc:", src)
print("\tdst:", dst)
print("\tdepth:", str(arguments.depth))
print("\tcurrent_depth:", str(current_depth))
print("\tsymlinks:", str(symlinks))
print("\tdry_run:", str(arguments.dry_run))
if current_depth >= arguments.depth:
return
if not arguments.dry_run:
made_dir = make_directory(end_site, dst, arguments.protocol,
arguments.debug)
if not made_dir:
raise Exception(
"ERROR::copytree::Unable to make the destination directory.")
files = get_list_of_files(arguments.protocol, start_site, arguments.sample,
src, arguments.debug)
if arguments.debug:
print("copytree:")
print("\tList of files:", files)
if arguments.diff:
files = do_diff(files, arguments.protocol, end_site, arguments.sample,
dst, arguments.debug)
if ignore is not None:
ignored_names = ignore(src, files)
else:
ignored_names = set()
errors = []
for file in files:
if file in ignored_names:
continue
srcname = os.path.join(src, file) \
if not (os.path.isfile(src) or (start_site.alias != 'local' \
and not remote_is_dir(start_site, src))) else src
dstname = os.path.join(dst, file)
if arguments.debug:
print("copytree:")
print("\tsrcname:", srcname)
print("\tdstname:", dstname)
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname) or \
(arguments.protocol == "xrootd" and start_site.alias != 'local' and remote_is_dir(start_site, srcname)):
copytree(start_site, srcname, end_site, dstname,
current_depth + 1, symlinks, ignore, arguments)
else:
srel = os.path.relpath(
src, src[:src.find(start_site.path) +
len(start_site.path)]) + "/"
if srel == "./":
srel = ""
copy_command, start_location, end_location = init_commands(
start_site, end_site, arguments)
print("Copying file " + file + " from " + srel)
if start_site.alias != 'local' and start_site.alias != 'local' and not remote_is_dir(
start_site, src):
command = copy_command + " " + start_location + " " + end_location
elif start_site.alias != 'local':
command = copy_command + " " + start_location + srel + file + " " + end_location + srel + file
elif os.path.isfile(start_location):
command = copy_command + " " + start_location + " " + end_location + srel + file
else:
command = copy_command + " " + start_location + srel + file + " " + end_location + srel + file
print("\tcopy command:", command)
if not arguments.dry_run:
os.system(command)
print("")
# catch the Error from the recursive copytree so that we can
# continue with other files
except Error as err:
errors.extend(err.args[0])
except EnvironmentError as why:
errors.append((srcname, dstname, str(why)))
if errors:
raise Error(errors)
def main():
module = AnsibleModule(
argument_spec=dict(
path=dict(type='path', required=True, aliases=['dest', 'name']),
follow=dict(type='bool', default=False),
get_md5=dict(type='bool', default=False),
get_checksum=dict(type='bool', default=True),
get_mime=dict(type='bool', default=True, aliases=['mime', 'mime_type', 'mime-type']),
get_attributes=dict(type='bool', default=True, aliases=['attr', 'attributes']),
checksum_algorithm=dict(type='str', default='sha1',
choices=['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
aliases=['checksum', 'checksum_algo']),
),
supports_check_mode=True,
)
path = module.params.get('path')
b_path = to_bytes(path, errors='surrogate_or_strict')
follow = module.params.get('follow')
get_mime = module.params.get('get_mime')
get_attr = module.params.get('get_attributes')
get_checksum = module.params.get('get_checksum')
checksum_algorithm = module.params.get('checksum_algorithm')
# NOTE: undocumented option since 2.9 to be removed at a later date if possible (3.0+)
# no real reason for keeping other than fear we may break older content.
get_md5 = module.params.get('get_md5')
# main stat data
try:
if follow:
st = os.stat(b_path)
else:
st = os.lstat(b_path)
except OSError as e:
if e.errno == errno.ENOENT:
output = {'exists': False}
module.exit_json(changed=False, stat=output)
module.fail_json(msg=e.strerror)
# process base results
output = format_output(module, path, st)
# resolved permissions
for perm in [('readable', os.R_OK), ('writeable', os.W_OK), ('executable', os.X_OK)]:
output[perm[0]] = os.access(b_path, perm[1])
# symlink info
if output.get('islnk'):
output['lnk_source'] = os.path.realpath(b_path)
output['lnk_target'] = os.readlink(b_path)
try: # user data
pw = pwd.getpwuid(st.st_uid)
output['pw_name'] = pw.pw_name
except (TypeError, KeyError):
pass
try: # group data
grp_info = grp.getgrgid(st.st_gid)
output['gr_name'] = grp_info.gr_name
except (KeyError, ValueError, OverflowError):
pass
# checksums
if output.get('isreg') and output.get('readable'):
# NOTE: see above about get_md5
if get_md5:
# Will fail on FIPS-140 compliant systems
try:
output['md5'] = module.md5(b_path)
except ValueError:
output['md5'] = None
if get_checksum:
output['checksum'] = module.digest_from_file(b_path, checksum_algorithm)
# try to get mime data if requested
if get_mime:
output['mimetype'] = output['charset'] = 'unknown'
mimecmd = module.get_bin_path('file')
if mimecmd:
mimecmd = [mimecmd, '--mime-type', '--mime-encoding', b_path]
try:
rc, out, err = module.run_command(mimecmd)
if rc == 0:
mimetype, charset = out.rsplit(':', 1)[1].split(';')
output['mimetype'] = mimetype.strip()
output['charset'] = charset.split('=')[1].strip()
except Exception:
pass
# try to get attr data
if get_attr:
output['version'] = None
output['attributes'] = []
output['attr_flags'] = ''
out = module.get_file_attributes(b_path)
for x in ('version', 'attributes', 'attr_flags'):
if x in out:
output[x] = out[x]
module.exit_json(changed=False, stat=output)
#
# License: see LICENSE file for details
#
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import random
import sys
# point to absolute path of peda.py
PEDAFILE = os.path.abspath(os.path.expanduser(__file__))
while os.path.islink(PEDAFILE):
PEDAFILE = os.path.abspath(os.path.join(os.path.dirname(PEDAFILE), os.path.expanduser(os.readlink(PEDAFILE))))
sys.path.insert(0, os.path.dirname(PEDAFILE))
from peda import *
try:
import zlib
except ImportError:
zlib = None
__version__ = 'alpha-1.0'
class Asm(AsmBase):
"""
Wrapper class for assemble/disassemble using as
def apport_excepthook(exc_type, exc_obj, exc_tb):
'''Catch an uncaught exception and make a traceback.'''
# create and save a problem report. Note that exceptions in this code
# are bad, and we probably need a per-thread reentrancy guard to
# prevent that happening. However, on Ubuntu there should never be
# a reason for an exception here, other than [say] a read only var
# or some such. So what we do is use a try - finally to ensure that
# the original excepthook is invoked, and until we get bug reports
# ignore the other issues.
# import locally here so that there is no routine overhead on python
# startup time - only when a traceback occurs will this trigger.
try:
# ignore 'safe' exit types.
if exc_type in (KeyboardInterrupt, ):
return
# do not do anything if apport was disabled
if not enabled():
return
try:
from cStringIO import StringIO
StringIO # pyflakes
except ImportError:
from io import StringIO
import re, traceback
from apport.fileutils import likely_packaged, get_recent_crashes
# apport will look up the package from the executable path.
try:
binary = os.path.realpath(os.path.join(os.getcwd(), sys.argv[0]))
except (TypeError, AttributeError, IndexError):
# the module has mutated sys.argv, plan B
try:
binary = os.readlink('/proc/%i/exe' % os.getpid())
except OSError:
return
# for interactive python sessions, sys.argv[0] == ''; catch that and
# other irregularities
if not os.access(binary, os.X_OK) or not os.path.isfile(binary):
return
# filter out binaries in user accessible paths
if not likely_packaged(binary):
return
import apport.report
pr = apport.report.Report()
# special handling of dbus-python exceptions
if hasattr(exc_obj, 'get_dbus_name'):
if exc_obj.get_dbus_name() == 'org.freedesktop.DBus.Error.NoReply':
# NoReply is an useless crash, we do not even get the method it
# was trying to call; needs actual crash from D-BUS backend (LP #914220)
return
if exc_obj.get_dbus_name(
) == 'org.freedesktop.DBus.Error.ServiceUnknown':
dbus_service_unknown_analysis(exc_obj, pr)
# append a basic traceback. In future we may want to include
# additional data such as the local variables, loaded modules etc.
tb_file = StringIO()
traceback.print_exception(exc_type, exc_obj, exc_tb, file=tb_file)
pr['Traceback'] = tb_file.getvalue().strip()
pr.add_proc_info(extraenv=['PYTHONPATH', 'PYTHONHOME'])
pr.add_user_info()
# override the ExecutablePath with the script that was actually running
pr['ExecutablePath'] = binary
if 'ExecutableTimestamp' in pr:
pr['ExecutableTimestamp'] = str(int(os.stat(binary).st_mtime))
try:
pr['PythonArgs'] = '%r' % sys.argv
except AttributeError:
pass
if pr.check_ignored():
return
mangled_program = re.sub('/', '_', binary)
# get the uid for now, user name later
user = os.getuid()
pr_filename = '%s/%s.%i.crash' % (os.environ.get(
'APPORT_REPORT_DIR', '/var/crash'), mangled_program, user)
crash_counter = 0
if os.path.exists(pr_filename):
if apport.fileutils.seen_report(pr_filename):
# flood protection
with open(pr_filename, 'rb') as f:
crash_counter = get_recent_crashes(f) + 1
if crash_counter > 1:
return
# remove the old file, so that we can create the new one with
# os.O_CREAT|os.O_EXCL
os.unlink(pr_filename)
else:
# don't clobber existing report
return
if crash_counter:
pr['CrashCounter'] = str(crash_counter)
with os.fdopen(
os.open(pr_filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o640), 'wb') as f:
pr.write(f)
finally:
# resume original processing to get the default behaviour,
# but do not trigger an AttributeError on interpreter shutdown.
if sys:
sys.__excepthook__(exc_type, exc_obj, exc_tb)
def zvol_device_name(self):
return os.path.split(os.readlink(self.zvol_dev))[-1]
def _create_and_check_link(self, source, link):
absolute_symlink(source, link)
self.assertTrue(os.path.islink(link))
self.assertEqual(source, os.readlink(link))
def copy(src, dst, hardlink=False, keep_symlink=True):
'''Copy a file to another location with a bunch of link handling'''
assert not P.isdir(src), 'Source path must not be a dir'
assert not P.isdir(dst), 'Destination path must not be a dir'
# There is nothing we can do about absolute sym links in system dirs. We just
# trace those to the source and copy.
if keep_symlink and P.islink(src):
if P.isabs(readlink(src)):
m = re.match('^/usr', readlink(src))
if m:
print("Resolving absolute link: ", src)
while P.islink(src) and \
P.isabs(readlink(src)) \
and os.path.basename(src) == os.path.basename(readlink(src)):
src = readlink(src)
print("Resolved to: ", src)
if keep_symlink and P.islink(src):
assert not P.isabs(readlink(src)), \
'Cannot copy symlink that points to an absolute path (%s)' % src
logger.debug('%8s %s -> %s' % ('symlink', src, dst))
# Some of the libraries are both in our install dir and in USGS conda's package.
# That is because they do not provide headers for cspice for example.
# So below we will run into trouble. Just overwrite any library we built
# with the one from conda.
if P.exists(dst):
is_soft_link = True
try:
link_val = readlink(dst)
except:
is_soft_link = False
if (not is_soft_link) or (readlink(dst) != readlink(src)):
print("Will overwrite " + dst + " with " + src)
os.remove(dst)
if not P.exists(dst):
try:
symlink(readlink(src), dst)
except:
pass
return
if P.exists(dst):
if hash_file(src) != hash_file(dst):
print("Will overwrite " + dst + " with " + src +
" having a different hash.")
else:
if hardlink:
try:
link(src, dst)
logger.debug('%8s %s -> %s' % ('hardlink', src, dst))
return
except OSError as o:
# Invalid cross-device link, not an error, fall back to copy
if o.errno != errno.EXDEV:
raise
logger.debug('%8s %s -> %s' % ('copy', src, dst))
shutil.copyfile(src, dst)
# Bugfix, make it writeable
mode = os.stat(dst)[stat.ST_MODE]
os.chmod(dst, mode | stat.S_IWUSR)
def formatted_filename(path, filename, mode):
if not stat.S_ISLINK(mode):
return filename
symlink_target = os.readlink(filename)
return filename + ' -> ' + symlink_target
def main():
module = AnsibleModule(
argument_spec=dict(
state=dict(choices=[
'file', 'directory', 'link', 'hard', 'touch', 'absent'
],
default=None),
path=dict(aliases=['dest', 'name'], required=True, type='path'),
original_basename=dict(
required=False), # Internal use only, for recursive ops
recurse=dict(default=False, type='bool'),
force=dict(required=False, default=False, type='bool'),
follow=dict(required=False, default=False, type='bool'),
diff_peek=dict(
default=None
), # Internal use only, for internal checks in the action plugins
validate=dict(
required=False,
default=None), # Internal use only, for template and copy
src=dict(required=False, default=None, type='path'),
),
add_file_common_args=True,
supports_check_mode=True)
params = module.params
state = params['state']
recurse = params['recurse']
force = params['force']
diff_peek = params['diff_peek']
src = params['src']
b_src = to_bytes(src, errors='surrogate_or_strict')
follow = params['follow']
# modify source as we later reload and pass, specially relevant when used by other modules.
path = params['path']
b_path = to_bytes(path, errors='surrogate_or_strict')
# short-circuit for diff_peek
if diff_peek is not None:
appears_binary = False
try:
f = open(b_path, 'rb')
head = f.read(8192)
f.close()
if b("\x00") in head:
appears_binary = True
except:
pass
module.exit_json(path=path,
changed=False,
appears_binary=appears_binary)
prev_state = get_state(b_path)
# state should default to file, but since that creates many conflicts,
# default to 'current' when it exists.
if state is None:
if prev_state != 'absent':
state = prev_state
elif recurse:
state = 'directory'
else:
state = 'file'
# source is both the source of a symlink or an informational passing of the src for a template module
# or copy module, even if this module never uses it, it is needed to key off some things
if src is None:
if state in ('link', 'hard'):
if follow and state == 'link':
# use the current target of the link as the source
src = to_native(os.path.realpath(b_path), errors='strict')
b_src = to_bytes(os.path.realpath(b_path), errors='strict')
else:
module.fail_json(
msg='src and dest are required for creating links')
# original_basename is used by other modules that depend on file.
if state not in ("link", "absent") and os.path.isdir(b_path):
basename = None
if params['original_basename']:
basename = params['original_basename']
elif src is not None:
basename = os.path.basename(src)
if basename:
params['path'] = path = os.path.join(path, basename)
b_path = to_bytes(path, errors='surrogate_or_strict')
prev_state = get_state(b_path)
# make sure the target path is a directory when we're doing a recursive operation
if recurse and state != 'directory':
module.fail_json(path=path,
msg="recurse option requires state to be 'directory'")
file_args = module.load_file_common_arguments(params)
changed = False
diff = {
'before': {
'path': path
},
'after': {
'path': path
},
}
state_change = False
if prev_state != state:
diff['before']['state'] = prev_state
diff['after']['state'] = state
state_change = True
if state == 'absent':
if state_change:
if not module.check_mode:
if prev_state == 'directory':
try:
shutil.rmtree(b_path, ignore_errors=False)
except Exception as e:
module.fail_json(msg="rmtree failed: %s" %
to_native(e))
else:
try:
os.unlink(b_path)
except Exception as e:
module.fail_json(path=path,
msg="unlinking failed: %s " %
to_native(e))
module.exit_json(path=path, changed=True, diff=diff)
else:
module.exit_json(path=path, changed=False)
elif state == 'file':
if state_change:
if follow and prev_state == 'link':
# follow symlink and operate on original
b_path = os.path.realpath(b_path)
path = to_native(b_path, errors='strict')
prev_state = get_state(b_path)
file_args['path'] = path
if prev_state not in ('file', 'hard'):
# file is not absent and any other state is a conflict
module.fail_json(path=path,
msg='file (%s) is %s, cannot continue' %
(path, prev_state))
changed = module.set_fs_attributes_if_different(file_args,
changed,
diff,
expand=False)
module.exit_json(path=path, changed=changed, diff=diff)
elif state == 'directory':
if follow and prev_state == 'link':
b_path = os.path.realpath(b_path)
path = to_native(b_path, errors='strict')
prev_state = get_state(b_path)
if prev_state == 'absent':
if module.check_mode:
module.exit_json(changed=True, diff=diff)
changed = True
curpath = ''
try:
# Split the path so we can apply filesystem attributes recursively
# from the root (/) directory for absolute paths or the base path
# of a relative path. We can then walk the appropriate directory
# path to apply attributes.
for dirname in path.strip('/').split('/'):
curpath = '/'.join([curpath, dirname])
# Remove leading slash if we're creating a relative path
if not os.path.isabs(path):
curpath = curpath.lstrip('/')
b_curpath = to_bytes(curpath, errors='surrogate_or_strict')
if not os.path.exists(b_curpath):
try:
os.mkdir(b_curpath)
except OSError as ex:
# Possibly something else created the dir since the os.path.exists
# check above. As long as it's a dir, we don't need to error out.
if not (ex.errno == errno.EEXIST
and os.path.isdir(b_curpath)):
raise
tmp_file_args = file_args.copy()
tmp_file_args['path'] = curpath
changed = module.set_fs_attributes_if_different(
tmp_file_args, changed, diff, expand=False)
except Exception as e:
module.fail_json(
path=path,
msg='There was an issue creating %s as requested: %s' %
(curpath, to_native(e)))
# We already know prev_state is not 'absent', therefore it exists in some form.
elif prev_state != 'directory':
module.fail_json(path=path,
msg='%s already exists as a %s' %
(path, prev_state))
changed = module.set_fs_attributes_if_different(file_args,
changed,
diff,
expand=False)
if recurse:
changed |= recursive_set_attributes(
module,
to_bytes(file_args['path'], errors='surrogate_or_strict'),
follow, file_args)
module.exit_json(path=path, changed=changed, diff=diff)
elif state in ('link', 'hard'):
if not os.path.islink(b_path) and os.path.isdir(b_path):
relpath = path
else:
b_relpath = os.path.dirname(b_path)
relpath = to_native(b_relpath, errors='strict')
absrc = os.path.join(relpath, src)
b_absrc = to_bytes(absrc, errors='surrogate_or_strict')
if not force and not os.path.exists(b_absrc):
module.fail_json(
path=path,
src=src,
msg=
'src file does not exist, use "force=yes" if you really want to create the link: %s'
% absrc)
if state == 'hard':
if not os.path.isabs(b_src):
module.fail_json(msg="absolute paths are required")
elif prev_state == 'directory':
if not force:
module.fail_json(
path=path,
msg='refusing to convert between %s and %s for %s' %
(prev_state, state, path))
elif os.listdir(b_path):
# refuse to replace a directory that has files in it
module.fail_json(
path=path,
msg='the directory %s is not empty, refusing to convert it'
% path)
elif prev_state in ('file', 'hard') and not force:
module.fail_json(
path=path,
msg='refusing to convert between %s and %s for %s' %
(prev_state, state, path))
if prev_state == 'absent':
changed = True
elif prev_state == 'link':
b_old_src = os.readlink(b_path)
if b_old_src != b_src:
diff['before']['src'] = to_native(b_old_src, errors='strict')
diff['after']['src'] = src
changed = True
elif prev_state == 'hard':
if not (state == 'hard'
and os.stat(b_path).st_ino == os.stat(b_src).st_ino):
changed = True
if not force:
module.fail_json(
dest=path,
src=src,
msg=
'Cannot link, different hard link exists at destination'
)
elif prev_state == 'file':
changed = True
if not force:
module.fail_json(dest=path,
src=src,
msg='Cannot link, %s exists at destination' %
prev_state)
elif prev_state == 'directory':
changed = True
if os.path.exists(b_path):
if state == 'hard' and os.stat(b_path).st_ino == os.stat(
b_src).st_ino:
module.exit_json(path=path, changed=False)
elif not force:
module.fail_json(
dest=path,
src=src,
msg=
'Cannot link, different hard link exists at destination'
)
else:
module.fail_json(dest=path,
src=src,
msg='unexpected position reached')
if changed and not module.check_mode:
if prev_state != 'absent':
# try to replace atomically
b_tmppath = to_bytes(os.path.sep).join([
os.path.dirname(b_path),
to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))
])
try:
if prev_state == 'directory' and state == 'link':
os.rmdir(b_path)
elif prev_state == 'directory' and state == 'hard':
if os.path.exists(b_path):
os.remove(b_path)
if state == 'hard':
os.link(b_src, b_tmppath)
else:
os.symlink(b_src, b_tmppath)
os.rename(b_tmppath, b_path)
except OSError as e:
if os.path.exists(b_tmppath):
os.unlink(b_tmppath)
module.fail_json(path=path,
msg='Error while replacing: %s' %
to_native(e, nonstring='simplerepr'))
else:
try:
if state == 'hard':
os.link(b_src, b_path)
else:
os.symlink(b_src, b_path)
except OSError as e:
module.fail_json(path=path,
msg='Error while linking: %s' %
to_native(e, nonstring='simplerepr'))
if module.check_mode and not os.path.exists(b_path):
module.exit_json(dest=path, src=src, changed=changed, diff=diff)
changed = module.set_fs_attributes_if_different(file_args,
changed,
diff,
expand=False)
module.exit_json(dest=path, src=src, changed=changed, diff=diff)
elif state == 'touch':
if not module.check_mode:
if prev_state == 'absent':
try:
open(b_path, 'wb').close()
except OSError as e:
module.fail_json(path=path,
msg='Error, could not touch target: %s' %
to_native(e, nonstring='simplerepr'))
elif prev_state in ('file', 'directory', 'hard'):
try:
os.utime(b_path, None)
except OSError as e:
module.fail_json(
path=path,
msg='Error while touching existing target: %s' %
to_native(e, nonstring='simplerepr'))
else:
module.fail_json(
msg=
'Cannot touch other than files, directories, and hardlinks (%s is %s)'
% (path, prev_state))
try:
module.set_fs_attributes_if_different(file_args,
True,
diff,
expand=False)
except SystemExit as e:
if e.code:
# We take this to mean that fail_json() was called from
# somewhere in basic.py
if prev_state == 'absent':
# If we just created the file we can safely remove it
os.remove(b_path)
raise e
module.exit_json(dest=path, changed=True, diff=diff)
module.fail_json(path=path, msg='unexpected position reached')
def _deleted_files():
'''
Iterates over /proc/PID/maps and /proc/PID/fd links and returns list of desired deleted files.
Returns:
List of deleted files to analyze, False on failure.
'''
deleted_files = []
for proc in psutil.process_iter(): # pylint: disable=too-many-nested-blocks
try:
pinfo = proc.as_dict(attrs=['pid', 'name'])
try:
with salt.utils.files.fopen('/proc/{0}/maps'.format(
pinfo['pid'])) as maps: # pylint: disable=resource-leakage
dirpath = '/proc/' + six.text_type(pinfo['pid']) + '/fd/'
listdir = os.listdir(dirpath)
maplines = maps.readlines()
except (OSError, IOError):
yield False
# /proc/PID/maps
mapline = re.compile(
r'^[\da-f]+-[\da-f]+ [r-][w-][x-][sp-] '
r'[\da-f]+ [\da-f]{2}:[\da-f]{2} (\d+) *(.+)( \(deleted\))?\n$'
)
for line in maplines:
line = salt.utils.stringutils.to_unicode(line)
matched = mapline.match(line)
if not matched:
continue
path = matched.group(2)
if not path:
continue
valid = _valid_deleted_file(path)
if not valid:
continue
val = (pinfo['name'], pinfo['pid'], path[0:-10])
if val not in deleted_files:
deleted_files.append(val)
yield val
# /proc/PID/fd
try:
for link in listdir:
path = dirpath + link
readlink = os.readlink(path)
filenames = []
if os.path.isfile(readlink):
filenames.append(readlink)
elif os.path.isdir(readlink) and readlink != '/':
for root, dummy_dirs, files in salt.utils.path.os_walk(
readlink, followlinks=True):
for name in files:
filenames.append(os.path.join(root, name))
for filename in filenames:
valid = _valid_deleted_file(filename)
if not valid:
continue
val = (pinfo['name'], pinfo['pid'], filename)
if val not in deleted_files:
deleted_files.append(val)
yield val
except OSError:
pass
except psutil.NoSuchProcess:
pass
def copy_tree(self,
src,
dst,
preserve_mode=1,
preserve_times=1,
preserve_symlinks=0,
update=0,
verbose=1,
dry_run=0):
"""Same as dir_util.copy_tree, but clobber existing symlinks."""
from distutils.file_util import copy_file
from distutils.dir_util import mkpath
if not dry_run and not os.path.isdir(src):
raise DistutilsFileError("cannot copy tree '%s': not a directory" %
src)
try:
names = os.listdir(src)
except OSError as e:
if dry_run:
names = []
else:
raise DistutilsFileError("error listing files in '%s': %s" %
(src, e.strerror))
if not dry_run:
mkpath(dst, verbose=verbose)
outputs = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if preserve_symlinks and os.path.islink(src_name):
link_dest = os.readlink(src_name)
if verbose >= 1:
log.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
if os.path.islink(dst_name):
os.unlink(dst_name)
os.symlink(link_dest, dst_name)
outputs.append(dst_name)
elif os.path.isdir(src_name):
outputs.extend(
self.copy_tree(src_name,
dst_name,
preserve_mode,
preserve_times,
preserve_symlinks,
update,
verbose=verbose,
dry_run=dry_run))
else:
copy_file(src_name,
dst_name,
preserve_mode,
preserve_times,
update,
verbose=verbose,
dry_run=dry_run)
outputs.append(dst_name)
return outputs
def ensure_hardlink(path, src, follow, force, timestamps):
b_path = to_bytes(path, errors='surrogate_or_strict')
b_src = to_bytes(src, errors='surrogate_or_strict')
prev_state = get_state(b_path)
file_args = module.load_file_common_arguments(module.params)
mtime = get_timestamp_for_time(timestamps['modification_time'], timestamps['modification_time_format'])
atime = get_timestamp_for_time(timestamps['access_time'], timestamps['access_time_format'])
# src is the source of a hardlink. We require it if we are creating a new hardlink.
# We require path in the argument_spec so we know it is present at this point.
if src is None:
raise AssibleModuleError(results={'msg': 'src is required for creating new hardlinks'})
if not os.path.exists(b_src):
raise AssibleModuleError(results={'msg': 'src does not exist', 'dest': path, 'src': src})
diff = initial_diff(path, 'hard', prev_state)
changed = False
if prev_state == 'absent':
changed = True
elif prev_state == 'link':
b_old_src = os.readlink(b_path)
if b_old_src != b_src:
diff['before']['src'] = to_native(b_old_src, errors='strict')
diff['after']['src'] = src
changed = True
elif prev_state == 'hard':
if not os.stat(b_path).st_ino == os.stat(b_src).st_ino:
changed = True
if not force:
raise AssibleModuleError(results={'msg': 'Cannot link, different hard link exists at destination',
'dest': path, 'src': src})
elif prev_state == 'file':
changed = True
if not force:
raise AssibleModuleError(results={'msg': 'Cannot link, %s exists at destination' % prev_state,
'dest': path, 'src': src})
elif prev_state == 'directory':
changed = True
if os.path.exists(b_path):
if os.stat(b_path).st_ino == os.stat(b_src).st_ino:
return {'path': path, 'changed': False}
elif not force:
raise AssibleModuleError(results={'msg': 'Cannot link: different hard link exists at destination',
'dest': path, 'src': src})
else:
raise AssibleModuleError(results={'msg': 'unexpected position reached', 'dest': path, 'src': src})
if changed and not module.check_mode:
if prev_state != 'absent':
# try to replace atomically
b_tmppath = to_bytes(os.path.sep).join(
[os.path.dirname(b_path), to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))]
)
try:
if prev_state == 'directory':
if os.path.exists(b_path):
try:
os.unlink(b_path)
except OSError as e:
if e.errno != errno.ENOENT: # It may already have been removed
raise
os.link(b_src, b_tmppath)
os.rename(b_tmppath, b_path)
except OSError as e:
if os.path.exists(b_tmppath):
os.unlink(b_tmppath)
raise AssibleModuleError(results={'msg': 'Error while replacing: %s'
% to_native(e, nonstring='simplerepr'),
'path': path})
else:
try:
os.link(b_src, b_path)
except OSError as e:
raise AssibleModuleError(results={'msg': 'Error while linking: %s'
% to_native(e, nonstring='simplerepr'),
'path': path})
if module.check_mode and not os.path.exists(b_path):
return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False)
changed |= update_timestamp_for_file(file_args['path'], mtime, atime, diff)
return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
def restartcheck(ignorelist=None, blacklist=None, excludepid=None, **kwargs):
'''
Analyzes files openeded by running processes and seeks for packages which need to be restarted.
Args:
ignorelist: string or list of packages to be ignored
blacklist: string or list of file paths to be ignored
excludepid: string or list of process IDs to be ignored
verbose: boolean, enables extensive output
timeout: int, timeout in minute
Returns:
Dict on error: { 'result': False, 'comment': '<reason>' }
String with checkrestart output if some package seems to need to be restarted or
if no packages need restarting.
.. versionadded:: 2015.8.3
CLI Example:
.. code-block:: bash
salt '*' restartcheck.restartcheck
'''
kwargs = salt.utils.args.clean_kwargs(**kwargs)
start_time = int(round(time.time() * 1000))
kernel_restart = True
verbose = kwargs.pop('verbose', True)
timeout = kwargs.pop('timeout', 5)
if __grains__.get('os_family') == 'Debian':
cmd_pkg_query = 'dpkg-query --listfiles '
systemd_folder = '/lib/systemd/system/'
systemd = '/bin/systemd'
kernel_versions = _kernel_versions_debian()
elif __grains__.get('os_family') == 'RedHat':
cmd_pkg_query = 'repoquery -l '
systemd_folder = '/usr/lib/systemd/system/'
systemd = '/usr/bin/systemctl'
kernel_versions = _kernel_versions_redhat()
elif __grains__.get('os_family') == NILRT_FAMILY_NAME:
cmd_pkg_query = 'opkg files '
systemd = ''
kernel_versions = _kernel_versions_nilrt()
else:
return {
'result':
False,
'comment':
'Only available on Debian, Red Hat and NI Linux Real-Time based systems.'
}
# Check kernel versions
kernel_current = __salt__['cmd.run']('uname -a')
for kernel in kernel_versions:
_check_timeout(start_time, timeout)
if kernel in kernel_current:
if __grains__.get('os_family') == 'NILinuxRT':
# Check kernel modules and hardware API's for version changes
if not _kernel_modules_changed_nilrt(
kernel) and not _sysapi_changed_nilrt():
kernel_restart = False
break
else:
kernel_restart = False
break
packages = {}
running_services = {}
restart_services = []
if ignorelist:
if not isinstance(ignorelist, list):
ignorelist = [ignorelist]
else:
ignorelist = ['screen', 'systemd']
if blacklist:
if not isinstance(blacklist, list):
blacklist = [blacklist]
else:
blacklist = []
if excludepid:
if not isinstance(excludepid, list):
excludepid = [excludepid]
else:
excludepid = []
for service in __salt__['service.get_running']():
_check_timeout(start_time, timeout)
service_show = __salt__['service.show'](service)
if 'ExecMainPID' in service_show:
running_services[service] = int(service_show['ExecMainPID'])
owners_cache = {}
for deleted_file in _deleted_files():
if deleted_file is False:
return {
'result':
False,
'comment':
'Could not get list of processes.'
' (Do you have root access?)'
}
_check_timeout(start_time, timeout)
name, pid, path = deleted_file[0], deleted_file[1], deleted_file[2]
if path in blacklist or pid in excludepid:
continue
try:
readlink = os.readlink('/proc/{0}/exe'.format(pid))
except OSError:
excludepid.append(pid)
continue
try:
packagename = owners_cache[readlink]
except KeyError:
packagename = __salt__['pkg.owner'](readlink)
if not packagename:
packagename = name
owners_cache[readlink] = packagename
for running_service in running_services:
_check_timeout(start_time, timeout)
if running_service not in restart_services and pid == running_services[
running_service]:
if packagename and packagename not in ignorelist:
restart_services.append(running_service)
name = running_service
if packagename and packagename not in ignorelist:
program = '\t' + six.text_type(
pid) + ' ' + readlink + ' (file: ' + six.text_type(path) + ')'
if packagename not in packages:
packages[packagename] = {
'initscripts': [],
'systemdservice': [],
'processes': [program],
'process_name': name
}
else:
if program not in packages[packagename]['processes']:
packages[packagename]['processes'].append(program)
if not packages and not kernel_restart:
return 'No packages seem to need to be restarted.'
for package in packages:
_check_timeout(start_time, timeout)
cmd = cmd_pkg_query + package
paths = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
while True:
_check_timeout(start_time, timeout)
line = salt.utils.stringutils.to_unicode(paths.stdout.readline())
if not line:
break
pth = line[:-1]
if pth.startswith('/etc/init.d/') and not pth.endswith('.sh'):
packages[package]['initscripts'].append(pth[12:])
if os.path.exists(systemd) and pth.startswith(systemd_folder) and pth.endswith('.service') and \
pth.find('.wants') == -1:
is_oneshot = False
try:
servicefile = salt.utils.files.fopen(pth) # pylint: disable=resource-leakage
except IOError:
continue
sysfold_len = len(systemd_folder)
for line in servicefile.readlines():
line = salt.utils.stringutils.to_unicode(line)
if line.find('Type=oneshot') > 0:
# scripts that does a single job and then exit
is_oneshot = True
continue
servicefile.close()
if not is_oneshot:
packages[package]['systemdservice'].append(
pth[sysfold_len:])
sys.stdout.flush()
paths.stdout.close()
# Alternatively, find init.d script or service that match the process name
for package in packages:
_check_timeout(start_time, timeout)
if not packages[package]['systemdservice'] and not packages[package][
'initscripts']:
service = __salt__['service.available'](
packages[package]['process_name'])
if service:
if os.path.exists('/etc/init.d/' +
packages[package]['process_name']):
packages[package]['initscripts'].append(
packages[package]['process_name'])
else:
packages[package]['systemdservice'].append(
packages[package]['process_name'])
restartable = []
nonrestartable = []
restartinitcommands = []
restartservicecommands = []
for package in packages:
_check_timeout(start_time, timeout)
if packages[package]['initscripts']:
restartable.append(package)
restartinitcommands.extend([
'service ' + s + ' restart'
for s in packages[package]['initscripts']
])
elif packages[package]['systemdservice']:
restartable.append(package)
restartservicecommands.extend([
'systemctl restart ' + s
for s in packages[package]['systemdservice']
])
else:
nonrestartable.append(package)
if packages[package]['process_name'] in restart_services:
restart_services.remove(packages[package]['process_name'])
for restart_service in restart_services:
_check_timeout(start_time, timeout)
restartservicecommands.extend(['systemctl restart ' + restart_service])
ret = _format_output(kernel_restart, packages, verbose, restartable,
nonrestartable, restartservicecommands,
restartinitcommands)
return ret
def compare_link(link_path, dst_path):
return os.path.abspath(os.readlink(link_path)) == os.path.abspath(dst_path)
def bind_or_link(src, target):
if path.islink(src):
symlink(readlink(src), target)
else:
bind_mount(src, target)
def visit(
self,
obj: CWLObjectType,
stagedir: str,
basedir: str,
copy: bool = False,
staged: bool = False,
) -> None:
stagedir = cast(Optional[str], obj.get("dirname")) or stagedir
tgt = os.path.join(
stagedir,
cast(str, obj["basename"]),
)
if obj["location"] in self._pathmap:
return
if obj["class"] == "Directory":
location = cast(str, obj["location"])
if location.startswith("file://"):
resolved = uri_file_path(location)
else:
resolved = location
self._pathmap[location] = MapperEnt(
resolved, tgt, "WritableDirectory" if copy else "Directory", staged
)
if location.startswith("file://"):
staged = False
self.visitlisting(
cast(List[CWLObjectType], obj.get("listing", [])),
tgt,
basedir,
copy=copy,
staged=staged,
)
elif obj["class"] == "File":
path = cast(str, obj["location"])
ab = abspath(path, basedir)
if "contents" in obj and path.startswith("_:"):
self._pathmap[path] = MapperEnt(
obj["contents"],
tgt,
"CreateWritableFile" if copy else "CreateFile",
staged,
)
else:
with SourceLine(
obj,
"location",
ValidationException,
_logger.isEnabledFor(logging.DEBUG),
):
deref = ab
if urllib.parse.urlsplit(deref).scheme in ["http", "https"]:
deref = downloadHttpFile(path)
else:
# Dereference symbolic links
st = os.lstat(deref)
while stat.S_ISLNK(st.st_mode):
rl = os.readlink(deref)
deref = (
rl
if os.path.isabs(rl)
else os.path.join(os.path.dirname(deref), rl)
)
st = os.lstat(deref)
self._pathmap[path] = MapperEnt(
deref, tgt, "WritableFile" if copy else "File", staged
)
self.visitlisting(
cast(List[CWLObjectType], obj.get("secondaryFiles", [])),
stagedir,
basedir,
copy=copy,
staged=staged,
)
def ensure_symlink(path, src, follow, force, timestamps):
b_path = to_bytes(path, errors='surrogate_or_strict')
b_src = to_bytes(src, errors='surrogate_or_strict')
prev_state = get_state(b_path)
mtime = get_timestamp_for_time(timestamps['modification_time'], timestamps['modification_time_format'])
atime = get_timestamp_for_time(timestamps['access_time'], timestamps['access_time_format'])
# source is both the source of a symlink or an informational passing of the src for a template module
# or copy module, even if this module never uses it, it is needed to key off some things
if src is None:
if follow:
# use the current target of the link as the source
src = to_native(os.path.realpath(b_path), errors='strict')
b_src = to_bytes(src, errors='surrogate_or_strict')
if not os.path.islink(b_path) and os.path.isdir(b_path):
relpath = path
else:
b_relpath = os.path.dirname(b_path)
relpath = to_native(b_relpath, errors='strict')
absrc = os.path.join(relpath, src)
b_absrc = to_bytes(absrc, errors='surrogate_or_strict')
if not force and not os.path.exists(b_absrc):
raise AssibleModuleError(results={'msg': 'src file does not exist, use "force=yes" if you'
' really want to create the link: %s' % absrc,
'path': path, 'src': src})
if prev_state == 'directory':
if not force:
raise AssibleModuleError(results={'msg': 'refusing to convert from %s to symlink for %s'
% (prev_state, path),
'path': path})
elif os.listdir(b_path):
# refuse to replace a directory that has files in it
raise AssibleModuleError(results={'msg': 'the directory %s is not empty, refusing to'
' convert it' % path,
'path': path})
elif prev_state in ('file', 'hard') and not force:
raise AssibleModuleError(results={'msg': 'refusing to convert from %s to symlink for %s'
% (prev_state, path),
'path': path})
diff = initial_diff(path, 'link', prev_state)
changed = False
if prev_state in ('hard', 'file', 'directory', 'absent'):
changed = True
elif prev_state == 'link':
b_old_src = os.readlink(b_path)
if b_old_src != b_src:
diff['before']['src'] = to_native(b_old_src, errors='strict')
diff['after']['src'] = src
changed = True
else:
raise AssibleModuleError(results={'msg': 'unexpected position reached', 'dest': path, 'src': src})
if changed and not module.check_mode:
if prev_state != 'absent':
# try to replace atomically
b_tmppath = to_bytes(os.path.sep).join(
[os.path.dirname(b_path), to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))]
)
try:
if prev_state == 'directory':
os.rmdir(b_path)
os.symlink(b_src, b_tmppath)
os.rename(b_tmppath, b_path)
except OSError as e:
if os.path.exists(b_tmppath):
os.unlink(b_tmppath)
raise AssibleModuleError(results={'msg': 'Error while replacing: %s'
% to_native(e, nonstring='simplerepr'),
'path': path})
else:
try:
os.symlink(b_src, b_path)
except OSError as e:
raise AssibleModuleError(results={'msg': 'Error while linking: %s'
% to_native(e, nonstring='simplerepr'),
'path': path})
if module.check_mode and not os.path.exists(b_path):
return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
# Now that we might have created the symlink, get the arguments.
# We need to do it now so we can properly follow the symlink if needed
# because load_file_common_arguments sets 'path' according
# the value of follow and the symlink existence.
file_args = module.load_file_common_arguments(module.params)
# Whenever we create a link to a nonexistent target we know that the nonexistent target
# cannot have any permissions set on it. Skip setting those and emit a warning (the user
# can set follow=False to remove the warning)
if follow and os.path.islink(b_path) and not os.path.exists(file_args['path']):
module.warn('Cannot set fs attributes on a non-existent symlink target. follow should be'
' set to False to avoid this.')
else:
changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False)
changed |= update_timestamp_for_file(file_args['path'], mtime, atime, diff)
return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
def is_broken_link(path):
"""Returns True if the path given as is a broken symlink"""
path = os.readlink(path)
return not os.path.exists(path)
def move_repos_into_place(src, dst):
# Find all the stuff in src/*, move it to a freshly-created
# directory beside dst, then play some games with symlinks so that
# dst is a name the new stuff and dst+".old" names the previous
# one. This feels like a lot of hooey for something so trivial.
# First, make a crispy fresh new directory to put the stuff in.
i = 0
while True:
date_suffix = time.strftime("%Y-%m-%d")
dname = dst + ".%s.%d" % (date_suffix, i)
try:
os.mkdir(dname)
break
except OSError:
exc = sys.exc_value
if exc.errno == errno.EEXIST:
pass
else:
raise exc
i = i + 1
# Put the stuff in our new directory.
for r in os.listdir(src):
sysassert(["cp", "-rv", src + "/" + r, dname])
# Make a symlink to the new directory; the symlink will be renamed
# to dst shortly.
i = 0
while True:
tmpnam = dst + ".TMP.%d" % i
try:
os.symlink(dname, tmpnam)
break
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
if exc.errno == errno.EEXIST:
pass
else:
raise exc
i = i + 1
# Make a symlink to the old directory; this symlink will be
# renamed shortly, too.
oldnam = None
if os.path.exists(dst):
i = 0
while True:
oldnam = dst + ".old.%d" % i
try:
os.symlink(os.readlink(dst), oldnam)
break
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
if exc.errno == errno.EEXIST:
pass
else:
raise exc
os.rename(tmpnam, dst)
if oldnam:
os.rename(oldnam, dst + ".old")
