def sftp_walk(self, remotepath):
# Kindof a stripped down version of os.walk, implemented for
# sftp. Tried running it flat without the yields, but it really
# chokes on big directories.
path = remotepath
files = []
folders = []
for f in self.sftp.listdir_attr(remotepath):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else:
files.append(f.filename)
#print (path,folders,files)
yield path, folders, files
for folder in folders:
new_path = os.path.join(remotepath, folder)
for x in self.sftp_walk(new_path):
yield x
def _sftp_walk(self, remotepath):
"""
https://techtalkontv.wordpress.com/2016/11/05/python-pramiko-sftp-copydownload-all-files-in-a-folder-recursively-from-remote-server/
"""
path = remotepath
files = []
folders = []
for f in self._sftp.listdir_attr(remotepath):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else:
files.append(f.filename)
if files:
yield path, files
for folder in folders:
new_path = os.path.join(remotepath, folder)
for x in self._sftp_walk(new_path):
yield x
def walk_files(self, directory):
from stat import S_ISLNK, S_ISDIR, S_ISREG
self._sftp_connect()
for entry in self._sftp.listdir_attr(directory):
path = posixpath.join(directory, entry.filename)
if S_ISLNK(entry.st_mode):
path = self._sftp.readlink(directory)
entry = self._sftp.stat(path)
if S_ISDIR(entry.st_mode):
for inner_path in self.walk_files(path):
yield inner_path
elif S_ISREG(entry.st_mode):
yield path
def check_init_script(configurator, question, answer):
try:
from stat import ST_MODE, S_ISDIR
if not os.path.lexists(answer):
os.makedirs(answer)
elif not S_ISDIR(os.stat(answer)[ST_MODE]):
raise ValidationError(
"Invalid path (%s): path must point to a directory not a file."
% answer)
with open(os.path.join(answer, 'plume'), 'w'):
answer = os.path.realpath(answer)
configurator.target_directory = answer
return answer
except OSError as e:
raise ValidationError(e)
except IOError as e:
raise ValidationError(e)
def make_writable(path):
try:
mode = stat(path).st_mode
if S_ISDIR(mode):
chmod(path, S_IMODE(mode) | S_IWRITE | S_IEXEC)
elif S_ISREG(mode) or S_ISLNK(mode):
chmod(path, S_IMODE(mode) | S_IWRITE)
else:
log.debug("path cannot be made writable: %s", path)
except Exception as e:
eno = getattr(e, 'errno', None)
if eno in (ENOENT, ):
raise
elif eno in (EACCES, EPERM):
log.debug("tried make writable but failed: %s\n%r", path, e)
else:
log.error("Error making path writable: %s\n%r", path, e)
raise
def _walk(self, remote_path):
path=remote_path
files=[]
folders=[]
for f in self.__sftp.listdir_attr(remote_path):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else:
files.append(f.filename)
if files:
yield path, files
for folder in folders:
new_path = os.path.join(remote_path, folder)
for x in self._walk(new_path):
yield x
def add_files(self, basePath, path, app):
if self.debug:
print(("add files " + path))
for f in os.listdir(path):
pathname = os.path.join(path, f)
mode = os.stat(pathname).st_mode
if S_ISREG(mode):
filename = os.path.join(os.path.relpath(path, basePath), f)
if self.debug:
print(("add " + pathname))
print(("name " + filename))
fileBuffer = open(pathname, 'rb').read()
appFile = app.file.add()
appFile.name = filename
appFile.encoding = CLEARTEXT
appFile.blob = fileBuffer
elif S_ISDIR(mode):
self.add_files(basePath, pathname, app)
def walk(self):
#logger.debug('self path=%s, called path=%s', self.path, path)
#npath = abspath_re.sub('', path)
dirs = list()
files = list()
#npath = self.path_join(self.path)
npath = self.path
logger.debug('walking path %s', npath)
for f in self.conn.listdir_attr(npath):
if S_ISDIR(f.st_mode):
dirs.append(f.filename)
else:
files.append(f.filename)
yield self.path, dirs, files
for d in dirs:
_path = os.path.join(path, d)
for f in self.walk(_path):
yield f
def check_access_rights(top):
for f in os.listdir(top):
pathname = os.path.join(top, f)
mode = os.stat(pathname).st_mode
if S_ISDIR(mode):
# directory, recurse into it
check_access_rights(pathname)
elif S_ISREG(mode):
# file, check permissions
permissions = oct(os.stat(pathname)[ST_MODE])
if PY3:
self.assertEqual("0o100775", permissions)
else:
self.assertEqual("0100775", permissions)
else:
# unknown file type
pass
def copystat(src, dst):
"""Copy stat bits from src to dst
This should be used when shutil.copystat would be used on directories
on win32 because win32 does not support utime() for directories.
According to the official docs written by Microsoft, it returns ENOACCES if the
supplied filename is a directory. Looks like a trainee implemented the function.
"""
if sys.platform == 'win32' and S_ISDIR(os.stat(dst)[ST_MODE]):
if os.name == 'nt':
st = os.stat(src)
mode = S_IMODE(st[ST_MODE])
if hasattr(os, 'chmod'):
os.chmod(dst, mode) # KEEP THIS ONE!
#else: pass # we are on Win9x,ME - no chmod here
else:
shutil.copystat(src, dst)
def generateSolution(team, folder):
print("Getting solutions for...", folder, team)
solutions = os.listdir(folder)
result = []
for solution in solutions:
if S_ISDIR(os.stat("%s/%s" % (folder, solution)).st_mode):
solution_page = open(
"%s/%s_%s.md" % (solutionsFolder, team, solution), 'w')
solution_page.write("---\n")
solution_page.write("layout: solution\n")
solution_page.write("team: %s\n" % (team, ))
solution_page.write("sol: %s\n" % (solution, ))
indexPreview = htmlPageIn("%s/%s" % (folder, solution))
if indexPreview:
solution_page.write("demo: %s\n" % (indexPreview))
solution_page.write("---\n")
readmeContent = getReadmeContent("%s/%s" % (folder, solution))
# copy folder
try:
shutil.copytree("%s/%s" % (folder, solution),
"%s/%s_%s" % (solutionsFolder, team, solution))
except:
pass
if readmeContent:
solution_page.write(readmeContent)
else:
solution_page.write(solution + "\n")
solution_page.close()
result.append("%s_%s" % (team, solution))
return result
def view_results():
dirpath = "output"
entries = (
co2wui_fpath(dirpath, fn) for fn in os.listdir(co2wui_fpath(dirpath))
)
entries = ((os.stat(path), path) for path in entries)
entries = (
(stat[ST_CTIME], path) for stat, path in entries if S_ISDIR(stat[ST_MODE])
)
results = []
for cdate, path in sorted(entries):
dirname = osp.basename(path)
output_files = [f.name for f in listdir_outputs("output", dirname)]
summary = get_summary(dirname)
outcome = "KO" if (summary is None or not summary or len(summary[0].keys()) <= 2) else "OK"
results.append(
{
"datetime": time.ctime(cdate),
"name": dirname,
"files": output_files,
"outcome": outcome,
}
)
running = False
if ("active_pid" in session) and (session["active_pid"] is not None):
running = True
return render_template(
"layout.html",
action="view_results",
data={
"breadcrumb": ["Co2mpas", _("View results")],
"props": {
"active": {"run": "active", "sync": "", "doc": "", "expert": ""}
},
"results": reversed(results),
"running": running,
"texts": co2wui_texts,
"globals": co2wui_globals,
},
)
def walktree(self,
remotepath,
fcallback,
dcallback,
ucallback,
recurse=True):
'''recursively descend, depth first, the directory tree rooted at
remotepath, calling discreet callback functions for each regular file,
directory and unknown file type.
:param str remotepath:
root of remote directory to descend, use '.' to start at
:attr:`.pwd`
:param callable fcallback:
callback function to invoke for a regular file.
(form: ``func(str)``)
:param callable dcallback:
callback function to invoke for a directory. (form: ``func(str)``)
:param callable ucallback:
callback function to invoke for an unknown file type.
(form: ``func(str)``)
:param bool recurse: *Default: True* - should it recurse
:returns: None
:raises:
'''
self._sftp_connect()
for entry in self.listdir(remotepath):
pathname = posixpath.join(remotepath, entry)
mode = self._sftp.stat(pathname).st_mode
if S_ISDIR(mode):
# It's a directory, call the dcallback function
dcallback(pathname)
if recurse:
# now, recurse into it
self.walktree(pathname, fcallback, dcallback, ucallback)
elif S_ISREG(mode):
# It's a file, call the fcallback function
fcallback(pathname)
else:
# Unknown file type
ucallback(pathname)
def local_file_create(host_obj, directory_path, dir_node, filename, db):
# type: (HostController, str, FileNode, str, SimpleDB) -> [(int, str)]
# where (int, str): (FILE_CREATE, full_path)
_log = get_mylog()
_log.debug('Adding {} to filenode for the directory node {}'.format(
filename, dir_node.name))
# print '\t\tAdding',filename,'to filenode for',dir_node.name
file_pathname = os.path.join(directory_path, filename)
file_stat = os.stat(file_pathname)
file_modified = file_stat.st_mtime
file_created = file_stat.st_ctime
mode = file_stat.st_mode
filenode = FileNode()
db.session.add(filenode)
filenode.name = filename
filenode.created_on = datetime.utcfromtimestamp(file_created)
filenode.last_modified = datetime.utcfromtimestamp(file_modified)
# DO NOT try and set the new node's `cloud` setting. If that's set, then
# we'll treat that node as a child of the cloud itself - as a child of
# the host.models.Cloud. That's not what we want.
# Appending the new filenode to the dir_node WILL work, because that will
# append it to either the Cloud (if this is a child of the root)
# or the FileNode correctly.
dir_node.children.append(filenode)
db.session.commit()
# Theoretically, the private data should have already been loaded or
# created. I guess the first time that the private data is generated, this
# happens. Might actually cause the host to write the .nebs, then read it
# right back in.
# Not really the worst.
if host_obj.is_private_data_file(file_pathname, filenode.cloud):
host_obj.reload_private_data(filenode.cloud)
updates = [(FILE_CREATE, file_pathname)]
if S_ISDIR(mode): # It's a directory, recurse into it
# use the directory's node as the new root.
rec_updates = recursive_local_modifications_check(
host_obj, file_pathname, filenode, db)
updates.extend(rec_updates)
db.session.commit()
return updates
def __fetch_info(self):
# lstat() does not follow sym links.
try:
info = os.lstat(self.__abs_path)
is_symlink = True if S_ISLNK(info.st_mode) != 0 else False
if Platform.isWindows:
is_symlink = jaraco.windows.filesystem.is_reparse_point(
self.__abs_path)
dst_info = info
if is_symlink:
dst_info = os.stat(self.__abs_path)
if info is not None:
self.__dict.update({
'exists': os.path.exists(self.__abs_path),
'size': info.st_size,
'r_size': 0,
'gid': info.st_gid,
'uid': info.st_uid,
'posix_perms': S_IMODE(info.st_mode),
'dir': S_ISDIR(dst_info.st_mode),
'create_date': info.st_ctime,
'mod_date': info.st_mtime,
'sym_link': is_symlink
})
# if its a directory - collecting the size makes no sense, and in fact confuses the markup of checked/partially
# modified scans - because changes to the posix perms can modify it? at least unit tests show that - anyway, we've
# got no present use for directories with a size attribute.
if self.is_dir:
self.__dict.update({'size': 0})
else:
# if its Mac, it might have the r_size and others
if Platform.isMac and hasattr(info, 'r_size'):
self.__dict.update({'r_size': info.r_size})
else:
self.setDefaults()
except OSError, err:
# lookup on fs failed, fill in a blank directory
logger.warn(
"stat() on path '{0}' failed, using empty/blank defaults, error: {1}"
.format(self.__abs_path, err))
self.setDefaults()
def _walk(
self, directory: str
) -> Generator[Tuple[str, List[Any], List[str]], None, None]:
dirs = []
nondirs = []
for entry in self.conn.listdir_attr(directory):
if S_ISDIR(entry.st_mode or 0):
dirs.append(entry.filename)
else:
nondirs.append(str(Path(directory).joinpath(entry.filename)))
yield directory, dirs, nondirs
# Recurse into sub-directories
for dirname in dirs:
new_path = str(Path(directory).joinpath(dirname))
yield from self._walk(new_path)
def sftp_tree(self, path='.'):
main_tree = list()
file_list = self.sftp.listdir_attr(path)
for idx, entry in enumerate(file_list):
mode = entry.st_mode
filename = entry.filename
if idx == len(file_list) - 1:
start_str = '└─ '
sub_start_str = ' '
else:
start_str = '├─ '
sub_start_str = '│ '
main_tree.append('{}{}'.format(start_str, filename))
if S_ISDIR(mode):
sub_path = filename if path == '.' else '{}/{}'.format(
path, filename)
tree = self.sftp_sub_tree(sub_path, sub_start_str, 1)
main_tree.extend(tree)
return main_tree
def _tree_chunks(repo, tree, startofs):
"Tree should be a sequence of (name, mode, hash) as per tree_decode()."
assert (startofs >= 0)
# name is the chunk's hex offset in the original file
for mode, name, oid in _skip_chunks_before_offset(tree, startofs):
ofs = int(name, 16)
skipmore = startofs - ofs
if skipmore < 0:
skipmore = 0
it = repo.cat(hexlify(oid))
_, obj_t, size = next(it)
data = b''.join(it)
if S_ISDIR(mode):
assert obj_t == b'tree'
for b in _tree_chunks(repo, tree_decode(data), skipmore):
yield b
else:
assert obj_t == b'blob'
yield data[skipmore:]
def walktree(top):
'''parcourt récursivement le dossier et renvoit
une liste avec tous les fichiers normaux'''
liste = []
for f in os.listdir(top):
pathname = os.path.join(top, f)
mode = os.stat(pathname).st_mode # give a code that
# says whether it's a directory or a file
if S_ISDIR(mode):
# It's a directory, recurse into it
result = walktree(pathname)
for _ in result:
liste.append(_)
elif S_ISREG(mode):
liste.append(pathname.split('/')[-1])
else:
# Unknown file type, print a message
print('Skipping %s' % pathname)
return liste
def info(self, path, **kwargs):
wpath = _as_unc_path(self.host, path)
stats = smbclient.stat(wpath, **kwargs)
if S_ISDIR(stats.st_mode):
stype = "directory"
elif S_ISLNK(stats.st_mode):
stype = "link"
else:
stype = "file"
res = {
"name": path + "/" if stype == "directory" else path,
"size": stats.st_size,
"type": stype,
"uid": stats.st_uid,
"gid": stats.st_gid,
"time": stats.st_atime,
"mtime": stats.st_mtime,
}
return res
def sftp_walk(sftp, remotePath):
#建立一个sftp客户端对象，通过ssh transport操作远程文件
files = []
folders = []
# Copy a remote file (remotePath) from the SFTP server to the local host
try:
for f in sftp.listdir_attr(remotePath):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else:
files.append(f.filename)
yield remotePath, folders, files
for folder in folders:
new_path = os.path.join(remotePath, folder)
for x in sftp_walk(sftp, new_path):
yield x
except (Exception) as e:
print(e)
def __get_all_files_in_remote_dir(sftp, remote_dir):
# 保存所有文件的列表
all_files = list()
# 去掉路径字符串最后的字符'/'，如果有的话
if remote_dir[-1] == '/':
remote_dir = remote_dir[0:-1]
# 获取当前指定目录下的所有目录及文件，包含属性值
files = sftp.listdir_attr(remote_dir)
for x in files:
# remote_dir目录中每一个文件或目录的完整路径
filename = remote_dir + '/' + x.filename
# 如果是目录，则递归处理该目录，这里用到了stat库中的S_ISDIR方法，与linux中的宏的名字完全一致
if S_ISDIR(x.st_mode):
all_files.extend(__get_all_files_in_remote_dir(sftp, filename))
else:
all_files.append(filename)
return all_files
def get_ftype_and_perm(mode):
""" Returns a tuple of whether the file is: file(regular file)/dir/slink
/char. spec/block spec/pipe/socket and the permission bits of the file.
If it does not match any of the cases below (it will return "unknown" twice)"""
if S_ISREG(mode):
return "file", S_IMODE(mode)
if S_ISDIR(mode):
return "dir", S_IMODE(mode)
if S_ISLNK(mode):
return "slink", S_IMODE(mode)
if S_ISCHR(mode):
return "character special", S_IMODE(mode)
if S_ISBLK(mode):
return "block special", S_IMODE(mode)
if S_ISFIFO(mode):
return "pipe", S_IMODE(mode)
if S_ISSOCK(mode):
return "socket", S_IMODE(mode)
return "unknown", "unknown"
def sftp_walk(remote_path):
"""Walks the remote path.
Parameters: dir path on remote
Returns: path, sub-dir, list of files
"""
path = remote_path
files = []
folders = []
for file in SFTP.listdir_attr(remote_path):
if S_ISDIR(file.st_mode):
folders.append(file.filename)
else:
files.append(file.filename)
if files:
yield path, files
for folder in folders:
new_path = os.path.join(remote_path, folder)
for n_file in sftp_walk(new_path):
yield n_file
def _walk_dir(file_path, file_list=[]):
for f in os.listdir(file_path):
try:
path_name = os.path.join(file_path, f)
except UnicodeDecodeError:
continue
islink = os.path.islink(path_name)
is_exists = os.path.exists(path_name)
if not islink and is_exists:
mode = os.stat(path_name).st_mode
if S_ISDIR(mode):
_walk_dir(path_name, file_list)
elif S_ISREG(mode):
size = os.stat(path_name).st_size
file_list.append(size)
else:
#pass
logging.info('invalid path: %s' % path_name)
def walk(self, remotepath):
"""Taken from https://gist.github.com/johnfink8/2190472
A stripped down version of os.walk implemented for sftp which
yields tuples: (path, folders, files)
"""
path = remotepath
files = []
folders = []
for f in self.listdir_attr(remotepath):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else: # non-dir
files.append(f.filename)
yield path, folders, files
for folder in folders:
new_path = os.path.join(remotepath, folder)
for x in self.walk(new_path):
yield x
def sftp(files_from, files_to, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try:
t = paramiko.Transport(
(settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t)
if mode == 'export':
try:
sftp.chdir(files_to)
except IOError:
raise CommandError(
'SFTP destination path does not exist: {}'.
format(files_to))
for filename in os.listdir(files_from):
sftp.put(files_from + '/' + filename, filename)
if deleteAfterCopy:
os.remove(os.path.join(files_from, filename))
else:
try:
sftp.chdir(files_from)
except IOError:
raise CommandError(
'SFTP source path does not exist: {}'.format(
files_from))
for filename in sftp.listdir('.'):
# skip subdirectories
if not S_ISDIR(sftp.stat(filename).st_mode):
sftp.get(filename, files_to + '/' + filename)
# delete files from sftp server once they are
# successfully pulled off:
if deleteAfterCopy:
sftp.remove(filename)
except:
dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed',
alert_type='error')
raise
finally:
sftp.close()
t.close()
def stat(_endpoint, filepath, _userid):
'''Stat a file and returns (size, mtime) as well as other extended info. This method assumes that the given userid has access.'''
try:
tstart = time.time()
statInfo = os.stat(_getfilepath(filepath))
tend = time.time()
log.info('msg="Invoked stat" filepath="%s" elapsedTimems="%.1f"' %
(_getfilepath(filepath), (tend - tstart) * 1000))
if S_ISDIR(statInfo.st_mode):
raise IOError('Is a directory')
return {
'inode': str(statInfo.st_ino),
'filepath': filepath,
'userid': str(statInfo.st_uid) + ':' + str(statInfo.st_gid),
'size': statInfo.st_size,
'mtime': statInfo.st_mtime
}
except (FileNotFoundError, PermissionError) as e:
raise IOError(e)
def walk(self, root):
''' Упрощенный аналог os.walk.
Для каждой итерации возвращает кортеж - (root, folders, files)
Для всех folders, files возвращает полные пути на базе root '''
files = []
folders = []
for f in self.listdir_attr(root):
if S_ISDIR(f.st_mode):
folders.append(tools.join_unix(root, f.filename))
else:
files.append(tools.join_unix(root, f.filename))
yield root, folders, files
for folder in folders:
new_root = tools.join_unix(root, folder).replace('\\', '/')
for x in self.walk(new_root):
yield x
def BuildTestList():
"""Returns a list of all available system tests.
Returns:
A list of test_names, constructed from the set of directory names
in the current directory. All directories containing the substring
'test' are included in the returned set.
"""
testing_directory = os.listdir('.')
test_list = []
for test in testing_directory:
mode = os.stat(test)[ST_MODE]
if ('test' in test and test != 'unittest_data'
and test != 'bitmap_test' and test != 'conditioner_test_data'
and S_ISDIR(mode)):
test_list += [test]
return test_list
