# Fichier d'exemple du livre "Developpement Systeme sous Linux" # (C) 2015-2019 - Christophe BLAESS <*****@*****.**> # https://www.blaess.fr/christophe/ # ------------------------------------------------------------------ from __future__ import print_function import os import stat import sys sys.argv.pop(0) for f in sys.argv: try: s = os.stat(f) if (stat.S_ISBLK(s.st_mode)): print(f, "bloc") if (stat.S_ISCHR(s.st_mode)): print(f, "caractere") if (stat.S_ISDIR(s.st_mode)): print(f, "repertoire") if (stat.S_ISFIFO(s.st_mode)): print(f, "fifo") if (stat.S_ISLNK(s.st_mode)): print(f, "lien") if (stat.S_ISREG(s.st_mode)): print(f, "fichier") if (stat.S_ISSOCK(s.st_mode)): print(f, "socket") except: print(f, "erreur")
"""Utilities for comparing files and directories.
def serve_file(path, content_type=None, disposition=None, name=None): """Set status, headers, and body in order to serve the given file. The Content-Type header will be set to the content_type arg, if provided. If not provided, the Content-Type will be guessed by the file extension of the 'path' argument. If disposition is not None, the Content-Disposition header will be set to "<disposition>; filename=<name>". If name is None, it will be set to the basename of path. If disposition is None, no Content-Disposition header will be written. """ response = cherrypy.response # If path is relative, users should fix it by making path absolute. # That is, CherryPy should not guess where the application root is. # It certainly should *not* use cwd (since CP may be invoked from a # variety of paths). If using tools.static, you can make your relative # paths become absolute by supplying a value for "tools.static.root". if not os.path.isabs(path): raise ValueError("'%s' is not an absolute path." % path) try: st = os.stat(path) except OSError: raise cherrypy.NotFound() # Check if path is a directory. if stat.S_ISDIR(st.st_mode): # Let the caller deal with it as they like. raise cherrypy.NotFound() # Set the Last-Modified response header, so that # modified-since validation code can work. response.headers['Last-Modified'] = http.HTTPDate(st.st_mtime) cptools.validate_since() if content_type is None: # Set content-type based on filename extension ext = "" i = path.rfind('.') if i != -1: ext = path[i:].lower() content_type = mimetypes.types_map.get(ext, "text/plain") response.headers['Content-Type'] = content_type if disposition is not None: if name is None: name = os.path.basename(path) cd = '%s; filename="%s"' % (disposition, name) response.headers["Content-Disposition"] = cd # Set Content-Length and use an iterable (file object) # this way CP won't load the whole file in memory c_len = st.st_size bodyfile = open(path, 'rb') # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code if cherrypy.request.protocol >= (1, 1): response.headers["Accept-Ranges"] = "bytes" r = http.get_ranges(cherrypy.request.headers.get('Range'), c_len) if r == []: response.headers['Content-Range'] = "bytes */%s" % c_len message = "Invalid Range (first-byte-pos greater than Content-Length)" raise cherrypy.HTTPError(416, message) if r: if len(r) == 1: # Return a single-part response. start, stop = r[0] if stop > c_len: stop = c_len r_len = stop - start response.status = "206 Partial Content" response.headers['Content-Range'] = ("bytes %s-%s/%s" % (start, stop - 1, c_len)) response.headers['Content-Length'] = r_len bodyfile.seek(start) response.body = file_generator_limited(bodyfile, r_len) else: # Return a multipart/byteranges response. response.status = "206 Partial Content" import mimetools boundary = mimetools.choose_boundary() ct = "multipart/byteranges; boundary=%s" % boundary response.headers['Content-Type'] = ct if response.headers.has_key("Content-Length"): # Delete Content-Length header so finalize() recalcs it. del response.headers["Content-Length"] def file_ranges(): # Apache compatibility: yield "\r\n" for start, stop in r: yield "--" + boundary yield "\r\nContent-type: %s" % content_type yield ("\r\nContent-range: bytes %s-%s/%s\r\n\r\n" % (start, stop - 1, c_len)) bodyfile.seek(start) for chunk in file_generator_limited( bodyfile, stop - start): yield chunk yield "\r\n" # Final boundary yield "--" + boundary + "--" # Apache compatibility: yield "\r\n" response.body = file_ranges() else: response.headers['Content-Length'] = c_len response.body = bodyfile else: response.headers['Content-Length'] = c_len response.body = bodyfile return response.body
# Author : Not sure where I got this from # Created : 28th November 2011 # Last Modified : # Version : 1.0 # Modifications : # Description : Show file information for a given file # get file information using os.stat() # tested with Python24 vegsaeat 25sep2006 import os import stat # index constants for os.stat() import time # pick a file you have ... file_name = raw_input("Enter a file name: ") file_stats = os.stat(file_name) # create a dictionary to hold file info file_info = { 'fname': file_name, 'fsize': file_stats [stat.ST_SIZE], 'f_lm': time.strftime("%d/%m/%Y %I:%M:%S %p",time.localtime(file_stats[stat.ST_MTIME])), 'f_la': time.strftime("%d/%m/%Y %I:%M:%S %p",time.localtime(file_stats[stat.ST_ATIME])), 'f_ct': time.strftime("%d/%m/%Y %I:%M:%S %p",time.localtime(file_stats[stat.ST_CTIME])) } print print "file name = %(fname)s" % file_info print "file size = %(fsize)s bytes" % file_info print "last modified = %(f_lm)s" % file_info print "last accessed = %(f_la)s" % file_info print "creation time = %(f_ct)s" % file_info
def isdir(self, s): """Return true if the pathname refers to an existing directory.""" st = self.callstat(s) if not st: return False return statmod.S_ISDIR(st.st_mode)
def ensure_dirs(path, gid=-1, uid=-1, mode=0o777, minimal=True): """ensure dirs exist, creating as needed with (optional) gid, uid, and mode. Be forewarned- if mode is specified to a mode that blocks the euid from accessing the dir, this code *will* try to create the dir. :param path: directory to ensure exists on disk :param gid: a valid GID to set any created directories to :param uid: a valid UID to set any created directories to :param mode: permissions to set any created directories to :param minimal: boolean controlling whether or not the specified mode must be enforced, or is the minimal permissions necessary. For example, if mode=0755, minimal=True, and a directory exists with mode 0707, this will restore the missing group perms resulting in 757. :return: True if the directory could be created/ensured to have those permissions, False if not. """ try: st = os.stat(path) except OSError: base = os.path.sep try: um = os.umask(0) # if the dir perms would lack +wx, we have to force it force_temp_perms = ((mode & 0o300) != 0o300) resets = [] apath = normpath(os.path.abspath(path)) sticky_parent = False for directory in apath.split(os.path.sep): base = join(base, directory) try: st = os.stat(base) if not stat.S_ISDIR(st.st_mode): # one of the path components isn't a dir return False # if it's a subdir, we need +wx at least if apath != base: sticky_parent = (st.st_mode & stat.S_ISGID) except OSError: # nothing exists. try: if force_temp_perms: if not _safe_mkdir(base, 0o700): return False resets.append((base, mode)) else: if not _safe_mkdir(base, mode): return False if base == apath and sticky_parent: resets.append((base, mode)) if gid != -1 or uid != -1: os.chown(base, uid, gid) except OSError: return False try: for base, m in reversed(resets): os.chmod(base, m) if gid != -1 or uid != -1: os.chown(base, uid, gid) except OSError: return False finally: os.umask(um) return True else: if not os.path.isdir(path): # don't change perms for existing paths that aren't dirs return False try: if ((gid != -1 and gid != st.st_gid) or (uid != -1 and uid != st.st_uid)): os.chown(path, uid, gid) if minimal: if mode != (st.st_mode & mode): os.chmod(path, st.st_mode | mode) elif mode != (st.st_mode & 0o7777): os.chmod(path, mode) except OSError: return False return True
def is_dir(self, path): try: return stat.S_ISDIR(self.sftp.stat(str(path)).st_mode) except FileNotFoundError: return False
def icon(stat_entry: jobs_pb2.StatEntry) -> Text: if stat.S_ISDIR(stat_entry.st_mode): return '📂' elif _is_symlink(stat_entry): return '🔗' return '📄'
def info(self, path): st = self.sftp.stat(path) return { "size": st.st_size, "type": "dir" if stat.S_ISDIR(st.st_mode) else "file", }
hashvalid = already_saved(ent) wasmissing = ent.sha_missing() oldsize = ent.size if opt.verbose: if not exists: status = 'D' elif not hashvalid: if ent.sha == index.EMPTY_SHA: status = 'A' else: status = 'M' else: status = ' ' if opt.verbose >= 2: log('%s %-70s\n' % (status, path_msg(ent.name))) elif not stat.S_ISDIR(ent.mode) and lastdir != dir: if not lastdir.startswith(dir): log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b'')))) lastdir = dir if opt.progress: progress_report(0) fcount += 1 if not exists: continue if opt.smaller and ent.size >= opt.smaller: if exists and not hashvalid: if opt.verbose: log('skipping large file "%s"\n' % path_msg(ent.name)) lastskip_name = ent.name
def find_hardlink_target(hlink_db, ent): if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1: link_paths = hlink_db.node_paths(ent.dev, ent.ino) if link_paths: return link_paths[0]
def main(argv=None): """ Usage is: submit.py [id=<chargecode>] <url> <commandline> Run from the working dir of the job, which must contain (in addition to the job files) a file named scheduler.conf with scheduler properties for the job. id=<chargecode> if present gives the project to charge the job to. Url is url of the submitting website including taskid parameter. Returns 0 with "jobid=<jobid>" on stdout if job submitted ok Returns 1 with multiline error message on stdout if error. Returns 2 for the specific error of queue limit exceeded. """ if argv is None: argv = sys.argv print "argv (%s)" % (argv, ) debugfile = open('nsgdebug', "w") debugfile.write("argv (%s)\n" % (argv, )) splits = argv[1].split("=", 1) if (len(splits) == 2 and splits[0] == "id"): account = splits[1] url = argv[2] cmdline = argv[3:] elif (len(splits) == 1 and splits[0] == "--account"): account = argv[2] url = argv[4] cmdline = argv[6:] else: print "failed to account information!" sys.exit(1) job_properties = lib.getProperties('_JOBINFO.TXT') print " job_properties['Tool'] (%s)" % job_properties['Tool'] tooltype = lib.getToolType(job_properties['Tool']) print "tool type is %s" % tooltype # HTCondor does not have the notion of queues and/or partition. This # is simply a place holder value with no meaning. queue = "condor" # The inputfile archive must currently be unzipped locally prior to # job submission. Otherwise, a 'failed to find modeldir!' error will # be thrown. However, the inputfile archive must be transferred to a # remote OSG site and unzipped locally at the site to run job. As # such, it may make more sense to restructure when this 'modeldir!' # error is thrown. For example, perhaps at job runtime, not prior to # submission. print "New input file used" os.system('unzip -n inputfile > /dev/null') scheduler_properties = lib.getProperties("scheduler.conf") print scheduler_properties scheduler_info = lib.schedulerInfo(scheduler_properties, tooltype) print scheduler_info hocfile = scheduler_properties['fname'] if ('outfilename' in scheduler_properties): outfilename = scheduler_properties['outfilename'] else: outfilename = "output" runtime = int(scheduler_info["runtime"]) useLocalDisk = False cwdir = os.getcwd() # assume that the modeldir is the only dir in the working dir file_list = os.listdir('.') modeldir = None for filename in file_list: stat_tuple = os.lstat(filename) if stat.S_ISDIR(stat_tuple[stat.ST_MODE]) and filename != '__MACOSX': modeldir = filename if modeldir == None: print "failed to find modeldir!" sys.exit(1) if scheduler_properties.has_key('subdirname'): subdirname = scheduler_properties['subdirname'] else: subdirname = None if (tooltype != "singularityparametersearch"): if subdirname != None: modeldir = subdirname # What is this section used for? Compiling models/libs on the fly # before the job is run? Mostly for neuron only? if (tooltype == "neuron77" or (scheduler_properties.has_key('nrnivmodloption') and scheduler_properties['nrnivmodloption'] == "1")): nrnivmodl = '/projects/ps-nsg/home/nsguser/applications_centos7/neuron7.7/nrn-7.7/x86_64/bin/nrnivmodl' print "running makelib.sh in %s %s" % (modeldir, nrnivmodl) os.system( "/projects/ps-nsg/home/nsguser/ngbw/contrib/scripts/makelib.sh %s %s" % (modeldir, nrnivmodl)) else: print "tooltype (%s) not neuron, not running makelib.sh" % tooltype fullpathmodeldir = None # # # Create batch_command.cmdline file. # # # rfile = open(lib.cmdfile, "w") rfile.write("#!/usr/bin/env bash\n") # Again, for OSG, you will need to transfer inputfile to remote # system and unzip it locally on remote machine before starting # standard 'tooltype' commands. rfile.write("unzip -n inputfile > /dev/null\n") # # Why does each nsg job allow a custom directory name? Would it # not make sense to standardize name based on say a gateway job id? rfile.write("cd job_work_dir\n") # Record job's start date and time and then report job start to the # gateway frontend. rfile.write("echo Job starting at `date` > ../start.txt\n") rfile.writelines(("curl %s\&status=START" % (url), "\n")) rfile.writelines(("export CIPRES_THREADSPP=%d" % (int(scheduler_info["threads_per_process"])), "\n")) rfile.writelines( ("export CIPRES_NP=%d" % (int(scheduler_info["ppn"]) * int(scheduler_info["nodes"])), "\n")) if (tooltype == "bash"): rfile.writelines(("module purge\n", )) rfile.writelines(("module list\n", )) rfile.writelines(("printenv\n", )) rfile.writelines(("lscpu\n", )) fname, interp, args = cmdline[0].split(" ", 2) rfile.writelines(("time -p bash %s %s" % (hocfile, args), "\n")) elif (tooltype == "freesurf"): rfile.writelines(("module purge\n", )) rfile.writelines(("module list\n", )) rfile.writelines(("printenv\n")) rfile.writelines(("lscpu\n", )) rfile.writelines( (". /opt/setup.sh\n", )) # Setup osgvo-neuroimaging environment rfile.writelines(('export FS_LICENSE="${PWD}/license.txt"\n', )) rfile.writelines(('export SUBJECTS_DIR="${PWD}/subjects"\n', )) rfile.writelines(('mkdir -p "${SUBJECTS_DIR}"\n', )) fname, interp, args = cmdline[0].split(" ", 2) rfile.writelines(("time -p recon-all %s" % (args), "\n")) elif (tooltype == "python"): if (scheduler_properties['pythonversion'] == "2.7.15"): rfile.writelines(("module purge\n", )) rfile.writelines(("module load python/2.7.15\n", )) rfile.writelines(("module load py-argparse/1.4.0-py2.7\n", )) rfile.writelines(("module load py-asn1crypto/0.22.0-py2.7\n", )) rfile.writelines(("module load py-bottleneck/1.2.1-py2.7\n", )) rfile.writelines(("module load py-cffi/1.11.5-py2.7\n", )) rfile.writelines(("module load py-cryptography/2.3.1-py2.7\n", )) rfile.writelines(("module load py-cycler/0.10.0-py2.7\n", )) rfile.writelines(("module load py-cython/0.29-py2.7\n", )) rfile.writelines(("module load py-dateutil/2.5.2-py2.7\n", )) rfile.writelines(("module load py-enum34/1.1.6-py2.7\n", )) rfile.writelines(("module load py-functools32/3.2.3-2-py2.7\n", )) rfile.writelines(("module load py-h5py/2.8.0-py2.7\n", )) rfile.writelines(("module load py-idna/2.5-py2.7\n", )) rfile.writelines(("module load py-ipaddress/1.0.18-py2.7\n", )) rfile.writelines(("module load py-kiwisolver/1.0.1-py2.7\n", )) rfile.writelines(("module load py-lit/0.5.0-py2.7\n", )) rfile.writelines(("module load py-mako/1.0.4-py2.7\n", )) rfile.writelines(("module load py-markupsafe/1.0-py2.7\n", )) rfile.writelines(("module load py-matplotlib/2.2.3-py2.7\n", )) rfile.writelines(("module load py-nose/1.3.7-py2.7\n", )) rfile.writelines(("module load py-numexpr/2.6.5-py2.7\n", )) rfile.writelines(("module load py-numpy/1.15.2-py2.7\n", )) rfile.writelines(("module load py-pandas/0.23.4-py2.7\n", )) rfile.writelines(("module load py-paramiko/2.1.2-py2.7\n", )) rfile.writelines(("module load py-pillow/5.1.0-py2.7\n", )) rfile.writelines(("module load py-pkgconfig/1.2.2-py2.7\n", )) rfile.writelines(("module load py-pyasn1/0.2.3-py2.7\n", )) rfile.writelines(("module load py-pycparser/2.18-py2.7\n", )) rfile.writelines(("module load py-pyparsing/2.2.0-py2.7\n", )) rfile.writelines(("module load py-pytz/2017.2-py2.7\n", )) rfile.writelines(("module load py-scikit-learn/0.20.0-py2.7\n", )) rfile.writelines(("module load py-scipy/1.1.0-py2.7\n", )) rfile.writelines(("module load py-setuptools/40.4.3-py2.7\n", )) rfile.writelines(("module load py-six/1.11.0-py2.7\n", )) rfile.writelines(("module load py-subprocess32/3.2.7-py2.7\n", )) rfile.writelines(("module load pybullet/2.3.5-py2.7\n", )) rfile.writelines(("module list\n", )) rfile.writelines(("printenv\n", )) rfile.writelines(("lscpu\n", )) fname, interp, args = cmdline[0].split(" ", 2) rfile.writelines(("time -p python %s %s" % (hocfile, args), "\n")) elif (scheduler_properties['pythonversion'] == "3.7.0"): rfile.writelines(("module purge\n", )) rfile.writelines(("module load python/3.7.0\n", )) rfile.writelines(("module load py-asn1crypto/0.22.0-py3.7\n", )) rfile.writelines(("module load py-bottleneck/1.2.1-py3.7\n", )) rfile.writelines(("module load py-cffi/1.11.5-py3.7\n", )) rfile.writelines(("module load py-cryptography/1.8.1-py3.7\n", )) rfile.writelines(("module load py-cryptography/2.3.1-py3.7\n", )) rfile.writelines(("module load py-cycler/0.10.0-py3.7\n", )) rfile.writelines(("module load py-cython/0.29-py3.7\n", )) rfile.writelines(("module load py-dateutil/2.5.2-py3.7\n", )) rfile.writelines(("module load py-idna/2.5-py3.7\n", )) rfile.writelines(("module load py-kiwisolver/1.0.1-py3.7\n", )) rfile.writelines(("module load py-matplotlib/3.0.0-py3.7\n", )) rfile.writelines(("module load py-numexpr/2.6.5-py3.7\n", )) rfile.writelines(("module load py-numpy/1.15.2-py3.7\n", )) rfile.writelines(("module load py-pandas/0.23.4-py3.7\n", )) rfile.writelines(("module load py-paramiko/2.1.2-py3.7\n", )) rfile.writelines(("module load py-pillow/5.1.0-py3.7\n", )) rfile.writelines(("module load py-pyasn1/0.2.3-py3.7\n", )) rfile.writelines(("module load py-pycparser/2.18-py3.7\n", )) rfile.writelines(("module load py-pyparsing/2.2.0-py3.7\n", )) rfile.writelines(("module load py-pytz/2017.2-py3.7\n", )) rfile.writelines(("module load py-scikit-learn/0.20.0-py3.7\n", )) rfile.writelines(("module load py-scipy/1.1.0-py3.7\n", )) rfile.writelines(("module load py-setuptools/40.4.3-py3.7\n", )) rfile.writelines(("module load py-six/1.11.0-py3.7\n", )) rfile.writelines(("module list\n", )) rfile.writelines(("printenv\n", )) rfile.writelines(("lscpu\n", )) fname, interp, args = cmdline[0].split(" ", 2) rfile.writelines(("time -p python3 %s %s" % (hocfile, args), "\n")) else: rfile.writelines(("module purge\n", )) rfile.writelines(("module list\n", )) rfile.writelines(("printenv\n", )) rfile.writelines(("lscpu\n", )) rfile.writelines(("time -p python3 %s %s" % (hocfile, args), "\n")) elif (tooltype == "tensorflow"): rfile.writelines(("module purge\n", )) rfile.writelines(("module list\n", )) rfile.writelines(("printenv\n", )) rfile.writelines(("lscpu\n", )) if (scheduler_info['gpus'] == 1): rfile.writelines(("nvidia-smi\n", )) fname, interp, args = cmdline[0].split(" ", 2) rfile.writelines(("time -p python3 %s %s" % (hocfile, args), "\n")) else: print('ERROR :: tooltype unknown!') sys.exit(1) # Record job's end date and time. rfile.write("echo Job finished at `date` > ../done.txt\n") # Once the main set of job commands have been run, move up and out # of the primary job_work_dir ... rfile.write("cd ../\n") # ... then create a compressed tarball of the job_work_dir and all # other files created during the job. For all compressed tarballs # less than 1GB, the tarball may be transferred back to the job's # HTCondor submit node via HTCondor's native file transfer # mechanism. Large file transfers greater than 1GB should use an # alternative mechanism such as via globus-url-copy or stachcp. # i.e., eventually there will need to be a rewrite of the logic here # to switch between different file transfer mechanisms depending on # job type and/or job attributes. For now, we assume only HTCondor # file transfer mechanism will be utilized. if (tooltype == "eeglab_tg" or tooltype == "dynasim_tg"): rfile.writelines(( "/bin/tar -cvzf %s.tar.gz ./%s ./scheduler_stderr.txt ./scheduler_stdout.txt ./stderr.txt ./stdout.txt" % (outfilename, modeldir), "\n")) else: rfile.writelines( ("/bin/tar -cvzf %s.tar.gz ./*" % (outfilename), "\n")) # Report job completion to the gateway frontend. rfile.write("curl %s\&status=DONE\n" % url) # # # batch_command.cmdline file creation complete. Close file # # # # # # and set file permissions. rfile.close() os.chmod(lib.cmdfile, 0744) # There is no inherent concept of a max wallclock time in HTCondor. # While one could be implemented, on OSG, we should assume all jobs # are preemptible. As such, setting wallclock time for jobs can be # eliminated for OSG-bound jobs. The following timestring settings # were left in the OSG version of the submit.py script for # completeness to compare with the submit.py script for Comet. The # lines can be eliminated, if desired. days, remainderminutes = divmod(runtime, 60 * 24) hours, remainderminutes = divmod(remainderminutes, 60) timestring = '{}-{:02d}:{:02d}:00'.format(days, hours, remainderminutes) # # # Create the HTCondor submit description file # # # rfile = open(lib.runfile, 'w') rfile.writelines(('universe = vanilla\n', )) rfile.writelines(('executable = batch_command.cmdline\n', )) #rfile.writelines(('arguments = %s' % (if_batch_command.cmdline_took_args), '\n')) rfile.writelines( ('request_cpus = %s' % (scheduler_info['threads_per_process']), '\n')) rfile.writelines( ('request_memory = %s' % (scheduler_info['memory']), '\n')) rfile.writelines(('request_disk = %s' % (scheduler_info['disk']), '\n')) if (scheduler_info['gpus'] == 1 ): # OSG currently only supports 1 GPU per job rfile.writelines(('request_gpus = 1\n', )) rfile.writelines( ('requirements = ' '(Arch == "X86_64") && ' '(OpSys == "LINUX") && ' '(OSGVO_OS_STRING == "RHEL 6" || OSGVO_OS_STRING == "RHEL 7") && ' '(HAS_MODULES =?= True) && ' '(HAS_SINGULARITY == True) && ' '(CUDACapability >= 3)', '\n')) else: rfile.writelines( ('requirements = ' '(Arch == "X86_64") && ' '(OpSys == "LINUX") && ' '(OSGVO_OS_STRING == "RHEL 6" || OSGVO_OS_STRING == "RHEL 7") && ' '(HAS_MODULES =?= True) && ' '(HAS_SINGULARITY == True)', '\n')) rfile.writelines(('input = %s' % (fname), '\n')) #rfile.writelines(('transfer_input_files = %s' % (if_there_were_other_input_files), '\n')) rfile.writelines(('output = scheduler_stdout.txt\n', )) #rfile.writelines(('transfer_output_files = %s' % (if_wanted_only_subset_of_output_files), '\n')) rfile.writelines(('error = scheduler_stderr.txt\n', )) rfile.writelines(('log = scheduler_stdlog.txt\n', )) rfile.writelines(('should_transfer_files = YES\n', )) rfile.writelines(('when_to_transfer_output = ON_EXIT_OR_EVICT\n', )) rfile.writelines(('notify_user = = %s' % (lib.email), '\n')) rfile.writelines(('notification = Always\n', )) if (tooltype == "freesurf"): rfile.writelines(( '+SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-freesurfer:latest"', '\n')) elif ((tooltype == "tensorflow") and (scheduler_info['gpus'] == 0)): rfile.writelines(( '+SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow:latest"', '\n')) elif ((tooltype == "tensorflow") and (scheduler_info['gpus'] == 1)): rfile.writelines(( '+SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow-gpu:latest"', '\n')) rfile.writelines(('+ProjectName = %s' % (account), '\n')) rfile.writelines(('queue 1\n', )) rfile.close() # # # Create epilog file. Is this file needed for OSG integration? # # # rfile = open('./epilog', "w") rfile.write("#!/usr/bin/env bash\n") rfile.writelines(("curl %s\&status=DONE" % (url), "\n")) rfile.close() os.chmod('./epilog', 0755) # Close nsgdebug file. debugfile.close() # Submit job to HTCondor. return lib.submitJob(partition=queue) return 0
def DoUnaryOp(op_id, s): # type: (Id_t, str) -> bool # Only use lstat if we're testing for a symlink. if op_id in (Id.BoolUnary_h, Id.BoolUnary_L): try: mode = posix.lstat(s).st_mode except OSError: # TODO: simple_test_builtin should this as status=2. #e_die("lstat() error: %s", e, word=node.child) return False return stat.S_ISLNK(mode) try: st = posix.stat(s) except OSError as e: # TODO: simple_test_builtin should this as status=2. # Problem: we really need errno, because test -f / is bad argument, # while test -f /nonexistent is a good argument but failed. Gah. # ENOENT vs. ENAMETOOLONG. #e_die("stat() error: %s", e, word=node.child) return False mode = st.st_mode if op_id in (Id.BoolUnary_e, Id.BoolUnary_a): # -a is alias for -e return True if op_id == Id.BoolUnary_f: return stat.S_ISREG(mode) if op_id == Id.BoolUnary_d: return stat.S_ISDIR(mode) if op_id == Id.BoolUnary_b: return stat.S_ISBLK(mode) if op_id == Id.BoolUnary_c: return stat.S_ISCHR(mode) if op_id == Id.BoolUnary_p: return stat.S_ISFIFO(mode) if op_id == Id.BoolUnary_S: return stat.S_ISSOCK(mode) if op_id == Id.BoolUnary_x: return posix.access(s, posix.X_OK) if op_id == Id.BoolUnary_r: return posix.access(s, posix.R_OK) if op_id == Id.BoolUnary_w: return posix.access(s, posix.W_OK) if op_id == Id.BoolUnary_s: return st.st_size != 0 if op_id == Id.BoolUnary_O: return st.st_uid == posix.geteuid() if op_id == Id.BoolUnary_G: return st.st_gid == posix.getegid() e_die("%s isn't implemented", op_id) # implicit location
def _batchCheckBasic(self, fn): assert self.__validPath(fn) fullfn = self.__fn2fullfn(fn) s = os.lstat(fullfn) # common check if True: try: pwd.getpwuid(s.st_uid) except KeyError: self.p._checkResult.append("\"%s\" has an invalid owner." % (fn)) try: grp.getgrgid(s.st_gid) except KeyError: self.p._checkResult.append("\"%s\" has an invalid group." % (fn)) # common check if True: if not (s.st_mode & stat.S_IRUSR): self.p._checkResult.append("\"%s\" is not readable by owner." % (fn)) if not (s.st_mode & stat.S_IWUSR): # FIXME: there're so many files violates this rule, strange # self.p._checkResult.append("\"%s\" is not writeable by owner." % (fn)) pass if not (s.st_mode & stat.S_IRGRP) and (s.st_mode & stat.S_IWGRP): self.p._checkResult.append( "\"%s\" is not readable but writable by group." % (fn)) if not (s.st_mode & stat.S_IROTH) and (s.st_mode & stat.S_IWOTH): self.p._checkResult.append( "\"%s\" is not readable but writable by other." % (fn)) if not (s.st_mode & stat.S_IRGRP) and ( (s.st_mode & stat.S_IROTH) or (s.st_mode & stat.S_IWOTH)): self.p._checkResult.append( "\"%s\" is not readable by group but readable/writable by other." % (fn)) if not (s.st_mode & stat.S_IWGRP) and (s.st_mode & stat.S_IWOTH): self.p._checkResult.append( "\"%s\" is not writable by group but writable by other." % (fn)) # common check if True: if (s.st_mode & stat.S_ISVTX): self.p._checkResult.append( "\"%s\" should not have sticky bit set." % (fn)) # dedicated check for symlink if stat.S_ISLNK(s.st_mode): if not os.path.exists(fullfn): self.p._checkResult.append("\"%s\" is a broken symlink." % (fn)) if stat.S_IMODE(s.st_mode) != 0o0777: self.p._checkResult.append("\"%s\" has invalid permission." % (fn)) return # dedicated check for directory if stat.S_ISDIR(s.st_mode): if (s.st_mode & stat.S_ISUID): self.p._checkResult.append( "\"%s\" should not have SUID bit set." % (fn)) if (s.st_mode & stat.S_ISGID): # if showdn.startswith("/var/lib/portage"): # pass # FIXME, portage set SGID for these directories? # elif showdn.startswith("/var/log/portage"): # pass # FIXME, portage set SGID for these directories? # elif showdn.startswith("/var/log/journal"): # pass # FIXME, systemd set SGID for these directories? # else: # self.p._checkResult.append("\"%s\" should not have SGID bit set." % (showdn)) pass return # dedicated check for regular file if stat.S_ISREG(s.st_mode): if (s.st_mode & stat.S_ISUID): bad = False if not (s.st_mode & stat.S_IXUSR): bad = True if not (s.st_mode & stat.S_IXGRP) and ( (s.st_mode & stat.S_IRGRP) or (s.st_mode & stat.S_IWGRP)): bad = True if not (s.st_mode & stat.S_IXOTH) and ( (s.st_mode & stat.S_IROTH) or (s.st_mode & stat.S_IWOTH)): bad = True if bad: self.p._checkResult.append( "\"%s\" is not appropriate for SUID bit." % (fn)) if (s.st_mode & stat.S_ISGID): # FIXME # self.infoPrinter.printError("File \"%s\" should not have SGID bit set." % (showfn)) pass return # all other file types are invalid for batch check self.p._checkResult.append("Type of \"%s\" is invalid." % (fn))
def put(local_path, remote_path, mode=None): """ Upload one or more files to a remote host. ``local_path`` may be a relative or absolute local file path, and may contain shell-style wildcards, as understood by the Python ``glob`` module. Tilde expansion (as implemented by ``os.path.expanduser``) is also performed. ``remote_path`` may also be a relative or absolute location, but applied to the remote host. Relative paths are relative to the remote user's home directory, but tilde expansion (e.g. ``~/.ssh/``) will also be performed if necessary. By default, `put` preserves file modes when uploading. However, you can also set the mode explicitly by specifying the ``mode`` keyword argument, which sets the numeric mode of the remote file. See the ``os.chmod`` documentation or ``man chmod`` for the format of this argument. Examples:: put('bin/project.zip', '/tmp/project.zip') put('*.py', 'cgi-bin/') put('index.html', 'index.html', mode=0755) """ ftp = connections[env.host_string].open_sftp() with closing(ftp) as ftp: # Expand tildes (assumption: default remote cwd is user $HOME) remote_path = remote_path.replace('~', ftp.normalize('.')) # Get remote mode for directory-vs-file detection try: rmode = ftp.lstat(remote_path).st_mode except: # sadly, I see no better way of doing this rmode = None # Expand local tildes and get globs globs = glob(os.path.expanduser(local_path)) # Deal with bad local_path if not globs: raise ValueError, "'%s' is not a valid local path or glob." \ % local_path # Iterate over all given local files for lpath in globs: # If remote path is directory, tack on the local filename _remote_path = remote_path if rmode is not None and stat.S_ISDIR(rmode): _remote_path = os.path.join( remote_path, os.path.basename(lpath) ) # Print if output.running: prefix = "[%s] " % env.host_string msg = "put: %s -> %s" % (lpath, _remote_path) if env.colors: prefix = env.color_settings['host_prefix'](prefix) print(prefix + msg) # Try to catch raised exceptions (which is the only way to tell if # this operation had problems; there's no return code) during upload try: # Actually do the upload rattrs = ftp.put(lpath, _remote_path) # and finally set the file mode lmode = mode or os.stat(lpath).st_mode if lmode != rattrs.st_mode: ftp.chmod(_remote_path, lmode) except Exception, e: msg = "put() encountered an exception while uploading '%s'" _handle_failure(message=msg % lpath, exception=e)
def isdir(self, path): return stat.S_ISDIR(self.st_mode(path))
def _queue_dir_transfer(self, task_cls, src_path, dst_path): """ Handles the folder transfer by stopping bad transfers, creating tasks for transfers, and placing them in the queue. :param task_cls: The type of task the transfer will be. :param str src_path: The path to be uploaded or downloaded. :param str dst_path: The path to be created. """ if issubclass(task_cls, tasks.DownloadTask): src, dst = self.remote, self.local if not os.access(dst.path_mod.dirname(dst_path), os.W_OK): gui_utilities.show_dialog_error('Permission Denied', self.application.get_active_window(), 'Can not write to the destination directory.') return task = task_cls.dir_cls(dst_path, src_path, size=0) elif issubclass(task_cls, tasks.UploadTask): if not os.access(src_path, os.R_OK): gui_utilities.show_dialog_error('Permission Denied', self.application.get_active_window(), 'Can not read the source directory.') return src, dst = self.local, self.remote task = task_cls.dir_cls(src_path, dst_path, size=0) if not stat.S_ISDIR(dst.path_mode(dst_path)): try: dst.make_dir(dst_path) except (IOError, OSError): gui_utilities.show_dialog_error('Permission Denied', self.application.get_active_window(), 'Can not create the destination directory.') return else: raise ValueError('unknown task class') queued_tasks = [] parent_directory_tasks = collections.OrderedDict({src_path: task}) for dir_cont in src.walk(src_path): dst_base_path = dst.path_mod.normpath(dst.path_mod.join(dst_path, src.get_relpath(dir_cont.dirpath, start=src_path))) src_base_path = dir_cont.dirpath parent_task = parent_directory_tasks.pop(src_base_path, None) if parent_task is None: continue queued_tasks.append(parent_task) new_task_count = 0 if issubclass(task_cls, tasks.DownloadTask): local_base_path, remote_base_path = (dst_base_path, src_base_path) else: local_base_path, remote_base_path = (src_base_path, dst_base_path) for filename in dir_cont.filenames: if not self.config['transfer_hidden'] and src.path_is_hidden(src.path_mod.join(src_base_path, filename)): continue try: file_size = src.get_file_size(src.path_mod.join(dir_cont.dirpath, filename)) except (IOError, OSError): continue # skip this file if we can't get it's size task = task_cls( self.local.path_mod.join(local_base_path, filename), self.remote.path_mod.join(remote_base_path, filename), parent=parent_task, size=file_size ) queued_tasks.append(task) new_task_count += 1 for dirname in dir_cont.dirnames: if not self.config['transfer_hidden'] and src.path_is_hidden(src.path_mod.join(src_base_path, dirname)): continue task = task_cls.dir_cls( self.local.path_mod.join(local_base_path, dirname), self.remote.path_mod.join(remote_base_path, dirname), parent=parent_task, size=0 ) parent_directory_tasks[src.path_mod.join(src_base_path, dirname)] = task new_task_count += 1 parent_task.size += new_task_count for grandparent_task in parent_task.parents: grandparent_task.size += new_task_count for task in queued_tasks: self.queue.put(task) self.status_display.sync_view(queued_tasks)
def test_vsiaz_extra_1(): if not gdaltest.built_against_curl(): pytest.skip() az_resource = gdal.GetConfigOption('AZ_RESOURCE') if az_resource is None: pytest.skip('Missing AZ_RESOURCE') if '/' not in az_resource: path = '/vsiaz/' + az_resource statres = gdal.VSIStatL(path) assert statres is not None and stat.S_ISDIR(statres.mode), \ ('%s is not a valid bucket' % path) readdir = gdal.ReadDir(path) assert readdir is not None, 'ReadDir() should not return empty list' for filename in readdir: if filename != '.': subpath = path + '/' + filename assert gdal.VSIStatL(subpath) is not None, \ ('Stat(%s) should not return an error' % subpath) unique_id = 'vsiaz_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id in readdir, \ ('ReadDir(%s) should contain %s' % (path, unique_id)) ret = gdal.Mkdir(subpath, 0) assert ret != 0, ('Mkdir(%s) repeated should return an error' % subpath) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id not in readdir, \ ('ReadDir(%s) should not contain %s' % (path, unique_id)) ret = gdal.Rmdir(subpath) assert ret != 0, ('Rmdir(%s) repeated should return an error' % subpath) ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') assert f is not None gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) assert ret != 0, \ ('Rmdir(%s) on non empty directory should return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) assert gdal.Rename(subpath + '/test.txt', subpath + '/test2.txt') == 0 f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test2.txt') assert ret >= 0, \ ('Unlink(%s) should not return an error' % (subpath + '/test2.txt')) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) return f = open_for_read('/vsiaz/' + az_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 # Same with /vsiaz_streaming/ f = open_for_read('/vsiaz_streaming/' + az_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 if False: # pylint: disable=using-constant-test # we actually try to read at read() time and bSetError = false # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsiaz/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert gdal.VSIGetLastErrorMsg() != '' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz') assert f is None, gdal.VSIGetLastErrorMsg() # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1
def get(self, request, token): """ Only used for get dirents in a folder share link. Permission checking: 1, If enable SHARE_LINK_LOGIN_REQUIRED, user must have been authenticated. 2, If enable ENABLE_SHARE_LINK_AUDIT, user must have been authenticated, or have been audited. 3, If share link is encrypted, share link password must have been checked. """ # argument check thumbnail_size = request.GET.get('thumbnail_size', 48) try: thumbnail_size = int(thumbnail_size) except ValueError: error_msg = 'thumbnail_size invalid.' return api_error(status.HTTP_400_BAD_REQUEST, error_msg) # permission check # check if login required if SHARE_LINK_LOGIN_REQUIRED and \ not request.user.is_authenticated(): error_msg = 'Permission denied.' return api_error(status.HTTP_403_FORBIDDEN, error_msg) # check share link audit if is_pro_version() and ENABLE_SHARE_LINK_AUDIT and \ not request.user.is_authenticated() and \ not request.session.get('anonymous_email'): error_msg = 'Permission denied.' return api_error(status.HTTP_403_FORBIDDEN, error_msg) # resource check try: share_link = FileShare.objects.get(token=token) except FileShare.DoesNotExist: error_msg = 'Share link %s not found.' % token return api_error(status.HTTP_404_NOT_FOUND, error_msg) # check share link password if share_link.is_encrypted() and not check_share_link_access( request, token): error_msg = 'Permission denied.' return api_error(status.HTTP_403_FORBIDDEN, error_msg) if share_link.s_type != 'd': error_msg = 'Share link %s is not a folder share link.' % token return api_error(status.HTTP_400_BAD_REQUEST, error_msg) repo_id = share_link.repo_id repo = seafile_api.get_repo(repo_id) if not repo: error_msg = 'Library %s not found.' % repo_id return api_error(status.HTTP_404_NOT_FOUND, error_msg) share_link_path = share_link.path request_path = request.GET.get('path', '/') if request_path == '/': path = share_link_path else: path = posixpath.join(share_link_path, request_path.strip('/')) path = normalize_dir_path(path) dir_id = seafile_api.get_dir_id_by_path(repo_id, path) if not dir_id: error_msg = 'Folder %s not found.' % request_path return api_error(status.HTTP_404_NOT_FOUND, error_msg) try: current_commit = seafile_api.get_commit_list(repo_id, 0, 1)[0] dirent_list = seafile_api.list_dir_by_commit_and_path( repo_id, current_commit.id, path, -1, -1) except Exception as e: logger.error(e) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) result = [] for dirent in dirent_list: # don't return parent folder(share link path) info to user # so use request_path here dirent_path = posixpath.join(request_path, dirent.obj_name) dirent_info = {} dirent_info['size'] = dirent.size dirent_info['last_modified'] = timestamp_to_isoformat_timestr( dirent.mtime) if stat.S_ISDIR(dirent.mode): dirent_info['is_dir'] = True dirent_info['folder_path'] = normalize_dir_path(dirent_path) dirent_info['folder_name'] = dirent.obj_name else: dirent_info['is_dir'] = False dirent_info['file_path'] = normalize_file_path(dirent_path) dirent_info['file_name'] = dirent.obj_name file_type, file_ext = get_file_type_and_ext(dirent.obj_name) if file_type in (IMAGE, XMIND) or \ file_type == VIDEO and ENABLE_VIDEO_THUMBNAIL: if os.path.exists( os.path.join(THUMBNAIL_ROOT, str(thumbnail_size), dirent.obj_id)): req_image_path = posixpath.join( request_path, dirent.obj_name) src = get_share_link_thumbnail_src( token, thumbnail_size, req_image_path) dirent_info['encoded_thumbnail_src'] = urlquote(src) result.append(dirent_info) return Response({'dirent_list': result})
def get_type(mode): if stat.S_ISDIR(mode) or stat.S_ISLNK(mode): type = 'dir' else: type = 'file' return type
def handle_webdata_request(request_handler: tornado.web.RequestHandler, ): request_path = request_handler.request.path # normalize request path try: normalized_request_path = normalize_request_path(request_path) except InvalidCharacterInPath as ex: print(ex) # unauthorize request_handler.set_status(403) request_handler.set_header('Content-Type', 'application/json') response_obj = { 'message': 'invalid character in path', 'path': request_path, } response_str = json.dumps(response_obj) request_handler.write(response_str) return # join with webdata directory if len(normalized_request_path) == 0: local_path = WEBDATA_DIRECTORY elif normalized_request_path == '/': local_path = WEBDATA_DIRECTORY else: local_path = os.path.join(WEBDATA_DIRECTORY, normalized_request_path) if local_path == WEBDATA_DIRECTORY: pass elif not is_child_path(WEBDATA_DIRECTORY, local_path): # unauthorize request_handler.set_status(403) request_handler.set_header('Content-Type', 'application/json') response_obj = { 'message': 'unauthorized access', 'path': request_path, } response_str = json.dumps(response_obj) request_handler.write(response_str) return # check if file exists if not os.path.exists(local_path): # not found request_handler.set_status(404) request_handler.set_header('Content-Type', 'application/json') response_obj = { 'message': 'file not found', 'path': request_path, } response_str = json.dumps(response_obj) request_handler.write(response_str) return file_stat = os.stat(local_path) if stat.S_ISDIR(file_stat.st_mode): # automatically serve index.html # directory # check if index.html exists child_filename_list = os.listdir(local_path) for child_filename in child_filename_list: lowered_child_filename = child_filename.lower() if lowered_child_filename in VALID_HTML_INDEX_FILENAME_LIST: child_filepath = os.path.join(local_path, child_filename) # send modified time os.path.getmtime(child_filepath) # TODO request_handler.set_status(200) request_handler.set_header('Content-Type', 'text/html') filesize = os.stat(child_filepath).st_size if filesize > 0: send_file_data(request_handler, child_filepath) return # not found # return directory listing # TODO option to disable directory listing # contruct static html page html_str = render_static_directory_listing_html( normalized_request_path, child_filename_list, ) request_handler.set_status(200) request_handler.set_header('Content-Type', 'text/html') request_handler.write(html_str) return if not stat.S_ISREG(file_stat.st_mode): # unauthorize # this is not a regular file request_handler.set_status(403) request_handler.set_header('Content-Type', 'application/json') response_obj = { 'message': 'this is not a regular file', 'path': request_path, } response_str = json.dumps(response_obj) request_handler.write(response_str) return # regular file # TODO parse and support Range header request_handler.set_status(200) mime_type = get_mime_type_by_filename(local_path) if mime_type in TEXT_MIME_TYPE_LIST: # set mime type and charset to utf-8 header_value = f'{mime_type}; charset=utf-8' request_handler.set_header('Content-Type', header_value) else: request_handler.set_header('Content-Type', mime_type) filesize = os.stat(local_path).st_size request_handler.set_header('Content-Length', str(filesize)) send_file_data(request_handler, local_path)
def run(self): # Walk the tests hierarchy looking for tests dirs = self.testdirs.split(':') tests = [] while dirs: dir = dirs.pop(0) if self.trace_tests: print 'Searching for tests in %s' % (dir, ) for f in os.listdir(dir): fn = os.path.join(dir, f) statb = os.stat(fn) if stat.S_ISDIR(statb[0]): dirs.append(fn) elif self.__TestFile_re.match(f): tests.append(fn) number = 0 import sys import traceback import new import unittest import types # Import each test into its own module, then add the test # cases in it to a complete suite. loader = unittest.defaultTestLoader suite = unittest.TestSuite() used_names = set() for fn in tests: stage = 'compile' try: # Assign a unique name for this test test_name = os.path.basename(fn).split('.')[0] test_name = test_name.replace('-', '_') number = 2 base_name = test_name while test_name in used_names: test_name = '%s%d' % (base_name, number) number += 1 # Read the test source in and compile it rv = compile(file(fn).read(), test_name, 'exec') state = 'evaluate' # Make a copy of the globals array so we don't # contaminate this environment. g = globals().copy() # The test cases use __file__ to determine the path to # the schemas g['__file__'] = fn # Create a module into which the test will be evaluated. module = new.module(test_name) # The generated code uses __name__ to look up the # containing module in sys.modules. g['__name__'] = test_name sys.modules[test_name] = module # Import the test into the module, making sure the created globals look like they're in the module. eval(rv, g) module.__dict__.update(g) # Find all subclasses of unittest.TestCase that were # in the test source and add them to the suite. for (nm, obj) in g.items(): if (type == type(obj)) and issubclass( obj, unittest.TestCase): suite.addTest(loader.loadTestsFromTestCase(obj)) if self.trace_tests: print '%s imported' % (fn, ) except Exception, e: print '%s failed in %s: %s' % (fn, stage, e) raise
def __write(self, fp, arcname=None, compress_type=None, st=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") if isinstance(fp, basestring): filename, fp = (fp, None) st = st or os.stat(filename) else: filename = '<stream>' st = st or os.stat(0) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) if (mtime.tm_year < 1980): mtime = time.localtime() date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo yield self.fp.write(zinfo.FileHeader(False)) return cmpr = _get_compressor(zinfo.compress_type) fp = fp or open(filename, 'rb') with fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ zinfo.file_size * 1.05 > ZIP64_LIMIT yield self.fp.write(zinfo.FileHeader(zip64)) file_size = 0 while 1: sz = 1024 * 8 if zinfo.file_size > 0: # known size, read only that much if zinfo.file_size == file_size: break sz = min(zinfo.file_size - file_size, sz) buf = fp.read(sz) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) yield self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) yield self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC if zinfo.file_size > 0 and zinfo.file_size != file_size: raise RuntimeError('File size changed during compressing') else: zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError('File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError('Compressed size larger than uncompressed size') # Seek backwards and write file header (which will now include # correct CRC and file sizes) # position = self.fp.tell() # Preserve current position in file # self.fp.seek(zinfo.header_offset, 0) # self.fp.write(zinfo.FileHeader(zip64)) # self.fp.seek(position, 0) yield self.fp.write(zinfo.DataDescriptor()) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def process(path): s = os.lstat(path) if stat.S_ISDIR(s.st_mode): update_hash('d') elif stat.S_ISCHR(s.st_mode): update_hash('c') elif stat.S_ISBLK(s.st_mode): update_hash('b') elif stat.S_ISSOCK(s.st_mode): update_hash('s') elif stat.S_ISLNK(s.st_mode): update_hash('l') elif stat.S_ISFIFO(s.st_mode): update_hash('p') else: update_hash('-') def add_perm(mask, on, off='-'): if mask & s.st_mode: update_hash(on) else: update_hash(off) add_perm(stat.S_IRUSR, 'r') add_perm(stat.S_IWUSR, 'w') if stat.S_ISUID & s.st_mode: add_perm(stat.S_IXUSR, 's', 'S') else: add_perm(stat.S_IXUSR, 'x') add_perm(stat.S_IRGRP, 'r') add_perm(stat.S_IWGRP, 'w') if stat.S_ISGID & s.st_mode: add_perm(stat.S_IXGRP, 's', 'S') else: add_perm(stat.S_IXGRP, 'x') add_perm(stat.S_IROTH, 'r') add_perm(stat.S_IWOTH, 'w') if stat.S_ISVTX & s.st_mode: update_hash('t') else: add_perm(stat.S_IXOTH, 'x') if include_owners: update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) update_hash(" ") if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) else: update_hash(" " * 9) update_hash(" ") if stat.S_ISREG(s.st_mode): update_hash("%10d" % s.st_size) else: update_hash(" " * 10) update_hash(" ") fh = hashlib.sha256() if stat.S_ISREG(s.st_mode): # Hash file contents with open(path, 'rb') as d: for chunk in iter(lambda: d.read(4096), b""): fh.update(chunk) update_hash(fh.hexdigest()) else: update_hash(" " * len(fh.hexdigest())) update_hash(" %s" % path) if stat.S_ISLNK(s.st_mode): update_hash(" -> %s" % os.readlink(path)) update_hash("\n")
def is_unarchived(self): cmd = '%s -ZT -s "%s"' % (self.cmd_path, self.src) if self.excludes: cmd += ' -x "' + '" "'.join(self.excludes) + '"' rc, out, err = self.module.run_command(cmd) old_out = out diff = '' out = '' if rc == 0: unarchived = True else: unarchived = False # Get some information related to user/group ownership umask = os.umask(0) os.umask(umask) # Get current user and group information groups = os.getgroups() run_uid = os.getuid() run_gid = os.getgid() try: run_owner = pwd.getpwuid(run_uid).pw_name except: run_owner = run_uid try: run_group = grp.getgrgid(run_gid).gr_name except: run_group = run_gid # Get future user ownership fut_owner = fut_uid = None if self.file_args['owner']: try: tpw = pwd.getpwname(self.file_args['owner']) except: try: tpw = pwd.getpwuid(self.file_args['owner']) except: tpw = pwd.getpwuid(run_uid) fut_owner = tpw.pw_name fut_uid = tpw.pw_uid else: try: fut_owner = run_owner except: pass fut_uid = run_uid # Get future group ownership fut_group = fut_gid = None if self.file_args['group']: try: tgr = grp.getgrnam(self.file_args['group']) except: try: tgr = grp.getgrgid(self.file_args['group']) except: tgr = grp.getgrgid(run_gid) fut_group = tgr.gr_name fut_gid = tgr.gr_gid else: try: fut_group = run_group except: pass fut_gid = run_gid for line in old_out.splitlines(): change = False pcs = line.split() if len(pcs) != 8: continue ztype = pcs[0][0] permstr = pcs[0][1:10] version = pcs[0][1] ostype = pcs[0][2] size = int(pcs[3]) path = pcs[7] # Skip excluded files if path in self.excludes: out += 'Path %s is excluded on request\n' % path continue # Itemized change requires L for symlink if path[-1] == '/': if ztype != 'd': err += 'Path %s incorrectly tagged as "%s", but is a directory.\n' % ( path, ztype) ftype = 'd' elif ztype == 'l': ftype = 'L' elif ztype == '-': ftype = 'f' elif ztype == '?': ftype = 'f' # Some files may be storing FAT permissions, not Unix permissions if len(permstr) == 6: if path[-1] == '/': permstr = 'rwxrwxrwx' elif permstr == 'rwx---': permstr = 'rwxrwxrwx' else: permstr = 'rw-rw-rw-' # Test string conformity if len(permstr) != 9 or not ZIP_FILE_MODE_RE.match(permstr): raise UnarchiveError('ZIP info perm format incorrect, %s' % permstr) # DEBUG # err += "%s%s %10d %s\n" % (ztype, permstr, size, path) dest = os.path.join(self.dest, path) try: st = os.lstat(dest) except: change = True self.includes.append(path) err += 'Path %s is missing\n' % path diff += '>%s++++++.?? %s\n' % (ftype, path) continue # Compare file types if ftype == 'd' and not stat.S_ISDIR(st.st_mode): change = True self.includes.append(path) err += 'File %s already exists, but not as a directory\n' % path diff += 'c%s++++++.?? %s\n' % (ftype, path) continue if ftype == 'f' and not stat.S_ISREG(st.st_mode): change = True unarchived = False self.includes.append(path) err += 'Directory %s already exists, but not as a regular file\n' % path diff += 'c%s++++++.?? %s\n' % (ftype, path) continue if ftype == 'L' and not stat.S_ISLNK(st.st_mode): change = True self.includes.append(path) err += 'Directory %s already exists, but not as a symlink\n' % path diff += 'c%s++++++.?? %s\n' % (ftype, path) continue itemized = list('.%s.......??' % ftype) dt_object = datetime.datetime( *(time.strptime(pcs[6], '%Y%m%d.%H%M%S')[0:6])) timestamp = time.mktime(dt_object.timetuple()) # Compare file timestamps if stat.S_ISREG(st.st_mode): if self.module.params['keep_newer']: if timestamp > st.st_mtime: change = True self.includes.append(path) err += 'File %s is older, replacing file\n' % path itemized[4] = 't' elif stat.S_ISREG(st.st_mode) and timestamp < st.st_mtime: # Add to excluded files, ignore other changes out += 'File %s is newer, excluding file\n' % path continue else: if timestamp != st.st_mtime: change = True self.includes.append(path) err += 'File %s differs in mtime (%f vs %f)\n' % ( path, timestamp, st.st_mtime) itemized[4] = 't' # Compare file sizes if stat.S_ISREG(st.st_mode) and size != st.st_size: change = True err += 'File %s differs in size (%d vs %d)\n' % (path, size, st.st_size) itemized[3] = 's' # Compare file checksums if stat.S_ISREG(st.st_mode): crc = crc32(dest) if crc != self._crc32(path): change = True err += 'File %s differs in CRC32 checksum (0x%08x vs 0x%08x)\n' % ( path, self._crc32(path), crc) itemized[2] = 'c' # Compare file permissions # Do not handle permissions of symlinks if ftype != 'L': # Only special files require no umask-handling if ztype == '?': mode = self._permstr_to_octal(permstr, 0) else: mode = self._permstr_to_octal(permstr, umask) if self.file_args['mode'] and self.file_args[ 'mode'] != stat.S_IMODE(st.st_mode): change = True err += 'Path %s differs in permissions (%o vs %o)\n' % ( path, self.file_args['mode'], stat.S_IMODE(st.st_mode)) itemized[5] = 'p' elif mode != stat.S_IMODE(st.st_mode): change = True itemized[5] = 'p' err += 'Path %s differs in permissions (%o vs %o)\n' % ( path, mode, stat.S_IMODE(st.st_mode)) # Compare file user ownership owner = uid = None try: owner = pwd.getpwuid(st.st_uid).pw_name except: uid = st.st_uid # If we are not root and requested owner is not our user, fail if run_uid != 0 and (fut_owner != run_owner or fut_uid != run_uid): raise UnarchiveError( 'Cannot change ownership of %s to %s, as user %s' % (path, fut_owner, run_owner)) if owner and owner != fut_owner: change = True err += 'Path %s is owned by user %s, not by user %s as expected\n' % ( path, owner, fut_owner) itemized[6] = 'o' elif uid and uid != fut_uid: change = True err += 'Path %s is owned by uid %s, not by uid %s as expected\n' % ( path, uid, fut_uid) itemized[6] = 'o' # Compare file group ownership group = gid = None try: group = grp.getgrgid(st.st_gid).gr_name except: gid = st.st_gid if run_uid != 0 and fut_gid not in groups: raise UnarchiveError( 'Cannot change group ownership of %s to %s, as user %s' % (path, fut_group, run_owner)) if group and group != fut_group: change = True err += 'Path %s is owned by group %s, not by group %s as expected\n' % ( path, group, fut_group) itemized[6] = 'g' elif gid and gid != fut_gid: change = True err += 'Path %s is owned by gid %s, not by gid %s as expected\n' % ( path, gid, fut_gid) itemized[6] = 'g' # Register changed files and finalize diff output if change: if path not in self.includes: self.includes.append(path) diff += '%s %s\n' % (''.join(itemized), path) if self.includes: unarchived = False # DEBUG # out = old_out + out return dict(unarchived=unarchived, rc=rc, out=out, err=err, cmd=cmd, diff=diff)
async def test_getattr_root(ctx): assert stat.S_ISDIR((await ctx.server.getattr(ROOT_INODE, some_ctx)).st_mode) await fsck(ctx)
def _restore_file(self, file_meta, restore_path, data_stream, data_gen, backup_level): file_abs_path = os.path.join(restore_path, file_meta['path']) inode = file_meta.get('inode', {}) file_mode = inode.get('mode') if os.path.exists(file_abs_path): if backup_level == 0: self._remove_file(file_abs_path) else: if file_meta.get('deleted'): self._remove_file(file_abs_path) return data_stream elif file_meta.get( 'new_level') and not stat.S_ISREG(file_mode): self._set_inode(file_abs_path, inode) return data_stream if not file_mode: return data_stream if stat.S_ISREG(file_mode): data_stream = self._restore_reg_file(file_abs_path, file_meta, data_gen, data_stream) elif stat.S_ISDIR(file_mode): try: os.makedirs(file_abs_path, file_mode) except (OSError, IOError) as error: LOG.warning('Directory {0} creation error: {1}'.format( file_abs_path, error)) elif stat.S_ISBLK(file_mode): try: self._make_dev_file(file_abs_path, file_meta['dev'], file_mode) except (OSError, IOError) as error: LOG.warning('Block file {0} creation error: {1}'.format( file_abs_path, error)) elif stat.S_ISCHR(file_mode): try: self._make_dev_file(file_abs_path, file_meta['dev'], file_mode) except (OSError, IOError) as error: LOG.warning('Character file {0} creation error: {1}'.format( file_abs_path, error)) elif stat.S_ISFIFO(file_mode): try: os.mkfifo(file_abs_path) except (OSError, IOError) as error: LOG.warning('FIFO or Pipe file {0} creation error: {1}'.format( file_abs_path, error)) elif stat.S_ISLNK(file_mode): try: os.symlink(file_meta.get('lname', ''), file_abs_path) except (OSError, IOError) as error: LOG.warning('Link file {0} creation error: {1}'.format( file_abs_path, error)) if not stat.S_ISLNK(file_mode): self._set_inode(file_abs_path, inode) return data_stream
def entries(cls, path): """directory entries in an array""" # prevent symlinks being followed if not stat.S_ISDIR(os.lstat(path).st_mode): raise OSError(ENOTDIR, os.strerror(ENOTDIR)) return os.listdir(path)
def get_repo_dirents(request, repo, commit, path, offset=-1, limit=-1): """List repo dirents based on commit id and path. Use ``offset`` and ``limit`` to do paginating. Returns: A tupple of (file_list, dir_list, dirent_more) TODO: Some unrelated parts(file sharing, stars, modified info, etc) need to be pulled out to multiple functions. """ dir_list = [] file_list = [] dirent_more = False if commit.root_id == EMPTY_SHA1: return ([], [], False) if limit == -1 else ([], [], False) else: try: dirs = seafile_api.list_dir_by_commit_and_path( commit.repo_id, commit.id, path, offset, limit) if not dirs: return ([], [], False) except SearpcError as e: logger.error(e) return ([], [], False) if limit != -1 and limit == len(dirs): dirent_more = True username = request.user.username starred_files = get_dir_starred_files(username, repo.id, path) fileshares = FileShare.objects.filter(repo_id=repo.id).filter( username=username) uploadlinks = UploadLinkShare.objects.filter(repo_id=repo.id).filter( username=username) view_dir_base = reverse("view_common_lib_dir", args=[repo.id, '']) dl_dir_base = reverse('repo_download_dir', args=[repo.id]) file_history_base = reverse('file_revisions', args=[repo.id]) for dirent in dirs: dirent.last_modified = dirent.mtime dirent.sharelink = '' dirent.uploadlink = '' if stat.S_ISDIR(dirent.props.mode): dpath = posixpath.join(path, dirent.obj_name) if dpath[-1] != '/': dpath += '/' for share in fileshares: if dpath == share.path: dirent.sharelink = gen_dir_share_link(share.token) dirent.sharetoken = share.token break for link in uploadlinks: if dpath == link.path: dirent.uploadlink = gen_shared_upload_link(link.token) dirent.uploadtoken = link.token break p_dpath = posixpath.join(path, dirent.obj_name) dirent.view_link = view_dir_base + '?p=' + urlquote(p_dpath) dirent.dl_link = dl_dir_base + '?p=' + urlquote(p_dpath) dir_list.append(dirent) else: file_list.append(dirent) if repo.version == 0: dirent.file_size = get_file_size(repo.store_id, repo.version, dirent.obj_id) else: dirent.file_size = dirent.size dirent.starred = False fpath = posixpath.join(path, dirent.obj_name) p_fpath = posixpath.join(path, dirent.obj_name) dirent.view_link = reverse('view_lib_file', args=[repo.id, p_fpath]) dirent.dl_link = get_file_download_link( repo.id, dirent.obj_id, p_fpath) dirent.history_link = file_history_base + '?p=' + urlquote( p_fpath) if fpath in starred_files: dirent.starred = True for share in fileshares: if fpath == share.path: dirent.sharelink = gen_file_share_link(share.token) dirent.sharetoken = share.token break return (file_list, dir_list, dirent_more)
def Eval(self, node): #print('!!', node.tag) if node.tag == bool_expr_e.WordTest: s = self._EvalCompoundWord(node.w) return bool(s) if node.tag == bool_expr_e.LogicalNot: b = self.Eval(node.child) return not b if node.tag == bool_expr_e.LogicalAnd: # Short-circuit evaluation if self.Eval(node.left): return self.Eval(node.right) else: return False if node.tag == bool_expr_e.LogicalOr: if self.Eval(node.left): return True else: return self.Eval(node.right) if node.tag == bool_expr_e.BoolUnary: op_id = node.op_id s = self._EvalCompoundWord(node.child) # Now dispatch on arg type arg_type = BOOL_ARG_TYPES[ op_id.enum_id] # could be static in the LST? if arg_type == bool_arg_type_e.Path: # Only use lstat if we're testing for a symlink. if op_id in (Id.BoolUnary_h, Id.BoolUnary_L): try: mode = posix.lstat(s).st_mode except OSError: return False return stat.S_ISLNK(mode) try: st = posix.stat(s) except OSError: # TODO: Signal extra debug information? #log("Error from stat(%r): %s" % (s, e)) return False mode = st.st_mode if op_id in (Id.BoolUnary_e, Id.BoolUnary_a): # -a is alias for -e return True if op_id == Id.BoolUnary_f: return stat.S_ISREG(mode) if op_id == Id.BoolUnary_d: return stat.S_ISDIR(mode) if op_id == Id.BoolUnary_x: return posix.access(s, posix.X_OK) if op_id == Id.BoolUnary_r: return posix.access(s, posix.R_OK) if op_id == Id.BoolUnary_w: return posix.access(s, posix.W_OK) if op_id == Id.BoolUnary_s: return st.st_size != 0 raise NotImplementedError(op_id) if arg_type == bool_arg_type_e.Str: if op_id == Id.BoolUnary_z: return not bool(s) if op_id == Id.BoolUnary_n: return bool(s) raise NotImplementedError(op_id) if arg_type == bool_arg_type_e.Other: if op_id == Id.BoolUnary_t: try: fd = int(s) except ValueError: # TODO: Need location information of [ e_die('Invalid file descriptor %r', s) return posix.isatty(fd) raise NotImplementedError(op_id) raise NotImplementedError(arg_type) if node.tag == bool_expr_e.BoolBinary: op_id = node.op_id s1 = self._EvalCompoundWord(node.left) # Whether to glob escape do_fnmatch = op_id in (Id.BoolBinary_GlobEqual, Id.BoolBinary_GlobDEqual, Id.BoolBinary_GlobNEqual) do_ere = (op_id == Id.BoolBinary_EqualTilde) s2 = self._EvalCompoundWord(node.right, do_fnmatch=do_fnmatch, do_ere=do_ere) # Now dispatch on arg type arg_type = BOOL_ARG_TYPES[op_id.enum_id] if arg_type == bool_arg_type_e.Path: st1 = posix.stat(s1) st2 = posix.stat(s2) # TODO: test newer than (mtime) if op_id == Id.BoolBinary_nt: return st1[stat.ST_MTIME] > st2[stat.ST_MTIME] if op_id == Id.BoolBinary_ot: return st1[stat.ST_MTIME] < st2[stat.ST_MTIME] raise NotImplementedError(op_id) if arg_type == bool_arg_type_e.Int: # NOTE: We assume they are constants like [[ 3 -eq 3 ]]. # Bash also allows [[ 1+2 -eq 3 ]]. i1 = self._StringToIntegerOrError(s1, blame_word=node.left) i2 = self._StringToIntegerOrError(s2, blame_word=node.right) if op_id == Id.BoolBinary_eq: return i1 == i2 if op_id == Id.BoolBinary_ne: return i1 != i2 if op_id == Id.BoolBinary_gt: return i1 > i2 if op_id == Id.BoolBinary_ge: return i1 >= i2 if op_id == Id.BoolBinary_lt: return i1 < i2 if op_id == Id.BoolBinary_le: return i1 <= i2 raise NotImplementedError(op_id) if arg_type == bool_arg_type_e.Str: if op_id in (Id.BoolBinary_GlobEqual, Id.BoolBinary_GlobDEqual): #log('Matching %s against pattern %s', s1, s2) # TODO: Respect extended glob? * and ! and ? are quoted improperly. # But @ and + are OK. return libc.fnmatch(s2, s1) if op_id == Id.BoolBinary_GlobNEqual: return not libc.fnmatch(s2, s1) if op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual): return s1 == s2 if op_id == Id.BoolBinary_NEqual: return s1 != s2 if op_id == Id.BoolBinary_EqualTilde: # TODO: This should go to --debug-file #log('Matching %r against regex %r', s1, s2) try: matches = libc.regex_match(s2, s1) except RuntimeError: # Status 2 indicates a regex parse error. This is fatal in OSH but # not in bash, which treats [[ like a command with an exit code. e_die("Invalid regex %r", s2, word=node.right, status=2) if matches is None: return False self._SetRegexMatches(matches) return True if op_id == Id.Redir_Less: # pun return s1 < s2 if op_id == Id.Redir_Great: # pun return s1 > s2 raise NotImplementedError(op_id) raise AssertionError(node.tag)