Ejemplo n.º 1
0
def path_ensdf_file():
  data = []
  dirs = path_ensdf_dir()
  for dr in dirs:
    for fr in _listdir(dr):
      data.append(_path.join(dr, fr))
  return data
Ejemplo n.º 2
0
def cleanup_files(root_path: str, ttl: int) -> tuple:
    """Remove obsolete files and empty directories
    """
    now = _current_time()
    success = []
    failed = []

    for root, d_names, f_names in _walk(root_path):
        for f_name in f_names:
            f_path = _path.join(root, f_name)
            m_time = _path.getmtime(f_path)
            if m_time + ttl < now:
                try:
                    _unlink(f_path)
                    success.append(f_path)

                except Exception as e:
                    failed.append((f_path, e))

        # Remove empty directories
        for d_name in d_names:
            d_path = _path.join(root, d_name)
            if not _listdir(d_path):
                _rmdir(d_path)

    return success, failed
Ejemplo n.º 3
0
def ListFullDir(dirname):
    """Sometimes it is inconvenient to have to
    type '\\'.join(basename, filename) when calling listdir
    to get the full path, so here's a shortcut func.

    """
    return ['\\'.join((dirname, filename)) for filename in _listdir(dirname)]
Ejemplo n.º 4
0
def _get_lib_path_parital_qualname(name, base, search_dirs, splitext=_splitext):
    """
    Internal function to search for partially qualified names
    @param name: filename
    @type name: str
    @param base: parially qualified base to join with name
    @type base: str
    @param search_dirs: search dirs
    @type search_dirs: list[str] or tuple[str]
    @return: str
    @rtype: str
    """
    error = FileNotFoundError
    ext = splitext(name)[1]
    if not (base.startswith('\\') or base.startswith('/')):
        base = '\\' + base
    if ext:
        for fldr in search_dirs:
            path = fldr + base

            try:
                files = _listdir(path)
            except error:
                continue

            for file in files:
                if file == name:
                    return '\\'.join((path, name))

    else:
        for fldr in search_dirs:
            path = fldr + base

            try:
                files = _listdir(path)
            except error:
                continue

            for file in files:
                file, ext = splitext(file)
                # test presence of ext to exclude dirs
                if ext and file == name:
                    return ''.join((path, "\\", name, ext))

    raise FileNotFoundError("Partially qualified name %s not found" % '\\'.join((base, name)))
Ejemplo n.º 5
0
def change_location(src, tgt, move=False, verbose=True):
    '''
    Copies/moves/deletes locations

    :param src:
        Source location where to copy from
    :param tgt:
        Target location where to copy to

        * To backup `src`, set `tgt` explicitly to ``True``. \
        `tgt` will be set to `src` + '_backup_' + \
        :func:`util.system.get_timestamp` then

    :param move:
        Deletes original location after copy (a.k.a. move)

        * To delete `src` , set `tgt` explicitly to ``False`` \
        and `move` to ``True`` (be careful!!1!)

    :param verbose:
        Show warnings
    '''

    from photon.util.system import shell_notify

    if _path.exists(src):
        if tgt:
            if _path.isfile(src):
                _copy2(src, search_location(
                    tgt, create_in=_path.dirname(tgt), verbose=verbose)
                )
            else:
                for l in _listdir(src):
                    change_location(
                        _path.abspath(_path.join(src, l)),
                        _path.abspath(_path.join(tgt, l))
                    )
        if move:
            if _path.isdir(src) and not _path.islink(src):
                _rmtree(src)
            else:
                _remove(src)
        if verbose:
            shell_notify(
                '%s location' % (
                    'deleted'
                    if not tgt and move else
                    'moved'
                    if move else
                    'copied'
                ),
                more=dict(src=src, tgt=tgt)
            )
def list_files(dir_paths, endswith=None, contains=None, startswith=None, contains_not=None):
    """ endswith may be a sting like '.jpg' """
    files=[]
    if type(dir_paths)!=type([]):
        dir_paths=[dir_paths]
    for path in dir_paths:#'/home/nkrasows/phd/data/graham/Neurons/4dBinNeuronVolume/h5/',
        try:
            gg= [ (_join(path,f) if path!="." else f) for f in _listdir(path) if _isfile(_join(path,f)) and (startswith == None or f.startswith(startswith)) and (endswith == None or f.endswith(endswith)) and (contains == None or contains in f)  and (contains_not == None or (not (contains_not in f))) ]
            files+=gg
        except:
            print("path",path,"invalid")
    files.sort()
    return files
Ejemplo n.º 7
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

    dllpath: string of the exact path of the DLL
    root: string of the directory to start walking/searching to find the DLL
    """
    global _dll
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return
    try:
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)  # regex match function
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile:  # try the current dir first
                    rel.add(_curdir + _sep + nfile.string)
            if not rel:
                for root, dirs, files in _walk(
                        root):  # no luck, walk the dir tree
                    for file in map(matcher, files):
                        if file:
                            rel.add(root + _sep + file.string)
                if not rel:  # if still nothing throw
                    raise TOSDB_Error(" could not locate DLL")
            if len(rel) > 1:  # only use the most recent version(s)
                ver = _compile('-[\d]{1,2}.[\d]{1,2}-')
                vers = tuple(
                    zip(map(lambda x: _search(ver, x).group().strip('-'), rel),
                        rel))
                vers_max = max(vers)[0].split('.')[0]
                mtup = tuple((x[0].split('.')[1], x[1]) for x in vers
                             if x[0].split('.')[0] == vers_max)
                mtup_max = max(mtup)[0]
                rel = set(x[1] for x in mtup if x[0] == mtup_max)
            # find the most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
            rec = max(d)
            dllpath = d[rec]
        _dll = _WinDLL(dllpath)
        print("+ Using Module ", dllpath)
        print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")
        else:
            print("- Failed to Connect to Service \ Engine")
        return True  # indicate the lib was loaded
    except Exception as e:
        raise TOSDB_CLibError("unable to initialize library", e)
Ejemplo n.º 8
0
    def run_command():
        """Sign packages"""
        from os import environ
        from argparse import ArgumentParser

        parser = ArgumentParser(prog="sign_packages",
                                description="Sign RPM or DEB packages.")
        parser.add_argument("packages_dir", help="Input packages directory")
        parser.add_argument("--quiet",
                            "-q",
                            help="Disable verbosity",
                            action="store_true")
        args = parser.parse_args()

        private_key = environ.get("GPG_PRIVATE_KEY", "")
        if not private_key:
            parser.exit(1, message="No private key\n")
            return

        packages_dir = _realpath(args.packages_dir)

        for file in _listdir(packages_dir):
            ext = _splitext(file)[1].lower()
            if ext == ".rpm":
                sign = sign_rpm_packages
                break
            elif ext == ".deb":
                sign = sign_deb_packages
                break
        else:
            parser.exit(1, "No package to sign\n")
            return

        try:
            sign(
                packages_dir=packages_dir,
                private_key=private_key,
                public_key=environ.get("GPG_PUBLIC_KEY", ""),
                pass_phrase=environ.get("GPG_PASS_PHRASE", ""),
                quiet=args.quiet,
            )
        except _CalledProcessError as exception:
            parser.exit(exception.returncode, exception.stdout)
        except RuntimeError as exception:
            parser.exit(1, str(exception))

        if not args.quiet:
            parser.exit(message="Signature successful\n")
Ejemplo n.º 9
0
def init(dllpath = None, root = "C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

    dllpath: string of the exact path of the DLL
    root: string of the directory to start walking/searching to find the DLL
    """  
    global _dll
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return
    try:
        if dllpath is None:
            matcher = _partial( _match, _REGEX_DLL_NAME)  # regex match function
            for nfile in map( matcher, _listdir( _curdir )):
                if nfile: # try the current dir first             
                    rel.add( _curdir+ _sep + nfile.string )                    
            if not rel:                
                for root,dirs, files in _walk(root): # no luck, walk the dir tree 
                    for file in map( matcher, files):  
                        if file:                           
                            rel.add( root + _sep + file.string )                           
                if not rel: # if still nothing throw
                    raise TOSDB_Error(" could not locate DLL")    
            if len(rel) > 1:  # only use the most recent version(s)
                ver = _compile('-[\d]{1,2}.[\d]{1,2}-')
                vers = tuple( zip( map( 
                    lambda x: _search(ver,x).group().strip('-'), rel), rel) )
                vers_max = max(vers)[0].split('.')[0]
                mtup = tuple( (x[0].split('.')[1],x[1]) 
                              for x in vers if x[0].split('.')[0] == vers_max)         
                mtup_max = max(mtup)[0]
                rel = set( x[1] for x in mtup if x[0] == mtup_max )                      
            # find the most recently updated
            d = dict( zip(map( lambda x : _stat(x).st_mtime, rel), rel ) )
            rec = max(d)
            dllpath = d[ rec ]
        _dll = _WinDLL( dllpath )
        print( "+ Using Module ", dllpath )
        print( "+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")       
        else:
            print("- Failed to Connect to Service \ Engine")
        return True # indicate the lib was loaded
    except Exception as e:
        raise TOSDB_CLibError( "unable to initialize library", e )        
Ejemplo n.º 10
0
    def run_command():
        """Sign packages"""
        from argparse import ArgumentParser
        parser = ArgumentParser(prog='sign_packages',
                                description='Sign RPM or DEB packages.')
        parser.add_argument('packages_dir', help='Input packages directory')
        parser.add_argument('--private_key', '-k', help='Private GPG key')
        parser.add_argument('--public_key', '-b', help='Public GPG key')
        parser.add_argument('--pass_phrase', '-p', help='GPG key pass phrase')
        parser.add_argument('--quiet',
                            '-q',
                            help='Disable verbosity',
                            action='store_true')
        args = parser.parse_args()

        packages_dir = _realpath(args.packages_dir)

        if args.private_key is not None and not args.private_key:
            # Passed and empty value: command called from build script but
            # not private key is defined. In this case, signature is disabled.
            return

        for file in _listdir(packages_dir):
            ext = _splitext(file)[1].lower()
            if ext == '.rpm':
                sign = sign_rpm_packages
                break
            elif ext == '.deb':
                sign = sign_deb_packages
                break
        else:
            parser.exit(1, 'No package to sign.')
            return

        try:
            sign(packages_dir=packages_dir,
                 private_key=args.private_key,
                 public_key=args.public_key,
                 pass_phrase=args.pass_phrase,
                 quiet=args.quiet)
        except _CalledProcessError as exception:
            parser.exit(exception.returncode, exception.stdout)
        except RuntimeError as exception:
            parser.exit(1, str(exception))

        if not args.quiet:
            parser.exit(message='Signature successful')
def list_directories(dir_paths, endswith=None, contains=None, startswith=None, contains_not=None):
    """ endswith may be a sting like '.jpg' """
    files=[]
    N_OK=0
    if type(dir_paths)!=type([]):
        dir_paths=[dir_paths]
    for path in dir_paths:
        try:
            gg= [ (_join(path,f) if path!="." else f) for f in _listdir(path) if _isfile(_join(path,f))==False and (startswith == None or f.startswith(startswith)) and (endswith == None or f.endswith(endswith)) and (contains == None or contains in f)  and (contains_not == None or (not (contains_not in f))) ]
            files+=gg
            N_OK+=1
        except:
            print("path <",path,"> is invalid")
    if N_OK==0:
        print('list_directories():: All paths were invalid!')
        raise ValueError()
    files.sort()
    return files
Ejemplo n.º 12
0
def find_file_by(_path, prefix=None, sufix=None):
    """

    :param _path:
    :param prefix:
    :param sufix:
    :return:
    """
    filename = None
    for _filename in _listdir(_path):
        _filepath = _join(_path, _filename)
        if _isfile(_filepath) and (_filename.startswith(prefix)
                                   if prefix else _filename.endswith(sufix)):
            if not filename:
                filename = _filename
            else:
                raise Exception('Multiple benchmark files in a same folder.')
    return filename
def list_directories(dir_paths, endswith=None, contains=None, startswith=None, contains_not=None):
    """ endswith may be a sting like '.jpg' """
    files=[]
    N_OK=0
    if type(dir_paths)!=type([]):
        dir_paths=[dir_paths]
    for path in dir_paths:
        try:
            gg= [ (_join(path,f) if path!="." else f) for f in _listdir(path) if _isfile(_join(path,f))==False and (startswith == None or f.startswith(startswith)) and (endswith == None or f.endswith(endswith)) and (contains == None or contains in f)  and (contains_not == None or (not (contains_not in f))) ]
            files+=gg
            N_OK+=1
        except:
            print("path <",path,"> is invalid")
    if N_OK==0:
        print('list_directories():: All paths were invalid!')
        raise ValueError()
    files.sort()
    return files
Ejemplo n.º 14
0
 def clear(self, path):
     for _file in _listdir(path):
         full_path = path_join(path, _file)
         if isdir(full_path):
             if _file in self.extensions_to_delete:
                 try:
                     rmtree(full_path)
                     print('Removed', ''.join((full_path, '/')))
                 except OSError:
                     print('Permission denied', full_path)
             else:
                 self.clear(full_path)
         else:
             try:
                 if _file.split('.')[::-1][0] in self.extensions_to_delete:
                     remove(full_path)
                     print('Removed', full_path)
             except OSError:
                 print('Permission denied', full_path)
Ejemplo n.º 15
0
 def clear(self, path):
     for _file in _listdir(path):
         full_path = path_join(path, _file)
         if isdir(full_path):
             if _file in self.extensions_to_delete:
                 try:
                     rmtree(full_path)
                     print('Removed', ''.join((full_path, '/')))
                 except OSError:
                     print('Permission denied', full_path)
             else:
                 self.clear(full_path)
         else:
             try:
                 if _file.split('.')[::-1][0] in self.extensions_to_delete:
                     remove(full_path)
                     print('Removed', full_path)
             except OSError:
                 print('Permission denied', full_path)
Ejemplo n.º 16
0
def nbconvert():
    """helper function to convert all .ipynb files in the current directory to 
    html files.

    The output html files are in a directory named `html` under the current 
    directory. Existing html files with the same name are overwritten.

    If no directory named `html` is present, it will be created (jupyter nbconvert
    does that automatically)
    """
    # Get the current directory
    #cdir = path.dirname(path.realpath(__file__))
    cdir =  _getcwd()
    # Get a list of .ipynb files
    files2convert = [f for f in _listdir(cdir) if f.endswith('.ipynb')]
    # Convert the files within a try-except block
    count_files_successfully_converted = 0
    failedFiles = []
    for i, f in enumerate(files2convert):
        _getMD5(f)
        print(">>> [{}] Converting file ... ".format(i+1))
        if _convert(f):
            count_files_successfully_converted += 1
        else:
            failedFiles.append(f)

    # Print some human readable feedback
    print("\n")
    print("*******************************************")
    print("                  REPORT                   ")
    print("*******************************************")
    print("\nCurrent Directory: ", cdir)
    print("Number of IPython notebooks found: ", len(files2convert))
    print("Number of files successfully converted to html:",
    count_files_successfully_converted)
    print("Number of files failed to convert to html:", len(failedFiles))
    if failedFiles:
        print("Files that failed to convert:")
        for f in failedFiles:
            print(f)
    print("\nDONE!")
    raw_input("Press ENTER to close the appliation ...")
Ejemplo n.º 17
0
def _get_lib_path_no_extension(filepath, splitext=_splitext):
    """Internal function to find file when given basepath but
    no file extension.

    Scan the directory of the file and try to match the head of
    the filepath to an entry in that directory.

    @param filepath: a filename with extension but no basename
            To make function not require xl param, just use
            xl = newExcel(False, False) and close at the end.
    @type filepath: str
    @rtype: str
    """
    base, head = _split(filepath)
    for entry in _listdir(base):
        filename, ext = splitext(entry)
        # if not ext, then we have a dir, not filename
        # test ext first to short circuit non-files
        if ext and filename == head:
            return '\\'.join((base, entry))
    raise FileNotFoundError("File '%s' not found via no-extension search" % filepath)
 def create_compiled_schedule(self, networkjson, weightdir, inps):
     import numpy as np
     weights = _listdir(weightdir)
     weights = [_join(weightdir, wt) for wt in weights]
     const = {}
     for wtfile in weights:
         with open(wtfile, 'r') as wts:
             line = wts.readline()
             toks = line.strip().split()
             print len(toks)
             if len(toks) > 4:
                 print toks[0], toks[1], toks[2], toks[3], len(toks[4:])
                 if toks[0] not in const:
                     const[toks[0]] = {}
                 if "bias" in wtfile[wtfile.rfind('/'):]:
                     const[toks[0]]['bias'] = np.array(
                         [float(x) for x in toks[4:]])
                 else:
                     const[toks[0]]['weights'] = np.array(
                         [float(x) for x in toks[4:]])
     schedule = []
     for layer in networkjson['network']:
         if layer['type'] in _available_layers:
             print layer['type'], layer['name'], layer[
                 'bottoms'], layer['type'] not in [
                     'Convolution', 'InnerProduct'
                 ] or (layer['name'] in const
                       and len(const[layer['name']]) == 2)
             xdlf_layer = copy.deepcopy(
                 _available_layers(layer['type'], mode='NCHW'))
             if layer['name'] in const:
                 xdlf_layer.set_layer_params(layer, const[layer['name']])
             else:
                 xdlf_layer.set_layer_params(layer)
             schedule.append(xdlf_layer)
         elif layer['name'] in inps:
             print "Detected input : ", layer['name'], layer['type'], layer[
                 'outputshapes']
     print schedule
     return schedule
def list_files(dir_paths,
               endswith=None,
               contains=None,
               startswith=None,
               contains_not=None):
    """ endswith may be a sting like '.jpg' """
    files = []
    if type(dir_paths) != type([]):
        dir_paths = [dir_paths]
    for path in dir_paths:  #'/home/nkrasows/phd/data/graham/Neurons/4dBinNeuronVolume/h5/',
        try:
            gg = [(_join(path, f) if path != "." else f)
                  for f in _listdir(path) if _isfile(_join(path, f)) and (
                      startswith == None or f.startswith(startswith)) and
                  (endswith == None or f.endswith(endswith)) and (
                      contains == None or contains in f) and (
                          contains_not == None or (not (contains_not in f)))]
            files += gg
        except:
            print("path", path, "invalid")
    files.sort()
    return files
Ejemplo n.º 20
0
def _build_buffers() -> None:
    global _buffer_map_pids_to_comms
    global _buffer_map_pids_to_cmdlines
    global _buffer_running_pids
    global _buffer_list_of_comms
    global _buffer_map_comms_to_pids
    global _buffer_map_comms_to_pids

    _buffer_map_pids_to_comms = {}  # Dict[str, str]
    _buffer_map_pids_to_cmdlines = {}  # Dict[str, List[str]]
    _buffer_running_pids = [pid for pid in _listdir('/proc') if pid.isdigit()]
    _buffer_list_of_comms = []  # List[str]
    _buffer_map_comms_to_pids = {}  # type: Dict[str, List[str]]
    _buffer_map_comms_to_pids = dict()  # type: Dict[str, List[str]]

    for index, pid in enumerate(_buffer_running_pids):
        try:
            comm_file = open(_join('/proc', pid, 'comm'), 'r')
            comm = comm_file.read().rstrip('\n')
            comm_file.close()
            _buffer_map_pids_to_comms[pid] = comm

            cmd_file = open(_join('/proc', pid, 'cmdline'), 'r')
            cmd = cmd_file.read().rstrip('\n')
            cmd_file.close()
            cmdline = cmd.split('\x00')

            _buffer_map_pids_to_cmdlines[pid] = cmdline

            if comm not in _buffer_list_of_comms:
                _buffer_list_of_comms.append(comm)
                _buffer_map_comms_to_pids[comm] = list()

            _buffer_map_comms_to_pids[comm].append(pid)

        except FileNotFoundError:
            _buffer_running_pids.pop(index)
Ejemplo n.º 21
0
def sign_deb_packages(packages_dir,
                      private_key=None,
                      public_key=None,
                      pass_phrase=None,
                      quiet=False):
    """
    Sign all DEB packages in a directory.

    Args:
        packages_dir (str): Directory containing packages.
        private_key (str): Path to GPG private key to use.
            If no private key specified, use current GPG configuration.
        public_key (str): Path to GPG public key to use.
        pass_phrase (str): GPG key pass phrase.
        quiet (bool): Hide packages manager output.
    """
    _init_gpg_configuration(private_key)

    # Sign packages
    packages = [
        package for package in _listdir(packages_dir)
        if _splitext(package)[1].lower() == ".deb"
    ]

    command = ["dpkg-sig"]
    if pass_phrase:
        command += ["-g", '--passphrase "%s"' % pass_phrase]

    _run(command + ["--sign", "builder"] + packages,
         quiet=quiet,
         cwd=packages_dir)

    # Verify signatures
    _run(["dpkg-sig", "--verify"] + packages, quiet=quiet, cwd=packages_dir)

    if not quiet:
        print("Signed packages:\n - %s" % "\n- ".join(packages))
Ejemplo n.º 22
0
def load_files(data_file_location):
    data = []

    startswith = None
    endswith = None
    contains = None
    contains_not = None

    for path in data_file_location:
        gg = [(_join(path, f) if path != "." else f) for f in _listdir(path) if
              _isfile(_join(path, f)) and (startswith == None or f.startswith(startswith)) and (
              endswith == None or f.endswith(endswith)) and (contains == None or contains in f) and (
              contains_not == None or (not (contains_not in f)))]
        data.append(gg)

    combined_list = []
    # Sort the lists:
    for i in range(len(data)):
        elem = sorted(data[i])
        combined_list = combined_list + elem

    combined_list = strip_files_of_doubles(combined_list)
    combined_list = sorted(combined_list, key=sort_func)
    return combined_list
Ejemplo n.º 23
0
def install_accelize_drm_library(packages_dir, quiet=False):
    """
    Install Accelize DRM library packages.

    Args:
        packages_dir (str): Directory containing packages.
        quiet (bool): Hide packages manager output.
    """
    # Get packages
    packages_dir = _realpath(packages_dir)
    packages = [
        _join(packages_dir, package_file)
        for package_file in _listdir(packages_dir)
        if (_splitext(package_file)[-1].lower() in ('.deb', '.rpm') and
            '-dev' not in package_file)]

    # Run command
    if quiet:
        run_kwargs = dict(stdout=_PIPE, stderr=_PIPE, universal_newlines=True)
    else:
        run_kwargs = dict()

    _run(detect_package_manager() + ' '.join(packages), shell=True,
         **run_kwargs).check_returncode()
Ejemplo n.º 24
0
    ## get_status: converts a solver specific status flag to a cobra pie flag.

    ## set_parameter: takes solver specific parameter strings and sets them.

    ## solve: solves the optimization problem.  this is where one should put
    ## in logic on what to try if the problem
    ## isn't optimal

    ## solve_problem: dumb and fast which will set parameters, if provided
    ##note that for some solvers

    ## update_problem: changes bounds and linear objective coefficient of the
    ## solver specific problem file, given the complementary cobra.mod
    from os import listdir as _listdir
    from os import path as _path
    for i in _listdir(_path.split(_path.abspath(__file__))[0]):
        if i.startswith("_") or i.startswith(".") or i == 'legacy.py':
            continue
        if not i.endswith(".py"):
            continue
        try:
            m = i.strip(".py")
            exec("from . import %s" % m)
            solver_name = m
            if solver_name.endswith('_solver'):
                solver_name = solver_name[:-len('_solver')]
            solver_dict[solver_name] = eval(m)
        except Exception, e:
            pass
    del _path
    del _listdir
Ejemplo n.º 25
0
def sign_rpm_packages(packages_dir,
                      private_key=None,
                      public_key=None,
                      pass_phrase=None,
                      quiet=False):
    """
    Sign all RPM packages in a directory.

    Args:
        packages_dir (str): Directory containing packages.
        private_key (str): Path to GPG private key to use.
            If no private key specified, use current GPG configuration.
        public_key (str): Path to GPG public key to use.
        pass_phrase (str): GPG key pass phrase.
        quiet (bool): Hide commands output.
    """
    _init_gpg_configuration(private_key)

    # Import public key
    if public_key:
        _run(["rpm", "--import", public_key], quiet=quiet)

    # Sign packages
    packages = [
        package for package in _listdir(packages_dir)
        if _splitext(package)[1].lower() == ".rpm"
    ]

    gpg_info = _run("gpg --export | gpg --list-packets",
                    shell=True,
                    quiet=True).stdout
    for line in gpg_info.strip().splitlines():
        if ":user ID packet" in line:
            gpg_user_id = line.rsplit(":", 1)[1].strip().strip('"')
            break
    else:
        raise RuntimeError("Unable to read GPG User ID")

    macros = [
        "_signature gpg",
        "_gpg_path %s" % _expanduser("~/.gnupg"),
        "_gpg_name %s" % gpg_user_id,
    ]

    if pass_phrase:
        macros += [
            "_gpgbin /usr/bin/gpg",
            " ".join((
                "__gpg_sign_cmd %{__gpg}",
                "gpg",
                "--force-v3-sigs",
                "--batch",
                "--verbose",
                "--no-armor",
                '--passphrase "%s"' % pass_phrase,
                "--no-secmem-warning",
                "-u",
                '"%{_gpg_name}"',
                "-sbo",
                "%{__signature_filename}",
                "--digest-algo",
                "sha256",
                "%{__plaintext_filename}",
            )),
        ]
    define = []
    for macro in macros:
        define.extend(["--define", macro])

    _run(["rpm", "--addsign"] + define + packages,
         quiet=True,
         cwd=packages_dir)

    # Verify signatures
    result = _run(["rpm", "--checksig"] + packages,
                  quiet=True,
                  cwd=packages_dir)
    for line in result.stdout.splitlines():
        line = line.rstrip()
        if (not line.endswith("gpg OK") and not line.endswith("pgp md5 OK")
                and not line.endswith("digests signatures OK")):
            raise RuntimeError("Package signature verification failure: %s" %
                               line)

    if not quiet:
        print("Signed packages:\n - %s" % "\n- ".join(packages))
Ejemplo n.º 26
0
def sign_rpm_packages(packages_dir,
                      private_key=None,
                      public_key=None,
                      pass_phrase=None,
                      quiet=False):
    """
    Sign all RPM packages in a directory.

    Args:
        packages_dir (str): Directory containing packages.
        private_key (str): Path to GPG private key to use.
            If no private key specified, use current GPG configuration.
        public_key (str): Path to GPG public key to use.
        pass_phrase (str): GPG key pass phrase.
        quiet (bool): Hide commands output.
    """
    _init_gpg_configuration(private_key)

    # Import public key
    if public_key:
        _run(['rpm', '--import', public_key], quiet=quiet)

    # Sign packages
    packages = [
        package for package in _listdir(packages_dir)
        if _splitext(package)[1].lower() == '.rpm'
    ]

    gpg_info = _run('gpg --export | gpg --list-packets',
                    shell=True,
                    quiet=True).stdout
    for line in gpg_info.strip().splitlines():
        if ':user ID packet' in line:
            gpg_user_id = line.rsplit(':', 1)[1].strip().strip('"')
            break
    else:
        raise RuntimeError('Unable to read GPG User ID')

    macros = [
        '_signature gpg',
        '_gpg_path %s' % _expanduser("~/.gnupg"),
        '_gpg_name %s' % gpg_user_id
    ]

    if pass_phrase:
        macros += [
            '_gpgbin /usr/bin/gpg', ' '.join(
                ('__gpg_sign_cmd %{__gpg}', 'gpg', '--force-v3-sigs',
                 '--batch', '--verbose', '--no-armor',
                 '--passphrase "%s"' % pass_phrase, '--no-secmem-warning',
                 '-u', '"%{_gpg_name}"', '-sbo', '%{__signature_filename}',
                 '--digest-algo', 'sha256', '%{__plaintext_filename}'))
        ]
    define = []
    for macro in macros:
        define.extend(["--define", macro])

    _run(['rpm', '--addsign'] + define + packages,
         quiet=True,
         cwd=packages_dir)

    # Verify signatures
    result = _run(['rpm', '--checksig'] + packages,
                  quiet=True,
                  cwd=packages_dir)
    for line in result.stdout.splitlines():
        line = line.rstrip()
        if (not line.endswith('gpg OK') and not line.endswith('pgp md5 OK')
                and not line.endswith('digests signatures OK')):
            raise RuntimeError('Package signature verification failure: %s' %
                               line)
Ejemplo n.º 27
0
    rect = Image.new("RGBA", rect_sz, 4 * (0, ))
    removed = img.copy()
    removed.paste(rect, pos)

    if not reverse:
        return removed
    else:
        return _inv(_sub(img, removed))


def _scale(section, dims):
    return tuple(b * d for b, d in zip(section, 2 * dims))


npath = "nouns/"
for noun in (_f for _f in _listdir(npath) if _isfile(npath + _f)):
    resized = _resize(Image.open((npath + noun)).convert("RGBA"), (100, 100))
    resized.save(((npath + "processed/") + noun))

vpath = "verbs/"
vmark = Image.open("verb_mark.png").convert("RGBA")
for verb in (_f for _f in _listdir(vpath) if _isfile(vpath + _f)):
    resized = _resize(Image.open((vpath + verb)).convert("RGBA"), (100, 100))
    _add(resized, vmark).save(((vpath + "processed/") + verb))

apath = "adjectives/"
amark = Image.open("adjective_mark.png").convert("RGBA")
for adj in (_f for _f in _listdir(apath) if _isfile(apath + _f)):
    resized = _resize(Image.open((apath + adj)).convert("RGBA"), (100, 100))
    _add(resized, amark).save(((apath + "processed/") + adj))
 def listdir(path):
     items = os._listdir(path)
     if isinstance(path, unicode):
         items = [unicode(String(i).toString()) for i in items]
     return items
Ejemplo n.º 29
0
 def os_listdir(path):
     return os._listdir(longpathify(uni(path)))
Ejemplo n.º 30
0
def listdir(dirpath):
    return [join(dirpath, fname) for fname in _listdir(dirpath)
            if fname[0] != '.' and not fname.endswith('.pyc')
            and not fname.endswith('~')
            and not isdir(join(dirpath, fname))]
Ejemplo n.º 31
0
from __future__ import unicode_literals

import logging
from os import listdir as _listdir  # To not conflict with operator
from os.path import dirname, basename, join, isdir, abspath, expanduser
from abc import ABCMeta, abstractmethod
from PyInquirer import prompt

import cookiecutter as cc
from cookiecutter.context_manager import work_in

logger = logging.getLogger(__name__)

# TODO: Allow for imports of custom operators and subdirectories.
__all__ = [
    basename(f)[:-3] for f in _listdir(dirname(__file__))
    if f[-3:] == ".py" and not f.endswith("__init__.py")
]


class BaseOperator(metaclass=ABCMeta):
    """Base operator mixin class."""
    def __init__(
        self,
        operator_dict=None,
        context=None,
        context_key=None,
        no_input=False,
        cc_dict=None,
        env=None,
        key=None,
Ejemplo n.º 32
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL and try to connect.

    Returns True if library was successfully loaded, not necessarily that
    it was also able to connect. Details are sent to stdout.
    
    init(dllpath=None, root="C:\\", bypass_check=False)
    
    dllpath      :: str  :: exact path of the DLL -or-
    root         :: str  :: directory to start walking/searching to find the DLL
    bypass_check :: bool :: used by virtual layer implemenation (DO NOT SET)

    returns -> bool 

    throws TOSDB_InitError TOSDB_Error
    """
    global _dll, _dll_depend1
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return False

    def _remove_older_versions():
        nonlocal rel
        getver = lambda x: _search(_REGEX_VER_SFFX, x).group().strip('-')
        vers = tuple(zip(map(getver, rel), rel))
        vers_max = max(vers)[0].split('.')[0]
        mtup = tuple(( x[0].split('.')[1],x[1]) \
                       for x in vers if x[0].split('.')[0] == vers_max )
        mtup_max = max(mtup)[0]
        rel = set(x[1] for x in mtup if x[0] == mtup_max)

    def _get_depends1_dll_path(dllpath):
        d = _path.dirname(dllpath)
        dbg = _match(_REGEX_DBG_DLL_PATH, dllpath)
        base = d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE
        path = base + ("_d.dll" if dbg else ".dll")
        return path

    try:
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile:  # try the current dir first
                    rel.add(_curdir + _sep + nfile.string)
            if not rel:  # no luck, walk the dir tree
                for root, dirs, files in _walk(root):
                    for file in map(matcher, files):
                        if file:
                            rel.add(root + _sep + file.string)
                if not rel:  # if still nothing throw
                    raise TOSDB_InitError(" could not locate DLL")
            if len(rel) > 1:  # only use the most recent version(s)
                _remove_older_versions()
            # most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
            rec = max(d)
            dllpath = d[rec]

        dllpath_depends1 = _get_depends1_dll_path(dllpath)
        _dll_depend1 = _CDLL(dllpath_depends1)
        _dll = _CDLL(dllpath)
        print("+ Using Module(s) ", dllpath)
        print("                  ", dllpath_depends1)
        print("+ Last Update:", _asctime(_localtime(_stat(dllpath).st_mtime)))
        print("+ Process ID:", str(_getpid()))
        if connect():
            print("+ Succesfully Connected to Service\Engine")
            if connected():
                print("+ Succesfully Connected to TOS")
            else:
                print("- Failed to Connect to TOS")
        else:
            print("- Failed to Connect to Service\Engine")
            print("- Failed to Connect to TOS")
        return True  # indicate the lib was loaded (but not if connect succeeded)
    except Exception as e:
        raise TOSDB_InitError("unable to initialize library:", e)
Ejemplo n.º 33
0
from os import listdir as _listdir
from os.path import dirname as _dirname
from importlib import import_module as _import_module

base_package = __name__.replace('.base', '')


def _not_implemented_conversion(item):
    raise NotImplementedError("This conversion has not been implemented yet")


list_api_forms = [
    filename.split('.')[0] for filename in _listdir(_dirname(__file__))
    if filename.startswith('api')
]

dict_api_forms = {}
list_forms = []
dict_converter = {}
dict_is_form = {}

for api_form in list_api_forms:
    module_api_form = _import_module('.' + api_form, base_package)
    form_name = module_api_form.form_name
    list_forms.append(form_name)
    dict_api_forms[form_name] = module_api_form
    dict_is_form.update(module_api_form.is_form)

for form_name in list_forms:
    dict_converter[form_name] = {}
    for method in dict_api_forms[form_name].__dict__.keys():
Ejemplo n.º 34
0
from datetime import date as _date
from json import load as _load, loads as _loads
from os import listdir as _listdir
from os.path import dirname as _dirname, join as _join, splitext as _splitext

from pybars import Compiler as _Compiler
from ssl_config._helpers import HELPERS as _HELPERS
from ssl_config._versions import Version as _Version

_DATA_DIR = _join(_dirname(__file__), '_data')

#: Supported server software
SERVERS = tuple(
    sorted(
        _splitext(name)[0]
        for name in _listdir(_join(_DATA_DIR, 'templates'))))

with open(_join(_DATA_DIR, 'guidelines.json'), 'rt') as json_file:
    #: Guidelines information as dict
    GUIDELINES = _load(json_file)

#: Mozilla SSL configuration levels
#:
#: Modern:
#:     Services with clients that support TLS 1.3 and don't need
#:     backward compatibility.
#:
#: Intermediate
#:     General-purpose servers with a variety of clients, recommended for
#:     almost all systems.
#:
Ejemplo n.º 35
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

  dllpath: string of the exact path of the DLL
  root: string of the directory to start walking/searching to find the DLL
  """
    global _dll, _dll_depend1
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return False

    def _remove_older_versions():
        nonlocal rel
        getver = lambda x: _search(_REGEX_VER_SFFX, x).group().strip('-')
        vers = tuple(zip(map(getver, rel), rel))
        vers_max = max(vers)[0].split('.')[0]
        mtup = tuple(( x[0].split('.')[1],x[1]) \
                       for x in vers if x[0].split('.')[0] == vers_max )
        mtup_max = max(mtup)[0]
        rel = set(x[1] for x in mtup if x[0] == mtup_max)

    def _get_depends1_dll_path(dllpath):
        d = _path.dirname(dllpath)
        return d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE + ".dll"

    try:
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)  # regex match function
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile:  # try the current dir first
                    rel.add(_curdir + _sep + nfile.string)
            if not rel:  # no luck, walk the dir tree
                for root, dirs, files in _walk(root):
                    for file in map(matcher, files):
                        if file:
                            rel.add(root + _sep + file.string)
                if not rel:  # if still nothing throw
                    raise TOSDB_InitError(" could not locate DLL")
            if len(rel) > 1:  # only use the most recent version(s)
                _remove_older_versions()
            # most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
            rec = max(d)
            dllpath = d[rec]

        dllpath_depends1 = _get_depends1_dll_path(dllpath)
        _dll_depend1 = _CDLL(dllpath_depends1)
        _dll = _CDLL(dllpath)

        print("+ Using Module(s) ", dllpath)
        print("                  ", dllpath_depends1)
        print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")
        else:
            print("- Failed to Connect to Service \ Engine")
        return True  # indicate the lib was loaded (but not if connect succeeded)
    except TOSDB_Error:
        raise
    except Exception as e:
        raise TOSDB_InitError("unable to initialize library", e)
Ejemplo n.º 36
0
def listdir(directory):
    return sorted(_listdir(directory))
Ejemplo n.º 37
0
from os import listdir as _listdir
__all__ = [
    _module.replace(".py", "") for _module in _listdir(__path__[0])
    if not _module.endswith(".pyc") and not _module.startswith("__")
]
Ejemplo n.º 38
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

    dllpath: string of the exact path of the DLL
    root: string of the directory to start walking/searching to find the DLL
    """  
    global _dll, _dll_depend1
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return False

    def _remove_older_versions():
        nonlocal rel  
        getver = lambda x: _search(_REGEX_VER_SFFX,x).group().strip('-')
        vers = tuple(zip(map(getver, rel), rel))
        vers_max = max(vers)[0].split('.')[0]
        mtup = tuple(( x[0].split('.')[1],x[1]) \
                       for x in vers if x[0].split('.')[0] == vers_max )         
        mtup_max = max(mtup)[0]
        rel = set(x[1] for x in mtup if x[0] == mtup_max)

    def _get_depends1_dll_path(dllpath):
        d = _path.dirname(dllpath)
        return d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE + ".dll"
    
    try:   
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)  
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile: # try the current dir first             
                    rel.add(_curdir+ _sep + nfile.string)                    
            if not rel: # no luck, walk the dir tree              
                for root, dirs, files in _walk(root):  
                    for file in map(matcher, files):  
                        if file:                           
                            rel.add(root + _sep + file.string)                           
                if not rel: # if still nothing throw
                    raise TOSDB_InitError(" could not locate DLL")          
            if len(rel) > 1:  # only use the most recent version(s)
                _remove_older_versions()              
            # most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel)) 
            rec = max(d)
            dllpath = d[rec]

        dllpath_depends1 = _get_depends1_dll_path(dllpath)
        _dll_depend1 = _CDLL(dllpath_depends1)
        _dll = _CDLL(dllpath)
        
        print("+ Using Module(s) ", dllpath)
        print("                  ", dllpath_depends1)
        print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")       
        else:
            print("- Failed to Connect to Service \ Engine")
        return True # indicate the lib was loaded (but not if connect succeeded)
    except TOSDB_Error:
        raise
    except Exception as e:
        raise TOSDB_InitError("unable to initialize library", e)        
Ejemplo n.º 39
0
 def listdir(path):
     items = os._listdir(path)
     if isinstance(path, unicode):
         items = [unicode(String(i).toString()) for i in items]
     return items
 def os_listdir(path):
   return os._listdir(longpathify(uni(path)))
Ejemplo n.º 41
0
def listdir(dirpath):
    return [
        join(dirpath, fname) for fname in _listdir(dirpath)
        if fname[0] != '.' and not fname.endswith('.pyc')
        and not fname.endswith('~') and not isdir(join(dirpath, fname))
    ]
Ejemplo n.º 42
0
def path_ensdf_dir():
  regex = _re.compile("ensdf_[\d]{6}_[\d]{3}")
  root = path_rawdata()
  return [_path.join(root, i) for i in _listdir(root) if regex.match(i)]