示例#1
0
def makedirs_fromfile(f, verbose=False, allow_fail=False):
    """
    Extract the directory from a file name and create it if it doesn't exist.

    parameters
    ----------
    filename: string
        The file name
    verbose: boolean, optional
        Optionally print that the dir is being created.
    """
    import errno
    from esutil import hdfs

    d = os.path.dirname(f)
    if d == '':
        return

    if hdfs.is_in_hdfs(f):
        if not hdfs.exists(d):
            if verbose:
                print('creating dir:', d)
            hdfs.mkdir(d)
    else:
        if not os.path.exists(d):
            if verbose:
                print('creating dir:', d)
            try:
                os.makedirs(d)
            except OSError as ex:
                if ex.errno == errno.EEXIST and os.path.isdir(path):
                    pass
                else:
                    if not allow_fail:
                        raise
示例#2
0
文件: io.py 项目: esheldon/esutil
def _get_fname_ftype_from_inputs(fileobj, **keywords):
    """
    Get filename, file type, and file system
    """

    fs='local'

    try:
        fname = fileobj.name
        fobj = fileobj
    except AttributeError:
        if is_in_hdfs(fileobj):
            fs='hdfs'

        # make sure we expand all ~username and other variables
        fname=ostools.expand_filename(fileobj)
        fobj = fname

    ftype=None
    if 'type' in keywords:
        ftype=keywords['type']
    elif 'typ' in keywords:
        ftype = keywords['typ']

    if ftype is None:
        ftype=get_ftype(fname)
    ftype = ftype.lower()

    return fname, fobj, ftype, fs
示例#3
0
def _get_fname_ftype_from_inputs(fileobj, **keywords):
    """
    Get filename, file type, and file system
    """

    fs='local'

    if isinstance(fileobj, file):
        fname = fileobj.name
        fobj = fileobj
    elif isinstance(fileobj, (str,unicode)):
        if is_in_hdfs(fileobj):
            fs='hdfs'

        # make sure we expand all ~username and other variables
        fname=ostools.expand_filename(fileobj)
        fobj = fname
    else:
        raise ValueError("Input must be a string or file object, or a "
                         "list thereof")


    ftype=None
    if 'type' in keywords:
        ftype=keywords['type']
    elif 'typ' in keywords:
        ftype = keywords['typ']

    if ftype is None:
        ftype=get_ftype(fname)
    ftype = ftype.lower()

    return fname, fobj, ftype, fs
示例#4
0
def makedirs_fromfile(f, verbose=False, allow_fail=False):
    """
    Extract the directory from a file name and create it if it doesn't exist.

    parameters
    ----------
    filename: string
        The file name
    verbose: boolean, optional
        Optionally print that the dir is being created.
    """
    import errno
    from esutil import hdfs

    d=os.path.dirname(f)
    if d=='':
        return

    if hdfs.is_in_hdfs(f):
        if not hdfs.exists(d):
            if verbose:
                print('creating dir:',d)
            hdfs.mkdir(d)
    else:
        if not os.path.exists(d):
            if verbose:
                print('creating dir:',d)
            try:
                os.makedirs(d)
            except OSError as ex:
                if ex.errno == errno.EEXIST and os.path.isdir(d):
                    pass
                else:
                    if not allow_fail:
                        raise
示例#5
0
def _get_fname_ftype_from_inputs(fileobj, **keywords):
    """
    Get filename, file type, and file system
    """

    fs = 'local'

    try:
        fname = fileobj.name
        fobj = fileobj
    except AttributeError:
        if is_in_hdfs(fileobj):
            fs = 'hdfs'

        # make sure we expand all ~username and other variables
        fname = ostools.expand_filename(fileobj)
        fobj = fname

    ftype = None
    if 'type' in keywords:
        ftype = keywords['type']
    elif 'typ' in keywords:
        ftype = keywords['typ']

    if ftype is None:
        ftype = get_ftype(fname)
    ftype = ftype.lower()

    return fname, fobj, ftype, fs
示例#6
0
    def read_data(self):
        c=self.config
        s2vals=sample_s2(c['mins2'],c['maxs2'],c['ns2'])
        #s2vals=s2vals[1:]
        self.alldata = []
        for s2 in s2vals:
            #print('-'*72)
            #print("s2:",s2)

            pattern=get_simfile(self['run'], self['objmodel'], self['psfmodel'],
                            s2, '*', self['psf_ellip'])
            print(pattern)
            if hdfs.is_in_hdfs(pattern):
                flist = hdfs.ls(pattern,full=True)
            else:
                flist = glob.glob(pattern)
            nf=len(flist)
            if nf != 0:
                st = self.struct(nf)
                for i in xrange(nf):
                    f=flist[i]
                    #print("    Found %s" % f)
                    t=eu.io.read(f)

                    for n in st.dtype.names:
                        if n in t.dtype.names:
                            st[n][i] = t[n].mean()
                self.alldata.append(st)
            else:
                print("FOUND NO FILES")
示例#7
0
def makedirs_fromfile(f, verbose=False):
    """
    Extract the directory from a file name and create it if it doesn't exist.

    parameters
    ----------
    filename: string
        The file name
    verbose: boolean, optional
        Optionally print that the dir is being created.
    """
    from esutil import hdfs
    d=os.path.dirname(f)
    if hdfs.is_in_hdfs(f):
        if not hdfs.exists(d):
            if verbose:
                print 'creating dir:',d
            hdfs.mkdir(d)
    else:
        if not os.path.exists(d):
            if verbose:
                print 'creating dir:',d
            os.makedirs(d)
示例#8
0
def makedirs_fromfile(f, verbose=False):
    """
    Extract the directory from a file name and create it if it doesn't exist.

    parameters
    ----------
    filename: string
        The file name
    verbose: boolean, optional
        Optionally print that the dir is being created.
    """
    from esutil import hdfs
    d = os.path.dirname(f)
    if hdfs.is_in_hdfs(f):
        if not hdfs.exists(d):
            if verbose:
                print 'creating dir:', d
            hdfs.mkdir(d)
    else:
        if not os.path.exists(d):
            if verbose:
                print 'creating dir:', d
            os.makedirs(d)
示例#9
0
文件: io.py 项目: esheldon/esutil
def fexists(fname):
    if is_in_hdfs(fname):
        return hdfs.exists(fname)
    else:
        return os.path.exists(fname)
示例#10
0
def fexists(fname):
    if is_in_hdfs(fname):
        return hdfs.exists(fname)
    else:
        return os.path.exists(fname)