def makedirs_fromfile(f, verbose=False, allow_fail=False):
"""
Extract the directory from a file name and create it if it doesn't exist.
parameters
----------
filename: string
The file name
verbose: boolean, optional
Optionally print that the dir is being created.
"""
import errno
from esutil import hdfs
d = os.path.dirname(f)
if d == '':
return
if hdfs.is_in_hdfs(f):
if not hdfs.exists(d):
if verbose:
print('creating dir:', d)
hdfs.mkdir(d)
else:
if not os.path.exists(d):
if verbose:
print('creating dir:', d)
try:
os.makedirs(d)
except OSError as ex:
if ex.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
if not allow_fail:
raise
def _get_fname_ftype_from_inputs(fileobj, **keywords):
"""
Get filename, file type, and file system
"""
fs='local'
try:
fname = fileobj.name
fobj = fileobj
except AttributeError:
if is_in_hdfs(fileobj):
fs='hdfs'
# make sure we expand all ~username and other variables
fname=ostools.expand_filename(fileobj)
fobj = fname
ftype=None
if 'type' in keywords:
ftype=keywords['type']
elif 'typ' in keywords:
ftype = keywords['typ']
if ftype is None:
ftype=get_ftype(fname)
ftype = ftype.lower()
return fname, fobj, ftype, fs
def _get_fname_ftype_from_inputs(fileobj, **keywords):
"""
Get filename, file type, and file system
"""
fs='local'
if isinstance(fileobj, file):
fname = fileobj.name
fobj = fileobj
elif isinstance(fileobj, (str,unicode)):
if is_in_hdfs(fileobj):
fs='hdfs'
# make sure we expand all ~username and other variables
fname=ostools.expand_filename(fileobj)
fobj = fname
else:
raise ValueError("Input must be a string or file object, or a "
"list thereof")
ftype=None
if 'type' in keywords:
ftype=keywords['type']
elif 'typ' in keywords:
ftype = keywords['typ']
if ftype is None:
ftype=get_ftype(fname)
ftype = ftype.lower()
return fname, fobj, ftype, fs
def makedirs_fromfile(f, verbose=False, allow_fail=False):
"""
Extract the directory from a file name and create it if it doesn't exist.
parameters
----------
filename: string
The file name
verbose: boolean, optional
Optionally print that the dir is being created.
"""
import errno
from esutil import hdfs
d=os.path.dirname(f)
if d=='':
return
if hdfs.is_in_hdfs(f):
if not hdfs.exists(d):
if verbose:
print('creating dir:',d)
hdfs.mkdir(d)
else:
if not os.path.exists(d):
if verbose:
print('creating dir:',d)
try:
os.makedirs(d)
except OSError as ex:
if ex.errno == errno.EEXIST and os.path.isdir(d):
pass
else:
if not allow_fail:
raise
def _get_fname_ftype_from_inputs(fileobj, **keywords):
"""
Get filename, file type, and file system
"""
fs = 'local'
try:
fname = fileobj.name
fobj = fileobj
except AttributeError:
if is_in_hdfs(fileobj):
fs = 'hdfs'
# make sure we expand all ~username and other variables
fname = ostools.expand_filename(fileobj)
fobj = fname
ftype = None
if 'type' in keywords:
ftype = keywords['type']
elif 'typ' in keywords:
ftype = keywords['typ']
if ftype is None:
ftype = get_ftype(fname)
ftype = ftype.lower()
return fname, fobj, ftype, fs
def read_data(self):
c=self.config
s2vals=sample_s2(c['mins2'],c['maxs2'],c['ns2'])
#s2vals=s2vals[1:]
self.alldata = []
for s2 in s2vals:
#print('-'*72)
#print("s2:",s2)
pattern=get_simfile(self['run'], self['objmodel'], self['psfmodel'],
s2, '*', self['psf_ellip'])
print(pattern)
if hdfs.is_in_hdfs(pattern):
flist = hdfs.ls(pattern,full=True)
else:
flist = glob.glob(pattern)
nf=len(flist)
if nf != 0:
st = self.struct(nf)
for i in xrange(nf):
f=flist[i]
#print(" Found %s" % f)
t=eu.io.read(f)
for n in st.dtype.names:
if n in t.dtype.names:
st[n][i] = t[n].mean()
self.alldata.append(st)
else:
print("FOUND NO FILES")
def makedirs_fromfile(f, verbose=False):
"""
Extract the directory from a file name and create it if it doesn't exist.
parameters
----------
filename: string
The file name
verbose: boolean, optional
Optionally print that the dir is being created.
"""
from esutil import hdfs
d=os.path.dirname(f)
if hdfs.is_in_hdfs(f):
if not hdfs.exists(d):
if verbose:
print 'creating dir:',d
hdfs.mkdir(d)
else:
if not os.path.exists(d):
if verbose:
print 'creating dir:',d
os.makedirs(d)
def makedirs_fromfile(f, verbose=False):
"""
Extract the directory from a file name and create it if it doesn't exist.
parameters
----------
filename: string
The file name
verbose: boolean, optional
Optionally print that the dir is being created.
"""
from esutil import hdfs
d = os.path.dirname(f)
if hdfs.is_in_hdfs(f):
if not hdfs.exists(d):
if verbose:
print 'creating dir:', d
hdfs.mkdir(d)
else:
if not os.path.exists(d):
if verbose:
print 'creating dir:', d
os.makedirs(d)
def fexists(fname):
if is_in_hdfs(fname):
return hdfs.exists(fname)
else:
return os.path.exists(fname)
def fexists(fname):
if is_in_hdfs(fname):
return hdfs.exists(fname)
else:
return os.path.exists(fname)
