def PGStuffFileInfo(fileclass, flist_filename, outdir=None):
import npypg
import es_util as eu
# the output table is the files table
files_table = conf['files_table']
# info for this file class
finfo = conf['files'][fileclass]
# info about this table
tabinfo = conf['tables'][files_table]
# read the file list
flist = open(flist_filename).readlines()
nf = len(flist)
# loop over the file names
for fnum in range(nf):
if True and (fnum > 0):
return
f = flist[fnum].strip()
stdout.write( 'File (%d/%d): %s\n' % (fnum+1,nf,f))
try:
if not os.path.exists(f):
raise RuntimeError('path does not exist: %s\n' % f)
except:
stderr.write('%s %s\n' % sys.exc_info()[0:2])
continue
# get some path information
pathinfo = ExtractPathInfo(f)
info = numpy.zeros(1, dtype=tabinfo['numpy_desc'])
info['fileclass'] = finfo['fileclass']
info['fileformat'] = finfo['fileformat']
info['relpath'] = pathinfo['relpath']
info['filedesc'] = pathinfo['filedesc']
info['basename'] = pathinfo['basename']
info['dirname'] = pathinfo['dirname']
stdout.write('Stuffing file info\n')
try:
npypg.array2table(info, files_table,
unique=tabinfo['unique_cols'],
serial_extra=tabinfo['serial_cols'],
outdir=outdir)
except:
stderr.write('Error stuffing metadata for file %s\n' % f)
stderr.write('%s %s\n' % sys.exc_info()[0:2])
continue
def PGCatStuff(flist, stufftype, ftype='red_cat'):
"""
Stuff the file info into the files table or the data into the obj table
"""
import npypg
import es_util as eu
allf = open(flist).readlines()
nf = len(allf)
#for f in open(flist):
fnum = 0
finfo = conf["files"][ftype]
tables = conf['tables']
files_table = "files"
for f in allf:
if False and (fnum > 0):
return
f=f.strip()
stdout.write( 'File (%d/%d): %s\n' % (fnum+1,nf,f))
stdout.flush()
try:
if not os.path.exists(f):
raise RuntimeError('path does not exist: %s\n' % f)
except:
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum+=1
continue
stdout.write( 'Reading data\n')
try:
if finfo['format'].lower() == 'fits':
if finfo['type'] == 'bintable':
t,h = eu.fits2array(f, finfo['hdu'])
elif finfo['type'] == 'header':
t = pyfits.getheader(f, finfo['hdu'])
else:
raise ValueError("Don't support typoe: %s yet\n" %
finfo['type'])
else:
raise ValueError('Unsupported file format: %s\n' % finfo['format'])
except:
stderr.write('Error reading from file: %s\n' % f)
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum+=1
continue
fnameinfo = ExtractPathInfo(f)
if stufftype.lower() == 'files':
info = numpy.zeros(1, dtype=tables[files_table]['desc'])
info['type'] = finfo['type']
info['format'] = finfo['format']
info['relpath'] = fnameinfo['relpath']
info['desc'] = fnameinfo['desc']
info['basename'] = fnameinfo['basename']
info['dirname'] = fnameinfo['dirname']
stdout.write('Stuffing metadata\n')
try:
npypg.array2table(meta, files_table,
unique=['fileid','filedesc','filepath'],
serial_extra=['fileid'])
except:
stderr.write('Error stuffing metadata for file %s\n' % f)
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum+=1
continue
elif stufftype.lower() == 'data':
stdout.write( 'Stuffing cat data\n')
try:
q="SELECT fileid FROM %s WHERE filedesc = '%s'" % \
(files_table, finfo['desc'])
stdout.write(q+'\n')
res = npypg.query(q)
sdef = 'S'+max_string_length
#newdesc = [('filedesc',sdef),('objdesc',sdef)]
newdesc = [('fileid','i8'),('objdesc',sdef)]
new = eu.add_fields(t, newdesc)
new['fileid'] = res[0]['fileid']
for i in range(len(new)):
new[i]['objdesc'] = ObjDesc(finfo['desc'], new[i]['NUMBER'])
npypg.array2table(new, objtable, unique=['objid','objdesc'],
serial_extra=['objid'])
except ValueError:
stderr.write("Error stuffing file '%s' %s\n" % (f,err))
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum += 1
continue
else:
raise ValueError('Unsupported stuff type: %s\n' % stufftype)
fnum+=1
if stufftype.lower() == 'files':
npypg.query('analyze %s' % files_table)
elif stufftype.lower() == 'data':
npypg.query('analyze %s' % objtable)
stdout.write('\n')