def describe(table, conninfo=None):
"""
Could do this with psycopg
"""
q="select * from %s limit 1" % table
res = npypg.query(q, conninfo=conninfo)
dtypes=[]
max_lens = {'name':0, 'type':0, 'shape':0}
for d in res.dtype.descr:
info = {}
info['name'] = d[0]
info['type'] = d[1][1:]
if len(d) > 2:
info['shape'] = d[2]
else:
info['shape'] = None
for lentype in max_lens:
try:
if len(info[lentype]) > max_lens[lentype]:
max_lens[lentype] = len(info[lentype])
except:
# usually due to None
pass
dtypes.append(info)
padding = 3
forms = {}
forms['name'] = '%-'+str(max_lens['name']+padding)+'s'
forms['type'] = '%-'+str(max_lens['type']+padding)+'s'
forms['shape'] = '%-'+str(max_lens['shape']+padding)+'s'
for n in ['name','type','shape']:
stdout.write(forms[n] % n)
stdout.write('\n')
#line = '-'*len(list(forms.keys()))
for d in dtypes:
for n in ['name','type','shape']:
stdout.write(forms[n] % d[n])
stdout.write('\n')
def fetch_arrays(query_string,conninfo=None, user=userdef, database=dbnamedef, dtype=None,textmax=None,lendict={}):
if have_npypg:
res = npypg.query(query_string)
elif have_psycopg:
conn = psycopg2.connect(conninfo, user=user, database=database)
cur = conn.cursor()
cur.execute(query_string)
# For queries that return nothing
if cur.description is None or cur.rowcount == 0:
return None
# If the user sent dtype, we don't have to compute the dtype can
# set has_arrays to false
has_arrays=False
if dtype is None:
dtype, has_arrays = PgDescr2NumpyDescr(cur.description,
textmax=textmax,
lendict=lendict)
if has_arrays:
#print 'Result contains arrays. We must prefetch the data into'
#print 'lists to determine the array sizes; this is less memory '
#print 'efficient. Suggestion: select a single row to get the dtype, '
#print 'then get the full query sending dtype as a keyword'
t = cur.fetchall()
newdtype = PgGetDescrWithArrays(dtype, t)
res = numpy.array(t, dtype=newdtype)
else:
res = numpy.fromiter(cur, dtype=dtype)
conn.close()
else:
raise RuntimeError('Either psycopg or npypg must be available')
return res
def pgqueryln(query, conninfo=None):
npypg.query(query, conninfo=conninfo)
sys.stdout.write('\n')
def PGCatStuff(flist, stufftype, ftype='red_cat'):
"""
Stuff the file info into the files table or the data into the obj table
"""
import npypg
import es_util as eu
allf = open(flist).readlines()
nf = len(allf)
#for f in open(flist):
fnum = 0
finfo = conf["files"][ftype]
tables = conf['tables']
files_table = "files"
for f in allf:
if False and (fnum > 0):
return
f=f.strip()
stdout.write( 'File (%d/%d): %s\n' % (fnum+1,nf,f))
stdout.flush()
try:
if not os.path.exists(f):
raise RuntimeError('path does not exist: %s\n' % f)
except:
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum+=1
continue
stdout.write( 'Reading data\n')
try:
if finfo['format'].lower() == 'fits':
if finfo['type'] == 'bintable':
t,h = eu.fits2array(f, finfo['hdu'])
elif finfo['type'] == 'header':
t = pyfits.getheader(f, finfo['hdu'])
else:
raise ValueError("Don't support typoe: %s yet\n" %
finfo['type'])
else:
raise ValueError('Unsupported file format: %s\n' % finfo['format'])
except:
stderr.write('Error reading from file: %s\n' % f)
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum+=1
continue
fnameinfo = ExtractPathInfo(f)
if stufftype.lower() == 'files':
info = numpy.zeros(1, dtype=tables[files_table]['desc'])
info['type'] = finfo['type']
info['format'] = finfo['format']
info['relpath'] = fnameinfo['relpath']
info['desc'] = fnameinfo['desc']
info['basename'] = fnameinfo['basename']
info['dirname'] = fnameinfo['dirname']
stdout.write('Stuffing metadata\n')
try:
npypg.array2table(meta, files_table,
unique=['fileid','filedesc','filepath'],
serial_extra=['fileid'])
except:
stderr.write('Error stuffing metadata for file %s\n' % f)
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum+=1
continue
elif stufftype.lower() == 'data':
stdout.write( 'Stuffing cat data\n')
try:
q="SELECT fileid FROM %s WHERE filedesc = '%s'" % \
(files_table, finfo['desc'])
stdout.write(q+'\n')
res = npypg.query(q)
sdef = 'S'+max_string_length
#newdesc = [('filedesc',sdef),('objdesc',sdef)]
newdesc = [('fileid','i8'),('objdesc',sdef)]
new = eu.add_fields(t, newdesc)
new['fileid'] = res[0]['fileid']
for i in range(len(new)):
new[i]['objdesc'] = ObjDesc(finfo['desc'], new[i]['NUMBER'])
npypg.array2table(new, objtable, unique=['objid','objdesc'],
serial_extra=['objid'])
except ValueError:
stderr.write("Error stuffing file '%s' %s\n" % (f,err))
stderr.write('%s %s\n' % sys.exc_info()[0:2])
fnum += 1
continue
else:
raise ValueError('Unsupported stuff type: %s\n' % stufftype)
fnum+=1
if stufftype.lower() == 'files':
npypg.query('analyze %s' % files_table)
elif stufftype.lower() == 'data':
npypg.query('analyze %s' % objtable)
stdout.write('\n')
