def Save( self, hdf5name ) :
'''
hdf5name:
(1) ==None/'': hdf5name = 'jizhipy.PCA.hdf5'
(2) ==filename of .hdf5
'''
from jizhipy.Basic import Path
import h5py
outname = hdf5name
if(outname in [None, '']): outname ='jizhipy.PCA.hdf5'
else :
if (outname[-5:] != '.hdf5') : outname += '.hdf5'
outname = Path.AbsPath(outname)
Path.ExistsPath(outname, True)
fo = h5py.File(outname, 'w')
if (self.verbose) : print(' Saving to: ', Path.AbsPath(outname, True))
self.hdf5name = outname
keys = self.__dict__.keys()
fo['class'] = 'jizhipy.PCA'
fo['stagger'] = stagger
fo['stagger'].attrs['comment'] = 'y1, y2 = y[:,:-stagger], y[:,stagger:]'
if ('v' in keys) :
fo['eigenvalue'] = v
fo['eigenvalue'].attrs['comment']='eign values of R matrix'
if ('P' in keys) :
fo['eigenvector'] = P
fo['eigenvector'].attrs['comment'] = 'each columns are eigen vectors of R matrix'
# if ('a' in keys) :
# fo['a'] = a
# fo['a'].attrs['comment'] = 'z=P*a, y=(P*a)*rms'
# if ('rms' in keys) :
# fo['rms'] = rms
# fo['rms'].attrs['comment'] = 'rms of each row of y-map'
fo.close()
return self.hdf5name
def Use(which, tf=True):
'''
which:
(1) which='Agg'
(2) which='tex', tf=True
(3) which is LatexPackageName
'''
import matplotlib as mpl
which = str(which).lower()
if (which == 'agg'): mpl.use('Agg')
elif (which == 'tex'):
# Some systems,
# set usetex=True , use tex
# set usetex=False, don't use tex
# But also some systems,
# set usetex=True , raise error
# set usetex=False, use tex!
# Therefore, set True/False dependenting on your system
from jizhipy.Basic import Path
import os
texcache = Path.AbsPath('~/.cache/matplotlib/tex.cache')
if (Path.ExistsPath(texcache)):
os.system('rm -r ' + texcache)
mpl.rcParams['text.usetex'] = bool(tf)
# import matplotlib.pyplot as plt
# plt.rc('text', usetex=True)
else: # which is packagename
mpl.rcParams['text.latex.preamble'] = [r'\usepackage{' + which + r'}']
def Fits2Hdf5( inputwhat, hduname=None, verbose=False ) :
'''
inputwhat:
(1) str, path of .fits
(2) one of the HDU: inputwhat = fo[i]
hduname:
if exists hdu.header['extname'] or inputwhat.name, use them, otherwise, use hduname+str(i)
'''
from jizhipy.Basic import Path, IsType
import h5py, pyfits
def _Fits2Hdf5( hname, hdu, outhdf5 ) :
try : name = hdu.header['extname']
except : name = hdu.name
if (name == '') :
if (hduname in ['', None]) : name = str(hname)
else : name = str(hduname)+str(hname)[3:]
outhdf5[name] = hdu.data #@#@
hdr = hdu.header
keys, values, comments = hdr.keys(), hdr.values(), hdr.comments
for i in range(len(keys)) :
outhdf5[name].attrs[keys[i]] = values[i]
if (comments[i] != '') :
outhdf5[name].attrs[keys[i]+'_comment'] = comments[i]
#--------------------------------------------------
if (IsType.isstr(inputwhat)) :
inname = Path.AbsPath(inputwhat)
outname = inname[:-5] + '.hdf5'
if (verbose) : print('Saving "'+outname+'"')
Path.ExistsPath(outname, 'True')
h = h5py.File(outname, 'w')
f = pyfits.open(inname)
for i in range(len(f.info(0))) :
_Fits2Hdf5( 'hdu'+str(i), f[i], h )
#--------------------------------------------------
else :
if (hduname not in ['', None]) : name = str(hduname)
else :
name = inputwhat.name
if (name == '') : name = 'hdu'
outname = name + '.hdf5'
if (verbose) : print('Saving "'+outname+'"')
Path.ExistsPath(outname, 'True')
h = h5py.File(outname, 'w')
_Fits2Hdf5( hduname, inputwhat, h )
#--------------------------------------------------
h.close()
def Linecolor(self, num, cmap='jet'):
'''
Return is from Blue to Red
num:
(1) color name:
in ['red', 'orange', 'yellow', 'green', 'cyan', 'blue', 'purple', 'pink', 'brown', 'black', 'white']
return () of this color
(2) int, number of colors:
return () of colors
(3) == 'png' :
open 'jizhipy_Plt_LineColor.png'
cmap:
Use which cmap to generate the linecolor
(1) None=>'gist_rainbow_r' | Name of cmap (see plt_color.cmap(None))
(2) <matplotlib.colors.LinearSegmentedColormap> (cmap instance)
(3) cmap='my': use my cmap
'''
from jizhipy.Basic import Raise, IsType, ShellCmd, Path
import numpy as np
#----------------------------------------
if (IsType.isstr(num)):
num = num.lower()
if ('png' in num):
uname = ShellCmd('uname')[0].lower()
figpath = Path.jizhipyPath(
'jizhipy_tool/jizhipy_Plt_Color_Linecolor.png')
operate = 'eog ' if (uname == 'linux') else 'open '
ShellCmd(operate + figpath)
return
#----------------------------------------
if (num in self.mycolor.keys()): return self.mycolor[num]
else: Raise(Exception, num + ' NOT in ' + str(self.mycolor.keys()))
#----------------------------------------
if (IsType.isstr(cmap) and cmap == 'my'):
_jet_data_my = np.array([(255, 0, 0), (255, 128, 0), (200, 200, 0),
(0, 200, 0), (0, 250, 250), (0, 0, 255),
(160, 0, 255)]) / 255.
r, g, b = _jet_data_my.T
from scipy.interpolate import interp1d
r, g, b = np.array([r, g, b]) * ratio
x = np.linspace(0, 1, r.size)
fr = interp1d(x, r)
fg = interp1d(x, g)
fb = interp1d(x, b)
x = np.linspace(0, 1, num)
r, g, b = fr(x), fg(x), fb(x)
color = []
for i in range(num):
color += [(r[i], g[i], b[i])]
return tuple(color)
#----------------------------------------
else:
cmap = self.Cmap(cmap)
color = [cmap(x)[:3] for x in np.linspace(0., 1, num)]
for i in range(len(color)):
color[i] = (color[i][0], color[i][1], color[i][2])
return tuple(color)
def islaohu( self ) :
from jizhipy.Basic import Path
tf = (Path.AbsPath('~')[:17] == '/public/home/wufq')
if (tf) :
import sys
from jizhipy.Plot import Plt
Plt.Style()
sys.path = ['/public/home/wufq/.mysoftware/lib/python2.7/site-packages'] + sys.path
return tf
def Save( self ) :
'''
Save instance to .hdf5
'''
from jizhipy.Basic import Path, IsType, Raise
import numpy as np
import h5py
self.outname = str(self.outname)
if (self.outname[-5:] !='.hdf5') : self.outname += '.hdf5'
Path.ExistsPath(self.outname, old=True)
if (self.verbose) : print('jizhipy.ClassHdf5: Saveing to "'+self.outname+'"')
fo = h5py.File(self.outname, 'w')
if (self.value is None or self.keys is None): self.Values()
fo['classinstance'] = str(type(self.classinstance))
for i in range(len(self.keys)) :
if (self.values[i] is None) :
fo[self.keys[i]] = '__None__'
fo[self.keys[i]].attrs['type'] = 'is None, not str'
#--------------------------------------------------
elif (IsType.isdtype(self.values[i])) :
fo[self.keys[i]]=np.dtype(self.values[i]).name
fo[self.keys[i]].attrs['type'] = 'numpy.dtype'
#--------------------------------------------------
elif (IsType.ismatrix(self.values[i])) :
fo[self.keys[i]] = np.array(self.values[i])
fo[self.keys[i]].attrs['type'] = 'numpy.matrix'
#--------------------------------------------------
elif (IsType.ismaskedarray(self.values[i])) :
fo[self.keys[i]] = self.values[i].data
fo[self.keys[i]+'.mask'] = self.values[i].mask
fo[self.keys[i]].attrs['type'] = 'numpy.MaskedArray .data'
fo[self.keys[i]+'.mask'].attrs['type'] = 'numpy.MaskedArray .mask'
#--------------------------------------------------
elif (IsType.isdict(self.values[i])) :
self._OperaDict(self.values[i])
try : fo[self.keys[i]] =self.values[i].values()
except : fo[self.keys[i]] = str(self.values[i].values())
fo[self.keys[i]+'.keys'] = self.values[i].keys()
fo[self.keys[i]].attrs['type']='dict .values()'
fo[self.keys[i]+'.keys'].attrs['type']='dict .keys()'
self._OperaDict(self.values[i])
#--------------------------------------------------
else :
# inside list and tuple, must not contain dict
try : fo[self.keys[i]] = self.values[i] # convert to np.ndarray with same dtype, if contains string, will conver to string array
except :
strv = str([self.values[i]])[1:-1]
Raise(Exception, "fo['"+self.keys[i]+"'] = "+strv)
if (IsType.islist(self.values[i])) :
fo[self.keys[i]].attrs['type'] = 'list'
elif (IsType.istuple(self.values[i])) :
fo[self.keys[i]].attrs['type'] = 'tuple'
else : fo[self.keys[i]].attrs['type'] = 'numpy.array'
#--------------------------------------------------
fo.flush()
fo.close()
def Read( self, hdf5name ) :
'''
Return instance of jp.PCA
'''
from jizhipy.Basic import Path
import h5py
hdf5name = Path.AbsPath(hdf5name)
if (self.verbose) : print(' Reading from: ', Path.AbsPath(hdf5name, True))
fo = h5py.File(hdf5name, 'r')
keys = fo.keys()
pca = PCA()
try :
if (fo['class'] != 'jizhipy.PCA') : raise
except :
if (self.verbose) : print(' NOT .hdf5 for class: jizhipy.PCA')
return pca
if ('stagger' in keys) : self.stagger = fo['stagger'].value
if ('eigenvalue' in keys) : self.v = fo['eigenvalue'].value
if ('eigenvector' in keys) : self.P = fo['eigenvector'].value
# if ('a' in keys) : self.a = fo['a'].value
# if ('rms' in keys) : self.rms = fo['rms'].value
fo.close()
return pca
def CatalogDecSign(cls, catalogname):
'''
In astronomy catalog, Dec is from +90 to -90, but for Dec = -00 23 15, if we np.loadtxt() it, it will miss the "-", treat -00 as 00, and make Dec=-0deg23arcmin15arcsec to +0deg23arcmin15arcsec.
This function is used to get the sign of Dec
catalogname:
the path/name of the input catalog
Return:
1D np.array with the same size of the input catalog
NOTE THAT:
In this catalog, must only Dec has sign !
'''
from jizhipy.Basic import Path
instr = open(Path.AbsPath(catalogname)).readlines()
sign = np.ones(len(instr), np.float32)
for i in range(len(sign)):
if ('-' in instr[i]): sign[i] = -1
return sign
def GSM(fwhm,
freq,
lon=None,
lat=None,
nside=None,
coordsys='Galactic',
gsmpath='gsm_parameter.out',
save=False):
'''
Already -2.726
fwhm:
[degree]
fwhm==None: don't smooth the map
freq, dfreq:
[MHz]
Must be one/isnum, NOT list
* freq: center frequency
* dfreq: averaged from freq-dfreq/2 (include) to freq+dfreq/2 (include). For GSM(), dfreq effect is very small, so remove this argument
lon, lat:
[degree]
lon: RA | l
lat: Dec | b
Must can be broadcast
Use lon, lat first, otherwise, use nside
nside:
Healpix full sky map instead of lon, lat
coordsys:
'Equatorial' | 'Galactic' | 'Ecliptic'
gsmpath:
str, where is the program "gsm_parameter.out"
./gsm_parameter.out freq outpath gsmdir
save:
True | False
Save gsm_xxx.npy or not?
Compile:
gfortran -ffixed-line-length-none gsm_parameter.f -o gsm_parameter.out
return:
gsm, ndarray
'''
import os
import healpy as hp
import numpy as np
from jizhipy.Transform import CoordTrans
from jizhipy.Array import Asarray
from jizhipy.Basic import Raise, Path, ShellCmd, IsType
if (gsmpath is None): gsmpath = 'gsm_parameter.out'
freq = Asarray(freq).flatten()
if (freq.size != 1):
Raise(Exception,
'freq must isnum, but now freq.size=' + str(freq.size))
freq = freq[0]
gsmpath = Path.AbsPath(str(gsmpath))
if (not Path.ExistsPath(gsmpath)):
gsmpath = Path.jizhipyPath('jizhipy_tool/GSM/gsm_parameter.out')
if (not Path.ExistsPath(gsmpath)):
Raise(Exception, 'gsmpath="' + gsmpath + '" NOT exists')
n = gsmpath.rfind('/')
if (n < 0): gsmdir = ''
else: gsmdir = gsmpath[:n + 1]
if (lon is not None and lat is not None):
if (np.sum((lat < -90) + (lat > 90)) > 0):
Raise(Exception, 'lat out of [-90, 90] degree')
nside = None
else:
islist = True
try:
nside = int(nside)
except:
nside = 512
#--------------------------------------------------
# list local "gsm_*.npy"
gsmlist = ShellCmd('ls gsm_*.npy')
ngsm = None
for i in range(len(gsmlist)):
try:
f = np.sort([float(gsmlist[i][4:-4]), freq])
except:
continue
r = (f[1] / f[0])**3
if (r < 1.01):
ngsm = i
break
if (ngsm is None):
freqstr = ('%.2f' % freq)
if (freqstr[-1] == '0'): freqstr = freqstr[:-1]
if (freqstr[-1] == '0'): freqstr = freqstr[:-2]
outname = 'gsm_' + freqstr
os.system(gsmpath + ' ' + freqstr + ' ' + outname + '.dat ' + gsmdir)
gsm = np.float32(np.loadtxt(outname + '.dat'))
os.system('rm ' + outname + '.dat')
if (save): np.save(outname + '.npy', gsm)
else:
gsm = np.float32(np.load(gsmlist[ngsm]))
if (not save): os.system('rm ' + gsmlist[ngsm])
if (Path.ExistsPath('qaz_cols.dat')): os.system('rm qaz_cols.dat')
nsidegsm = hp.get_nside(gsm)
#--------------------------------------------------
coordsys = str(coordsys).lower()
if (nside is None):
if (fwhm is not None and fwhm > 1):
fwhm = (fwhm**2 - 1)**0.5
gsm = hp.smoothing(gsm, fwhm * np.pi / 180, verbose=False)
gsm[gsm < 0] = 0
islist = False if (IsType.isnum(lon + lat)) else True
lon, lat = lon + lat * 0, lon * 0 + lat # same shape
lon %= 360
lon, lat = CoordTrans.Celestial(lon * np.pi / 180, lat * np.pi / 180,
coordsys, 'galactic') # rad
npix = hp.ang2pix(nsidegsm, np.pi / 2 - lat, lon)
del lon, lat
gsm = gsm[npix]
del npix
else:
if (nside != nsidegsm): gsm = hp.ud_grade(gsm, nside)
if (coordsys != 'galactic'):
gsm = CoordTrans.CelestialHealpix(gsm, 'RING', 'galactic',
coordsys)[0]
if (fwhm is not None and fwhm > 1):
fwhm = (fwhm**2 - 1)**0.5
gsm = hp.smoothing(gsm, fwhm * np.pi / 180, verbose=False)
gsm[gsm < 0] = 0
gsm = np.float32(gsm)
if (not islist): gsm = gsm[0]
return gsm
def Ass2Txt( pos, path, fmt=None ) :
#'''
#Read and write Chinese
#pos:
# str to judge what are the real texts
# For example:
# pos = [('0000,0000,0000,,', '\N{'), ('shad1}', '\n')]
#path:
# Must be one
# If path is one file, handle it
# If path is directory, use fmt to judge which files will be handled
#fmt:
# with or without '.' is OK: 'ass' | '.ass'
#'''
from jizhipy.Basic import ShellCmd, Path, IsType
import chardet, codecs
path = Path.AbsPath(path)
if (IsType.isfile(path)) : path = [path]
else :
indir = path
if (fmt is None) : path = ShellCmd('ls '+indir)
else :
fmt = str(fmt).lower()
if (fmt[0] != '.') : fmt = '.' + fmt
path = ShellCmd('ls '+indir+'*'+fmt)
fmt = fmt.upper()
path += ShellCmd('ls '+indir+'*'+fmt)
for i in range(len(path)) : path[i] = indir+path[i]
#---------------------------------------------
outnamelist = []
for i in range(len(path)) :
filename = path[i]
try : a = open(filename).read()
except : continue
code = chardet.detect(a)['encoding']
a = a.decode(code)
txt = ''
while ('\n' in a) :
n = a.index('\n')
b = a[:n+1]
for i in range(len(pos)) :
n1 = n2 = -1
if (pos[i][0] in b) :
n1 = b.rfind(pos[i][0])+len(pos[i][0])
if (pos[i][1] in b) : n2 = b.rfind(pos[i][1])
if (n1<0 or n2<0) : break
c = b[n1:n2]
if (c[0] == '\r') : c = c[1:]
if (c[-1] == '\r') : c = c[:-1]
txt += c + '\n'
if (txt != '') : txt += '\n'
a = a[n+1:]
outname = filename + '.txt'
b = codecs.open(outname, 'w', code)
b.write(txt)
b.close()
outnamelist.append(outname)
return outnamelist
def Array2FitsImage( arraylist, outname, names=None, keys=None, values=None, comments=None, verbose=False ) :
'''
arraylist:
(1) isndarray
(2) list/tuple of ndarray: [array1, array2, ...]
Otherwise, raise error
Examples:
Array2FitsImage( [1,2,3] ), means there are 3 arrays, array1=1, array2=2, array3=3
FITS can save int/float number to ImageHDU, but will raise error "IOError: Header missing END card." when pyfits.open() it.
You must convert to
Array2FitsImage( np.array([1,2,3]), 'test.fits' )
OR Array2FitsImage( [[1,2,3]], 'test.fits' )
names:
None | Same shape as arraylist
Name of each ImageHDU in fo.info()
keys:
None | Same shape as arraylist
(1) arraylist isndarray: keys=['key1', 'key2', ...], list of str
(2) arraylist is list/tuple of ndarray: [array1, array2, ...], then keys=[arraykey1, arraykey2, ...], arraykey1=['key1', 'key2', ...] (list of list)
values:
Same shape as keys
comments:
None | same shape as keys even though is ''
'''
from jizhipy.Basic import Path, IsType
import pyfits
outname = Path.AbsPath(outname)
if (outname[-5:].lower() != '.fits') : outname += '.fits'
Path.ExistsPath(outname, True)
if (verbose) : print('Saving "'+outname+'"')
#--------------------------------------------------
if (IsType.isstr(keys)) :
keys, values = [keys], [values]
if (comments is not None) : comments = [comments]
else : comments = ['']
#--------------------------------------------------
if (not IsType.islist(arraylist) and not IsType.istuple(arraylist)) :
arraylist = [arraylist]
if (names is not None) : names = [names]
keys = [keys for i in range(len(arraylist))]
values = [values for i in range(len(arraylist))]
comments = [comments for i in range(len(arraylist))]
#--------------------------------------------------
hdulist = pyfits.HDUList()
for i in range(len(arraylist)) :
data = arraylist[i]
try : name = str(names[i])
except : name = 'hdu'+str(i)
if (data.dtype.name[:7] == 'complex') :
hdu = pyfits.ImageHDU( data.real )
hdu.name = name+'REAL'
hdr = hdu.header
for j in range(len(keys[i])) :
key, value = keys[i][j], values[i][j]
try : comment = comments[i][j]
except : comment = ''
hdr.set(key, value, comment)
hdulist.append(hdu) # Real part
hdu = pyfits.ImageHDU( data.imag )
hdu.name = name+'IMAG'
hdr = hdu.header
try :
for j in range(len(keys[i])) :
key, value = keys[i][j], values[i][j]
try : comment = comments[i][j]
except : comment = ''
hdr.set(key, value, comment)
except : pass
hdulist.append(hdu) # Imag part
#--------------------------------------------------
else :
hdu = pyfits.ImageHDU( data )
hdu.name = name
hdr = hdu.header
try :
for j in range(len(keys[i])) :
key, value = keys[i][j], values[i][j]
try : comment = comments[i][j]
except : comment = ''
hdr.set(key, value, comment)
except : pass
hdulist.append(hdu)
#--------------------------------------------------
hdulist.writeto( outname )
def Hdf52Fits( inputwhat, hduname='', verbose=False ) :
'''
inputwhat:
(1) str, path of .hdf5
(2) one of the dataset: inputwhat = fo[keys[i]]
'''
from jizhipy.Basic import Path, IsType
import numpy as np
import h5py, pyfits
def _Hdf52Fits( dataset, hduname='' ) :
if (hduname in ['', None]) :
name = str(dataset.name)
name = name[name.rfind('/')+1:]
else : name = str(hduname)
attrs = dataset.attrs.items()
data = dataset.value
hdulist = []
try :
if (data.dtype.name[:7] == 'complex') :
hdu = pyfits.ImageHDU( data.real )
hdu.name = name+'REAL'
hdr = hdu.header
for i in range(len(attrs)) :
hdr[str(attrs[i][0])] = attrs[i][1]
hdulist.append(hdu) #@#@
hdu = pyfits.ImageHDU( data.imag )
hdu.name = name+'IMAG'
hdr = hdu.header
for i in range(len(attrs)) :
hdr[str(attrs[i][0])] = attrs[i][1]
hdulist.append(hdu) #@#@
else : raise
except :
if (IsType.isndarray(data) or IsType.islist(data) or IsType.istuple(data)) :
try : hdu = pyfits.ImageHDU( data )
except : hdu = pyfits.ImageHDU( np.array(data, np.float32) )
hdu.name = name
hdr = hdu.header
for i in range(len(attrs)) :
hdr[str(attrs[i][0])[:8]] = attrs[i][1]
hdulist.append(hdu)
else : other.header[name[:8]] = data
return hdulist
#--------------------------------------------------
hdulist = pyfits.HDUList()
other = pyfits.ImageHDU([])
other.name = 'other'
hdrother = other.header.keys()
if (IsType.isstr(inputwhat)) :
inname = Path.AbsPath(inputwhat)
outname = inname[:-5] + '.fits'
if (verbose) : print('Saving "'+outname+'"')
fo = h5py.File(inname, 'r')
keys = fo.keys()
for i in range(len(keys)) :
hdu = _Hdf52Fits( fo[keys[i]], hduname )
for j in range(len(hdu)) : hdulist.append( hdu[j] )
#--------------------------------------------------
else :
if (hduname in ['', None]) :
name = str(inputwhat.name)
name = name[name.rfind('/')+1:]
else : name = str(hduname)
outname = name + '.fits'
if (verbose) : print('Saving "'+outname+'"')
hdu = _Hdf52Fits( inputwhat, hduname )
for j in range(len(hdu)) : hdulist.append( hdu[j] )
#--------------------------------------------------
hdr = other.header.keys()
if (hdr != hdrother) : hdulist.append(other)
Path.ExistsPath(outname, 'True')
hdulist.writeto( outname )
hdulist.close()
def __init__(self,
freq_unit='MHz',
unit='TCMB',
resolution='hi',
pygsmpath='gsm_components.hdf5'):
""" Global sky model (GSM) class for generating sky models.
Upon initialization, the map PCA data are loaded into memory and interpolation
functions are pre-computed.
Parameters
----------
freq_unit: 'Hz', 'MHz', or 'GHz'
Unit of frequency. Defaults to 'MHz'.
unit: 'MJysr', 'TCMB', 'TRJ'
Unit of output data. MJy/Steradian, T_CMB in Kelvin, or T_RJ.
resolution: 'hi' or 'low'
Resolution of output map. Either 300 arcmin (low) or 24 arcmin (hi).
For frequencies under 10 GHz, output is 48 arcmin.
Notes
-----
"""
import numpy as np
import h5py
if (pygsmpath is None): pygsmpath = 'gsm2016_components.hdf5'
if (not Path.ExistsPath(pygsmpath)):
pygsmpath = Path.jizhipyPath(
'jizhipy_tool/pyGSM_component/gsm2016_components.hdf5')
if (not Path.ExistsPath(pygsmpath)):
Raise(Exception, 'pygsmpath="' + pygsmpath + '" NOT exists')
self.GSM2016_FILEPATH = pygsmpath
#----------------------------------------
if unit not in ['MJysr', 'TCMB', 'TRJ']:
raise RuntimeError(
"UNIT ERROR: %s not supported. Only MJysr, TCMB, TRJ are allowed."
% unit)
if resolution.lower() in ('hi', 'high', 'h'):
resolution = 'hi'
elif resolution.lower() in ('low', 'lo', 'l'):
resolution = 'low'
else:
raise RuntimeError(
"RESOLUTION ERROR: Must be either hi or low, not %s" %
resolution)
self.h5 = h5py.File(self.GSM2016_FILEPATH, "r")
self.freq_unit = freq_unit
self.unit = unit
self.resolution = resolution
# Map data to load
labels = ['Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2', 'Free-Free']
self.n_comp = len(labels)
self.map_ni_hr = np.array(
[self.h5['highres_%s_map' % lb][:] for lb in labels])
self.map_ni_lr = self.h5['lowres_maps']
self.spec_nf = self.h5['spectra'][:]
self.generated_map_data = None
self.generated_map_freqs = None
def __init__(self,
freq_unit='MHz',
basemap='haslam',
interpolation='pchip',
pygsmpath='gsm_components.hdf5'):
"""
Global sky model (GSM) class for generating sky models.
Upon initialization, the map PCA data are loaded into memory and interpolation
functions are pre-computed.
Parameters
----------
freq_unit: 'Hz', 'MHz', or 'GHz'
Unit of frequency. Defaults to 'MHz'.
gsm_version: 'haslam', 'wmap' or '5deg'
GSM version to generate. The 5deg map has 5.1 degree resolution.
This is a synthesized map made of all the maps considered in
the de Oliveira-Costa et. al. paper
At frequencies below 1GHz, haslam is preferred; you get higher
resolution (1 degree) by locking to the Haslam 408 MHz map.
At CMB frequencies, it is best to lock to the WMAP 23 GHz
map, which is presented denoised with 2 degree resolution.
interpolation: 'cubic' or 'pchip'
Choose whether to use cubic spline interpolation or
piecewise cubic hermitian interpolating polynomial (PCHIP).
PCHIP is designed to never locally overshoot data, whereas
splines are designed to have smooth first and second derivatives.
Notes
-----
The scipy `interp1d` function does not allow one to explicitly
set second derivatives to zero at the endpoints, as is done in
the original GSM. As such, results will differ. Further, we default
to use PCHIP interpolation.
"""
import h5py
from jizhipy.Basic import Path
if (pygsmpath is None): pygsmpath = 'gsm_components.hdf5'
if (not Path.ExistsPath(pygsmpath)):
pygsmpath = Path.jizhipyPath(
'jizhipy_tool/pyGSM_component/gsm_components.hdf5')
if (not Path.ExistsPath(pygsmpath)):
Raise(Exception, 'pygsmpath="' + pygsmpath + '" NOT exists')
self.GSM_FILEPATH = pygsmpath
#----------------------------------------
try:
assert basemap in {'5deg', 'wmap', 'haslam'}
except AssertionError:
raise RuntimeError(
"GSM basemap unknown: %s. Choose '5deg', 'haslam' or 'wmap'" %
basemap)
try:
assert interpolation in {'cubic', 'pchip'}
except AssertionError:
raise RuntimeError(
"Interpolation must be set to either 'cubic' or 'pchip'")
self.h5 = h5py.File(self.GSM_FILEPATH, "r")
self.basemap = basemap
self.interpolation_method = interpolation
self.freq_unit = freq_unit
self.pca_map_data = None
self.interp_comps = None
self.update_interpolants()
self.generated_map_data = None
self.generated_map_freqs = None
def CrawlerBaiduWenku(website,
page50,
outdir,
outname,
browserDriverPath,
logger='INFO',
timeout=5):
'''
browserDriverPath:
How to download and use browserDriver ?
Using Chrome for example.
(1) Check the Chrome version:
input chrome://version on the website bar
You can see something like:
Google Chrome 69.0.3497.92
It means that the version is 69
(2) Open website http://chromedriver.storage.googleapis.com/index.html
There are some documents from 2.0 to 2.9
Open one of them, for example 2.46
Then open notes.txt, then you will see:
Supports Chrome v......
Find that matches your Chrome, here 2.40 to 2.44 are OK for Chrome 69
Then go the document 2.44, download the chromedriver for your system (linux, mac, win)
(3) Unzip the downloaded file, then you will obtain an application named "chromedriver"
(4) @browserDriverPath is the path of this "chromedriver"
website:
The url/website/link of the document on the wenku.baidu.com
e.g. the website of document "人脸检测与识别关键技术研究" is "https://wenku.baidu.com/view/f267c923dd36a32d737581df.html?sxts=1552644845367"
page50:
For a thesis or some document which are very long: exceed 50 pages, some times default website is only for the front 50 page, for the next 50 pages, the website is
https://wenku.baidu.com/view/f267c923dd36a32d737581df.html?sxts=1552644845367&pn=51
(added "&pn=51" behind the normal website)
The parameter @page50 here is [int]:
page50=False: only crawl @website is enough
page50=51: crawl @website and @website+'&pn=51'
page50=101: crawl @website, @website+'&pn=51', @website+'&pn=101'
NOTE THAT &pn= is only 51, 101, 151, 201, ...
outdir:
[str] path of the output directory
outname:
[str] main_name of the picture without format
NOTE THAT outname[-1] != '_'
logger:
=False (not print) | ='DEBUG' (print all) | 'INFo' (print count)
timeout:
How many seconds to be timeout (Default timeout=3)
'''
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time, re, requests
from jizhipy.Basic import Logger, Path
#----------------------------------------
# Check arguments
browserDriverPath = Path.AbsPath(browserDriverPath)
Path.ExistsPath(browserDriverPath, stop=True)
try:
page50 = int(page50)
except:
page50 = False
if (page50 not in range(51, 10000, 50)): page50 = ['']
else: page50 = [''] + range(51, page50 + 2, 50)
if (outdir[-1] != '/'): outdir += '/'
outdirAbs = Path.AbsPath(outdir)
Path.ExistsPath(outdirAbs, mkdir=True)
outname = str(outname)
if (outname[-1] == '_'): outname = outname[:-1]
try:
timeout = float(timeout)
except:
timeout = 5
logger = Logger(logger, True)
#----------------------------------------
# __init__
browser = webdriver.Chrome(browserDriverPath)
wait = WebDriverWait(browser, timeout)
# 匹配url(网址)信息的模板，url\("(.*?)"\) 相当于url("...")，其中"..."就是用来打开图片的网址
pattern = re.compile('.*?url\("(.*?)"\)', re.S)
#----------------------------------------
count = 0
for p5 in page50:
# Get the website
if (p5 != ''): p5 = '&pn=' + str(p5)
browser.get(website + p5)
# 定位到:鼠标'banner-more-btn'
submit = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, 'banner-more-btn')))
# 滑轮滑倒看得到鼠标的位置，不然鼠标会点不到
browser.execute_script("arguments[0].scrollIntoViewIfNeeded(true);",
submit)
# 让爬虫点击一下鼠标
submit.click()
# 定位到第一个节点
elem = wait.until(
EC.presence_of_element_located(
(By.ID, 'reader-container-inner-1')))
# 获取图片link
for i in elem.find_elements_by_class_name('bd'):
count += 1
try:
browser.execute_script(
"arguments[0].scrollIntoViewIfNeeded(true);", i)
# time.sleep(0.6) #@#@
# 定位到存放图片信息的节点
n = i.find_element_by_class_name('reader-pic-item')
js = n.get_attribute('style')
# findall，用模板匹配url(网址)
href = pattern.findall(js)
link = href[0]
# 保存图片
html = requests.get(link).content
stri = (4 - len(str(count))) * '0' + str(count)
imageName = outdirAbs + outname + '_' + stri + '.jpg'
fo = open(imageName, 'wb')
fo.write(html)
fo.close()
print(count)
except:
pass
def ObjectDetector(image,
framework,
modelPath,
txtPath=None,
labelMap=None,
minConfidence=None,
scaleFactor=None,
size=None,
mean=None,
crop=False,
**kwargs):
'''
Parameters
----------
image:
Must be 3D/2D array/matrix, NOT path
framework:
'Tensorflow' | 'Caffe' | 'Torch' | 'Darknet' | 'ONNX' | 'ModelOptimizer' | 'Haar' (OpenCV .xml)
modelPath, txtPath:
(1) for Tensorflow:
* modelPath: path of .pb (binary frozen graph) file with binary protobuf description of the network architecture
* txtPath: path of .pbtxt file that contains text graph definition in protobuf format
(2) for Caffe:
* modelPath: path of .caffemodel file with learned network
* txtPath: path of .prototxt file with text description of the network architecture
(3) for OpenCV.xml:
* modelPath: path of .xml file
* txtPath: set txtPath=None
minConfidence:
(1) for NOT Haar:
* Detect objects with confidence >= minConfidence
(2) for Haar:
* minConfidence=minNeighbors, if a candidate target is consist of >= minNeighbors windows, it may be a real target, otherwise, drop it
scaleFactor:
(1) for NOT Haar:
* Use what image to train this model(network)
* scaleFactor=1: use original image range 0~255
* scaleFactor=1/255: use normalized image range 0~1
* For Caffe, use scaleFactor=1
* For Tensorflow, use scaleFactor=1/127.5
* (effect: image *= scaleFactor)
(2) for Haar:
* scaleFactor = NextWindowSize / CurrentWindowSize
(default scaleFactor=1.1)
size:
(1) for NOT Haar:
* size=(height, width) for output image
(2) for Haar:
* size=flags, usually use 3 cases:
flags=0 (default)
flags=cv2.CV_HAAR_DO_CANNY_PRUNING
flags=cv2.CASCADE_SCALE_IMAGE
mean:
(only for NOT Haar)
scalar with mean values which are subtracted from channels. Values are intended to be in (mean-R, mean-G, mean-B) order if @p image is BGR (NOT RGB) and @p swapRB=True
swapRB:
(only for NOT Haar)
True/False, swap R(red) and B(blue) or not
crop:
(only for NOT Haar)
True/False, crop the image or not after resize
@p size is used to resize @p image:
(1) crop=False: resize directly and don't preserve aspect ratio
(2) crop=True: first, resize @p image preserving aspect ratio, so that @p image.shape >= size; then, crop the longer axis from the image center, make @p image.shape==size
'''
from jizhipy.Basic import Path, Raise
import numpy as np
import cv2
# Check input parameters
framework_valid = [
'tensorflow', 'caffe', 'torch', 'darknet', 'onnx', 'modeloptimizer',
'haar'
]
framework, fw0 = str(framework).lower(), framework
if (framework not in framework_valid):
Raise(
Exception, 'framework="' + framework +
'" is NOT valid, must be one of ' + str(framework_valid))
#----------
if (minConfidence is None):
if (framework != 'haar'): minConfidence = 0.7
else: minConfidence = 3
#----------
if (scaleFactor is not None):
scaleFactor = float(scaleFactor)
else:
if (framework == 'tensorflow'): scaleFactor = 1 / 127.5
elif (framework == 'haar'): scaleFactor = 1.1
else: scaleFactor = 1.0
#----------
if (size is not None):
try:
size = tuple(size)[:2]
except:
pass
else:
if (framework == 'haar'): size = 0
#----------
swapRB = False
swapRB, crop = bool(swapRB), bool(crop)
#----------
modelPath = Path.AbsPath(modelPath)
Path.ExistsPath(modelPath, stop=True)
if (txtPath is not None):
txtPath = Path.AbsPath(txtPath)
Path.ExistsPath(txtPath, stop=True)
height, width = image.shape[:2]
#----------------------------------------
#----------------------------------------
if (framework != 'haar'):
# Creates 4-dimensional blob from image
blob = cv2.dnn.blobFromImage(image, scaleFactor, size, mean, swapRB,
crop)
# blob = cv2.dnn.blobFromImage(image)
# Read the model(weight)
if (framework == 'tensorflow'):
net = cv2.dnn.readNetFromTensorflow(modelPath, txtPath)
elif (framework == 'caffe'):
net = cv2.dnn.readNetFromCaffe(txtPath, modelPath)
elif (framework == 'torch'):
isBinary = txtPath
net = cv2.dnn.readNetFromTorch(modelPath, isBinary)
elif (framework == 'darknet'):
net = cv2.dnn.readNetFromDarknet(txtPath, modelPath)
elif (framework == 'onnx'):
net = cv2.dnn.readNetFromONNX(modelPath)
elif (framework == 'ModelOptimizer'):
net = cv2.dnn.readNetFromModelOptimizer(txtPath, modelPath)
else:
Raise(Exception, 'Not support framework: ' + fw0)
# Pass the blob through the network
net.setInput(blob)
# detection[0,0].shape(num_object, 7)
# 7: 0:? 1:ClassIndex 2:Confidence 3,4,5,6:Box
detection = net.forward()[0, 0]
# Select >= minConfidence
n = np.arange(len(detection))[detection[:, 2] >= minConfidence]
detection = detection[n]
# Detection confidence and rects
confidence = detection[:, 2]
rects = (detection[:, 3:] *
np.array([[width, height, width, height]])).astype(
int) # xmin, ymin, xmax, ymax
# Confidence to text to draw
if (labelMap is None): label = len(rects) * ['']
else: label = labelMap[detection[:, 1].astype(int)]
text = []
for i in range(len(rects)):
c = ('%.1f' % (100 * confidence[i])) + '%'
if (label[i] != ''): c = str(label[i]) + ': ' + c
text.append(c)
#----------------------------------------
#----------------------------------------
else:
cascade = cv2.CascadeClassifier(modelPath)
rects = cascade.detectMultiScale(image,
scaleFactor,
minConfidence,
minSize=(5, 5),
flags=size)
if (len(rects) > 0): rects[:, 2:] += rects[:, :2]
confidence = np.zeros(len(rects), int)
text = len(rects) * ['']
#----------------------------------------
#----------------------------------------
# from imutils.object_detection import non_max_suppression
# rects = non_max_suppression(rects, probs=None, overlapThresh=0.7)
return (rects, confidence, text)
def NVSSSUMSS(fwhm,
lon=None,
lat=None,
nside=None,
coordsys='Galactic',
galcenter=True,
nvsssumsspath='nvss-sumss_1.4GHz_15mJy_fullsky.hdf5'):
'''
return radio source map
* merged from NVSS and SUMSS
* healpix, RING, full sky, 1.4GHz, 15mJ limit for completeness
* fill the missing region
fwhm:
[degree]
fwhm==None: don't smooth the map
lon, lat:
[degree]
lon: RA | l
lat: Dec | b
Must can be broadcast
Use lon, lat first, otherwise, use nside
nside:
Healpix full sky map instead of lon, lat
nside0 = 1024
coordsys:
'Equatorial' | 'Galactic' | 'Ecliptic'
galcenter:
True | False
==True: Retain the galactic center
==False: discard the galactic center
nvsssumsspath:
str, where is the file 'nvss-sumss_1.4GHz_15mJy_fullsky.hdf5'
'''
import numpy as np
import healpy as hp
import h5py
from jizhipy.Array import Asarray
from jizhipy.Basic import Raise, Path, IsType
from jizhipy.Transform import CoordTrans
if (nvsssumsspath is None):
nvsssumsspath = 'nvss-sumss_1.4GHz_15mJy_fullsky.hdf5'
nvsssumsspath = Path.AbsPath(str(nvsssumsspath))
if (not Path.ExistsPath(nvsssumsspath)):
nvsssumsspath = Path.jizhipyPath(
'jizhipy_tool/nvss-sumss_1.4GHz_15mJy_fullsky.hdf5')
if (not Path.ExistsPath(nvsssumsspath)):
Raise(Exception,
'nvsssumsspath="' + nvsssumsspath + '" NOT exists')
#--------------------------------------------------
nside0 = 1024
if (lon is not None and lat is not None):
if (np.sum((lat < -90) + (lat > 90)) > 0):
Raise(Exception, 'lat out of [-90, 90] degree')
nside = None
else:
islist = True
try:
nside = int(nside)
except:
nside = nside0
#--------------------------------------------------
fo = h5py.File(nvsssumsspath, 'r')
a = fo['nvsum'].value
n, void = fo['void'].value
a[n.astype(int)] = void
resol = 0.06 # degree
if (not galcenter):
theta, phi = hp.pix2ang(nside0, np.arange(a.size))
theta, phi = 90 - theta * 180 / np.pi, phi * 180 / np.pi
tf = (-5 < theta) * (theta < 5) * ((phi < 55) + (phi > 345))
n = np.arange(a.size)[tf]
del theta, phi, tf
m = (np.random.random(n.size) * void.size).astype(int)
a[n] = void[m]
#--------------------------------------------------
coordsys = str(coordsys).lower()
if (nside is None): # use lon and lat to select region
if (fwhm is not None and fwhm > resol):
a = hp.smoothing(a, fwhm * np.pi / 180, verbose=False)
a[a < 0] = 0
islist = False if (IsType.isnum(lon + lat)) else True
lon, lat = lon + lat * 0, lon * 0 + lat # same shape
lon %= 360
lon, lat = CoordTrans.Celestial(lon * np.pi / 180, lat * np.pi / 180,
coordsys, 'galactic') # rad
npix = hp.ang2pix(nside0, np.pi / 2 - lat, lon)
del lon, lat
a = a[npix]
del npix
else: # full sky healpix map
if (coordsys != 'galactic'):
a = CoordTrans.CelestialHealpix(a, 'RING', 'galactic', coordsys)[0]
if (fwhm is not None and fwhm > resol):
a = hp.smoothing(a, fwhm * np.pi / 180, verbose=False)
a[a < 0] = 0
if (nside != nside0): a = hp.ud_grade(a, nside)
a = np.float32(a)
if (not islist): a = a[0]
return a
def BoxText(image,
bbox=None,
text=None,
color=(0, 255, 0),
lw=None,
style='box',
boxScale=1,
fontSize=None,
loc=None,
**kwargs):
'''
Draw the box/cross on the image
image:
np.ndarray or matrix, NOT path
Can be gray or RGB image
bbox:
1D array.shape=(4,) | 2D array.shape=(num_person, 4)
bbox | bbox.T = xmin, ymin, xmax, ymax
boxScale:
Scale the area of the box
boxScale=1: face
boxScale=1.2: the whole head
text:
Put text on the image
str | list of str:
bbox is 1D: len(text)==1
bbox is 2D: len(text)==len(bbox)==num_person
color:
color=(red, green, blue), 0~255
style:
'cross' | 'box' | 'circle' | 'point'
loc:
(1) Location of text:
None | [int] 0 ~ 11 (see below)
None: find the best loc automatically, try the order 0, 3, 1, 2, 4, 5, 6, 8, 7
-----------------------
| 6 |
| |
| 0 |
| ----------- |
| | 1 | |
|7 | 4 | 5 |
| | 2 | |
| ----------- |
| 3 |
| |
| 8 |
-----------------------
fontSize:
None | [int]
width=1000 for image, fontSize=40
if (fontSize is None):
fontSize = image.shape[1]/1000*40
fontFace:
cv2.FONT_HERSHEY_SIMPLEX # normal size sans-serif
cv2.FONT_HERSHEY_PLAIN # small size sans-serif
cv2.FONT_HERSHEY_DUPLEX # normal size sans-serif (more complex than FONT_HERSHEY_SIMPLEX)
cv2.FONT_HERSHEY_COMPLEX # normal size serif
cv2.FONT_HERSHEY_TRIPLEX # normal size serif (more complex than FONT_HERSHEY_COMPLEX)
cv2.FONT_HERSHEY_COMPLEX_SMALL # smaller version of FONT_HERSHEY_COMPLEX
cv2.FONT_HERSHEY_SCRIPT_SIMPLEX # Hand-writing style
cv2.FONT_HERSHEY_SCRIPT_COMPLEX # More complex variant of FONT_HERSHEY_SCRIPT_SIMPLEX
cv2.FONT_ITALIC # Flag for italic
'''
if (bbox is None or len(bbox) == 0): bbox, style = None, None
if (text is None or len(text) == 0): text = None
if (bbox is None and text is None): return image
import cv2
from jizhipy.Basic import IsType, Path
from PIL import Image, ImageDraw, ImageFont
import numpy as np
image, loc0 = np.array(image), loc
here = Path.DirPath(__file__, 1)
width, height = image.shape[:2][::-1]
if (fontSize is None): fontSize = int(40 * width / 1000)
if (lw is None): lw = 2. * width / 1000
if (lw < 1): lw = 1
if (bbox is not None):
bbox = np.array(bbox)
if (len(bbox.shape) == 1): bbox = bbox[None, :]
if (boxScale != 1):
dx = ((bbox[:, 2] - bbox[:, 0]) * (boxScale - 1) / 2).astype(int)
dy = ((bbox[:, 3] - bbox[:, 1]) * (boxScale - 1) / 2).astype(int)
bbox[:, 0] -= dx
bbox[:, 2] += dx
bbox[:, 1] -= dy
bbox[:, 3] += dy
xmin, ymin, xmax, ymax = bbox.T
xmin[xmin < 0] = 0
xmax[xmax >= width] = width - 1
ymin[ymin < 0] = 0
ymax[ymax >= height] = height - 1
bbox[:, 0], bbox[:, 1], bbox[:, 2], bbox[:, 3] = xmin, ymin, xmax, ymax
if (text is not None):
if (IsType.isstr(text)):
if (bbox is not None): text = len(bbox) * [text]
else: text = [text]
else:
text = list(text)
if (bbox is not None and len(text) < len(bbox)):
text += (len(bbox) - len(text)) * ['']
if (loc is None): loc = 'up'
lw = int(round(lw))
textlw = 1 if (lw == 1) else lw - 1
#----------------------------------------
def _loc(xmin, xmax, ymin, ymax, loc):
if (loc == 1): x, y = xmin + lw, ymin + lw
elif (loc == 2): x, y = xmin + lw, ymax - fontSize - lw
elif (loc == 3): x, y = xmin + lw, ymax + lw
elif (loc == 4): x, y = xmin + lw, int((ymax + ymin - fontSize) / 2)
elif (loc == 5): x, y = xmax + lw, int((ymax + ymin - fontSize) / 2)
elif (loc == 6): x, y = int(width / 2 - 1.5 * fontSize), lw
elif (loc == 7): x, y = lw, int((height - fontSize) / 2)
elif (loc == 8):
x, y = int(width / 2 - 1.5 * fontSize), height - fontSize - lw
else:
x, y = xmin, ymin - fontSize - lw # loc=0
return (x, y)
#----------------------------------------
def _putText(image, xmin, xmax, ymin, ymax, text):
try:
text = text.decode('utf8') # python2
except:
pass # python3
if (loc0 is not None):
x, y = _loc(xmin, xmax, ymin, ymax, int(loc0))
else:
x, y = _loc(xmin, xmax, ymin, ymax, 0)
locOrder, repeat = [3, 1, 2, 4, 5, 6, 8, 7, 1], True
for loc in locOrder:
repeat = (len(text) * fontSize >
width - x) or (y < 0) or (fontSize > height - y)
if (not repeat): break
x, y = _loc(xmin, xmax, ymin, ymax, loc)
try: # Use PIL, can put chinese
font = ImageFont.truetype(here + '/simsun.ttc',
fontSize,
encoding='utf-8')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = Image.fromarray(image)
draw = ImageDraw.Draw(image)
draw.text((x, y), text, color, font)
image = np.array(image)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
except: # Use cv2.putText(), but can NOT put chinese
image = cv2.putText(image, text, (x, y + fontSize - lw),
cv2.FONT_HERSHEY_SIMPLEX, 1, color, lw)
return image
#----------------------------------------
if (str(style).lower() == 'box'):
for i in range(len(bbox)):
xmin, ymin, xmax, ymax = bbox[i] # means bbox.shape=(n, 4)
image = cv2.rectangle(image, (int(xmin), int(ymin)),
(int(xmax), int(ymax)), color, lw)
if (text is not None):
image = _putText(image, xmin, xmax, ymin, ymax, text[i])
#----------------------------------------
elif (str(style).lower() == 'cross'):
for i in range(len(bbox)):
xmin, ymin, xmax, ymax = bbox[i]
w, h = xmax - xmin, ymax - ymin
x1, w1 = xmin + (w - lw) / 2, lw
y1, h1 = ymin + (h - lw) / 2, lw
image = cv2.rectangle(image, (x1, ymin), (x1 + w1, ymax), color,
lw)
image = cv2.rectangle(image, (xmin, y1), (xmax, y1 + h1), color,
lw)
if (text is not None):
image = _putText(image, xmin, xmax, ymin, ymax, text[i])
#----------------------------------------
elif (str(style).lower() == 'point'):
x, y = bbox.T.reshape(2, -1)
for i in range(len(x)):
image = cv2.circle(image, (x[i], y[i]), lw, color, lw)
#----------------------------------------
elif (style is None and text is not None):
xmin, xmax, ymin, ymax = 0, 0, 0, 0
for i in range(len(text)):
image = _putText(image, xmin, xmax, ymin, ymax, text[i])
#----------------------------------------
try:
image = image.get()
except:
pass
return image
def LAB(fwhm,
freqmin,
freqmax,
lon=None,
lat=None,
nside=None,
coordsys='Galactic',
labpath='labh.fits',
verbose=True):
'''
fwhm:
[degree]
fwhm==None: don't smooth the map
freqmin, freqmax:
[MHz]
Average from freqmin (include) to freqmax (include)
LAB freq resol: 0.00488281 MHz
Must be isnum, NOT list
If freqmax=None, then freqmax=freqmin
lon, lat:
[degree]
lon: RA | l
lat: Dec | b
Must can be broadcast
Use lon, lat first, otherwise, use nside
nside:
Healpix full sky map instead of lon, lat
coordsys:
'Equatorial' | 'Galactic'
labpath:
str, where is the file "labh.fits"
LAB: [1418.2329, 1422.5786] MHz
'''
import pyfits
import healpy as hp
import numpy as np
from jizhipy.Array import Asarray
from jizhipy.Basic import Raise, IsType, Path
from jizhipy.Astro import DopplerEffect
from jizhipy.Transform import CoordTrans
if (freqmin is not None):
freqmin = Asarray(freqmin).flatten()
if (freqmin.size != 1):
Raise(
Exception, 'freqmin must isnum, but now freqmin.size=' +
str(freqmin.size))
freqmin = freqmin[0]
if (freqmax is not None):
freqmax = Asarray(freqmax)
if (freqmax.size != 1):
Raise(
Exception, 'freqmax must isnum, but now freqmax.size=' +
str(freqmax.size))
freqmax = freqmax[0]
if (freqmin is not None and freqmax is None):
freqmax = freqmin
elif (freqmin is None and freqmax is not None):
freqmin = freqmax
elif (freqmin is None and freqmax is None):
# if (verboase) : Raise(Warning, 'freqmin = freqmax = None')
if (verboase): print('Warning: jp.LAB(), freqmin = freqmax = None')
return np.array(0)
if (freqmin < 1418.2329 or freqmax > 1422.5786):
if (verbose):
print(
'Warning: jp.LAB(), freqmin=%.3f, freqmax=%.3f out of [1418.2329, 1422.5786] MHz'
% (freqmin, freqmax))
return np.array(0)
#--------------------------------------------------
if (labpath is None): labpath = 'labh.fits'
if (not Path.ExistsPath(labpath)):
labpath = Path.jizhipyPath('jizhipy_tool/labh.fits')
if (not Path.ExistsPath(labpath)):
Raise(Exception, 'labath="' + labpath + '" NOT exists')
fo = pyfits.open(labpath)
#--------------------------------------------------
if (lon is not None and lat is not None):
if (np.sum((lat < -90) + (lat > 90)) > 0):
Raise(Exception, 'lat out of [-90, 90] degree')
nside = None
else:
nside, islist = int(nside), True
#--------------------------------------------------
hdr = fo[0].header
bscale, bzero, blank = hdr['BSCALE_'], hdr['BZERO_'], hdr['BLANK_']
freq0 = hdr['FREQ0'] * 1e-6 # MHz
Nlon, Nlat, Nfreq = hdr['NAXIS1'], hdr['NAXIS2'], hdr['NAXIS3']
vmin, dv = hdr['CRVAL3'] / 1000., hdr['CDELT3'] / 1000. # km/s
#--------------------------------------------------
vlist = np.arange(vmin, vmin + Nfreq * dv, dv)
freqlist = freq0 + DopplerEffect(freq0, None, vlist)
df = freqlist[Nfreq / 2] - freqlist[Nfreq / 2 - 1]
if (freqmin < freqlist.min() or freqmax > freqlist.max()):
# if (verbose) : Raise(Warning, 'freqmin=%.3f, freqmax=%.3f out of [%.4f, %.4f] MHz' % (freqmin, freqmax, freqlist.min(), freqlist.max()))
if (verbose):
print(
'Warning: jp.LAB(), freqmin=%.3f, freqmax=%.3f out of [%.4f, %.4f] MHz'
% (freqmin, freqmax, freqlist.min(), freqlist.max()))
return np.array(0)
nfreqmin = abs(freqlist - freqmin)
nfreqmin = np.where(nfreqmin == nfreqmin.min())[0][0]
nfreqmax = abs(freqlist - freqmax)
nfreqmax = np.where(nfreqmax == nfreqmax.min())[0][0]
nfreqmin, nfreqmax = np.sort([nfreqmin, nfreqmax])
nfreq = np.arange(nfreqmin, nfreqmax + 1)
#--------------------------------------------------
# Take healpix map in Galactic
lab = fo[0].data[nfreq] + 0
tf = (lab == blank)
lab = np.float32(lab * bscale + bzero)
lab[tf] = np.nan
lab = np.ma.MaskedArray(lab, tf)
lab = np.float32(lab.mean(0).data) # (361, 721)
del tf
#--------------------------------------------------
nsidelab = 256 # 512
latlab, lonlab = hp.pix2ang(nsidelab, np.arange(12 * nsidelab**2))
latlab, lonlab = 90 - latlab * 180 / np.pi, lonlab * 180 / np.pi
lonlab %= 360
lonlab[lonlab > 180] = lonlab[lonlab > 180] - 360
nlon = ((lonlab - 180) / -0.5).round().astype(int)
nlon[nlon >= Nlon] = Nlon - 1
del lonlab
nlat = ((latlab + 90) / 0.5).round().astype(int)
nlat[nlat >= Nlat] = Nlat - 1
del latlab
npix = nlat * Nlon + nlon
del nlat, nlon
lab = lab.flatten()[npix] # healpix map
del npix
if (fwhm not in [0, None] and fwhm > 0.5):
fwhm = (fwhm**2 - 0.5**2)**0.5
lab = np.float32(
hp.smoothing(lab, fwhm * np.pi / 180,
verbose=False)) # hp.smoothing() is slow !
#--------------------------------------------------
coordsys = str(coordsys).lower()
if (nside is None):
islist = False if (IsType.isnum(lon + lat)) else True
lon, lat = lon + lat * 0, lon * 0 + lat # same shape
lon %= 360
lon, lat = CoordTrans.Celestial(lon * np.pi / 180, lat * np.pi / 180,
coordsys, 'Galactic') # rad
npix = hp.ang2pix(nsidelab, np.pi / 2 - lat, lon)
del lon, lat
lab = lab[npix]
else:
if (nside != nsidelab): lab = hp.ud_grade(lab, nside)
if (coordsys != 'galactic'):
lab = CoordTrans.CelestialHealpix(lab, 'RING', 'Galactic',
coordsys)[0]
lab = np.float32(lab)
if (not islist): lab = lab[0]
return lab
def GSM2016(freq=None,
outname=None,
save=False,
nside=None,
coordsys='Galactic',
unit='K',
lowresol=False,
dtype=None,
verbose=False,
rmnegative=False,
rmsource=False,
gsm2016path='GSM2016_data/',
example=False):
'''
An improved model of diffuse galactic radio emission from 10 MHz to 5 THz; arxiv:1605.04920; 2017MNRAS.464.3486Z
Already remove CMB: -2.726
example:
True | False
==True: read and return 'gsm2016_example.hdf5'
jp.GSM2016(example=True)
Only diffuse map, had remove CMB, 408MHz, K, nside=512
compname = ['Total', 'Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2', 'Free-Free'] # 'Dust1'=cold dust, 'Dust2'=warm dust
freq:
in MHz
Must be one value, int | float, NOT list
outname:
Force to use .hdf5 format
(1) ==str: complete output name
(2) else: outname = 'gsm2016_'+strfreq+'.hdf5'
save:
==False:
exists, read, NOT save
NOT exists, generate, NOT save
==True:
exists, generate, save
NOT exists, generate, save
==None:
exists, read, NOT save
NOT exists, generate, save
lowresol:
True | False
(1) ==False:
nside=1024
* freq< 10GHz, force resol=56arcmin
** freq<=10GHz, force resol=24arcmin
(2) ==False
nside=64, resol=5deg=300arcmin
Use './data/lowres_maps.txt'
unit:
'K'/'Trj' | 'Tcmb' | 'MJysr'
dtype:
'float64' | 'float32'
nside:
Default nside=1024
rmnegative:
reset negative
True | False
rmsource:
remove sources from the output map
0/False | 1 | 2 | 3 | 4
0/False: NOT remove sources
1: remove CygA, CasA, Crab
2: remove sources outside center plane
3: remove sources inside center plane
return:
gsm.shape = (6, 12*nside**2)
compname = ['Total', 'Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2', 'Free-Free'] # 'Dust1'=cold dust, 'Dust2'=warm dust
'''
import numpy as np
import healpy as hp
import h5py
from jizhipy.Astro import RemoveSource
from jizhipy.Basic import Raise, SysFrame, Mkdir, Path, Time, IsType
if (verbose): print('jizhipy.GSM2016:')
if (nside is not None):
n = int(round(np.log(nside) / np.log(2)))
nside = 2**n
else:
nside = 512
coordsys = str(coordsys)
if (verbose):
print(' freq =', freq, 'MHz')
print(' coordsys =', coordsys)
print(' nside =', nside)
#----------------------------------------
if (gsm2016path is None): gsm2016path = 'GSM2016_data/'
if (not Path.ExistsPath(gsm2016path)):
gsm2016path = Path.jizhipyPath('jizhipy_tool/GSM2016_data/')
if (not Path.ExistsPath(gsm2016path)):
Raise(Exception, 'gsmp2016path="' + gsm2016path + '" NOT exists')
fdir = Path.AbsPath(gsm2016path)
#----------------------------------------
if (example):
if (verbose): print(' Use "gsm2016_example.hdf5"')
fo = h5py.File(fdir + 'gsm2016_example.hdf5', 'r')
gsm = fo['gsm2016'].value
fo.close()
# ud_grade to nside
if (nside is not None and gsm[0].size != 12 * nside**2):
gsm2 = []
for i in range(len(gsm)):
gsm2.append(hp.ud_grade(gsm[i], nside))
gsm = np.array(gsm2)
if (coordsys.lower() != 'galactic'):
for i in range(len(gsm)):
gsm[i] = CoordTrans.CelestialHealpix(gsm[i], 'ring',
'Galactic', coordsys)[0]
return gsm
#---------------------------------------------
#---------------------------------------------
freq, unit, lowresol, verbose, rmnegative, rmsource = float(freq), str(
unit).lower(), bool(lowresol), bool(verbose), bool(rmnegative), int(
rmsource)
vb = '123' if (verbose) else False
if (verbose):
print(
' Total, Synchrotron, CMB, HI, Cold dust, Warm dust, Free-Free')
if (freq <= 0):
print('freq = ' + str(freq) + ' <=0')
exit()
dtype = np.dtype(dtype)
#---------------------------------------------
#---------------------------------------------
if (IsType.isstr(outname)): outname = str(outname)
else:
strfreq = str(freq).split('.')
while (strfreq[1] != '' and strfreq[1][-1] == '0'):
strfreq[1] = strfreq[1][:-1]
if (strfreq[1] == ''): strfreq = strfreq[0]
else: strfreq = strfreq[0] + '.' + strfreq[1]
outname = 'gsm2016_' + strfreq + '.hdf5'
if (save is not False and '/' in outname):
for i in range(len(outname) - 1, -1, -1):
if (outname[i] == '/'): break
Mkdir(outname[:i + 1])
#--------------------------------------------------
#--------------------------------------------------
freq /= 1000. # GHz
kB, C, h, T = 1.38065e-23, 2.99792e8, 6.62607e-34, 2.725
hoverk = h / kB
def K_CMB2MJysr(K_CMB, nu): # in Kelvin and Hz
B_nu = 2 * (h * nu) * (nu / C)**2 / (np.exp(hoverk * nu / T) - 1)
conversion_factor = (B_nu * C / nu / T)**2 / 2 * np.exp(
hoverk * nu / T) / kB
return K_CMB * conversion_factor * 1e20 # 1e-26 for Jy | 1e6 for MJy
def K_RJ2MJysr(K_RJ, nu): # in Kelvin and Hz
conversion_factor = 2 * (nu / C)**2 * kB
return K_RJ * conversion_factor * 1e20 # 1e-26 for Jy | 1e6 for MJy
mjysr2k = 1. / K_RJ2MJysr(1., 1e9 * freq)
if (unit == 'tcmb'):
unit = 'Tcmb'
conversion = 1. / K_CMB2MJysr(1., 1e9 * freq)
elif (unit == 'mjysr'):
unit = 'MJysr'
conversion = 1.
if (verbose):
print((' Unit Conversion at %.3f MHz: 1 MJysr == %f ' + unit) %
(freq * 1000, mjysr2k))
else: # (unit in ['k', 'trj']) :
unit = 'K'
conversion = mjysr2k
#--------------------------------------------------
#--------------------------------------------------
exist = True if (Path.ExistsPath(outname)) else False
# if (exist and save is not True) :
if (exist):
try:
fo = h5py.File(outname, 'r')
gsm = fo['gsm2016'].value
fo.close()
if (verbose): print(' Exists: ', outname)
return gsm
except:
pass
#--------------------------------------------------
exist = False
if (save is not False): save = True
if (not exist): # generate
spec_nf = np.loadtxt(fdir + 'spectra.txt')
nfreq = spec_nf.shape[1]
left_index = -1
for i in range(nfreq - 1):
if (freq >= spec_nf[0, i] and freq <= spec_nf[0, i + 1]):
left_index = i
break
if (left_index < 0):
print(
"FREQUENCY ERROR: %.3f MHz is outside supported frequency range of %.3f MHz to %.3f MHz."
% (freq * 1000, spec_nf[0, 0] * 1000, spec_nf[0, -1] * 1000))
exit()
#--------------------------------------------------
# compname must be this order, do NOT modify
compname = ['Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2',
'Free-Free'] # 'Dust1'=cold dust, 'Dust2'=warm dust
#--------------------------------------------------
if (lowresol):
nside_data, op_resolution = 64, 300.
else:
nside_data = 1024 # forced because of *.bin files
op_resolution = 56. if (freq < 10) else 24.
if (not lowresol):
gsm = np.array([
np.fromfile(fdir + 'highres_' + cname + '_map.bin',
dtype='float32') for cname in compname
])
else:
gsm = np.loadtxt(fdir + 'lowres_maps.txt')
#--------------------------------------------------
interp_spec_nf = np.copy(spec_nf)
interp_spec_nf[0:2] = np.log10(interp_spec_nf[0:2])
x1 = interp_spec_nf[0, left_index]
x2 = interp_spec_nf[0, left_index + 1]
y1 = interp_spec_nf[1:, left_index]
y2 = interp_spec_nf[1:, left_index + 1]
x = np.log10(freq)
interpolated_vals = (x * (y2 - y1) + x2 * y1 - x1 * y2) / (x2 - x1)
gsm = 10.**interpolated_vals[0] * interpolated_vals[1:, None] * gsm
gsm = gsm * conversion
#--------------------------------------------------
if (len(gsm.shape) == 1): gsm = gsm[None, :]
else: gsm = np.concatenate([gsm.sum(0)[None, :], gsm], 0)
#--------------------------------------------------
#--------------------------------------------------
# generate is nest, convert to ring
# First ud_grade so that faster
for i in range(len(gsm)):
gsm[i] = hp.reorder(gsm[i], n2r=True)
# ud_grade to nside
if (gsm[0].size != 12 * nside**2):
gsm2 = []
for i in range(len(gsm)):
gsm2.append(hp.ud_grade(gsm[i], nside))
gsm = np.array(gsm2)
#--------------------------------------------------
#--------------------------------------------------
# remove negative value in the hpmaps
for i in range(len(gsm)):
if (not rmnegative and i != 0): break
tf = gsm[i] < 0
if (tf[tf].size == 0): continue
g = gsm[i].copy()
# g[tf] = g.mean()
g[tf] = g[g > 0].min()
g = hp.smoothing(g, np.pi / 180 * 10, verbose=False)
gsm[i][tf] = g[tf]
#--------------------------------------------------
#--------------------------------------------------
casa = np.array([[(111.44, -1.37), (93.89, 2.37)]]) # fwhm=7
casa1 = np.array([[(118.12, 4.78), (121.83, 1.09)]])
cyga = np.array([[(81, 2), (26.33, 8.38)]]) # fwhm=7
cyga1 = np.array([[(84.9, -0.55), (71.8, 0.4)]]) # fwhm=3
cyga2 = np.array([[(76.6, 0.4), (71.8, 0.4)]]) # fwhm=5
cyga3 = np.array([[(88.82, 4.69), (88.69, 2)]]) # fwhm=3
crab = np.array([[(-96, -2), (26.33, 8.38)]]) # fwhm=4.5
crab1 = np.array([[(-91.82, -1), (26.33, 8.38)]]) # fwhm=3
crab2 = np.array([[(-99.31, -3.43), (26.33, 8.38)]]) # fwhm=3
lb1 = np.concatenate(
[casa, casa1, cyga, crab, cyga1, cyga2, cyga3, crab1, crab2], 0)
fwhm1 = np.array([7, 3, 7, 4.5, 3, 5, 3, 2, 2])
lb10, lb11 = lb1[:, 0], lb1[:, 1]
#--------------------------------------------------
other = np.array([[(-153.41, -1.91), (-163.58, -9.29)],
[(-170.31, 2.08), (-163.58, -9.29)],
[(-175.26, -5.88), (-163.58, -9.29)],
[(-150.64, -19.38), (-157, -20)],
[(-153.34, -16.39), (-157, -20)],
[(133.73, 1.07), (126.89, 0.83)],
[(160.4, 2.78), (157.61, -1.24)],
[(73.2, -9.06), (68.38, -7.56)],
[(63.39, -5.98), (68.38, -7.56)]])
fwhm2 = np.array([4, 4, 4, 1, 1, 2, 2, 3, 2])
lb20, lb21 = other[:, 0], other[:, 1]
#--------------------------------------------------
center = np.array([[(-54.49, 0.08), (-58.03, -0.04)],
[(-61.38, -0.18), (-58.03, -0.04)],
[(-69.23, -0.59), (-58.03, -0.04)],
[(-72.15, -0.73), (-58.03, -0.04)],
[(-75.88, -0.42), (-58.03, -0.04)],
[(-77.63, -1.2), (-58.03, -0.04)],
[(-65.21, -1.2), (-58.03, -0.04)],
[(-63.1, 9.8), (-60.6, 8.89)],
[(-6.93, 16.9), (-9.35, 16.06)]])
fwhm3 = np.array([2, 2, 3, 3, 2, 2, 2, 2, 2])
lb30, lb31 = center[:, 0], center[:, 1]
if (rmsource in [1, 2, 3, 4]):
if (rmsource == 4):
lb0 = np.concatenate([lb30, lb20, lb10], 0)
lb1 = np.concatenate([lb31, lb21, lb11], 0)
fwhm = np.concatenate([fwhm3, fwhm2, fwhm1])
elif (rmsource == 1):
lb0, lb1, fwhm = lb10, lb11, fwhm1
elif (rmsource == 2):
lb0, lb1, fwhm = lb20, lb21, fwhm2
elif (rmsource == 3):
lb0, lb1, fwhm = lb30, lb31, fwhm3
gsm[0] = np.log10(gsm[0])
times, same, onebyone = 1, False, True
pix, revalue = RemoveSource(gsm[0], lb0, lb1, fwhm, times, same,
onebyone, vb)
gsm[0][pix] = revalue
gsm[0] = hp.smoothing(gsm[0], np.pi / 180, verbose=False)
gsm[0] = 10.**gsm[0]
gsm = gsm.astype(dtype)
#--------------------------------------------------
#--------------------------------------------------
if (coordsys.lower() != 'galactic'):
for i in range(len(gsm)):
gsm[i] = CoordTrans.CelestialHealpix(gsm[i], 'ring', 'Galactic',
coordsys)[0]
if (save):
if (verbose): print(' Saving to: ', outname)
Path.ExistsPath(outname, old=True)
fo = h5py.File(outname, 'w')
fo['gsm2016'] = gsm
fo['gsm2016'].attrs['freq'] = freq * 1000
fo['gsm2016'].attrs['unit'] = unit
fo['gsm2016'].attrs['MJysr2K'] = mjysr2k
fo['gsm2016'].attrs['lowresol'] = int(lowresol)
fo['gsm2016'].attrs['nside'] = nside
fo['gsm2016'].attrs['pixtype'] = 'healpix'
fo['gsm2016'].attrs['ordering'] = 'ring'
fo['gsm2016'].attrs['coordsys'] = 'Galactic'
fo['gsm2016'].attrs['rmnegative'] = int(rmnegative)
fo['gsm2016'].attrs['rmsource'] = int(rmsource)
fo['gsm2016'].attrs['creadate'] = Time(0)
fo.close()
return gsm
