def Save( self, hdf5name ) :
'''
hdf5name:
(1) ==None/'': hdf5name = 'jizhipy.PCA.hdf5'
(2) ==filename of .hdf5
'''
from jizhipy.Basic import Path
import h5py
outname = hdf5name
if(outname in [None, '']): outname ='jizhipy.PCA.hdf5'
else :
if (outname[-5:] != '.hdf5') : outname += '.hdf5'
outname = Path.AbsPath(outname)
Path.ExistsPath(outname, True)
fo = h5py.File(outname, 'w')
if (self.verbose) : print(' Saving to: ', Path.AbsPath(outname, True))
self.hdf5name = outname
keys = self.__dict__.keys()
fo['class'] = 'jizhipy.PCA'
fo['stagger'] = stagger
fo['stagger'].attrs['comment'] = 'y1, y2 = y[:,:-stagger], y[:,stagger:]'
if ('v' in keys) :
fo['eigenvalue'] = v
fo['eigenvalue'].attrs['comment']='eign values of R matrix'
if ('P' in keys) :
fo['eigenvector'] = P
fo['eigenvector'].attrs['comment'] = 'each columns are eigen vectors of R matrix'
# if ('a' in keys) :
# fo['a'] = a
# fo['a'].attrs['comment'] = 'z=P*a, y=(P*a)*rms'
# if ('rms' in keys) :
# fo['rms'] = rms
# fo['rms'].attrs['comment'] = 'rms of each row of y-map'
fo.close()
return self.hdf5name
def Use(which, tf=True):
'''
which:
(1) which='Agg'
(2) which='tex', tf=True
(3) which is LatexPackageName
'''
import matplotlib as mpl
which = str(which).lower()
if (which == 'agg'): mpl.use('Agg')
elif (which == 'tex'):
# Some systems,
# set usetex=True , use tex
# set usetex=False, don't use tex
# But also some systems,
# set usetex=True , raise error
# set usetex=False, use tex!
# Therefore, set True/False dependenting on your system
from jizhipy.Basic import Path
import os
texcache = Path.AbsPath('~/.cache/matplotlib/tex.cache')
if (Path.ExistsPath(texcache)):
os.system('rm -r ' + texcache)
mpl.rcParams['text.usetex'] = bool(tf)
# import matplotlib.pyplot as plt
# plt.rc('text', usetex=True)
else: # which is packagename
mpl.rcParams['text.latex.preamble'] = [r'\usepackage{' + which + r'}']
def islaohu( self ) :
from jizhipy.Basic import Path
tf = (Path.AbsPath('~')[:17] == '/public/home/wufq')
if (tf) :
import sys
from jizhipy.Plot import Plt
Plt.Style()
sys.path = ['/public/home/wufq/.mysoftware/lib/python2.7/site-packages'] + sys.path
return tf
def Read( self, hdf5name ) :
'''
Return instance of jp.PCA
'''
from jizhipy.Basic import Path
import h5py
hdf5name = Path.AbsPath(hdf5name)
if (self.verbose) : print(' Reading from: ', Path.AbsPath(hdf5name, True))
fo = h5py.File(hdf5name, 'r')
keys = fo.keys()
pca = PCA()
try :
if (fo['class'] != 'jizhipy.PCA') : raise
except :
if (self.verbose) : print(' NOT .hdf5 for class: jizhipy.PCA')
return pca
if ('stagger' in keys) : self.stagger = fo['stagger'].value
if ('eigenvalue' in keys) : self.v = fo['eigenvalue'].value
if ('eigenvector' in keys) : self.P = fo['eigenvector'].value
# if ('a' in keys) : self.a = fo['a'].value
# if ('rms' in keys) : self.rms = fo['rms'].value
fo.close()
return pca
def Fits2Hdf5( inputwhat, hduname=None, verbose=False ) :
'''
inputwhat:
(1) str, path of .fits
(2) one of the HDU: inputwhat = fo[i]
hduname:
if exists hdu.header['extname'] or inputwhat.name, use them, otherwise, use hduname+str(i)
'''
from jizhipy.Basic import Path, IsType
import h5py, pyfits
def _Fits2Hdf5( hname, hdu, outhdf5 ) :
try : name = hdu.header['extname']
except : name = hdu.name
if (name == '') :
if (hduname in ['', None]) : name = str(hname)
else : name = str(hduname)+str(hname)[3:]
outhdf5[name] = hdu.data #@#@
hdr = hdu.header
keys, values, comments = hdr.keys(), hdr.values(), hdr.comments
for i in range(len(keys)) :
outhdf5[name].attrs[keys[i]] = values[i]
if (comments[i] != '') :
outhdf5[name].attrs[keys[i]+'_comment'] = comments[i]
#--------------------------------------------------
if (IsType.isstr(inputwhat)) :
inname = Path.AbsPath(inputwhat)
outname = inname[:-5] + '.hdf5'
if (verbose) : print('Saving "'+outname+'"')
Path.ExistsPath(outname, 'True')
h = h5py.File(outname, 'w')
f = pyfits.open(inname)
for i in range(len(f.info(0))) :
_Fits2Hdf5( 'hdu'+str(i), f[i], h )
#--------------------------------------------------
else :
if (hduname not in ['', None]) : name = str(hduname)
else :
name = inputwhat.name
if (name == '') : name = 'hdu'
outname = name + '.hdf5'
if (verbose) : print('Saving "'+outname+'"')
Path.ExistsPath(outname, 'True')
h = h5py.File(outname, 'w')
_Fits2Hdf5( hduname, inputwhat, h )
#--------------------------------------------------
h.close()
def CatalogDecSign(cls, catalogname):
'''
In astronomy catalog, Dec is from +90 to -90, but for Dec = -00 23 15, if we np.loadtxt() it, it will miss the "-", treat -00 as 00, and make Dec=-0deg23arcmin15arcsec to +0deg23arcmin15arcsec.
This function is used to get the sign of Dec
catalogname:
the path/name of the input catalog
Return:
1D np.array with the same size of the input catalog
NOTE THAT:
In this catalog, must only Dec has sign !
'''
from jizhipy.Basic import Path
instr = open(Path.AbsPath(catalogname)).readlines()
sign = np.ones(len(instr), np.float32)
for i in range(len(sign)):
if ('-' in instr[i]): sign[i] = -1
return sign
def Ass2Txt( pos, path, fmt=None ) :
#'''
#Read and write Chinese
#pos:
# str to judge what are the real texts
# For example:
# pos = [('0000,0000,0000,,', '\N{'), ('shad1}', '\n')]
#path:
# Must be one
# If path is one file, handle it
# If path is directory, use fmt to judge which files will be handled
#fmt:
# with or without '.' is OK: 'ass' | '.ass'
#'''
from jizhipy.Basic import ShellCmd, Path, IsType
import chardet, codecs
path = Path.AbsPath(path)
if (IsType.isfile(path)) : path = [path]
else :
indir = path
if (fmt is None) : path = ShellCmd('ls '+indir)
else :
fmt = str(fmt).lower()
if (fmt[0] != '.') : fmt = '.' + fmt
path = ShellCmd('ls '+indir+'*'+fmt)
fmt = fmt.upper()
path += ShellCmd('ls '+indir+'*'+fmt)
for i in range(len(path)) : path[i] = indir+path[i]
#---------------------------------------------
outnamelist = []
for i in range(len(path)) :
filename = path[i]
try : a = open(filename).read()
except : continue
code = chardet.detect(a)['encoding']
a = a.decode(code)
txt = ''
while ('\n' in a) :
n = a.index('\n')
b = a[:n+1]
for i in range(len(pos)) :
n1 = n2 = -1
if (pos[i][0] in b) :
n1 = b.rfind(pos[i][0])+len(pos[i][0])
if (pos[i][1] in b) : n2 = b.rfind(pos[i][1])
if (n1<0 or n2<0) : break
c = b[n1:n2]
if (c[0] == '\r') : c = c[1:]
if (c[-1] == '\r') : c = c[:-1]
txt += c + '\n'
if (txt != '') : txt += '\n'
a = a[n+1:]
outname = filename + '.txt'
b = codecs.open(outname, 'w', code)
b.write(txt)
b.close()
outnamelist.append(outname)
return outnamelist
def Array2FitsImage( arraylist, outname, names=None, keys=None, values=None, comments=None, verbose=False ) :
'''
arraylist:
(1) isndarray
(2) list/tuple of ndarray: [array1, array2, ...]
Otherwise, raise error
Examples:
Array2FitsImage( [1,2,3] ), means there are 3 arrays, array1=1, array2=2, array3=3
FITS can save int/float number to ImageHDU, but will raise error "IOError: Header missing END card." when pyfits.open() it.
You must convert to
Array2FitsImage( np.array([1,2,3]), 'test.fits' )
OR Array2FitsImage( [[1,2,3]], 'test.fits' )
names:
None | Same shape as arraylist
Name of each ImageHDU in fo.info()
keys:
None | Same shape as arraylist
(1) arraylist isndarray: keys=['key1', 'key2', ...], list of str
(2) arraylist is list/tuple of ndarray: [array1, array2, ...], then keys=[arraykey1, arraykey2, ...], arraykey1=['key1', 'key2', ...] (list of list)
values:
Same shape as keys
comments:
None | same shape as keys even though is ''
'''
from jizhipy.Basic import Path, IsType
import pyfits
outname = Path.AbsPath(outname)
if (outname[-5:].lower() != '.fits') : outname += '.fits'
Path.ExistsPath(outname, True)
if (verbose) : print('Saving "'+outname+'"')
#--------------------------------------------------
if (IsType.isstr(keys)) :
keys, values = [keys], [values]
if (comments is not None) : comments = [comments]
else : comments = ['']
#--------------------------------------------------
if (not IsType.islist(arraylist) and not IsType.istuple(arraylist)) :
arraylist = [arraylist]
if (names is not None) : names = [names]
keys = [keys for i in range(len(arraylist))]
values = [values for i in range(len(arraylist))]
comments = [comments for i in range(len(arraylist))]
#--------------------------------------------------
hdulist = pyfits.HDUList()
for i in range(len(arraylist)) :
data = arraylist[i]
try : name = str(names[i])
except : name = 'hdu'+str(i)
if (data.dtype.name[:7] == 'complex') :
hdu = pyfits.ImageHDU( data.real )
hdu.name = name+'REAL'
hdr = hdu.header
for j in range(len(keys[i])) :
key, value = keys[i][j], values[i][j]
try : comment = comments[i][j]
except : comment = ''
hdr.set(key, value, comment)
hdulist.append(hdu) # Real part
hdu = pyfits.ImageHDU( data.imag )
hdu.name = name+'IMAG'
hdr = hdu.header
try :
for j in range(len(keys[i])) :
key, value = keys[i][j], values[i][j]
try : comment = comments[i][j]
except : comment = ''
hdr.set(key, value, comment)
except : pass
hdulist.append(hdu) # Imag part
#--------------------------------------------------
else :
hdu = pyfits.ImageHDU( data )
hdu.name = name
hdr = hdu.header
try :
for j in range(len(keys[i])) :
key, value = keys[i][j], values[i][j]
try : comment = comments[i][j]
except : comment = ''
hdr.set(key, value, comment)
except : pass
hdulist.append(hdu)
#--------------------------------------------------
hdulist.writeto( outname )
def Hdf52Fits( inputwhat, hduname='', verbose=False ) :
'''
inputwhat:
(1) str, path of .hdf5
(2) one of the dataset: inputwhat = fo[keys[i]]
'''
from jizhipy.Basic import Path, IsType
import numpy as np
import h5py, pyfits
def _Hdf52Fits( dataset, hduname='' ) :
if (hduname in ['', None]) :
name = str(dataset.name)
name = name[name.rfind('/')+1:]
else : name = str(hduname)
attrs = dataset.attrs.items()
data = dataset.value
hdulist = []
try :
if (data.dtype.name[:7] == 'complex') :
hdu = pyfits.ImageHDU( data.real )
hdu.name = name+'REAL'
hdr = hdu.header
for i in range(len(attrs)) :
hdr[str(attrs[i][0])] = attrs[i][1]
hdulist.append(hdu) #@#@
hdu = pyfits.ImageHDU( data.imag )
hdu.name = name+'IMAG'
hdr = hdu.header
for i in range(len(attrs)) :
hdr[str(attrs[i][0])] = attrs[i][1]
hdulist.append(hdu) #@#@
else : raise
except :
if (IsType.isndarray(data) or IsType.islist(data) or IsType.istuple(data)) :
try : hdu = pyfits.ImageHDU( data )
except : hdu = pyfits.ImageHDU( np.array(data, np.float32) )
hdu.name = name
hdr = hdu.header
for i in range(len(attrs)) :
hdr[str(attrs[i][0])[:8]] = attrs[i][1]
hdulist.append(hdu)
else : other.header[name[:8]] = data
return hdulist
#--------------------------------------------------
hdulist = pyfits.HDUList()
other = pyfits.ImageHDU([])
other.name = 'other'
hdrother = other.header.keys()
if (IsType.isstr(inputwhat)) :
inname = Path.AbsPath(inputwhat)
outname = inname[:-5] + '.fits'
if (verbose) : print('Saving "'+outname+'"')
fo = h5py.File(inname, 'r')
keys = fo.keys()
for i in range(len(keys)) :
hdu = _Hdf52Fits( fo[keys[i]], hduname )
for j in range(len(hdu)) : hdulist.append( hdu[j] )
#--------------------------------------------------
else :
if (hduname in ['', None]) :
name = str(inputwhat.name)
name = name[name.rfind('/')+1:]
else : name = str(hduname)
outname = name + '.fits'
if (verbose) : print('Saving "'+outname+'"')
hdu = _Hdf52Fits( inputwhat, hduname )
for j in range(len(hdu)) : hdulist.append( hdu[j] )
#--------------------------------------------------
hdr = other.header.keys()
if (hdr != hdrother) : hdulist.append(other)
Path.ExistsPath(outname, 'True')
hdulist.writeto( outname )
hdulist.close()
def CrawlerBaiduWenku(website,
page50,
outdir,
outname,
browserDriverPath,
logger='INFO',
timeout=5):
'''
browserDriverPath:
How to download and use browserDriver ?
Using Chrome for example.
(1) Check the Chrome version:
input chrome://version on the website bar
You can see something like:
Google Chrome 69.0.3497.92
It means that the version is 69
(2) Open website http://chromedriver.storage.googleapis.com/index.html
There are some documents from 2.0 to 2.9
Open one of them, for example 2.46
Then open notes.txt, then you will see:
Supports Chrome v......
Find that matches your Chrome, here 2.40 to 2.44 are OK for Chrome 69
Then go the document 2.44, download the chromedriver for your system (linux, mac, win)
(3) Unzip the downloaded file, then you will obtain an application named "chromedriver"
(4) @browserDriverPath is the path of this "chromedriver"
website:
The url/website/link of the document on the wenku.baidu.com
e.g. the website of document "人脸检测与识别关键技术研究" is "https://wenku.baidu.com/view/f267c923dd36a32d737581df.html?sxts=1552644845367"
page50:
For a thesis or some document which are very long: exceed 50 pages, some times default website is only for the front 50 page, for the next 50 pages, the website is
https://wenku.baidu.com/view/f267c923dd36a32d737581df.html?sxts=1552644845367&pn=51
(added "&pn=51" behind the normal website)
The parameter @page50 here is [int]:
page50=False: only crawl @website is enough
page50=51: crawl @website and @website+'&pn=51'
page50=101: crawl @website, @website+'&pn=51', @website+'&pn=101'
NOTE THAT &pn= is only 51, 101, 151, 201, ...
outdir:
[str] path of the output directory
outname:
[str] main_name of the picture without format
NOTE THAT outname[-1] != '_'
logger:
=False (not print) | ='DEBUG' (print all) | 'INFo' (print count)
timeout:
How many seconds to be timeout (Default timeout=3)
'''
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time, re, requests
from jizhipy.Basic import Logger, Path
#----------------------------------------
# Check arguments
browserDriverPath = Path.AbsPath(browserDriverPath)
Path.ExistsPath(browserDriverPath, stop=True)
try:
page50 = int(page50)
except:
page50 = False
if (page50 not in range(51, 10000, 50)): page50 = ['']
else: page50 = [''] + range(51, page50 + 2, 50)
if (outdir[-1] != '/'): outdir += '/'
outdirAbs = Path.AbsPath(outdir)
Path.ExistsPath(outdirAbs, mkdir=True)
outname = str(outname)
if (outname[-1] == '_'): outname = outname[:-1]
try:
timeout = float(timeout)
except:
timeout = 5
logger = Logger(logger, True)
#----------------------------------------
# __init__
browser = webdriver.Chrome(browserDriverPath)
wait = WebDriverWait(browser, timeout)
# 匹配url(网址)信息的模板，url\("(.*?)"\) 相当于url("...")，其中"..."就是用来打开图片的网址
pattern = re.compile('.*?url\("(.*?)"\)', re.S)
#----------------------------------------
count = 0
for p5 in page50:
# Get the website
if (p5 != ''): p5 = '&pn=' + str(p5)
browser.get(website + p5)
# 定位到:鼠标'banner-more-btn'
submit = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, 'banner-more-btn')))
# 滑轮滑倒看得到鼠标的位置，不然鼠标会点不到
browser.execute_script("arguments[0].scrollIntoViewIfNeeded(true);",
submit)
# 让爬虫点击一下鼠标
submit.click()
# 定位到第一个节点
elem = wait.until(
EC.presence_of_element_located(
(By.ID, 'reader-container-inner-1')))
# 获取图片link
for i in elem.find_elements_by_class_name('bd'):
count += 1
try:
browser.execute_script(
"arguments[0].scrollIntoViewIfNeeded(true);", i)
# time.sleep(0.6) #@#@
# 定位到存放图片信息的节点
n = i.find_element_by_class_name('reader-pic-item')
js = n.get_attribute('style')
# findall，用模板匹配url(网址)
href = pattern.findall(js)
link = href[0]
# 保存图片
html = requests.get(link).content
stri = (4 - len(str(count))) * '0' + str(count)
imageName = outdirAbs + outname + '_' + stri + '.jpg'
fo = open(imageName, 'wb')
fo.write(html)
fo.close()
print(count)
except:
pass
def GSM2016(freq=None,
outname=None,
save=False,
nside=None,
coordsys='Galactic',
unit='K',
lowresol=False,
dtype=None,
verbose=False,
rmnegative=False,
rmsource=False,
gsm2016path='GSM2016_data/',
example=False):
'''
An improved model of diffuse galactic radio emission from 10 MHz to 5 THz; arxiv:1605.04920; 2017MNRAS.464.3486Z
Already remove CMB: -2.726
example:
True | False
==True: read and return 'gsm2016_example.hdf5'
jp.GSM2016(example=True)
Only diffuse map, had remove CMB, 408MHz, K, nside=512
compname = ['Total', 'Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2', 'Free-Free'] # 'Dust1'=cold dust, 'Dust2'=warm dust
freq:
in MHz
Must be one value, int | float, NOT list
outname:
Force to use .hdf5 format
(1) ==str: complete output name
(2) else: outname = 'gsm2016_'+strfreq+'.hdf5'
save:
==False:
exists, read, NOT save
NOT exists, generate, NOT save
==True:
exists, generate, save
NOT exists, generate, save
==None:
exists, read, NOT save
NOT exists, generate, save
lowresol:
True | False
(1) ==False:
nside=1024
* freq< 10GHz, force resol=56arcmin
** freq<=10GHz, force resol=24arcmin
(2) ==False
nside=64, resol=5deg=300arcmin
Use './data/lowres_maps.txt'
unit:
'K'/'Trj' | 'Tcmb' | 'MJysr'
dtype:
'float64' | 'float32'
nside:
Default nside=1024
rmnegative:
reset negative
True | False
rmsource:
remove sources from the output map
0/False | 1 | 2 | 3 | 4
0/False: NOT remove sources
1: remove CygA, CasA, Crab
2: remove sources outside center plane
3: remove sources inside center plane
return:
gsm.shape = (6, 12*nside**2)
compname = ['Total', 'Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2', 'Free-Free'] # 'Dust1'=cold dust, 'Dust2'=warm dust
'''
import numpy as np
import healpy as hp
import h5py
from jizhipy.Astro import RemoveSource
from jizhipy.Basic import Raise, SysFrame, Mkdir, Path, Time, IsType
if (verbose): print('jizhipy.GSM2016:')
if (nside is not None):
n = int(round(np.log(nside) / np.log(2)))
nside = 2**n
else:
nside = 512
coordsys = str(coordsys)
if (verbose):
print(' freq =', freq, 'MHz')
print(' coordsys =', coordsys)
print(' nside =', nside)
#----------------------------------------
if (gsm2016path is None): gsm2016path = 'GSM2016_data/'
if (not Path.ExistsPath(gsm2016path)):
gsm2016path = Path.jizhipyPath('jizhipy_tool/GSM2016_data/')
if (not Path.ExistsPath(gsm2016path)):
Raise(Exception, 'gsmp2016path="' + gsm2016path + '" NOT exists')
fdir = Path.AbsPath(gsm2016path)
#----------------------------------------
if (example):
if (verbose): print(' Use "gsm2016_example.hdf5"')
fo = h5py.File(fdir + 'gsm2016_example.hdf5', 'r')
gsm = fo['gsm2016'].value
fo.close()
# ud_grade to nside
if (nside is not None and gsm[0].size != 12 * nside**2):
gsm2 = []
for i in range(len(gsm)):
gsm2.append(hp.ud_grade(gsm[i], nside))
gsm = np.array(gsm2)
if (coordsys.lower() != 'galactic'):
for i in range(len(gsm)):
gsm[i] = CoordTrans.CelestialHealpix(gsm[i], 'ring',
'Galactic', coordsys)[0]
return gsm
#---------------------------------------------
#---------------------------------------------
freq, unit, lowresol, verbose, rmnegative, rmsource = float(freq), str(
unit).lower(), bool(lowresol), bool(verbose), bool(rmnegative), int(
rmsource)
vb = '123' if (verbose) else False
if (verbose):
print(
' Total, Synchrotron, CMB, HI, Cold dust, Warm dust, Free-Free')
if (freq <= 0):
print('freq = ' + str(freq) + ' <=0')
exit()
dtype = np.dtype(dtype)
#---------------------------------------------
#---------------------------------------------
if (IsType.isstr(outname)): outname = str(outname)
else:
strfreq = str(freq).split('.')
while (strfreq[1] != '' and strfreq[1][-1] == '0'):
strfreq[1] = strfreq[1][:-1]
if (strfreq[1] == ''): strfreq = strfreq[0]
else: strfreq = strfreq[0] + '.' + strfreq[1]
outname = 'gsm2016_' + strfreq + '.hdf5'
if (save is not False and '/' in outname):
for i in range(len(outname) - 1, -1, -1):
if (outname[i] == '/'): break
Mkdir(outname[:i + 1])
#--------------------------------------------------
#--------------------------------------------------
freq /= 1000. # GHz
kB, C, h, T = 1.38065e-23, 2.99792e8, 6.62607e-34, 2.725
hoverk = h / kB
def K_CMB2MJysr(K_CMB, nu): # in Kelvin and Hz
B_nu = 2 * (h * nu) * (nu / C)**2 / (np.exp(hoverk * nu / T) - 1)
conversion_factor = (B_nu * C / nu / T)**2 / 2 * np.exp(
hoverk * nu / T) / kB
return K_CMB * conversion_factor * 1e20 # 1e-26 for Jy | 1e6 for MJy
def K_RJ2MJysr(K_RJ, nu): # in Kelvin and Hz
conversion_factor = 2 * (nu / C)**2 * kB
return K_RJ * conversion_factor * 1e20 # 1e-26 for Jy | 1e6 for MJy
mjysr2k = 1. / K_RJ2MJysr(1., 1e9 * freq)
if (unit == 'tcmb'):
unit = 'Tcmb'
conversion = 1. / K_CMB2MJysr(1., 1e9 * freq)
elif (unit == 'mjysr'):
unit = 'MJysr'
conversion = 1.
if (verbose):
print((' Unit Conversion at %.3f MHz: 1 MJysr == %f ' + unit) %
(freq * 1000, mjysr2k))
else: # (unit in ['k', 'trj']) :
unit = 'K'
conversion = mjysr2k
#--------------------------------------------------
#--------------------------------------------------
exist = True if (Path.ExistsPath(outname)) else False
# if (exist and save is not True) :
if (exist):
try:
fo = h5py.File(outname, 'r')
gsm = fo['gsm2016'].value
fo.close()
if (verbose): print(' Exists: ', outname)
return gsm
except:
pass
#--------------------------------------------------
exist = False
if (save is not False): save = True
if (not exist): # generate
spec_nf = np.loadtxt(fdir + 'spectra.txt')
nfreq = spec_nf.shape[1]
left_index = -1
for i in range(nfreq - 1):
if (freq >= spec_nf[0, i] and freq <= spec_nf[0, i + 1]):
left_index = i
break
if (left_index < 0):
print(
"FREQUENCY ERROR: %.3f MHz is outside supported frequency range of %.3f MHz to %.3f MHz."
% (freq * 1000, spec_nf[0, 0] * 1000, spec_nf[0, -1] * 1000))
exit()
#--------------------------------------------------
# compname must be this order, do NOT modify
compname = ['Synchrotron', 'CMB', 'HI', 'Dust1', 'Dust2',
'Free-Free'] # 'Dust1'=cold dust, 'Dust2'=warm dust
#--------------------------------------------------
if (lowresol):
nside_data, op_resolution = 64, 300.
else:
nside_data = 1024 # forced because of *.bin files
op_resolution = 56. if (freq < 10) else 24.
if (not lowresol):
gsm = np.array([
np.fromfile(fdir + 'highres_' + cname + '_map.bin',
dtype='float32') for cname in compname
])
else:
gsm = np.loadtxt(fdir + 'lowres_maps.txt')
#--------------------------------------------------
interp_spec_nf = np.copy(spec_nf)
interp_spec_nf[0:2] = np.log10(interp_spec_nf[0:2])
x1 = interp_spec_nf[0, left_index]
x2 = interp_spec_nf[0, left_index + 1]
y1 = interp_spec_nf[1:, left_index]
y2 = interp_spec_nf[1:, left_index + 1]
x = np.log10(freq)
interpolated_vals = (x * (y2 - y1) + x2 * y1 - x1 * y2) / (x2 - x1)
gsm = 10.**interpolated_vals[0] * interpolated_vals[1:, None] * gsm
gsm = gsm * conversion
#--------------------------------------------------
if (len(gsm.shape) == 1): gsm = gsm[None, :]
else: gsm = np.concatenate([gsm.sum(0)[None, :], gsm], 0)
#--------------------------------------------------
#--------------------------------------------------
# generate is nest, convert to ring
# First ud_grade so that faster
for i in range(len(gsm)):
gsm[i] = hp.reorder(gsm[i], n2r=True)
# ud_grade to nside
if (gsm[0].size != 12 * nside**2):
gsm2 = []
for i in range(len(gsm)):
gsm2.append(hp.ud_grade(gsm[i], nside))
gsm = np.array(gsm2)
#--------------------------------------------------
#--------------------------------------------------
# remove negative value in the hpmaps
for i in range(len(gsm)):
if (not rmnegative and i != 0): break
tf = gsm[i] < 0
if (tf[tf].size == 0): continue
g = gsm[i].copy()
# g[tf] = g.mean()
g[tf] = g[g > 0].min()
g = hp.smoothing(g, np.pi / 180 * 10, verbose=False)
gsm[i][tf] = g[tf]
#--------------------------------------------------
#--------------------------------------------------
casa = np.array([[(111.44, -1.37), (93.89, 2.37)]]) # fwhm=7
casa1 = np.array([[(118.12, 4.78), (121.83, 1.09)]])
cyga = np.array([[(81, 2), (26.33, 8.38)]]) # fwhm=7
cyga1 = np.array([[(84.9, -0.55), (71.8, 0.4)]]) # fwhm=3
cyga2 = np.array([[(76.6, 0.4), (71.8, 0.4)]]) # fwhm=5
cyga3 = np.array([[(88.82, 4.69), (88.69, 2)]]) # fwhm=3
crab = np.array([[(-96, -2), (26.33, 8.38)]]) # fwhm=4.5
crab1 = np.array([[(-91.82, -1), (26.33, 8.38)]]) # fwhm=3
crab2 = np.array([[(-99.31, -3.43), (26.33, 8.38)]]) # fwhm=3
lb1 = np.concatenate(
[casa, casa1, cyga, crab, cyga1, cyga2, cyga3, crab1, crab2], 0)
fwhm1 = np.array([7, 3, 7, 4.5, 3, 5, 3, 2, 2])
lb10, lb11 = lb1[:, 0], lb1[:, 1]
#--------------------------------------------------
other = np.array([[(-153.41, -1.91), (-163.58, -9.29)],
[(-170.31, 2.08), (-163.58, -9.29)],
[(-175.26, -5.88), (-163.58, -9.29)],
[(-150.64, -19.38), (-157, -20)],
[(-153.34, -16.39), (-157, -20)],
[(133.73, 1.07), (126.89, 0.83)],
[(160.4, 2.78), (157.61, -1.24)],
[(73.2, -9.06), (68.38, -7.56)],
[(63.39, -5.98), (68.38, -7.56)]])
fwhm2 = np.array([4, 4, 4, 1, 1, 2, 2, 3, 2])
lb20, lb21 = other[:, 0], other[:, 1]
#--------------------------------------------------
center = np.array([[(-54.49, 0.08), (-58.03, -0.04)],
[(-61.38, -0.18), (-58.03, -0.04)],
[(-69.23, -0.59), (-58.03, -0.04)],
[(-72.15, -0.73), (-58.03, -0.04)],
[(-75.88, -0.42), (-58.03, -0.04)],
[(-77.63, -1.2), (-58.03, -0.04)],
[(-65.21, -1.2), (-58.03, -0.04)],
[(-63.1, 9.8), (-60.6, 8.89)],
[(-6.93, 16.9), (-9.35, 16.06)]])
fwhm3 = np.array([2, 2, 3, 3, 2, 2, 2, 2, 2])
lb30, lb31 = center[:, 0], center[:, 1]
if (rmsource in [1, 2, 3, 4]):
if (rmsource == 4):
lb0 = np.concatenate([lb30, lb20, lb10], 0)
lb1 = np.concatenate([lb31, lb21, lb11], 0)
fwhm = np.concatenate([fwhm3, fwhm2, fwhm1])
elif (rmsource == 1):
lb0, lb1, fwhm = lb10, lb11, fwhm1
elif (rmsource == 2):
lb0, lb1, fwhm = lb20, lb21, fwhm2
elif (rmsource == 3):
lb0, lb1, fwhm = lb30, lb31, fwhm3
gsm[0] = np.log10(gsm[0])
times, same, onebyone = 1, False, True
pix, revalue = RemoveSource(gsm[0], lb0, lb1, fwhm, times, same,
onebyone, vb)
gsm[0][pix] = revalue
gsm[0] = hp.smoothing(gsm[0], np.pi / 180, verbose=False)
gsm[0] = 10.**gsm[0]
gsm = gsm.astype(dtype)
#--------------------------------------------------
#--------------------------------------------------
if (coordsys.lower() != 'galactic'):
for i in range(len(gsm)):
gsm[i] = CoordTrans.CelestialHealpix(gsm[i], 'ring', 'Galactic',
coordsys)[0]
if (save):
if (verbose): print(' Saving to: ', outname)
Path.ExistsPath(outname, old=True)
fo = h5py.File(outname, 'w')
fo['gsm2016'] = gsm
fo['gsm2016'].attrs['freq'] = freq * 1000
fo['gsm2016'].attrs['unit'] = unit
fo['gsm2016'].attrs['MJysr2K'] = mjysr2k
fo['gsm2016'].attrs['lowresol'] = int(lowresol)
fo['gsm2016'].attrs['nside'] = nside
fo['gsm2016'].attrs['pixtype'] = 'healpix'
fo['gsm2016'].attrs['ordering'] = 'ring'
fo['gsm2016'].attrs['coordsys'] = 'Galactic'
fo['gsm2016'].attrs['rmnegative'] = int(rmnegative)
fo['gsm2016'].attrs['rmsource'] = int(rmsource)
fo['gsm2016'].attrs['creadate'] = Time(0)
fo.close()
return gsm
def ObjectDetector(image,
framework,
modelPath,
txtPath=None,
labelMap=None,
minConfidence=None,
scaleFactor=None,
size=None,
mean=None,
crop=False,
**kwargs):
'''
Parameters
----------
image:
Must be 3D/2D array/matrix, NOT path
framework:
'Tensorflow' | 'Caffe' | 'Torch' | 'Darknet' | 'ONNX' | 'ModelOptimizer' | 'Haar' (OpenCV .xml)
modelPath, txtPath:
(1) for Tensorflow:
* modelPath: path of .pb (binary frozen graph) file with binary protobuf description of the network architecture
* txtPath: path of .pbtxt file that contains text graph definition in protobuf format
(2) for Caffe:
* modelPath: path of .caffemodel file with learned network
* txtPath: path of .prototxt file with text description of the network architecture
(3) for OpenCV.xml:
* modelPath: path of .xml file
* txtPath: set txtPath=None
minConfidence:
(1) for NOT Haar:
* Detect objects with confidence >= minConfidence
(2) for Haar:
* minConfidence=minNeighbors, if a candidate target is consist of >= minNeighbors windows, it may be a real target, otherwise, drop it
scaleFactor:
(1) for NOT Haar:
* Use what image to train this model(network)
* scaleFactor=1: use original image range 0~255
* scaleFactor=1/255: use normalized image range 0~1
* For Caffe, use scaleFactor=1
* For Tensorflow, use scaleFactor=1/127.5
* (effect: image *= scaleFactor)
(2) for Haar:
* scaleFactor = NextWindowSize / CurrentWindowSize
(default scaleFactor=1.1)
size:
(1) for NOT Haar:
* size=(height, width) for output image
(2) for Haar:
* size=flags, usually use 3 cases:
flags=0 (default)
flags=cv2.CV_HAAR_DO_CANNY_PRUNING
flags=cv2.CASCADE_SCALE_IMAGE
mean:
(only for NOT Haar)
scalar with mean values which are subtracted from channels. Values are intended to be in (mean-R, mean-G, mean-B) order if @p image is BGR (NOT RGB) and @p swapRB=True
swapRB:
(only for NOT Haar)
True/False, swap R(red) and B(blue) or not
crop:
(only for NOT Haar)
True/False, crop the image or not after resize
@p size is used to resize @p image:
(1) crop=False: resize directly and don't preserve aspect ratio
(2) crop=True: first, resize @p image preserving aspect ratio, so that @p image.shape >= size; then, crop the longer axis from the image center, make @p image.shape==size
'''
from jizhipy.Basic import Path, Raise
import numpy as np
import cv2
# Check input parameters
framework_valid = [
'tensorflow', 'caffe', 'torch', 'darknet', 'onnx', 'modeloptimizer',
'haar'
]
framework, fw0 = str(framework).lower(), framework
if (framework not in framework_valid):
Raise(
Exception, 'framework="' + framework +
'" is NOT valid, must be one of ' + str(framework_valid))
#----------
if (minConfidence is None):
if (framework != 'haar'): minConfidence = 0.7
else: minConfidence = 3
#----------
if (scaleFactor is not None):
scaleFactor = float(scaleFactor)
else:
if (framework == 'tensorflow'): scaleFactor = 1 / 127.5
elif (framework == 'haar'): scaleFactor = 1.1
else: scaleFactor = 1.0
#----------
if (size is not None):
try:
size = tuple(size)[:2]
except:
pass
else:
if (framework == 'haar'): size = 0
#----------
swapRB = False
swapRB, crop = bool(swapRB), bool(crop)
#----------
modelPath = Path.AbsPath(modelPath)
Path.ExistsPath(modelPath, stop=True)
if (txtPath is not None):
txtPath = Path.AbsPath(txtPath)
Path.ExistsPath(txtPath, stop=True)
height, width = image.shape[:2]
#----------------------------------------
#----------------------------------------
if (framework != 'haar'):
# Creates 4-dimensional blob from image
blob = cv2.dnn.blobFromImage(image, scaleFactor, size, mean, swapRB,
crop)
# blob = cv2.dnn.blobFromImage(image)
# Read the model(weight)
if (framework == 'tensorflow'):
net = cv2.dnn.readNetFromTensorflow(modelPath, txtPath)
elif (framework == 'caffe'):
net = cv2.dnn.readNetFromCaffe(txtPath, modelPath)
elif (framework == 'torch'):
isBinary = txtPath
net = cv2.dnn.readNetFromTorch(modelPath, isBinary)
elif (framework == 'darknet'):
net = cv2.dnn.readNetFromDarknet(txtPath, modelPath)
elif (framework == 'onnx'):
net = cv2.dnn.readNetFromONNX(modelPath)
elif (framework == 'ModelOptimizer'):
net = cv2.dnn.readNetFromModelOptimizer(txtPath, modelPath)
else:
Raise(Exception, 'Not support framework: ' + fw0)
# Pass the blob through the network
net.setInput(blob)
# detection[0,0].shape(num_object, 7)
# 7: 0:? 1:ClassIndex 2:Confidence 3,4,5,6:Box
detection = net.forward()[0, 0]
# Select >= minConfidence
n = np.arange(len(detection))[detection[:, 2] >= minConfidence]
detection = detection[n]
# Detection confidence and rects
confidence = detection[:, 2]
rects = (detection[:, 3:] *
np.array([[width, height, width, height]])).astype(
int) # xmin, ymin, xmax, ymax
# Confidence to text to draw
if (labelMap is None): label = len(rects) * ['']
else: label = labelMap[detection[:, 1].astype(int)]
text = []
for i in range(len(rects)):
c = ('%.1f' % (100 * confidence[i])) + '%'
if (label[i] != ''): c = str(label[i]) + ': ' + c
text.append(c)
#----------------------------------------
#----------------------------------------
else:
cascade = cv2.CascadeClassifier(modelPath)
rects = cascade.detectMultiScale(image,
scaleFactor,
minConfidence,
minSize=(5, 5),
flags=size)
if (len(rects) > 0): rects[:, 2:] += rects[:, :2]
confidence = np.zeros(len(rects), int)
text = len(rects) * ['']
#----------------------------------------
#----------------------------------------
# from imutils.object_detection import non_max_suppression
# rects = non_max_suppression(rects, probs=None, overlapThresh=0.7)
return (rects, confidence, text)
def NVSSSUMSS(fwhm,
lon=None,
lat=None,
nside=None,
coordsys='Galactic',
galcenter=True,
nvsssumsspath='nvss-sumss_1.4GHz_15mJy_fullsky.hdf5'):
'''
return radio source map
* merged from NVSS and SUMSS
* healpix, RING, full sky, 1.4GHz, 15mJ limit for completeness
* fill the missing region
fwhm:
[degree]
fwhm==None: don't smooth the map
lon, lat:
[degree]
lon: RA | l
lat: Dec | b
Must can be broadcast
Use lon, lat first, otherwise, use nside
nside:
Healpix full sky map instead of lon, lat
nside0 = 1024
coordsys:
'Equatorial' | 'Galactic' | 'Ecliptic'
galcenter:
True | False
==True: Retain the galactic center
==False: discard the galactic center
nvsssumsspath:
str, where is the file 'nvss-sumss_1.4GHz_15mJy_fullsky.hdf5'
'''
import numpy as np
import healpy as hp
import h5py
from jizhipy.Array import Asarray
from jizhipy.Basic import Raise, Path, IsType
from jizhipy.Transform import CoordTrans
if (nvsssumsspath is None):
nvsssumsspath = 'nvss-sumss_1.4GHz_15mJy_fullsky.hdf5'
nvsssumsspath = Path.AbsPath(str(nvsssumsspath))
if (not Path.ExistsPath(nvsssumsspath)):
nvsssumsspath = Path.jizhipyPath(
'jizhipy_tool/nvss-sumss_1.4GHz_15mJy_fullsky.hdf5')
if (not Path.ExistsPath(nvsssumsspath)):
Raise(Exception,
'nvsssumsspath="' + nvsssumsspath + '" NOT exists')
#--------------------------------------------------
nside0 = 1024
if (lon is not None and lat is not None):
if (np.sum((lat < -90) + (lat > 90)) > 0):
Raise(Exception, 'lat out of [-90, 90] degree')
nside = None
else:
islist = True
try:
nside = int(nside)
except:
nside = nside0
#--------------------------------------------------
fo = h5py.File(nvsssumsspath, 'r')
a = fo['nvsum'].value
n, void = fo['void'].value
a[n.astype(int)] = void
resol = 0.06 # degree
if (not galcenter):
theta, phi = hp.pix2ang(nside0, np.arange(a.size))
theta, phi = 90 - theta * 180 / np.pi, phi * 180 / np.pi
tf = (-5 < theta) * (theta < 5) * ((phi < 55) + (phi > 345))
n = np.arange(a.size)[tf]
del theta, phi, tf
m = (np.random.random(n.size) * void.size).astype(int)
a[n] = void[m]
#--------------------------------------------------
coordsys = str(coordsys).lower()
if (nside is None): # use lon and lat to select region
if (fwhm is not None and fwhm > resol):
a = hp.smoothing(a, fwhm * np.pi / 180, verbose=False)
a[a < 0] = 0
islist = False if (IsType.isnum(lon + lat)) else True
lon, lat = lon + lat * 0, lon * 0 + lat # same shape
lon %= 360
lon, lat = CoordTrans.Celestial(lon * np.pi / 180, lat * np.pi / 180,
coordsys, 'galactic') # rad
npix = hp.ang2pix(nside0, np.pi / 2 - lat, lon)
del lon, lat
a = a[npix]
del npix
else: # full sky healpix map
if (coordsys != 'galactic'):
a = CoordTrans.CelestialHealpix(a, 'RING', 'galactic', coordsys)[0]
if (fwhm is not None and fwhm > resol):
a = hp.smoothing(a, fwhm * np.pi / 180, verbose=False)
a[a < 0] = 0
if (nside != nside0): a = hp.ud_grade(a, nside)
a = np.float32(a)
if (not islist): a = a[0]
return a
def GSM(fwhm,
freq,
lon=None,
lat=None,
nside=None,
coordsys='Galactic',
gsmpath='gsm_parameter.out',
save=False):
'''
Already -2.726
fwhm:
[degree]
fwhm==None: don't smooth the map
freq, dfreq:
[MHz]
Must be one/isnum, NOT list
* freq: center frequency
* dfreq: averaged from freq-dfreq/2 (include) to freq+dfreq/2 (include). For GSM(), dfreq effect is very small, so remove this argument
lon, lat:
[degree]
lon: RA | l
lat: Dec | b
Must can be broadcast
Use lon, lat first, otherwise, use nside
nside:
Healpix full sky map instead of lon, lat
coordsys:
'Equatorial' | 'Galactic' | 'Ecliptic'
gsmpath:
str, where is the program "gsm_parameter.out"
./gsm_parameter.out freq outpath gsmdir
save:
True | False
Save gsm_xxx.npy or not?
Compile:
gfortran -ffixed-line-length-none gsm_parameter.f -o gsm_parameter.out
return:
gsm, ndarray
'''
import os
import healpy as hp
import numpy as np
from jizhipy.Transform import CoordTrans
from jizhipy.Array import Asarray
from jizhipy.Basic import Raise, Path, ShellCmd, IsType
if (gsmpath is None): gsmpath = 'gsm_parameter.out'
freq = Asarray(freq).flatten()
if (freq.size != 1):
Raise(Exception,
'freq must isnum, but now freq.size=' + str(freq.size))
freq = freq[0]
gsmpath = Path.AbsPath(str(gsmpath))
if (not Path.ExistsPath(gsmpath)):
gsmpath = Path.jizhipyPath('jizhipy_tool/GSM/gsm_parameter.out')
if (not Path.ExistsPath(gsmpath)):
Raise(Exception, 'gsmpath="' + gsmpath + '" NOT exists')
n = gsmpath.rfind('/')
if (n < 0): gsmdir = ''
else: gsmdir = gsmpath[:n + 1]
if (lon is not None and lat is not None):
if (np.sum((lat < -90) + (lat > 90)) > 0):
Raise(Exception, 'lat out of [-90, 90] degree')
nside = None
else:
islist = True
try:
nside = int(nside)
except:
nside = 512
#--------------------------------------------------
# list local "gsm_*.npy"
gsmlist = ShellCmd('ls gsm_*.npy')
ngsm = None
for i in range(len(gsmlist)):
try:
f = np.sort([float(gsmlist[i][4:-4]), freq])
except:
continue
r = (f[1] / f[0])**3
if (r < 1.01):
ngsm = i
break
if (ngsm is None):
freqstr = ('%.2f' % freq)
if (freqstr[-1] == '0'): freqstr = freqstr[:-1]
if (freqstr[-1] == '0'): freqstr = freqstr[:-2]
outname = 'gsm_' + freqstr
os.system(gsmpath + ' ' + freqstr + ' ' + outname + '.dat ' + gsmdir)
gsm = np.float32(np.loadtxt(outname + '.dat'))
os.system('rm ' + outname + '.dat')
if (save): np.save(outname + '.npy', gsm)
else:
gsm = np.float32(np.load(gsmlist[ngsm]))
if (not save): os.system('rm ' + gsmlist[ngsm])
if (Path.ExistsPath('qaz_cols.dat')): os.system('rm qaz_cols.dat')
nsidegsm = hp.get_nside(gsm)
#--------------------------------------------------
coordsys = str(coordsys).lower()
if (nside is None):
if (fwhm is not None and fwhm > 1):
fwhm = (fwhm**2 - 1)**0.5
gsm = hp.smoothing(gsm, fwhm * np.pi / 180, verbose=False)
gsm[gsm < 0] = 0
islist = False if (IsType.isnum(lon + lat)) else True
lon, lat = lon + lat * 0, lon * 0 + lat # same shape
lon %= 360
lon, lat = CoordTrans.Celestial(lon * np.pi / 180, lat * np.pi / 180,
coordsys, 'galactic') # rad
npix = hp.ang2pix(nsidegsm, np.pi / 2 - lat, lon)
del lon, lat
gsm = gsm[npix]
del npix
else:
if (nside != nsidegsm): gsm = hp.ud_grade(gsm, nside)
if (coordsys != 'galactic'):
gsm = CoordTrans.CelestialHealpix(gsm, 'RING', 'galactic',
coordsys)[0]
if (fwhm is not None and fwhm > 1):
fwhm = (fwhm**2 - 1)**0.5
gsm = hp.smoothing(gsm, fwhm * np.pi / 180, verbose=False)
gsm[gsm < 0] = 0
gsm = np.float32(gsm)
if (not islist): gsm = gsm[0]
return gsm