def parser_html(self, response):
path_one = '/data01/html/huanping/中华人民共和国生态环境部/{}/'\
.format(time.strftime("%Y-%m-%d"))
if not os.path.exists(path_one):
os.makedirs(path_one)
path = path_one + make_index(response.save['url'])
with open(path, 'w+') as f:
f.write(response.text)
ob = BeautifulSoup(response.text, "html.parser")
obTime = ob.find('span', class_="wzxq_fbt2").findAll("p")[0].getText()
timeList = str(obTime).replace("\n", "").strip().split("-")
table = ob.findAll("table")
keys = []
flag = 1
objs = []
'''judge include table '''
if len(table) != 0:
'''table is include a lagel'''
if len(table[0].findAll("a")) != 0:
rows = table[0].findAll("tr")
for row in rows:
if flag == 1:
'''deal first row data'''
for cell in row.findAll(['td', 'th']):
keys.append(str(cell.get_text()).replace("\n", "").strip())
flag += 1
else:
if len(row.findAll("a")) != 0:
'''judge row is include a label'''
obj = {}
data = {}
try:
for i in range(0, len(keys)):
obj[keys[i]] = str(row.findAll(['td', 'th'])[i].get_text()).replace("\n",
"")
fileUrl = "http://hps.mee.gov.cn/jsxm/xmsl/" + timeList[0] + timeList[
1] + "/" + row.findAll("a")[0].attrs['oldsrc']
obj["项目名称"] = str(obj["项目名称"]).replace("/", "每").strip()
except:
logging.error({"error": traceback.format_exc(), "filePath": path,
"time": time.strftime("%Y-%m-%d %H:%M:%S")})
finally:
data["info"] = json.dumps(obj, ensure_ascii=False)
data["sub_url"] = ob.find(attrs={"name": "Url"})['content']
data["file_url"] = fileUrl
data["create_time"] = time.strftime("%Y-%m-%d %H:%M:%S")
data["source"] = "中华人民共和国生态环境部"
data["pdf_id"] = make_index(data["file_url"] + data["sub_url"])
data["file_name"] = obj["项目名称"]
data["path"] = "/data01/pdf/"
data["status"] = 0
data["topic"] = 'pdf_info'
objs.append(data)
sql = f"insert into {mysql_table_pdf_download}(info,sub_url,file_url," \
f"file_name,path,source,status,pdf_id,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
mysql_tool(sql, [str(data["info"]), data["sub_url"], data["file_url"],
data["file_name"], data["path"], data["source"],
data["status"], data["pdf_id"],data["create_time"]])
return objs
def zjhp_parse_remains(self, response):
data_list = []
path_one = '/data01/html/huanping/浙江省生态环境厅/{}/'\
.format(time.strftime("%Y-%m-%d"))
if not os.path.exists(path_one):
os.makedirs(path_one)
path = path_one + make_index(response.save['url'])
with open(path, 'w+') as f:
f.write(response.text)
oBject = BeautifulSoup(response.text, "html.parser")
x = oBject.find("div", class_='mainContainer;').findAll('a')
for i in range(len(x)):
data_info = {}
if 'Produced By 大汉网络 大汉版通发布系统' in x[i].getText():
data_info['file_name'] = "UnknowPdfFile" + time.strftime(
"%Y-%m-%d %H:%M:%S")
else:
data_info['file_name'] = x[i].getText()[:-4]
try:
j = oBject.find(
"div", class_='mainContainer;').findAll('a')[i]['href']
try:
url_part = re.findall(r"&filename=(.*?).pdf", j, re.S)[0]
if '/module/download/downfile.jsp?' in j:
data_info['file_url'] = (
'http://zjjcmspublic.oss-cn-hangzhou.aliyuncs.com/jcms_files'
'/jcms1/web1756/site/attach/-1/{}.pdf'.format(
url_part))
else:
data_info['file_url'] = url_part
except IndexError:
url_part = 'http://zjjcmspublic.oss-cn-hangzhou.aliyuncs.com/jcms_files/jcms1/web1756/site{}'\
.format(j)
data_info['file_url'] = url_part
finally:
data_info["info"] = "this html can not get html info"
data_info["sub_url"] = response.save['url']
data_info["create_time"] = time.strftime(
"%Y-%m-%d %H:%M:%S")
data_info["source"] = "浙江省环境生态厅"
data_info["pdf_id"] = make_index(data_info["file_url"] +
data_info["sub_url"])
data_info["path"] = "/data01/pdf/"
data_info["status"] = 0
data_info["topic"] = "pdf_info"
sql_query = f"insert into {mysql_table_pdf_download}(info,sub_url,file_url,file_name,path," \
f"source,status,pdf_id,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
mysql_tool(sql_query, [
str(data_info["info"]), data_info["sub_url"],
data_info["file_url"], data_info["file_name"],
data_info["path"], data_info["source"],
data_info["status"], data_info["pdf_id"],
data_info["create_time"]
])
except KeyError:
logger.warning('this html has no pdf file')
finally:
data_list.append(data_info)
return data_list
def getItems(self, response):
datas=[]
if response.status_code == 200:
ob = BeautifulSoup(response.text, "html.parser")
'''get list'''
items = ob.find('div', class_="main_rt_list").findAll("li")
for item in items:
html_info = {}
'''get a label'''
click = item.find("a")
if click != None:
href = str(click.attrs['href'])[1:]
if href.endswith(".shtml"):
url = "http://hps.mee.gov.cn/jsxm/xmsl" + str(click.attrs['href'])[1:]
self.crawl(url, save={'url': url}, timeout=6, callback=self.parser_html)
html_info["url"] = url
html_info["create_time"] = time.strftime("%Y-%m-%d %H:%M:%S")
html_info["source"] = "中华人民共和国生态环境部"
html_info["status"] = 0
html_info["type"] = "huanping"
path = "/data01/html/" + html_info["type"] + "/" + html_info["source"] + \
"/" + html_info["create_time"][0:10] + "/"
id = make_index(html_info["url"] + path)
html_info["path"] = path
html_info["html_id"] = id
html_info['topic'] = "html_info"
sql = f"insert into {mysql_table_html_download}" \
f"(html_id,url,path,status,`type`,create_time) values (%s,%s,%s,%s,%s,%s)"
mysql_tool(sql, [str(html_info["html_id"]), html_info["url"], html_info["path"],
html_info["status"], html_info["type"], html_info["create_time"]])
datas.append(html_info)
return datas
def zjhp_condition_remains(self, response):
html_detail_tbody = BeautifulSoup(response.text, 'lxml')
infos_list = []
tbodys_one = html_detail_tbody.findAll("tr",
class_='tr_main_value_even')
tbodys_two = html_detail_tbody.findAll("tr",
class_='tr_main_value_odd')
for infos in tbodys_one + tbodys_two:
html_info = {}
if infos.a is not None:
html_info["url"] = infos.a['href']
self.crawl(infos.a['href'],
save={'url': infos.a['href']},
timeout=6,
callback=self.zjhp_parse_remains)
html_info["create_time"] = time.strftime("%Y-%m-%d %H:%M:%S")
html_info["source"] = "浙江省生态环境厅"
html_info["type"] = "huanping"
path = "/data01/html/" + html_info["type"] + "/" + html_info["source"] +\
"/" + html_info["create_time"][0:10] + "/"
id = make_index(html_info["url"] + path)
html_info["path"] = path
html_info["html_id"] = id
html_info["status"] = 0
html_info['topic'] = "html_info"
sql = f"insert into {mysql_table_html_download}" \
f"(html_id,url,path,status,`type`,create_time) values (%s,%s,%s,%s,%s,%s)"
mysql_tool(sql, [
str(html_info["html_id"]), html_info["url"],
html_info["path"], html_info["status"], html_info["type"],
html_info["create_time"]
])
infos_list.append(html_info)
return infos_list
def arc_extraction(img,
orders,
extraction_width,
column_range,
gain=1,
readn=0,
dark=0,
plot=False,
**kwargs):
logging.info("Using arc extraction to produce spectrum.")
_, ncol = img.shape
nord, _ = orders.shape
if plot:
# Prepare output image
output = np.zeros((np.sum(extraction_width[1:-1]) + nord - 2, ncol))
pos = [0]
spectrum = np.zeros((nord - 2, ncol))
uncertainties = np.zeros((nord - 2, ncol))
x = np.arange(ncol)
for i, onum in enumerate(range(1, nord - 1)): # loop thru orders
x_left_lim = column_range[onum, 0] # First column to extract
x_right_lim = column_range[onum, 1] # Last column to extract
ycen = np.polyval(orders[onum], x).astype(int)
yb, yt = ycen - extraction_width[onum,
0], ycen + extraction_width[onum, 1]
index = make_index(yb, yt, x_left_lim, x_right_lim)
# Sum over the prepared index
arc = np.sum(img[index], axis=0)
spectrum[i, x_left_lim:x_right_lim] = arc # store total counts
uncertainties[i, x_left_lim:x_right_lim] = (
np.sqrt(np.abs(arc * gain + dark + readn**2)) / gain
) # estimate uncertainty
if plot:
output[pos[i]:pos[i] + index[0].shape[0],
x_left_lim:x_right_lim] = img[index]
pos += [pos[i] + index[0].shape[0]]
if plot:
plt.imshow(output,
vmin=0,
vmax=np.mean(output) + 5 * np.std(output),
origin="lower")
for i in range(nord - 2):
tmp = spectrum[i] - np.min(
spectrum[i, column_range[i + 1, 0]:column_range[i + 1, 1]])
tmp = tmp / np.max(tmp) * 0.9 * (pos[i + 1] - pos[i])
tmp += pos[i]
tmp[tmp < pos[i]] = pos[i]
plt.plot(x, tmp)
plt.show()
return spectrum, 0, uncertainties
def extract_spectrum(img,
ycen,
ylow,
yhigh,
xlow,
xhigh,
gain=1,
readn=0,
lambda_sf=0.1,
lambda_sp=0,
osample=1,
swath_width=None,
no_scatter=False,
telluric=False,
normalize=False,
scatter_below=0,
scatter_above=0,
yscatter_below=0,
yscatter_above=0,
threshold=0,
use_2d=False,
plot=False,
ord_num=0,
im_norm=None,
im_ordr=None,
**kwargs):
if use_2d:
raise NotImplementedError("Curved extraction not supported yet")
nrow, ncol = img.shape
noise = readn / gain
irow = np.arange(nrow)
ycene = ycen[xlow:xhigh]
nysf = yhigh + ylow + 1
yslitf = -ylow, yhigh
spec = np.zeros(ncol)
sunc = np.zeros(ncol)
no_scatter = no_scatter or (np.all(scatter_below == 0)
and np.all(scatter_above == 0))
if swath_width is None:
i = np.unique(ycen.astype(int)) # Points of row crossing
ni = len(
i) # This is how many times this order crosses to the next row
if len(i) > 1: # Curved order crosses rows
i = np.sum(i[1:] - i[:-1]) / (len(i) - 1)
nbin = np.clip(int(np.round(ncol / i)) // 3, 3,
20) # number of swaths along the order
else: # Perfectly aligned orders
nbin = np.clip(ncol // 400, 3,
None) # Still follow the changes in PSF
nbin = nbin * (xhigh -
xlow) // ncol # Adjust for the true order length
else:
nbin = np.clip(int(np.round((xhigh - xlow) / swath_width)), 1, None)
nslitf = osample * (ylow + yhigh + 2) + 1
slitf = np.zeros((2 * nbin, nslitf))
# Define order boundary
yc = ycen.astype(int)
ymin = ycen - ylow
ymax = ycen + yhigh
# Calculate boundaries of distinct slitf regions
# Here is the layout to understand the lines below
#
# 1st swath 3rd swath 5th swath ...
# /============|============|============|============|============|
#
# 2nd swath 4th swath 6th swath
# |------------|------------|------------|------------|
# |.....|
# overlap
#
# + ******* 1
# + *
# + *
# * weights (+) previous swath, (*) current swath
# * +
# * +
# * +++++++ 0
bins = np.linspace(xlow, xhigh, 2 * nbin + 1) # boundaries of bins
ibeg_half = np.ceil(bins[:-2]).astype(int) # beginning of each bin
iend_half = np.floor(bins[2:]).astype(int) # end of each bin
# Perform slit decomposition within each swath stepping through the order with
# half swath width. Spectra for each decomposition are combined with linear weights.
for ihalf in range(0, 2 * nbin - 1): # loop through swaths
ib = ibeg_half[ihalf] # left column
ie = iend_half[ihalf] + 1 # right column
nc = ie - ib # number of columns in swath
logging.debug("Extractiing Swath %i, Columns: %i - %i", ihalf, ib, ie)
# Weight for combining overlapping regions
weight = np.ones(nc)
if ihalf > 0:
weight[:nc // 2 + 1] = np.arange(nc // 2 + 1) / nc * 2
oweight = 1 - weight
# Cut out swath from image
index = make_index(yc - ylow, yc + yhigh, ib, ie)
sf = img[index]
# Telluric
if telluric:
tel_lim = (telluric if telluric > 5 and telluric < nysf / 2 else
min(5, nysf / 3))
tel = np.sum(sf, axis=0)
itel = np.arange(nysf)
itel = itel[np.abs(itel - nysf / 2) >= tel_lim]
tel = sf[itel, :]
sc = np.zeros(nc)
for itel in range(nc):
sc[itel] = np.median(tel[itel])
tell = sc
else:
tell = 0
if not no_scatter:
# y indices
index_y = np.array(
[np.arange(k, nysf + k) for k in yc[ib:ie] - yslitf[0]])
dy_scatter = (index_y.T - yscatter_below[None, ib:ie]) / (
yscatter_above[None, ib:ie] - yscatter_below[None, ib:ie])
scatter = (scatter_above[None, ib:ie] - scatter_below[None, ib:ie]
) * dy_scatter + scatter_below[None, ib:ie]
else:
scatter = 0
# Do Slitfunction extraction
sf -= scatter + tell
sf = np.clip(sf, 0, None)
# offset from the central line
y_offset = ycen[ib:ie] - yc[ib:ie]
# if use_2d:
# sp, sfsm, model, unc = slitfunc_curved(
# sf,
# y_offset,
# tilt,
# lambda_sp=lambda_sp,
# lambda_sl=lambda_sf,
# osample=osample,
# )
# delta_x = None # TODO get this from slitfunc_curved
# else:
sp, sfsm, model, unc = slitfunc(sf,
y_offset,
lambda_sp=lambda_sp,
lambda_sf=lambda_sf,
osample=osample)
if normalize:
# In case we do FF normalization replace the original image by the
# ratio of sf/sfbin where number of counts is larger than threshold
# and with 1 elsewhere
scale = 1
ii = np.where(model > threshold / gain)
sss = np.ones((nysf, nc))
ddd = np.copy(model)
sss[ii] = sf[ii] / model[ii]
if ihalf > 0:
overlap = iend_half[ihalf - 1] - ibeg_half[ihalf] + 1
sss[ii] /= scale
sp *= scale
else:
nc_old = nc
sss_old = np.zeros((nysf, nc))
ddd_old = np.zeros((nysf, nc))
overlap = ibeg_half[1] - ibeg_half[0]
# Combine new and old sections
ncc = overlap
index = make_index(yc + yslitf[0], yc + yslitf[1], ib, ib + ncc)
im_norm[index] = (sss_old[:, -ncc:] * oweight[:ncc] +
sss[:, :ncc] * weight[:ncc])
im_ordr[index] = (ddd_old[:, -ncc:] * oweight[:ncc] +
ddd[:, :ncc] * weight[:ncc])
if ihalf == 2 * nbin - 2:
# TODO check
index = make_index(yc + yslitf[0], yc + yslitf[1], ib + ncc,
ib + nc)
im_norm[index] = sss[:, ncc:nc]
im_ordr[index] = ddd[:, ncc:nc]
nc_old = nc
sss_old = np.copy(sss)
ddd_old = np.copy(ddd)
# Combine overlaping regions
# TODO
# if use_2d:
# if ihalf > 0 and ihalf < 2 * nbin - 2:
# spec[ib + delta_x : ie - delta_x] = (
# spec[ib + delta_x : ie - delta_x] * oweight[delta_x : nc - delta_x]
# + sp[delta_x:-delta_x] * weight[delta_x:-delta_x]
# )
# elif ihalf == 0:
# spec[ib : ie - delta_x] = sp[:-delta_x]
# elif ihalf == 2 * nbin - 2:
# spec[ib + delta_x : ie] = (
# spec[ib + delta_x : ie] * oweight[delta_x:-1]
# + sp[delta_x:-1] * weight[delta_x:-1]
# )
# else:
spec[ib:ie] = spec[ib:ie] * oweight + sp * weight
sunc[ib:ie] = sunc[ib:ie] * oweight + unc * weight
if plot:
plot_slitfunction(sf, sp, sfsm, model, y_offset, ord_num, ib, ie,
osample)
# TODO ????? is that really correct
sunc = np.sqrt(sunc + spec)
# TODO what to return ?
slitf = np.mean(slitf, axis=0)
model = spec[:, None] * slitf[None, :]
return spec, slitf, model, sunc
def make_scatter(im, orders, **kwargs):
"""
This subroutine is still far from perfect. NP wrote it in 2002 and since it
was a pain in the back. The problem is that that it works very smoothlu with
a good data set but once we get to the low S/N, overlapping orders and bad
cosmetics things starting to fail. Typical failures are connected to the
the identification of the walls of adjucent orders and the bottom.
History of modifications:
2006-09-26 (NP) the condition for local maxima ID is set to
(positive left der and non-positive right der) or
(non-negative left der and negative right der)
"""
# Get image size.
nrow, ncol = im.shape
first = 0 # first order
last = len(orders) + 1 # last order
nord = len(orders)
# Get kwargs data, TODO change names
ccd_gain = kwargs.get("gain", 1)
ccd_rdnoise = kwargs.get("readn", 0) / ccd_gain
osamp = kwargs.get("osample", 10)
lambda_sf = kwargs.get("lambda_sf", 1)
lambda_sp = kwargs.get("lambda_sp", 0)
extra_offset = kwargs.get("order_extra_width", 1)
width = kwargs.get("swath_width", 400)
column_range = kwargs.get("column_range", np.tile([0, ncol], (nord, 0)))
debug = kwargs.get("debug", False)
pol = kwargs.get("pol", False)
subtract = kwargs.get("subtract", False)
# Initialize arrays.
xcol = np.arange(ncol) # indices of all columns
back = np.zeros((nord + 1, ncol)) # fitted scattered light model
back_data = np.zeros((nord + 1, ncol)) # scattered light data
yback = np.zeros((nord + 1, ncol)) # scattered light coordinates
ycen1 = np.polyval(orders[0], xcol) # the shape of the starting order
# TODO DEBUG
# plt.ion()
# Loop over neighbouring orders, (0, 1), (1, 2), (2, 3), ...
for order0, order1 in zip(range(first, last), range(first + 1, last + 1)):
logging.debug("orders: %i, %i" % (order0, order1))
# Calculate shapes
ycen0 = np.polyval(orders[order0], xcol)
ycen1 = np.polyval(orders[order1], xcol)
cr0 = column_range[order0]
cr1 = column_range[order1]
ibeg = max(cr0[0], cr1[0]) # left boundary to consider
iend = min(cr0[1], cr1[1]) # right boundary to consider
width = np.clip(
(iend - ibeg) * 0.1, width,
None) # width to consider (swath < width < iend-ibeg+1)
width = int(width // 2) # half-width to avoid rounding errors
height = np.mean(ycen1[ibeg:iend] - ycen0[ibeg:iend]) # mean height
height = int(np.round(height * 0.5 * extra_offset))
height = np.clip(height, 3,
None) # half-height to avoid rounding errors
icen = (ibeg + iend) // 2 # central column
ibeg = np.clip(icen - width, ibeg,
None) # starting column of the region of interest
iend = np.clip(icen + width, None,
iend) # ending column of the region of interest
ycen = 0.5 * (ycen1 + ycen0) # central line of the troff
ymin = ycen.astype(
int) - height # bottom boundary of the region of interest
ymax = ycen.astype(
int) + height # top boundary of the region of interest
yback[order1, :] = ycen # scattered light coordinates
# Copy the region of interest to sf, j=0..nc
index = make_index(ymin, ymax, ibeg, iend)
sf = im[index]
sf = sf - np.min(sf)
tmp = ycen[ibeg:iend] - ycen[ibeg:iend].astype(int)
# TODO debug
# plt.imshow(sf)
# plt.plot(sf.shape[0]/2 + tmp)
# plt.show()
# TODO why is this so slow?
# TODO use passed parameters
osamp = 1
# TODO: Copying these ensures that nothing bad happens during slitfunc, which sometimes happens otherwise
sf = np.copy(sf)
tmp = np.copy(tmp)
sp, sfsm, model, unc = slitfunc(sf,
tmp,
lambda_sp=2,
lambda_sf=2,
osample=1)
nslitf = len(sfsm)
yslitf = (np.arange(-0.5, nslitf - 0.5, 1) / osamp - 1.5 - height
) # final subpixel scale
dev = (ccd_rdnoise / ccd_gain / np.sqrt(np.sum(sp))
) # typical pixel scatter normalized to counts
var = np.std(sfsm)
#
# This is an improved version which is near the final form
#
jback = bottom(np.clip(sfsm, None, np.median(sfsm)),
1,
eps=dev,
poly=True) # fit the bottom with a smooth curve
jback = np.clip(sfsm - jback, 0, None) # find all positive spikes
# ===========================================================================
k = np.where(sfsm > np.median(sfsm) + 0.5 * var)[0]
nback = k.shape[0]
if nback > 0:
m1 = np.where(yslitf[k] <
0) # part of the gap which is below the central line
m2 = np.where(yslitf[k] >
0) # part of the gap which is above the central line
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
else:
n1 = 0
n2 = 0
if n1 == 0 or n2 == 0: # this happens e.g. if the two adjacent
# orders have dramatically different level
# of signal
k = np.where(sfsm > middle(sfsm, 1, eps=dev, poly=True))[0]
m1 = np.where(yslitf[k] < 0)
m2 = np.where(yslitf[k] > 0)
nback = k.shape[0]
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
if n1 == 0 or n2 == 0: # this happens e.g. if the two adjacent
# orders have dramatically different level
# of signal
k = np.where(sfsm > middle(sfsm, 1e3, eps=dev))[0]
m1 = np.where(yslitf[k] < 0)
m2 = np.where(yslitf[k] > 0)
nback = k.shape[0]
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
ss = np.array([0, *jback, 0])
kk = np.where(((ss[k + 1] >= ss[k]) & (ss[k + 1] > ss[k + 2]))
| ((ss[k + 1] > ss[k]) & (ss[k + 1] >= ss[k + 2])))
k = k[kk]
m1 = np.where(
yslitf[k] < 0) # part of the gap which is below the central line
m2 = np.where(
yslitf[k] > 0) # part of the gap which is above the central line
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
# ===========================================================================
if n1 == 0 or n2 == 0: # ultimate method
logging.info(
"mkscatter: failed finding interorder boundaries. Using 5 pixels around central line."
)
k = np.sort(abs(yslitf))[0] + [-3, 3]
nback = len(k)
debug = True
if nback < 2 or (nback >= 2 and
(np.min(yslitf[k]) > height * 0.5
or np.max(yslitf[k]) < -height * 0.5)):
n1 = np.min(np.abs(yslitf +
height * 0.5)) # in this case just select
n2 = np.min(np.abs(yslitf - height * 0.5)) # the central half
k1 = n1.shape[0]
k2 = n2.shape[0]
k = [k1, k2]
nback = 2
if jback[0] >= jback[1]:
k = np.array([0, *k])
nback = nback + 1
if jback[nslitf - 1] >= jback[nslitf - 2]:
k = np.array([*k, nslitf - 1])
nback = nback + 1
if debug:
plt.plot(yslitf, sfsm)
plt.plot(yslitf, middle(sfsm, 1, eps=dev, poly=True))
plt.plot(yslitf, middle(sfsm, 1e3, eps=dev, poly=True))
plt.hline(np.median(sfsm) + 0.5 * var)
plt.plot(yslitf[k], sfsm[k])
plt.show()
jback = np.where(yslitf[k] < 0)[0]
n1 = jback.shape[0]
imax1 = np.argmax(yslitf[k[jback]])
m1 = yslitf[k[jback]][imax1]
imax1 = k[jback[imax1]]
jback = np.where(yslitf[k] > 0)[0]
n2 = jback.shape[0]
imax2 = np.argmin(yslitf[k[jback]])
m2 = yslitf[k[jback]][imax2]
imax2 = k[jback[imax2]]
k = [imax1, imax2]
if nback == 0:
print(
"mkscatter: failed to find the boundaries of the inter-order troff using default orders: %i-%i x-range: (%i,%i)"
% (order0, order1, ibeg, iend))
k = [0, len(nslitf)]
tmp = np.clip(sfsm[k[0]:k[1]], None, np.median(sfsm))
jback = bottom(sfsm[k[0]:k[1]], 1, eps=dev,
poly=True) # fit bottom with a straight line
iback = np.where(sfsm[k[0]:k[1]] <= jback +
ccd_rdnoise)[0] # find all the points below
nback = iback.shape[0]
if nback <= 5:
plt.plot(yslitf, sfsm)
plt.plot(
yslitf,
bottom(np.clip(sfsm, None, np.median(sfsm)),
1,
eps=dev,
poly=True),
)
plt.plot(yslitf[k], sfsm[k], "*")
raise Exception(
"mkscatter: major error in the order format: could not detect inter-order troff"
)
iback += k[0]
imax1 = np.min(iback)
imax2 = np.max(iback)
sf1 = sfsm[iback]
step = (max(sf1) - min(sf1)) / np.clip(len(sf1) / 10, 10, 200)
nh = int(1 / step)
h, _ = np.histogram(sf1, bins=nh)
ihmax = np.argmax(h)
hmax = h[ihmax]
i0 = np.where(h[:ihmax] < 0.1 * hmax)
n1 = i0[0].shape[0]
i0 = np.max(i0) if n1 > 0 else 0
i1 = np.where(h[ihmax:] < 0.1 * hmax)[0]
n2 = i1.shape[0]
i1 = np.min(i1) + ihmax if n2 > 0 else nh - 1
ii = (np.where((sf1 - min(sf1) >= step * i0)
& (sf1 - min(sf1) < step * i1))[0] + imax1)
nii1 = ii.shape[0]
if nii1 <= 0:
raise Exception(
"mkscatter: could not detect background points between orders %i and %i"
% (order0, order1))
y1 = np.ceil(np.min(yslitf[ii])) # bottom boundary of the troff
y2 = np.floor(np.max(yslitf[ii])) # top boundary of the troff
for j in range(ncol): # scattered light in col j
back_data[order1,
j] = interpolate_bad_pixels(ycen[j], y1, y2, im, j, nrow)
if order0 == first: # for the first order try
yy = ycen0[j] - (ycen[j] - ycen0[j]
) # to find background below
yback[0, j] = yy # scattered light coordinates
back_data[0,
j] = interpolate_bad_pixels(yy, y1, y2, im, j, nrow)
elif order1 == last: # for the last order try
yy = ycen1[j] + (ycen1[j] - ycen[j]
) # to find background above
yback[-1, j] = yy # scattered light coordinates
back_data[-1,
j] = interpolate_bad_pixels(yy, y1, y2, im, j, nrow)
if (order1 % 10) == 5 or (order1 % 10) == 0 or nord < 10:
logging.info("mkscatter: order %i of total %i was processed",
order1, last)
# bad pixels are masked out
back_data = np.ma.masked_array(back_data, mask=back_data == -1000)
# Interpolate missing data from the adjacent troffs
for i in range(ncol):
back_data[:, i] = np.interp(yback[:, i], yback[:, i], back_data[:, i])
# Filter out noise: 0th background troff
crange = column_range[0]
x = np.arange(crange[0], crange[1])
b = middle(back_data[0, x], 20., eps=dev)
if pol:
back[0, x] = middle(b, 11, poly=True, eps=dev, min=0)
else:
back[0, x] = middle(b, lambda_sp, eps=dev, min=0)
# All intermediate background troffs
if nord > 1:
for order0, order1 in zip(range(first, last),
range(first + 1, last + 1)):
cr0 = column_range[order0]
cr1 = column_range[order1]
crange = (
max(cr0[0], cr1[0]),
min(cr0[1], cr1[1]),
) # right boundary to consider
x = np.arange(crange[0], crange[1])
b = middle(back_data[order1, x], 20., eps=dev)
if pol:
back[order1, x] = middle(b,
11,
poly=True,
eps=dev,
min=0,
double=True)
else:
back[order1, x] = middle(b, lambda_sp, eps=dev, min=0)
# The last background troff
crange = column_range[last]
x = np.arange(crange[0], crange[1])
b = middle(back_data[last, x], 20., eps=dev)
if pol:
back[-1, x] = bottom(b, 11, poly=True, eps=dev, min=0)
else:
back[-1, x] = bottom(b, lambda_sp, eps=dev, min=0)
if subtract:
# Type conversion to avoid problems with UINT arrays when subtracting
im = im.astype(np.float32)
ycen = np.arange(nrow)
if pol:
ii = np.arange(0, nord, 2) # polarization: orders come in pairs
ii = np.array([*ii, np.max(ii) + 2
]) # do not use interpolarization space
else:
ii = np.arange(nord) # no polarization, count all orders
for j in range(ncol):
b = back[:, j]
b = b[ii]
y = yback[:, j]
y = y[ii]
im[:, j] -= np.interp(ycen, y, b)
return back, yback
def make_scatter(im, orders, **kwargs):
"""
This subroutine is still far from perfect. NP wrote it in 2002 and since it
was a pain in the back. The problem is that that it works very smoothlu with
a good data set but once we get to the low S/N, overlapping orders and bad
cosmetics things starting to fail. Typical failures are connected to the
the identification of the walls of adjucent orders and the bottom.
History of modifications:
2006-09-26 (NP) the condition for local maxima ID is set to
(positive left der and non-positive right der) or
(non-negative left der and negative right der)
"""
# Get image size.
nrow, ncol = im.shape
first = 0 # first order
last = len(orders) + 1 # last order
nord = len(orders)
# Get kwargs data, TODO change names
ccd_gain = kwargs.get("gain", 1)
ccd_rdnoise = kwargs.get("readn", 0) / ccd_gain
osamp = kwargs.get("osample", 10)
lambda_sf = kwargs.get("lambda_sf", 1)
lambda_sp = kwargs.get("lambda_sp", 0)
extra_offset = kwargs.get("order_extra_width", 1)
width = kwargs.get("swath_width", 400)
column_range = kwargs.get("column_range", np.tile([0, ncol], (nord, 0)))
debug = kwargs.get("debug", False)
pol = kwargs.get("pol", False)
subtract = kwargs.get("subtract", False)
# Initialize arrays.
xcol = np.arange(ncol) # indices of all columns
back = np.zeros((nord + 1, ncol)) # fitted scattered light model
back_data = np.zeros((nord + 1, ncol)) # scattered light data
yback = np.zeros((nord + 1, ncol)) # scattered light coordinates
ycen1 = np.polyval(orders[0], xcol) # the shape of the starting order
# TODO DEBUG
# plt.ion()
# Loop over neighbouring orders, (0, 1), (1, 2), (2, 3), ...
for order0, order1 in zip(range(first, last), range(first + 1, last + 1)):
logging.debug("orders: %i, %i" % (order0, order1))
# Calculate shapes
ycen0 = np.polyval(orders[order0], xcol)
ycen1 = np.polyval(orders[order1], xcol)
cr0 = column_range[order0]
cr1 = column_range[order1]
ibeg = max(cr0[0], cr1[0]) # left boundary to consider
iend = min(cr0[1], cr1[1]) # right boundary to consider
width = np.clip(
(iend - ibeg) * 0.1, width, None
) # width to consider (swath < width < iend-ibeg+1)
width = int(width // 2) # half-width to avoid rounding errors
height = np.mean(ycen1[ibeg:iend] - ycen0[ibeg:iend]) # mean height
height = int(np.round(height * 0.5 * extra_offset))
height = np.clip(height, 3, None) # half-height to avoid rounding errors
icen = (ibeg + iend) // 2 # central column
ibeg = np.clip(
icen - width, ibeg, None
) # starting column of the region of interest
iend = np.clip(
icen + width, None, iend
) # ending column of the region of interest
ycen = 0.5 * (ycen1 + ycen0) # central line of the troff
ymin = ycen.astype(int) - height # bottom boundary of the region of interest
ymax = ycen.astype(int) + height # top boundary of the region of interest
yback[order1, :] = ycen # scattered light coordinates
# Copy the region of interest to sf, j=0..nc
index = make_index(ymin, ymax, ibeg, iend)
sf = im[index]
sf = sf - np.min(sf)
tmp = ycen[ibeg:iend] - ycen[ibeg:iend].astype(int)
# TODO debug
# plt.imshow(sf)
# plt.plot(sf.shape[0]/2 + tmp)
# plt.show()
# TODO why is this so slow?
# TODO use passed parameters
osamp = 1
# TODO: Copying these ensures that nothing bad happens during slitfunc, which sometimes happens otherwise
sf = np.copy(sf)
tmp = np.copy(tmp)
sp, sfsm, model, unc = slitfunc(sf, tmp, lambda_sp=2, lambda_sf=2, osample=1)
nslitf = len(sfsm)
yslitf = (
np.arange(-0.5, nslitf - 0.5, 1) / osamp - 1.5 - height
) # final subpixel scale
dev = (
ccd_rdnoise / ccd_gain / np.sqrt(np.sum(sp))
) # typical pixel scatter normalized to counts
var = np.std(sfsm)
#
# This is an improved version which is near the final form
#
jback = bottom(
np.clip(sfsm, None, np.median(sfsm)), 1, eps=dev, poly=True
) # fit the bottom with a smooth curve
jback = np.clip(sfsm - jback, 0, None) # find all positive spikes
# ===========================================================================
k = np.where(sfsm > np.median(sfsm) + 0.5 * var)[0]
nback = k.shape[0]
if nback > 0:
m1 = np.where(
yslitf[k] < 0
) # part of the gap which is below the central line
m2 = np.where(
yslitf[k] > 0
) # part of the gap which is above the central line
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
else:
n1 = 0
n2 = 0
if n1 == 0 or n2 == 0: # this happens e.g. if the two adjacent
# orders have dramatically different level
# of signal
k = np.where(sfsm > middle(sfsm, 1, eps=dev, poly=True))[0]
m1 = np.where(yslitf[k] < 0)
m2 = np.where(yslitf[k] > 0)
nback = k.shape[0]
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
if n1 == 0 or n2 == 0: # this happens e.g. if the two adjacent
# orders have dramatically different level
# of signal
k = np.where(sfsm > middle(sfsm, 1e3, eps=dev))[0]
m1 = np.where(yslitf[k] < 0)
m2 = np.where(yslitf[k] > 0)
nback = k.shape[0]
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
ss = np.array([0, *jback, 0])
kk = np.where(
((ss[k + 1] >= ss[k]) & (ss[k + 1] > ss[k + 2]))
| ((ss[k + 1] > ss[k]) & (ss[k + 1] >= ss[k + 2]))
)
k = k[kk]
m1 = np.where(yslitf[k] < 0) # part of the gap which is below the central line
m2 = np.where(yslitf[k] > 0) # part of the gap which is above the central line
n1 = m1[0].shape[0]
n2 = m2[0].shape[0]
# ===========================================================================
if n1 == 0 or n2 == 0: # ultimate method
logging.info(
"mkscatter: failed finding interorder boundaries. Using 5 pixels around central line."
)
k = np.sort(abs(yslitf))[0] + [-3, 3]
nback = len(k)
debug = True
if nback < 2 or (
nback >= 2
and (np.min(yslitf[k]) > height * 0.5 or np.max(yslitf[k]) < -height * 0.5)
):
n1 = np.min(np.abs(yslitf + height * 0.5)) # in this case just select
n2 = np.min(np.abs(yslitf - height * 0.5)) # the central half
k1 = n1.shape[0]
k2 = n2.shape[0]
k = [k1, k2]
nback = 2
if jback[0] >= jback[1]:
k = np.array([0, *k])
nback = nback + 1
if jback[nslitf - 1] >= jback[nslitf - 2]:
k = np.array([*k, nslitf - 1])
nback = nback + 1
if debug:
plt.plot(yslitf, sfsm)
plt.plot(yslitf, middle(sfsm, 1, eps=dev, poly=True))
plt.plot(yslitf, middle(sfsm, 1e3, eps=dev, poly=True))
plt.hline(np.median(sfsm) + 0.5 * var)
plt.plot(yslitf[k], sfsm[k])
plt.show()
jback = np.where(yslitf[k] < 0)[0]
n1 = jback.shape[0]
imax1 = np.argmax(yslitf[k[jback]])
m1 = yslitf[k[jback]][imax1]
imax1 = k[jback[imax1]]
jback = np.where(yslitf[k] > 0)[0]
n2 = jback.shape[0]
imax2 = np.argmin(yslitf[k[jback]])
m2 = yslitf[k[jback]][imax2]
imax2 = k[jback[imax2]]
k = [imax1, imax2]
if nback == 0:
print(
"mkscatter: failed to find the boundaries of the inter-order troff using default orders: %i-%i x-range: (%i,%i)"
% (order0, order1, ibeg, iend)
)
k = [0, len(nslitf)]
tmp = np.clip(sfsm[k[0] : k[1]], None, np.median(sfsm))
jback = bottom(
sfsm[k[0] : k[1]], 1, eps=dev, poly=True
) # fit bottom with a straight line
iback = np.where(sfsm[k[0] : k[1]] <= jback + ccd_rdnoise)[
0
] # find all the points below
nback = iback.shape[0]
if nback <= 5:
plt.plot(yslitf, sfsm, xs=1)
plt.plot(
yslitf,
bottom(np.clip(sfsm, None, np.median(sfsm)), 1, eps=dev, poly=True),
)
plt.plot(yslitf[k], sfsm[k], psym=2)
raise Exception(
"mkscatter: major error in the order format: could not detect inter-order troff"
)
iback += k[0]
imax1 = np.min(iback)
imax2 = np.max(iback)
sf1 = sfsm[iback]
step = (max(sf1) - min(sf1)) / np.clip(len(sf1) / 10, 10, 200)
nh = int(1 / step)
h, _ = np.histogram(sf1, bins=nh)
ihmax = np.argmax(h)
hmax = h[ihmax]
i0 = np.where(h[:ihmax] < 0.1 * hmax)
n1 = i0[0].shape[0]
i0 = np.max(i0) if n1 > 0 else 0
i1 = np.where(h[ihmax:] < 0.1 * hmax)[0]
n2 = i1.shape[0]
i1 = np.min(i1) + ihmax if n2 > 0 else nh - 1
ii = (
np.where((sf1 - min(sf1) >= step * i0) & (sf1 - min(sf1) < step * i1))[0]
+ imax1
)
nii1 = ii.shape[0]
if nii1 <= 0:
raise Exception(
"mkscatter: could not detect background points between orders %i and %i"
% (order0, order1)
)
y1 = np.ceil(np.min(yslitf[ii])) # bottom boundary of the troff
y2 = np.floor(np.max(yslitf[ii])) # top boundary of the troff
for j in range(ncol): # scattered light in col j
back_data[order1, j] = interpolate_bad_pixels(ycen[j], y1, y2, im, j, nrow)
if order0 == first: # for the first order try
yy = ycen0[j] - (ycen[j] - ycen0[j]) # to find background below
yback[0, j] = yy # scattered light coordinates
back_data[0, j] = interpolate_bad_pixels(yy, y1, y2, im, j, nrow)
elif order1 == last: # for the last order try
yy = ycen1[j] + (ycen1[j] - ycen[j]) # to find background above
yback[-1, j] = yy # scattered light coordinates
back_data[-1, j] = interpolate_bad_pixels(yy, y1, y2, im, j, nrow)
if (order1 % 10) == 5 or (order1 % 10) == 0 or nord < 10:
logging.info("mkscatter: order %i of total %i was processed", order1, last)
# bad pixels are masked out
back_data = np.ma.masked_array(back_data, mask=back_data == -1000)
# Interpolate missing data from the adjacent troffs
for i in range(ncol):
back_data[:, i] = np.interp(yback[:, i], yback[:, i], back_data[:, i])
# Filter out noise: 0th background troff
crange = column_range[0]
x = np.arange(crange[0], crange[1])
b = middle(back_data[0, x], 20., eps=dev)
if pol:
back[0, x] = middle(b, 11, poly=True, eps=dev, min=0)
else:
back[0, x] = middle(b, lambda_sp, eps=dev, min=0)
# All intermediate background troffs
if nord > 1:
for order0, order1 in zip(range(first, last), range(first + 1, last + 1)):
cr0 = column_range[order0]
cr1 = column_range[order1]
crange = (
max(cr0[0], cr1[0]),
min(cr0[1], cr1[1]),
) # right boundary to consider
x = np.arange(crange[0], crange[1])
b = middle(back_data[order1, x], 20., eps=dev)
if pol:
back[order1, x] = middle(b, 11, poly=True, eps=dev, min=0, double=True)
else:
back[order1, x] = middle(b, lambda_sp, eps=dev, min=0)
# The last background troff
crange = column_range[last]
x = np.arange(crange[0], crange[1])
b = middle(back_data[last, x], 20., eps=dev)
if pol:
back[-1, x] = bottom(b, 11, poly=True, eps=dev, min=0)
else:
back[-1, x] = bottom(b, lambda_sp, eps=dev, min=0)
if subtract:
# Type conversion to avoid problems with UINT arrays when subtracting
im = im.astype(np.float32)
ycen = np.arange(nrow)
if pol:
ii = np.arange(0, nord, 2) # polarization: orders come in pairs
ii = np.array([*ii, np.max(ii) + 2]) # do not use interpolarization space
else:
ii = np.arange(nord) # no polarization, count all orders
for j in range(ncol):
b = back[:, j]
b = b[ii]
y = yback[:, j]
y = y[ii]
im[:, j] -= np.interp(ycen, y, b)
return back, yback
def make_slitfunction(
img,
ycen,
ylow,
yhigh,
xlow,
xhigh,
gain=1,
readn=0,
lambda_sf=0.1,
lambda_sp=0,
osample=1,
swath_width=None,
no_scatter=False,
telluric=False,
normalize=False,
scatter_below=0,
scatter_above=0,
yscatter_below=0,
yscatter_above=0,
threshold=0,
use_2d=False,
plot=False,
ord_num=0,
im_norm=None,
im_ordr=None,
**kwargs
):
nrow, ncol = img.shape
noise = readn / gain
irow = np.arange(nrow)
ycene = ycen[xlow:xhigh]
nysf = yhigh + ylow + 1
yslitf = -ylow, yhigh
spec = np.zeros(ncol)
sunc = np.zeros(ncol)
no_scatter = no_scatter or (
np.all(scatter_below == 0) and np.all(scatter_above == 0)
)
if swath_width is None:
i = np.unique(ycen.astype(int)) # Points of row crossing
ni = len(i) # This is how many times this order crosses to the next row
if len(i) > 1: # Curved order crosses rows
i = np.sum(i[1:] - i[:-1]) / (len(i) - 1)
nbin = np.clip(
int(np.round(ncol / i)) // 3, 3, 20
) # number of swaths along the order
else: # Perfectly aligned orders
nbin = np.clip(ncol // 400, 3, None) # Still follow the changes in PSF
nbin = nbin * (xhigh - xlow) // ncol # Adjust for the true order length
else:
nbin = np.clip(int(np.round((xhigh - xlow) / swath_width)), 1, None)
nslitf = osample * (ylow + yhigh + 2) + 1
slitf = np.zeros((2 * nbin, nslitf))
# Define order boundary
yc = ycen.astype(int)
ymin = ycen - ylow
ymax = ycen + yhigh
# Calculate boundaries of distinct slitf regions
# Here is the layout to understand the lines below
#
# 1st swath 3rd swath 5th swath ...
# /============|============|============|============|============|
#
# 2nd swath 4th swath 6th swath
# |------------|------------|------------|------------|
# |.....|
# overlap
#
# + ******* 1
# + *
# + *
# * weights (+) previous swath, (*) current swath
# * +
# * +
# * +++++++ 0
bins = np.linspace(xlow, xhigh, 2 * nbin + 1) # boundaries of bins
ibeg_half = np.ceil(bins[:-2]).astype(int) # beginning of each bin
iend_half = np.floor(bins[2:]).astype(int) # end of each bin
bincen = 0.5 * (ibeg_half + iend_half) # center of each bin
# Perform slit decomposition within each swath stepping through the order with
# half swath width. Spectra for each decomposition are combined with linear weights.
for ihalf in range(0, 2 * nbin - 1): # loop through swaths
ib = ibeg_half[ihalf] # left column
ie = iend_half[ihalf] + 1 # right column
nc = ie - ib # number of columns in swath
# Weight for combining overlapping regions
weight = np.ones(nc)
if ihalf > 0:
weight[: nc // 2 + 1] = np.arange(nc // 2 + 1) / nc * 2
oweight = 1 - weight
# Cut out swath from image
index = make_index(yc - ylow, yc + yhigh, ib, ie)
sf = img[index]
# Telluric
if telluric:
tel_lim = (
telluric if telluric > 5 and telluric < nysf / 2 else min(5, nysf / 3)
)
tel = np.sum(sf, axis=0)
itel = np.arange(nysf)
itel = itel[np.abs(itel - nysf / 2) >= tel_lim]
tel = sf[itel, :]
sc = np.zeros(nc)
for itel in range(nc):
sc[itel] = np.median(tel[itel])
tell = sc
else:
tell = 0
if not no_scatter:
# y indices
index_y = np.array([np.arange(k, nysf + k) for k in yc[ib:ie] - yslitf[0]])
dy_scatter = (index_y.T - yscatter_below[None, ib:ie]) / (
yscatter_above[None, ib:ie] - yscatter_below[None, ib:ie]
)
scatter = (
scatter_above[None, ib:ie] - scatter_below[None, ib:ie]
) * dy_scatter + scatter_below[None, ib:ie]
else:
scatter = 0
# Do Slitfunction extraction
sf -= scatter + tell
sf = np.clip(sf, 0, None)
# offset from the central line
y_offset = ycen[ib:ie] - yc[ib:ie]
if use_2d:
sp, sfsm, model, unc = slitfunc_curved(
sf,
y_offset,
tilt,
lambda_sp=lambda_sp,
lambda_sl=lambda_sf,
osample=osample,
)
delta_x = None # TODO get this from slitfunc_curved
else:
sp, sfsm, model, unc = slitfunc(
sf, y_offset, lambda_sp=lambda_sp, lambda_sf=lambda_sf, osample=osample
)
if normalize:
# In case we do FF normalization replace the original image by the
# ratio of sf/sfbin where number of counts is larger than threshold
# and with 1 elsewhere
scale = 1
ii = np.where(model > threshold / gain)
sss = np.ones((nysf, nc))
ddd = np.copy(model)
sss[ii] = sf[ii] / model[ii]
if ihalf > 0:
overlap = iend_half[ihalf - 1] - ibeg_half[ihalf] + 1
sss[ii] /= scale
sp *= scale
else:
nc_old = nc
sss_old = np.zeros((nysf, nc))
ddd_old = np.zeros((nysf, nc))
overlap = ibeg_half[1] - ibeg_half[0]
# Combine new and old sections
ncc = overlap
index = make_index(yc + yslitf[0], yc + yslitf[1], ib, ib + ncc)
im_norm[index] = (
sss_old[:, -ncc:] * oweight[:ncc] + sss[:, :ncc] * weight[:ncc]
)
im_ordr[index] = (
ddd_old[:, -ncc:] * oweight[:ncc] + ddd[:, :ncc] * weight[:ncc]
)
if ihalf == 2 * nbin - 2:
# TODO check
index = make_index(yc + yslitf[0], yc + yslitf[1], ib + ncc, ib + nc)
im_norm[index] = sss[:, ncc:nc]
im_ordr[index] = ddd[:, ncc:nc]
nc_old = nc
sss_old = np.copy(sss)
ddd_old = np.copy(ddd)
# Combine overlaping regions
if use_2d:
if ihalf > 0 and ihalf < 2 * nbin - 2:
spec[ib + delta_x : ie - delta_x] = (
spec[ib + delta_x : ie - delta_x] * oweight[delta_x : nc - delta_x]
+ sp[delta_x:-delta_x] * weight[delta_x:-delta_x]
)
elif ihalf == 0:
spec[ib : ie - delta_x] = sp[:-delta_x]
elif ihalf == 2 * nbin - 2:
spec[ib + delta_x : ie] = (
spec[ib + delta_x : ie] * oweight[delta_x:-1]
+ sp[delta_x:-1] * weight[delta_x:-1]
)
else:
spec[ib:ie] = spec[ib:ie] * oweight + sp * weight
sunc[ib:ie] = sunc[ib:ie] * oweight + unc * weight
if plot:
# TODO make this nice
plt.clf()
scale = 1
pscale = np.mean(sp)
sfplot = gaussian_filter1d(sfsm, osample)
sfflat = sfsm[:-2] * pscale
model = np.mean(model, axis=1)
if not no_scatter:
poffset = np.mean(scatter_below[ib:ie] + scatter_above[ib:ie]) * 0.5
else:
poffset = 0
# Plot 1: The observed slit
plt.subplot(221)
plt.title("Order %i, Columns %i through %i" % (ord_num, ib, ie))
# plt.plot(sfflat)
# plt.plot(ysfpnt[jbad], sfpnt[jbad] * pscale, "g+")
plt.plot(sfflat, "+")
plt.plot(model)
# Plot 2: The recovered slit function
plt.subplot(222)
plt.title("Order %i, Columns %i through %i" % (ord_num, ib, ie))
plt.plot(model)
# plt.plot(ysfpnt[jbad], sfpnt[jbad] * pscale, "g+")
# plt.plot(sfplot)
# Plot 3: Difference between observed and recovered
plt.subplot(223)
plt.title("Data - Fit")
plt.plot(sfflat - model)
# plt.plot(ysfpnt[jbad], (sfpnt - sfsm2)[jbad] * pscale, "g+")
plt.plot(np.sqrt((model + poffset + readn ** 2) / gain))
plt.plot(-np.sqrt((model + poffset + readn ** 2) / gain))
plt.subplot(224)
plt.title("Data - Fit")
plt.plot(sfflat - model)
# plt.plot(ysfpnt[jbad], (sf.flat - sfsm.flat)[jbad] * pscale, "g+")
plt.plot(np.sqrt((model + poffset + readn ** 2) / gain))
plt.plot(-np.sqrt((model + poffset + readn ** 2) / gain))
plt.draw()
plt.pause(0.001)
# TODO ????? is that really correct
sunc = np.sqrt(sunc + spec)
# TODO what to return ?
slitf = np.mean(slitf, axis=0)
model = spec[:, None] * slitf[None, :]
return spec, slitf, model, sunc
def getarc(img, orders, onum, awid, x_left_lim=0, x_right_lim=-1):
"""
# This subroutine extracts a curved arc (arc) from an image array (im). The
# curvature of the arc is determined from polynomial fit coefficients (orc)
# which usually trace the curvature of the echelle orders. The particular
# arc to extract is specified by an order number (onum), which need not be
# integral. Positions of nonintegral orders are interpolated from surrounding
# orders.
# im (input array (# columns , # rows)) image from which to extract arc.
# orc (input array (# of coeff per fit , # of orders)) coefficients from PIT
# fit of column number versus row number (of echelle orders, usually). The
# polynomials trace arcs (orders, usually) indexed by order number, begining
# with zero closest to row zero and increasing as row number increases.
# **Note** These are the extended order coefficients.
# onum (input scalar) order number of arc to extract - need not be integral.
# awid (input scalar) full width of arc to be extracted.
# Two specifications are possible:
# max(awid) <= 1, awid is fraction of the local distance between orders to mash.
# max(awid) > 1, awid is the specific number of pixels to mash.
# arc (output vector (# columns)) counts PER PIXEL in arc extracted from image.
# [pix (output vector (# columns)] returns the fractional number of pixels
# mashed in each column to make arc.
# 29-Nov-91 GB translated from ANA
# 22-Dec-91 GB made to return zeros if arc off image
# 05-Jul-94 JAV, CMJ Moved endelse to extract full arc instead of half when
# fraction of an arc is specified
"""
# ncol, nrow = img.shape
i1, i2 = x_left_lim, x_right_lim
# if(keyword_set(x_left_lim )) then i1 = x_left_lim else i1 = 0
# if(keyword_set(x_right_lim)) then i2 = x_right_lim else i2 = ncol-1
# Define useful quantities
# ncol = len(img[0, i1:i2]) # number of columns
nrow = len(img[:, i1]) # number of rows
ix = np.arange(i1, i2, 1, dtype=float) # vector of column indicies
arc = np.zeros_like(ix) # dimension arc vector
pix = 1.0 # define in case of trouble
# Interpolate polynomial coefficients for surrounding orders to get polynomial
# coefficients. Note that this is mathematically equivalent to interpolating
# the column indicies for surrounding orders, since the column indicies are
# linear functions of the polynomial coefficients. However, interpolating
# coefficients should be faster.
if np.max(awid) < 1: # awid is an order fraction
ob = onum - awid / 2 # order # of bottom edge of arc
obi = int(ob) # next lowest integral order #A
cb = orders[obi] + (ob - obi) * (orders[obi + 1] - orders[obi])
yb = np.polyval(cb, ix) # row # of bottom edge of swath
ot = onum + awid / 2 # order # of top edge of arc
oti = int(ot) # next lowest integral order #
ct = orders[oti] + (ot - oti) * (orders[oti + 1] - orders[oti])
yt = np.polyval(ct, ix) # row # of top edge of swath
else: # awid is number of pixels
c = np.copy(orders[onum])
yb = np.polyval(c, ix) - awid / 2 # row # of bottom edge of swath
yt = np.polyval(c, ix) + awid / 2 # row # of top edge of swath
if np.min(yb) < 0: # check if arc is off bottom
raise Exception(
"FORDS: Warning - requested arc is below bottom of image. %i" % onum
)
if np.max(yt) > nrow: # check if arc is off top of im
raise Exception(
"FORDS: Warning - requested arc is above top of image. %i" % onum
)
diff = np.round(np.mean(yt - yb) * 0.5)
yb = np.clip(np.round((yb + yt) * 0.5 - diff), 0, None).astype(int)
yt = np.clip(np.round((yb + yt) * 0.5 + diff), None, nrow).astype(int)
# Define the indices for the pixels
# in x: the rows between yb and yt
# in y: the column, but n times to match the x index
index = make_index(yb, yt, i1, i2, zero=True)
# Sum over the prepared index
arc = np.sum(img[index], axis=0)
return arc, pix