def get_new_entry():
"""
create a list of obsids which need to be proccessed
Input: amp_avg_list --- a list kept in <house_keeping> directory
new_entry --- a list newly created during this session
Output: a list of obsid which are not proccessed yet
"""
#
#--- prepare a data list file
#
cmd = 'cp ' + house_keeping + '/amp_avg_list ' + house_keeping + '/amp_avg_list~'
os.system(cmd)
#
#--- create a list of obsids which are already proccessed before
#
file = house_keeping + '/amp_avg_list'
amp = mcf.get_val(file, lst=0)
camp = []
for ent in amp:
atemp = re.split('\s+|\t+', ent)
camp.append(atemp[2])
#
#--- read a new obsid list
#
file = working_dir + '/new_entry'
test = mcf.get_val(file, lst=0)
#
#--- add a list of old obsids which have not been proccessed.
#
file = house_keeping + '/keep_entry'
test2 = mcf.get_val(file, lst=0)
test = test + test2
#
#--- find which obsids are new ones
#
return mcf.find_missing_elem(test, camp)
def test_photon_dir():
"""
check whether cti directories are correctly created
Input: none, but read from <temp_comp_are>/....
Outpu: 0 if it is OK
1 if there is a problem
"""
try:
cmd = 'ls ' + temp_comp_area + 'photons/acis/cti/*/ccd*/ccd*html > ' + zspace
os.system(cmd)
result_list = mcf.get_val(zspace)
mcf.rm_file(zspace)
if len(result_list) > 0:
return 0
else:
return 1
except:
return 1
import acis_cti_get_data as gdata
import create_cti_data_table as cdt
import clean_table as clean
import cti_detrend_factor as det
import create_adjusted_cti_tables as adj
#
#--- temp writing file name
#
rtail = int(
10000 * random.random()
) #---- put a romdom # tail so that it won't mix up with other scripts space
zspace = '/tmp/zspace' + str(rtail)
#
#--- a couple of things needed
#
dare = mcf.get_val('.dare', dir=bin_data, lst=1)
hakama = mcf.get_val('.hakama', dir=bin_data, lst=1)
working_dir = exc_dir + '/Working_dir/'
#---------------------------------------------------------------------------------------------------
#-- acis_cti_run_script: a master script to run all cti data extraction/manupulation scripts ---
#---------------------------------------------------------------------------------------------------
def acis_cti_run_script():
"""
this is a master script to run all cti data extraction/manupulation scripts
input: none
output: resulting cti tables save in /data/mta/Scripts/ACIS/CTI/Data/
"""
sys.path.append(mta_dir)
#
#--- import several functions
#
import convertTimeFormat as tcnv #---- contains MTA time conversion routines
import mta_common_functions as mcf #---- contains other functions c
#
#--- temp writing file name
#
rtail = int(10000 * random.random()) #---- put a romdom # tail so that it won't mix up with other scripts space
zspace = '/tmp/zspace' + str(rtail)
#
#--- a couple of things needed
#
bin_data = '/data/mta/MTA/data/'
dare = mcf.get_val('.dare', dir = bin_data, lst=1)
hakama = mcf.get_val('.hakama', dir = bin_data, lst=1)
web_dir = '/data/mta_www/mta_acis_sci_run/Corner_pix/'
script_dir = '/data/mta/Script/Corner_pix/Scripts/'
#---------------------------------------------------------------------------------------------------
#-- move_old_files: move older gif files to saving directory --
#---------------------------------------------------------------------------------------------------
def move_old_files():
"""
move older gif files to saving directory
input: none
output: none
"""
def find_cti_values_from_file():
"""
extract cti information from flt_run_pipe output
Input: none, but read from <temp_comp_area>/photons/acis/cit/*/ccd*/ccd*html
Output: full_list --- a list of lists which contains:
ccdno --- ccd #
obsid --- obsid
obsnum --- verion #
date_obs--- starting date
date_end--- ending date
al_cti --- a list of al cti
ti_cti --- a list of ti cti
mn_cti --- a list of mn cti
"""
cmd = 'ls ' + temp_comp_area + 'photons/acis/cti/*/ccd*/ccd*html > ' + zspace
os.system(cmd)
f = open(zspace, 'r')
result_list = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
full_list = []
#
#--- "file" is .../ccd<ccd#>.html
#--- and "data" is the content of the ccd<ccd#>.html
#
for file in result_list:
data = mcf.get_val(file)
ccdno = -9999
obsid = 9999
obsnum = 0
date_obs = ''
date_end = ''
si_read = 0
al_node = 0
ti_node = 0
mn_node = 0
al_cti = []
ti_cti = []
mn_cti = []
al_err = []
ti_err = []
mn_err = []
elm = ''
try:
for ent in data:
#
#--- detrmine which line we are reading
#
m1 = re.search('Al Ka', ent)
m2 = re.search('Mn Ka', ent)
m3 = re.search('Ti Ka', ent)
if m1 is not None:
elm = 'al'
elif m2 is not None:
elm = 'mn'
elif m3 is not None:
elm = 'ti'
#
#--- indicator for cti section
#
ms = re.search('S/Ix10', ent)
if ms is not None:
si_read = 1
#
#--- book keeping data
#
if ccdno == -9999:
m = re.search('MTA ACIS CTI Node Analysis Report', ent)
if m is not None:
atemp = re.split('CCD', ent)
btemp = re.split('<', atemp[1])
ccdno = btemp[0]
if obsid == 9999:
outval = find_value_from_line(ent, 'OBS_ID')
if outval != '':
obsid = outval
if obsnum == 0:
outval = find_value_from_line(ent, 'OBI_NUM')
if outval != '':
obsnum = outval
if date_obs == '':
date_obs = find_value_from_line(ent, 'DATE-OBS')
if date_end == '':
date_end = find_value_from_line(ent, 'DATE-END')
#
#--- read cti values (reading order must be al, mn, then ti)
#
if (si_read == 1) and (al_node < 4):
out = find_cti_from_line(ent)
if out != '':
al_cti.append(out)
al_node += 1
if al_node > 3:
si_read = 0
if (si_read == 1) and (al_node > 3) and (mn_node < 4):
out = find_cti_from_line(ent)
if out != '':
mn_cti.append(out)
mn_node += 1
if mn_node > 3:
si_read = 0
if (si_read == 1) and (mn_node > 3) and (ti_node < 4):
out = find_cti_from_line(ent)
if out != '':
ti_cti.append(out)
ti_node += 1
if ti_node > 3:
si_read = 0
plist = [
ccdno, obsid, obsnum, date_obs, date_end, al_cti, mn_cti,
ti_cti
]
full_list.append(plist)
except:
plist = []
full_list.append(plist)
return full_list
#
#--- mta common functions
#
import mta_common_functions as mtac
#
#--- Exposure related funcions shared
#
import exposureFunctions as expf
#
#--- a couple of things needed
#
dare = mtac.get_val('.dare', dir=bindata_dir, lst=1)
hakama = mtac.get_val('.hakama', dir=bindata_dir, lst=1)
#-----------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------
def acis_dose_test_run():
"""
test ska shell access
"""
start = '05/07/15,00:00:00'
stop = '05/15/15,00:00:00'
#
sys.path.append(bin_dir)
sys.path.append(mta_dir)
#
#--- converTimeFormat contains MTA time conversion routines
#
import convertTimeFormat as tcnv
import mta_common_functions as mcf
#
#--- sql realated settings
#
db_user = '******'
db_server = 'ocatsqlsrv'
file = bdata_dir + '/.targpass'
db_passwd = mcf.get_val(file)
#---------------------------------------------------------------------------------------------------
#-- hrc_gain_find_ar_lac: find new AL Lac observations and put in a list --
#---------------------------------------------------------------------------------------------------
def hrc_gain_find_ar_lac():
"""
find new AL Lac observations and put in a list
Input: none, but the data will be read from mp_reports and also hrc_obsid_list in <house_keeping>
Output: "./candidate_list" which lists obsids of new AR Lac observations
candidate_list it also returns the same list
"""
hrc_list = hrc_gain_find_hrc_obs()
import mta_common_functions as mcf #---- contains other functions commonly used in MTA scripts
import hrc_gain_find_ar_lac as arlist #---- get AR Lac candidate list
from kapteyn import kmpfit
#
#--- temp writing file name
#
rtail = int(
10000 * random.random()
) #---- put a romdom # tail so that it won't mix up with other scripts space
zspace = '/tmp/zspace' + str(rtail)
#
#--- a couple of things needed
#
dare = mcf.get_val('.dare', dir=bdata_dir, lst=1)
hakama = mcf.get_val('.hakama', dir=bdata_dir, lst=1)
working_dir = exc_dir + '/Working_dir/'
#
#--- AR Lac position
#
ra = 332.179975
dec = 45.7422544
#---------------------------------------------------------------------------------------------------
#--- hrc_gain_fit_gaus: extract hrc evt2 file and create pha distribution ---
#---------------------------------------------------------------------------------------------------
mta_dir = '/data/mta/Script/Python_script2.7/'
sys.path.append(mta_dir)
import convertTimeFormat as tcnv #---- contains MTA time conversion routines
import mta_common_functions as mcf #---- contains other functions commonly used in MTA scripts
#
#--- temp writing file name
#
rtail = int(10000 * random.random()) #---- put a romdom # tail so that it won't mix up with other scripts space
zspace = '/tmp/zspace' + str(rtail)
bindata_dir = '/data/mta/Script/Exposure/house_keeping/Info_dir/'
#
#--- a couple of things needed
#
dare = mcf.get_val('.dare', dir = bindata_dir, lst=1)
hakama = mcf.get_val('.hakama', dir = bindata_dir, lst=1)
#---------------------------------------------------------------------------------------------
#---------------------------------------------------------------------------------------------
#---------------------------------------------------------------------------------------------
def run_script():
for year in range(1999,2015):
leap = tcnv.isLeapYear(year)
if leap == 1:
dend = 367
else:
dend = 366
#
#--- append a path to a private folder to python directory
#
sys.path.append(bin_dir)
sys.path.append(mta_dir)
#
#--- converTimeFormat contains MTA time conversion routines
#
import convertTimeFormat as tcnv
import mta_common_functions as mcf
#
#--- a couple of things needed
#
dare = mcf.get_val('.dare', dir = dat_dir, lst=1)
hakama = mcf.get_val('.hakama', dir = dat_dir, lst=1)
#--------------------------------------------------------------------------------------
#-- run_grating: controlling function to run the script --
#--------------------------------------------------------------------------------------
def run_grating():
"""
controlling function to run the script
input: none
output: /data/mta/www/mta_grat/<dir> where dir is <Mon><yy>
"""
#
#--- set data collecting range
#
#--- import several functions
#
import convertTimeFormat as tcnv #---- contains MTA time conversion routines
import mta_common_functions as mcf #---- contains other functions commonly used in MTA scripts
#
#--- temp writing file name
#
rtail = int(
10000 * random.random()
) #---- put a romdom # tail so that it won't mix up with other scripts space
zspace = '/tmp/zspace' + str(rtail)
zspace2 = zspace + '_note'
#
#--- a couple of things needed
#
dare = mcf.get_val('.dare', dir=bindata_dir, lst=1)
#-----------------------------------------------------------------------------------------------------------
#-- run_sim_temp_script: run all scripts to update tsc_temps.txt data file --
#-----------------------------------------------------------------------------------------------------------
def run_sim_temp_script(year, sdate, edate):
"""
run all scripts to update tsc_temps.txt data file
input: year --- year of the data to be extracted
sdate --- stating ydate
edate --- ending ydate
these three can be <blank>. if that is the case, the period starts from the
day after the date of the last data entry to today
output: updated <date_dir>/tsc_temps.txt