def mk_backup(file):
"""
copy a file to file~ if file is not an enmpty file
Input: file --- original file
Output: file2 --- file with "~" at the end
"""
chk = mcf.isFileEmpty(file)
if chk == 1:
file2 = file + '~'
cmd = 'mv ' + file + ' ' + file2
os.system(cmd)
def completeTask(temp_dir, outdir, lname):
"""
mv a list to a proper location, and send out email if there is new obsid
input: temp_dir: a dir where a file is created
outdir: a dir where the files are kept, usually too_dir
lname: a file name, e.g., too_list, ddt_list, new_obs_list, and obs_in_30days
"""
#
#--- change the old file name to that with "~"
#
test = outdir + lname
if mcf.chkFile(test) == 1:
oname = lname + '~'
cmd = 'mv ' + outdir + lname + ' ' + outdir + oname
os.system(cmd)
#
#--- move the new one to the too_dir
#
test = temp_dir + lname
if mcf.chkFile(test) == 1:
cmd = 'mv ' + temp_dir + lname + ' ' + outdir + lname
os.system(cmd)
#
#--- if the file is empty or does not exist, copy back the old one
#
test = outdir + lname
if mcf.isFileEmpty(test) == 0:
cmd = 'cp ' + outdir + oname + ' ' + outdir + lname
os.system(cmd)
#
#--- check whether there are any new observations. if so, send out notification email
#
lname2 = outdir + lname
oname2 = outdir + oname
new_ent = tdfnc.comp_two_record_lists(lname2, oname2)
#
#--- only "unobserved" is truely new
#
cleaned_list = []
for ent in new_ent:
atemp = re.split('\s+', ent)
if atemp[3] == 'unobserved':
cleaned_list.append(ent)
if len(cleaned_list) > 0 and lname != 'obs_in_30days':
atemp = re.split('_list', lname)
tdfnc.send_email(atemp[0], cleaned_list)
def completeTask(temp_dir, outdir, lname):
"""
mv a list to a proper location, and send out email if there is new obsid
input: temp_dir: a dir where a file is created
outdir: a dir where the files are kept, usually too_dir
lname: a file name, e.g., too_list, ddt_list, new_obs_list, and obs_in_30days
"""
#
#--- change the old file name to that with "~"
#
test = outdir + lname
if mcf.chkFile(test) == 1:
oname = lname + '~'
cmd = 'mv ' + outdir + lname + ' ' + outdir + oname
os.system(cmd)
#
#--- move the new one to the too_dir
#
test = temp_dir + lname
if mcf.chkFile(test) == 1:
cmd = 'mv ' + temp_dir + lname + ' ' + outdir + lname
os.system(cmd)
#
#--- if the file is empty or does not exist, copy back the old one
#
test = outdir + lname
if mcf.isFileEmpty(test) == 0:
cmd = 'cp ' + outdir + oname + ' ' + outdir + lname
os.system(cmd)
#
#--- check whether there are any new observations. if so, send out notification email
#
lname2 = outdir + lname
oname2 = outdir + oname
new_ent = tdfnc.comp_two_record_lists(lname2, oname2)
#
#--- only "unobserved" is truely new
#
cleaned_list = []
for ent in new_ent:
atemp = re.split('\s+', ent)
if atemp[3] == 'unobserved':
cleaned_list.append(ent)
if len(cleaned_list) > 0 and lname != 'obs_in_30days':
atemp = re.split('_list', lname)
tdfnc.send_email(atemp[0], cleaned_list)
def send_mail(tag, email_list):
"""
sending email out
Input: tag --- user and machine name in the form of c3po-v_mat
email_list --- a string of name in the from of 'tisobecfa.harvard.edu,[email protected]'
Output: email sent out
"""
chk = mcf.isFileEmpty(zspace)
if chk > 0:
atemp = re.split('_', tag)
cmd = 'cat ' + zspace + ' | mailx -s "Subject: Cron Error : ' + atemp[1] + ' on ' + atemp[0] + '" ' + email_list
os.system(cmd)
mcf.rm_file(zspace)
def send_mail(tag, email_list):
"""
sending email out
Input: tag --- user and machine name in the form of c3po-v_mat
email_list --- a string of name in the from of 'tisobecfa.harvard.edu,[email protected]'
Output: email sent out
"""
chk = mcf.isFileEmpty(zspace)
if chk > 0:
atemp = re.split('_', tag)
cmd = 'cat ' + zspace + ' | mailx -s "Subject: Cron Error : ' + atemp[1] + ' on ' + atemp[0] + '" ' + email_list
os.system(cmd)
mcf.rm_file(zspace)
def send_mail(tag, email_list):
"""
sending email out
Input: tag --- user and machine name in the form of c3po-v_mat
email_list --- a list of email address
Output: email sent out
"""
chk = mcf.isFileEmpty(zspace)
if chk > 0:
atemp = re.split('_', tag)
for email_address in email_list:
cmd = 'cat ' + zspace + ' | mailx -s "Subject: Cron Error : ' + atemp[
1] + ' on ' + atemp[0] + '" ' + email_address
os.system(cmd)
mcf.rm_file(zspace)
def clean_cti_data_table(dir):
"""
remmove data points which are extrme outlyers and then clean up output data tables.
Input: dir --- the directory where the data files are kept
Output: updated data files in the directory <dir>
"""
dropped = data_dir + dir + '/dropped_data'
fo = open(dropped, 'w')
dropped_obsids = []
for elm in elm_list:
line = 'ELM: ' + elm + '\n'
fo.write(line)
for ccd in range(0, 10):
if ccd == 5 or ccd == 7:
drop_factor = 5.0 #--- drop_factor sets the boundray of the outlyer: how may signam away?
else:
drop_factor = 4.0
#
#--- check the input file exists
#
dname = data_dir + dir + '/' + elm + '_ccd' + str(ccd)
chk = mcf.isFileEmpty(dname)
if chk > 0:
line = 'CCD: ' + str(ccd) + '\n'
fo.write(line)
f = open(dname, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- separate data into separate array data sets
#
dcolumns = separate_data(data)
cti = ['' for x in range(4)]
cti[0] = dcolumns[0]
cti[1] = dcolumns[1]
cti[2] = dcolumns[2]
cti[3] = dcolumns[3]
obsid = dcolumns[10]
dom = []
for ent in dcolumns[8]:
time_list = tcnv.dateFormatConAll(ent)
dom.append(time_list[7])
#
#--- go around quads
#
drop_list = []
for i in range(0, 4):
line = "QUAD" + str(i)+ '\n'
fo.write(line)
#
#--- fit a lienar line
#
(intc, slope) = linear_fit(dom, cti[i])
sum = 0
#
#--- compute a deviation from the fitted line
#
diff_save = []
for j in range(0, len(dom)):
diff = float(cti[i][j]) - (intc + slope * float(dom[j]))
diff_save.append(diff)
sum += diff * diff
sigma = math.sqrt(sum/len(dom))
#
#--- find outlyers
#
out_val = drop_factor * sigma
for j in range(0, len(dom)):
if diff_save[j] > out_val:
drop_list.append(j)
fo.write(data[j])
fo.write('\n')
#
#--- clean up the list; removing duplicated lines
#
drop_list = mcf.removeDuplicate(drop_list, chk = 0)
cleaned_data = []
for i in range(0, len(dom)):
chk = 0
for comp in drop_list:
if i == comp:
chk = 1
break
if chk == 0:
cleaned_data.append(data[i])
cleaned_data = mcf.removeDuplicate(cleaned_data, chk = 0)
for ent in drop_list:
dropped_obsids.append(obsid[ent])
f = open(dname, 'w')
for ent in cleaned_data:
f.write(ent)
f.write('\n')
f.close()
fo.close()
dropped_obsids = mcf.removeDuplicate(dropped_obsids, chk = 0)
out = data_dir + dir + '/bad_data_obsid'
f = open(out, 'w')
for ent in dropped_obsids:
f.write(ent)
f.write('\n')
def updata_data_selection_page():
"""
update html gain file information and data_selection.html page
input: none but read two gain_selection files (from <house_keeping> and <html_dir>
output: <html_file>/Gain_files/gain_selection and gain fits file
<html_file>/data_selection.html
"""
#
#--- check two gain list files are different. if so, new gain file was found
#
gain_file1 = house_keeping + 'Gain_files/gain_selection'
gain_file2 = html_dir + 'Data_save/Gain_files/gain_selection'
cmd = 'diff ' + gain_file1 + ' ' + gain_file2 + ' >' + zspace
os.system(cmd)
if mcf.isFileEmpty(zspace) == 0:
mcf.rm_file(zspace)
else:
mcf.rm_file(zspace)
#
#--- find which gain files are added
#
data = hcf.read_file_data(gain_file1)
flist1 = []
for ent in data:
atemp = re.split('\s+', ent)
flist1.append(atemp[-1])
data2 = hcf.read_file_data(gain_file2)
flist2 = []
for ent in data2:
atemp = re.split('\s+', ent)
flist2.append(atemp[-1])
#
#--- add new gain file to <html_dir>/Data_save/Gain_files
#
missing = list(set(flist1) - set(flist2))
for ent in missing:
cmd = 'cp -f ' + house_keeping + 'Gain_files/' + ent + ' ' + html_dir + 'Data_save/Gain_files/.'
os.system(cmd)
cmd = 'cp -f ' + gain_file1 + ' ' + gain_file2
os.system(cmd)
#
#--- update data_selection.html page
#
line = ''
#
#--- create the table of the gain file list
#
for ent in data:
atemp = re.split('\s+', ent)
line = line + '<tr>\n\t<th>'
if atemp[0] != '0':
dtime1 = gformat_time(atemp[0])
line = line + 'After ' + str(dtime1)
if atemp[1] != '1.0e12':
dtime2 = gformat_time(atemp[1])
line = line + ' Before ' + str(dtime2)
line = line + '</th>\n\t<td><a href="./Data_sve/Gain_file/' + atemp[
2] + '">' + atemp[2] + '</a></td>\n</tr>\n'
#
#--- find the current date
#
current_time = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime())
#
#--- read the template and substitute the table and update date
#
template = house_keeping + 'Templates/data_selection_template'
f = open(template, 'r')
data = f.read()
f.close()
data = data.replace("#TABLE#", line)
data = data.replace("#UPDATE#", current_time)
#
#--- print out the file
#
out = html_dir + 'data_selection.html'
fo = open(out, 'w')
fo.write(data)
fo.close()