def mk_backup(file): """ copy a file to file~ if file is not an enmpty file Input: file --- original file Output: file2 --- file with "~" at the end """ chk = mcf.isFileEmpty(file) if chk == 1: file2 = file + '~' cmd = 'mv ' + file + ' ' + file2 os.system(cmd)
def completeTask(temp_dir, outdir, lname): """ mv a list to a proper location, and send out email if there is new obsid input: temp_dir: a dir where a file is created outdir: a dir where the files are kept, usually too_dir lname: a file name, e.g., too_list, ddt_list, new_obs_list, and obs_in_30days """ # #--- change the old file name to that with "~" # test = outdir + lname if mcf.chkFile(test) == 1: oname = lname + '~' cmd = 'mv ' + outdir + lname + ' ' + outdir + oname os.system(cmd) # #--- move the new one to the too_dir # test = temp_dir + lname if mcf.chkFile(test) == 1: cmd = 'mv ' + temp_dir + lname + ' ' + outdir + lname os.system(cmd) # #--- if the file is empty or does not exist, copy back the old one # test = outdir + lname if mcf.isFileEmpty(test) == 0: cmd = 'cp ' + outdir + oname + ' ' + outdir + lname os.system(cmd) # #--- check whether there are any new observations. if so, send out notification email # lname2 = outdir + lname oname2 = outdir + oname new_ent = tdfnc.comp_two_record_lists(lname2, oname2) # #--- only "unobserved" is truely new # cleaned_list = [] for ent in new_ent: atemp = re.split('\s+', ent) if atemp[3] == 'unobserved': cleaned_list.append(ent) if len(cleaned_list) > 0 and lname != 'obs_in_30days': atemp = re.split('_list', lname) tdfnc.send_email(atemp[0], cleaned_list)
def send_mail(tag, email_list): """ sending email out Input: tag --- user and machine name in the form of c3po-v_mat email_list --- a string of name in the from of 'tisobecfa.harvard.edu,[email protected]' Output: email sent out """ chk = mcf.isFileEmpty(zspace) if chk > 0: atemp = re.split('_', tag) cmd = 'cat ' + zspace + ' | mailx -s "Subject: Cron Error : ' + atemp[1] + ' on ' + atemp[0] + '" ' + email_list os.system(cmd) mcf.rm_file(zspace)
def send_mail(tag, email_list): """ sending email out Input: tag --- user and machine name in the form of c3po-v_mat email_list --- a list of email address Output: email sent out """ chk = mcf.isFileEmpty(zspace) if chk > 0: atemp = re.split('_', tag) for email_address in email_list: cmd = 'cat ' + zspace + ' | mailx -s "Subject: Cron Error : ' + atemp[ 1] + ' on ' + atemp[0] + '" ' + email_address os.system(cmd) mcf.rm_file(zspace)
def clean_cti_data_table(dir): """ remmove data points which are extrme outlyers and then clean up output data tables. Input: dir --- the directory where the data files are kept Output: updated data files in the directory <dir> """ dropped = data_dir + dir + '/dropped_data' fo = open(dropped, 'w') dropped_obsids = [] for elm in elm_list: line = 'ELM: ' + elm + '\n' fo.write(line) for ccd in range(0, 10): if ccd == 5 or ccd == 7: drop_factor = 5.0 #--- drop_factor sets the boundray of the outlyer: how may signam away? else: drop_factor = 4.0 # #--- check the input file exists # dname = data_dir + dir + '/' + elm + '_ccd' + str(ccd) chk = mcf.isFileEmpty(dname) if chk > 0: line = 'CCD: ' + str(ccd) + '\n' fo.write(line) f = open(dname, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- separate data into separate array data sets # dcolumns = separate_data(data) cti = ['' for x in range(4)] cti[0] = dcolumns[0] cti[1] = dcolumns[1] cti[2] = dcolumns[2] cti[3] = dcolumns[3] obsid = dcolumns[10] dom = [] for ent in dcolumns[8]: time_list = tcnv.dateFormatConAll(ent) dom.append(time_list[7]) # #--- go around quads # drop_list = [] for i in range(0, 4): line = "QUAD" + str(i)+ '\n' fo.write(line) # #--- fit a lienar line # (intc, slope) = linear_fit(dom, cti[i]) sum = 0 # #--- compute a deviation from the fitted line # diff_save = [] for j in range(0, len(dom)): diff = float(cti[i][j]) - (intc + slope * float(dom[j])) diff_save.append(diff) sum += diff * diff sigma = math.sqrt(sum/len(dom)) # #--- find outlyers # out_val = drop_factor * sigma for j in range(0, len(dom)): if diff_save[j] > out_val: drop_list.append(j) fo.write(data[j]) fo.write('\n') # #--- clean up the list; removing duplicated lines # drop_list = mcf.removeDuplicate(drop_list, chk = 0) cleaned_data = [] for i in range(0, len(dom)): chk = 0 for comp in drop_list: if i == comp: chk = 1 break if chk == 0: cleaned_data.append(data[i]) cleaned_data = mcf.removeDuplicate(cleaned_data, chk = 0) for ent in drop_list: dropped_obsids.append(obsid[ent]) f = open(dname, 'w') for ent in cleaned_data: f.write(ent) f.write('\n') f.close() fo.close() dropped_obsids = mcf.removeDuplicate(dropped_obsids, chk = 0) out = data_dir + dir + '/bad_data_obsid' f = open(out, 'w') for ent in dropped_obsids: f.write(ent) f.write('\n')
def updata_data_selection_page(): """ update html gain file information and data_selection.html page input: none but read two gain_selection files (from <house_keeping> and <html_dir> output: <html_file>/Gain_files/gain_selection and gain fits file <html_file>/data_selection.html """ # #--- check two gain list files are different. if so, new gain file was found # gain_file1 = house_keeping + 'Gain_files/gain_selection' gain_file2 = html_dir + 'Data_save/Gain_files/gain_selection' cmd = 'diff ' + gain_file1 + ' ' + gain_file2 + ' >' + zspace os.system(cmd) if mcf.isFileEmpty(zspace) == 0: mcf.rm_file(zspace) else: mcf.rm_file(zspace) # #--- find which gain files are added # data = hcf.read_file_data(gain_file1) flist1 = [] for ent in data: atemp = re.split('\s+', ent) flist1.append(atemp[-1]) data2 = hcf.read_file_data(gain_file2) flist2 = [] for ent in data2: atemp = re.split('\s+', ent) flist2.append(atemp[-1]) # #--- add new gain file to <html_dir>/Data_save/Gain_files # missing = list(set(flist1) - set(flist2)) for ent in missing: cmd = 'cp -f ' + house_keeping + 'Gain_files/' + ent + ' ' + html_dir + 'Data_save/Gain_files/.' os.system(cmd) cmd = 'cp -f ' + gain_file1 + ' ' + gain_file2 os.system(cmd) # #--- update data_selection.html page # line = '' # #--- create the table of the gain file list # for ent in data: atemp = re.split('\s+', ent) line = line + '<tr>\n\t<th>' if atemp[0] != '0': dtime1 = gformat_time(atemp[0]) line = line + 'After ' + str(dtime1) if atemp[1] != '1.0e12': dtime2 = gformat_time(atemp[1]) line = line + ' Before ' + str(dtime2) line = line + '</th>\n\t<td><a href="./Data_sve/Gain_file/' + atemp[ 2] + '">' + atemp[2] + '</a></td>\n</tr>\n' # #--- find the current date # current_time = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()) # #--- read the template and substitute the table and update date # template = house_keeping + 'Templates/data_selection_template' f = open(template, 'r') data = f.read() f.close() data = data.replace("#TABLE#", line) data = data.replace("#UPDATE#", current_time) # #--- print out the file # out = html_dir + 'data_selection.html' fo = open(out, 'w') fo.write(data) fo.close()