-
Notifications
You must be signed in to change notification settings - Fork 2
/
get_data.py
145 lines (126 loc) · 4.72 KB
/
get_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
import os
import sys
#import mwapy.get_observation_info
#from mwapy.obssched.base import schedule
from subprocess import Popen, PIPE
import glob
# A script to retrieve a list of observations from NGAS
# and convert them into ms files. This script will
# then run the SIP with the settings in parset.txt
####################################################
#Parset files path(s)
loc_parset_file = '/home/562/meb562/MWA_SIP/locs_parset.txt'
parset_file = '/home/562/meb562/MWA_SIP/parset.txt'
#####################################################
def read_parset(parset_file):
"""
Read the parameter file defined in parset file path and return args
"""
print 'Reading parameter set file'
f = open(parset_file,'r')
config_dict = {}
for line in f:
if line[0] == '#':
pass;
else:
items = line.split('=', 1)
config_dict[items[0]] = eval(items[1])
return config_dict
#####################################################
def find_cal(obs_id):
obs_id = obs_id.split('\n')[0]
obs_id = obs_id.strip()
cal_pref = 'HydA'
do_cal_pref = False
if do_cal_pref:
cal_output = Popen(["python", "/short/ek6/MWA_Code/MWA_extras/find_calibrator.py", "-v","--matchproject","--source="+cal_pref, str(obs_id)], stdout=PIPE).communicate()[0]
else:
cal_output = Popen(["python", "/short/ek6/MWA_Code/MWA_extras/find_calibrator.py", "-v","--matchproject", str(obs_id)], stdout=PIPE).communicate()[0]
try:
cal_id = cal_output.split()[7]
cal_name = cal_output.split()[10]
print str(obs_id)+": recommended cal is "+str(cal_id)+' = '+str(cal_name)
except:
print str(obs_id)+": cannot find suitable cal "
cal_id=None
pass;
os.chdir('/short/ek6/CALS/')
cal_num = None
return_cal = None
for cal in glob.glob('*.cal'):
cal_num = cal[0:10]
if cal_id == cal_num:
print "Found calibration file "+cal
return_cal = '/short/ek6/CALS/'+cal
if return_cal == None:
print "No calibrator file found, please generate it"
return return_cal
###############
def find_cal_mwats(obs_id):
obs_id = obs_id.split('\n')[0]
obs_id = obs_id.strip()
date_output = Popen(["python", "/short/ek6/MWA_Code/bin/get_observation_info.py", "-g", str(obs_id)], stdout=PIPE).communicate()[0]
date_output = date_output.split()[7]
date_output = date_output.replace('/', '-')
date_output = date_output[1:11]
print "Observation date "+date_output
os.chdir('/short/ek6/CALS/')
cal_list = open('MWATS_cal_list.txt', 'r')
return_cal = None
for line in cal_list:
sdate = line.split(',')[0]
cal_id = line.split(',')[1]
cal_id = cal_id.split(' ')[1]
if sdate == date_output:
print 'Recomended calibrator is '+cal_id
for cal in glob.glob('*.cal'):
cal_num = cal[0:10]
if cal_id.split('\n')[0] == cal_num:
print "Found calibration file "+cal
return_cal = '/short/ek6/CALS/'+cal
if return_cal == None:
print "No calibrator file found, please generate it"
return return_cal
####################################################
# Read the location parset to find the paths
locs = read_parset(loc_parset_file)
SIP_home = locs['SIP_home']
error_logs = locs['error_logs']
msgen_loc = locs['msgen_loc']
####################################################
def run_obsresolve(obs_id):
print "Getting DATA: Running obsresolve with GPS ID = "+str(obs_id)
#os.system("obsresolve.py -r ngas01.ivec.org -s ngas01.ivec.org -d ./ -o "+str(obs_id))
os.system('/short/ek6/MWA_Code/MWA_Tools/scripts/obsdownload.py -o '+str(obs_id))
############## Main code ######################
obs_id = sys.argv[1] # Get the obs_id we want to work on. Parsed to the script by run_pipe.py
parset = read_parset(parset_file)
do_parset_cal = parset['do_parset_cal']
do_MWATS_cal = parset['do_MWATS_cal']
if do_parset_cal:
cal = locs['cal_loc']
if do_MWATS_cal:
cal = find_cal_mwats(obs_id)
else:
cal = find_cal(obs_id) # Look for a calibrator
if cal == None:
print 'Error: No calibrator found, exciting processing'
os._exit(1)
else:
os.chdir(msgen_loc)
if os.path.exists(obs_id):
print 'Data folder already made, partial download might not have completed.'
os.chdir(obs_id)
run_obsresolve(obs_id)
else:
os.mkdir(obs_id)
os.chdir(obs_id)
run_obsresolve(obs_id) # Get the files
os.chdir(msgen_loc)
os.chdir(obs_id)
# Make the metafits for cotter
os.system('make_metafits.py --gps='+str(obs_id))
#########################
cal_file = open("cal.txt", "w")
cal_file.write(str(cal))
cal_file.close()