def insert_details(self, data, project_path, isghb, cycle_id, status, obs_no):
print("------------------==insert_details==----------------------")
print(data)
print(project_path)
dbutils = DBUtils()
for each_rec in data:
print("==each_rec==", each_rec)
lta_file = os.path.basename(each_rec)
if len(lta_file) == 1:
lta_file = lta_file[0]
try:
current_date_timestamp = datetime.datetime.fromtimestamp(currentTimeInSec).strftime('%Y-%m-%d %H:%M:%S')
lta_details = gS.get_naps_scangroup_details(lta_file)
utils = self
lta_details["ltacomb_size"] = int(utils.calculalate_file_sizse_in_MB(each_rec))
lta_details["status"] = "unprocessed"
lta_details["base_path"] = project_path
lta_details["start_time"] = current_date_timestamp
lta_details["proposal_dir"] = project_path.split('/')[-1]
lta_details["pipeline_id"] = 1
lta_details["comments"] = status
lta_details["counter"] = 0
lta_details["ltacomb_file"] = lta_file
lta_details["isghb"] = isghb
lta_details["cycle_id"] = cycle_id
if obs_no:
lta_details["observation_no"] = int(obs_no)
projectobsno_data = {}
for key in tableSchema.projectobsnoData.iterkeys():
if key in lta_details.iterkeys():
projectobsno_data[key] = lta_details[key]
ltadetails_data = {}
for key in tableSchema.ltadetailsData.iterkeys():
if key in lta_details.iterkeys():
ltadetails_data[key] = lta_details[key]
#print ltadetails_data
# print("ltadetails_data")
# print(ltadetails_data)
columnKeys = {"project_id"}
whereKeys = {"proposal_dir": lta_details["proposal_dir"], "cycle_id": cycle_id}
project_id = dbutils.select_test_table("projectobsno", columnKeys, whereKeys, 0)
if project_id:
project_id = project_id[0]
else:
project_id = dbutils.insert_into_table("projectobsno", projectobsno_data, tableSchema.projectobsnoId)
ltadetails_data["project_id"] = project_id
lta_id = dbutils.insert_into_table("ltadetails", ltadetails_data, tableSchema.ltadetailsId)
print lta_id
print("projectobsno")
print(projectobsno_data)
except Exception as e:
print e
def combining_lsb_usb(self):
print("Started Stage4: ")
cycle_id = self.pipeline_configuration()["cycle_id"]
spam.set_aips_userid(11)
dbutils = DBUtils()
fileutils = FileUtils()
status = "failed"
comments = "combine usb lsb failed"
# fileutils = FileUtils()
# query conditions for projectobsno
columnKeys = {"project_id", "base_path", "observation_no"}
whereKeys = {"isghb": True, "cycle_id": 16, "status": "now"}
project_data = dbutils.select_from_table("projectobsno", columnKeys,
whereKeys, 0)
print(project_data)
project_id = project_data[1]
base_path = project_data[2]
obsno = project_data[0]
start_time = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
# print(project_id, base_path, obsno)
# query conditions for calibrationinput
columnKeys = {"calibration_id", "uvfits_file"}
whereKeys = {"project_id": project_id, "status": "now"}
calibration_data = dbutils.select_from_table("calibrationinput",
columnKeys, whereKeys,
None)
print(calibration_data)
if not calibration_data:
print(
"All the data is processed ... OR \n ==> please check the DB for combinelsbusb"
)
spam.exit()
print(len(calibration_data))
if len(calibration_data) < 2:
status = "success"
comments = "single file combinelsbusb not required"
usb_lsb_file = glob.glob(base_path + "/PRECALIB/*GMRT*.UVFITS")
if calibration_data:
projectobsno_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"project_id": project_id
}
}
print("Updating the projectobsno ... ")
dbutils.update_table(projectobsno_update_data, "projectobsno")
if calibration_data:
calibration_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"calibration_id":
calibration_data[0]["calibration_id"]
}
}
dbutils.update_table(calibration_update_data,
"calibrationinput")
else:
projectobsno_update_data = {
"set": {
"status": "failed",
"comments": "Failed Error: Something went wrong"
},
"where": {
"project_id": project_id
}
}
print("Updating the projectobsno ... ")
dbutils.update_table(projectobsno_update_data, "projectobsno")
else:
print("Values > 2")
print("*************" + str(os.getcwd()))
for each_uvfits in calibration_data:
precalib_files = glob.glob(base_path + "/PRECALIB/*")
lsb_list = glob.glob(base_path + "/PRECALIB/*_LL_*.UVFITS")
usb_list = glob.glob(base_path + "/PRECALIB/*_RR_*.UVFITS")
if len(lsb_list) == 0 or len(usb_list) == 0:
print(len(lsb_list), len(usb_list))
lsb_list = glob.glob(base_path + "/PRECALIB/*LSB*.UVFITS")
usb_list = glob.glob(base_path + "/PRECALIB/*USB*.UVFITS")
projectobsno_update_data = {
"set": {
"status": "processing",
"comments": "combining_lsb_usb"
},
"where": {
"project_id": project_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
calibration_id = each_uvfits["calibration_id"]
uvfits_file = each_uvfits["uvfits_file"]
calibration_update_data = {
"set": {
"status": "processing",
"comments": "combining_lsb_usb",
"start_time": start_time
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(calibration_update_data,
"calibrationinput")
print("lsb_list : " + str(len(lsb_list)))
print("usb_list : " + str(len(usb_list)))
status = "failed"
comments = "combining lsb usb"
if len(lsb_list) == len(usb_list):
print(">>>>>>COMBINE_LSB_USB<<<<<<<")
usb_list.sort()
lsb_list.sort()
print(usb_list)
print(lsb_list)
to_spam = list(zip(usb_list, lsb_list))
file_size = 0
print(to_spam)
for each_pair in to_spam:
print("-------------------------")
comb = each_pair[0].replace('USB', 'COMB')
data = each_pair, comb
print("++++++++++++++++" + comb)
currentTimeInSec = time.time()
fits_comb = comb.split('/')[-1]
check_comb_file = glob.glob("fits/" + fits_comb)
if not check_comb_file:
status, comments = fileutils.run_spam_combine_usb_lsb(
data)
if status == 'success':
status = str(cycle_id)
print("__________________________________________")
print(glob.glob("fits/*"))
print("__________________________________________")
end_time = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
if not comments:
comments = "done combining usb lsb"
if glob.glob(comb):
file_size = fileutils.calculalate_file_sizse_in_MB(
comb)
imagininput_data = {
"project_id": project_id,
"calibration_id": calibration_id,
"calibrated_fits_file": os.path.basename(comb),
"file_size": file_size,
"start_time": start_time,
"end_time": end_time,
"comments": "c16 " + comments,
}
dbutils.insert_into_table("imaginginput",
imagininput_data,
"imaging_id")
print("-------------------------")
end_time = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
calibration_update_data = {
"set": {
"status": status,
"comments": comments,
"start_time": start_time,
"end_time": end_time
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(calibration_update_data,
"calibrationinput")
projectobsno_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"project_id": project_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
def running_gvfits(self):
print("Started Stage2: ")
cycle_id = self.pipeline_configuration()["cycle_id"]
dbutils = DBUtils()
spamutils = SpamUtils()
fileutils = FileUtils()
currentTimeInSec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
currentTimeInSec).strftime('%Y-%m-%d %H:%M:%S')
columnKeys = {"project_id", "ltacomb_file", "lta_id"}
whereKeys = {"comments": str(cycle_id)}
lta_details = dbutils.select_from_table("ltadetails", columnKeys,
whereKeys, None)
print(lta_details)
for each_lta in lta_details:
print(each_lta)
project_id = each_lta["project_id"]
# project_id = each_lta[0]
lta_file = each_lta["ltacomb_file"]
# lta_file = each_lta[1]
# lta_id = each_lta[2]
lta_id = each_lta["lta_id"]
columnKeys = {"base_path"}
whereKeys = {"project_id": project_id}
lta_path_details = dbutils.select_test_table(
"projectobsno", columnKeys, whereKeys, 0)
print(lta_path_details)
base_path = lta_path_details[0]
print(base_path)
uvfits_file = lta_file + '.UVFITS'
base_lta = base_path + '/' + lta_file
if os.path.exists(base_lta):
base_uvfits = base_path + '/' + uvfits_file
gvfits_status = spamutils.run_gvfits(base_lta, base_uvfits)
if os.path.exists(base_uvfits):
status = str(cycle_id)
else:
status = "failed"
calibration_data = {
"project_id":
project_id,
"lta_id":
lta_id,
"uvfits_file":
uvfits_file,
"status":
status,
"comments":
gvfits_status,
"uvfits_size":
fileutils.calculalate_file_sizse_in_MB(base_uvfits),
"start_time":
current_date_timestamp
}
dbutils.insert_into_table("calibrationinput", calibration_data,
"calibration_id")
else:
project_update_data = {
"set": {
"status": "failed",
"comments": "ltacomb failed"
},
"where": {
"project_id": project_id
}
}
lta_details_update_data = {
"set": {
"status": "failed",
"comments": "ltacomb failed"
},
"where": {
"lta_id": lta_id
}
}
dbutils.update_table(project_update_data, "projectobsno")
dbutils.update_table(lta_details_update_data, "ltadetails")
def stage3(self):
spam.set_aips_userid(33)
dbutils = DBUtils()
fileutils = FileUtils()
# while True:
# columnKeys = {"calibration_id"}
# whereData = {"comments": "c15", "status": "copying"}
# uncalibrated_uvfits = dbutils.select_from_table("calibrationinput", columnKeys, whereData, 0)
# if not uncalibrated_uvfits:
# break
# print("Waiting for bandwidth ... ")
# time.sleep(50)
columnKeys = {"calibration_id", "project_id", "uvfits_file"}
whereData = {"status": "cycle226"}
uncalibrated_uvfits = dbutils.select_from_table(
"calibrationinput", columnKeys, whereData, 0)
if not uncalibrated_uvfits:
print(
"All for the data is processed ... please check the DB for pre_calib"
)
spam.exit()
calibration_id = uncalibrated_uvfits[0]
project_id = uncalibrated_uvfits[1]
uvfits_file = uncalibrated_uvfits[2]
columnKeys = {"file_path", "observation_no"}
whereData = {"project_id": project_id}
project_details = dbutils.select_from_table("projectobsno", columnKeys,
whereData, 0)
base_path = project_details[1]
observation_no = project_details[0]
current_date_timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
projectobsno_update_data = {
"set": {
"status":
"processing",
"comments":
"running precalibrate_target, calibration_id = " +
str(calibration_id),
},
"where": {
"project_id": project_id
}
}
calibration_update_data = {
"set": {
"status": "copying",
"start_time": current_date_timestamp
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
dbutils.update_table(calibration_update_data, "calibrationinput")
UVFITS_FILE_NAME = uvfits_file
UVFITS_BASE_DIR = base_path
is_fits_dir = os.getcwd().split('/')
print(is_fits_dir)
SPAM_WORKING_DIR = os.getcwd()
print(SPAM_WORKING_DIR)
# for num in range(1, 3):
# SPAM_THREAD_DIR += "/" + is_fits_dir[num]
# if 'fits' not in is_fits_dir:
# SPAM_THREAD_DIR = os.getcwd()
SPAM_WORKING_DIR = os.getcwd() + "/fits/"
print(SPAM_WORKING_DIR, UVFITS_BASE_DIR, UVFITS_FILE_NAME)
UVFITS_FILE_PATH = UVFITS_BASE_DIR + "/" + UVFITS_FILE_NAME
print(UVFITS_FILE_PATH)
print(SPAM_WORKING_DIR)
fileutils.copy_files(UVFITS_FILE_PATH, SPAM_WORKING_DIR)
print("Copying done ==> Moving to pre_cal_target")
current_date_timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
calibration_update_data = {
"set": {
"status": "processing",
"start_time": current_date_timestamp
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(calibration_update_data, "calibrationinput")
fileutils.run_spam_precalibration_stage(UVFITS_BASE_DIR,
SPAM_WORKING_DIR,
UVFITS_FILE_NAME,
observation_no)
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
check_status_file = glob.glob(base_path + "/PRECALIB/failed_log.txt")
comments = "failed"
if check_status_file:
status = "failed"
comments = str(open(check_status_file[0], 'r').read())
else:
status = "success"
comments = "precalibrate_target done, calibration_id = " + str(
calibration_id)
projectobsno_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"project_id": project_id
}
}
calibration_update_data = {
"set": {
"status": status,
"end_time": current_date_timestamp,
"comments": comments
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
dbutils.update_table(calibration_update_data, "calibrationinput")
if not check_status_file:
split_files = glob.glob(base_path + "/PRECALIB/*GMRT*.UVFITS")
for each_split_file in split_files:
current_time_in_sec = time.time()
calibrated_uvfits = {
'start_time':
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'),
'end_time':
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'),
'project_id':
project_id,
'calibration_id':
calibration_id,
"file_size":
fileutils.calculalate_file_sizse_in_MB(each_split_file),
"calibrated_fits_file":
each_split_file.split('/')[-1],
"status":
"success"
}
print dbutils.insert_into_table("imaginginput",
calibrated_uvfits,
"project_id")
def __main__():
dbutils = DBUtils()
fileutils = FileUtils()
columnKeys = {"calibration_id", "project_id", "uvfits_file"}
whereData = {"comments": "c17", "status": "success"}
uncalibrated_uvfits = dbutils.select_from_table("calibrationinput",
columnKeys, whereData, 0)
calibration_id = uncalibrated_uvfits[0]
project_id = uncalibrated_uvfits[1]
uvfits_file = uncalibrated_uvfits[2]
columnKeys = {"file_path", "observation_no"}
whereData = {"project_id": project_id, "cycle_id": 17}
project_details = dbutils.select_from_table("projectobsno", columnKeys,
whereData, 0)
base_path = project_details[1]
observation_no = project_details[0]
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
projectobsno_update_data = {
"set": {
"status":
"processing",
"comments":
"running precalibrate_target, calibration_id = " +
str(calibration_id),
},
"where": {
"project_id": project_id,
"status": "unprocessed"
}
}
calibration_update_data = {
"set": {
"status": "processing",
"start_time": current_date_timestamp
},
"where": {
"calibration_id": calibration_id,
"status": "success"
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
dbutils.update_table(calibration_update_data, "calibrationinput")
EACH_UVFITS_FILE = base_path + '/' + uvfits_file
UVFITS_BASE_DIR = base_path + "/"
if not check_pipeline_flag(UVFITS_BASE_DIR):
set_flag(UVFITS_BASE_DIR, PRECAL_PROCESSING)
is_fits_dir = os.getcwd().split('/')
SPAM_WORKING_DIR = os.getcwd()
SPAM_THREAD_DIR = ""
for num in range(1, 4):
SPAM_THREAD_DIR += "/" + is_fits_dir[num]
if 'fits' not in is_fits_dir:
SPAM_THREAD_DIR = os.getcwd()
SPAM_WORKING_DIR = os.getcwd() + "/fits/"
copy_files(EACH_UVFITS_FILE, SPAM_WORKING_DIR)
print "Copying done ==> Moving to pre_cal_target"
run_spam_precalibration_stage(UVFITS_BASE_DIR, SPAM_WORKING_DIR,
uvfits_file)
check_status_file = glob.glob(base_path + "/PRECALIB/failed_log.txt")
if check_status_file:
status = "failed"
else:
status = "success"
projectobsno_update_data = {
"set": {
"status":
status,
"comments":
"precalibrate_target " + status + ", calibration_id = " +
str(calibration_id),
},
"where": {
"project_id": project_id
}
}
calibration_update_data = {
"set": {
"status": status,
"end_time": current_date_timestamp
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
dbutils.update_table(calibration_update_data, "calibrationinput")
if status == 'success':
calibrated_uvfits_list = glob.glob(base_path +
'/PRECALIB/*.UVFITS')
if calibrated_uvfits_list:
for each_uvfits in calibrated_uvfits_list:
imaging_data = {
"project_id":
project_id,
"calibration_id":
calibration_id,
"calibrated_fits_file":
os.path.basename(each_uvfits),
"status":
"unprocessed",
"comments":
"c17",
"file_size":
fileutils.calculalate_file_sizse_in_MB(each_uvfits)
}
dbutils.insert_into_table("imaginginput", imaging_data,
"imaging_id")
delete_dir(SPAM_THREAD_DIR)
spam.exit()
