def combining_lsb_usb(self):
print("Started Stage4: ")
cycle_id = self.pipeline_configuration()["cycle_id"]
spam.set_aips_userid(11)
dbutils = DBUtils()
fileutils = FileUtils()
status = "failed"
comments = "combine usb lsb failed"
# fileutils = FileUtils()
# query conditions for projectobsno
columnKeys = {"project_id", "base_path", "observation_no"}
whereKeys = {"isghb": True, "cycle_id": 16, "status": "now"}
project_data = dbutils.select_from_table("projectobsno", columnKeys,
whereKeys, 0)
print(project_data)
project_id = project_data[1]
base_path = project_data[2]
obsno = project_data[0]
start_time = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
# print(project_id, base_path, obsno)
# query conditions for calibrationinput
columnKeys = {"calibration_id", "uvfits_file"}
whereKeys = {"project_id": project_id, "status": "now"}
calibration_data = dbutils.select_from_table("calibrationinput",
columnKeys, whereKeys,
None)
print(calibration_data)
if not calibration_data:
print(
"All the data is processed ... OR \n ==> please check the DB for combinelsbusb"
)
spam.exit()
print(len(calibration_data))
if len(calibration_data) < 2:
status = "success"
comments = "single file combinelsbusb not required"
usb_lsb_file = glob.glob(base_path + "/PRECALIB/*GMRT*.UVFITS")
if calibration_data:
projectobsno_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"project_id": project_id
}
}
print("Updating the projectobsno ... ")
dbutils.update_table(projectobsno_update_data, "projectobsno")
if calibration_data:
calibration_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"calibration_id":
calibration_data[0]["calibration_id"]
}
}
dbutils.update_table(calibration_update_data,
"calibrationinput")
else:
projectobsno_update_data = {
"set": {
"status": "failed",
"comments": "Failed Error: Something went wrong"
},
"where": {
"project_id": project_id
}
}
print("Updating the projectobsno ... ")
dbutils.update_table(projectobsno_update_data, "projectobsno")
else:
print("Values > 2")
print("*************" + str(os.getcwd()))
for each_uvfits in calibration_data:
precalib_files = glob.glob(base_path + "/PRECALIB/*")
lsb_list = glob.glob(base_path + "/PRECALIB/*_LL_*.UVFITS")
usb_list = glob.glob(base_path + "/PRECALIB/*_RR_*.UVFITS")
if len(lsb_list) == 0 or len(usb_list) == 0:
print(len(lsb_list), len(usb_list))
lsb_list = glob.glob(base_path + "/PRECALIB/*LSB*.UVFITS")
usb_list = glob.glob(base_path + "/PRECALIB/*USB*.UVFITS")
projectobsno_update_data = {
"set": {
"status": "processing",
"comments": "combining_lsb_usb"
},
"where": {
"project_id": project_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
calibration_id = each_uvfits["calibration_id"]
uvfits_file = each_uvfits["uvfits_file"]
calibration_update_data = {
"set": {
"status": "processing",
"comments": "combining_lsb_usb",
"start_time": start_time
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(calibration_update_data,
"calibrationinput")
print("lsb_list : " + str(len(lsb_list)))
print("usb_list : " + str(len(usb_list)))
status = "failed"
comments = "combining lsb usb"
if len(lsb_list) == len(usb_list):
print(">>>>>>COMBINE_LSB_USB<<<<<<<")
usb_list.sort()
lsb_list.sort()
print(usb_list)
print(lsb_list)
to_spam = list(zip(usb_list, lsb_list))
file_size = 0
print(to_spam)
for each_pair in to_spam:
print("-------------------------")
comb = each_pair[0].replace('USB', 'COMB')
data = each_pair, comb
print("++++++++++++++++" + comb)
currentTimeInSec = time.time()
fits_comb = comb.split('/')[-1]
check_comb_file = glob.glob("fits/" + fits_comb)
if not check_comb_file:
status, comments = fileutils.run_spam_combine_usb_lsb(
data)
if status == 'success':
status = str(cycle_id)
print("__________________________________________")
print(glob.glob("fits/*"))
print("__________________________________________")
end_time = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
if not comments:
comments = "done combining usb lsb"
if glob.glob(comb):
file_size = fileutils.calculalate_file_sizse_in_MB(
comb)
imagininput_data = {
"project_id": project_id,
"calibration_id": calibration_id,
"calibrated_fits_file": os.path.basename(comb),
"file_size": file_size,
"start_time": start_time,
"end_time": end_time,
"comments": "c16 " + comments,
}
dbutils.insert_into_table("imaginginput",
imagininput_data,
"imaging_id")
print("-------------------------")
end_time = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
calibration_update_data = {
"set": {
"status": status,
"comments": comments,
"start_time": start_time,
"end_time": end_time
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(calibration_update_data,
"calibrationinput")
projectobsno_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"project_id": project_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
def process_targets(self):
print("Started Stage5: ")
"""
SPAM's process_target
:return:
"""
cycle_id = self.pipeline_configuration()["cycle_id"]
fileutils = FileUtils()
aips_id = int(random.random() * 100)
spam.set_aips_userid(11)
# Setting the Process Start Date Time
start_time = str(datetime.datetime.now())
# Taking system's in/out to backup variable
original_stdout = sys.stdout
original_stderr = sys.stderr
thread_dir = os.getcwd()
# Changing directory to fits/
os.chdir("fits/")
datfil_dir = thread_dir + "/datfil/"
fits_dir = thread_dir + "/fits/"
curr_dir = thread_dir + "/fits/"
process_status = False
db_model = DBUtils()
# Get random imaging_id & project_id
column_keys = [
tableSchema.imaginginputId, tableSchema.projectobsnoId,
"calibrated_fits_file"
]
where_con = {"status": str(cycle_id)}
to_be_processed = db_model.select_from_table("imaginginput",
column_keys, where_con,
None)
imaginginput_details = random.choice(to_be_processed)
print(imaginginput_details)
imaging_id = imaginginput_details["imaging_id"]
# Update status for imaginginput for selected imaging_id
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
update_data = {
"set": {
"status": "processing",
"start_time": current_date_timestamp,
"comments": "",
"end_time": current_date_timestamp
},
"where": {
"imaging_id": imaging_id,
}
}
db_model.update_table(update_data, "imaginginput")
project_id = imaginginput_details["project_id"]
calibrated_fits_file = imaginginput_details["calibrated_fits_file"]
# Using the above project_id, fetch base_path
column_keys = ["base_path"]
where_con = {"project_id": project_id}
process_target_log = open('process_target.log', 'a+')
process_target_log.write(
'\n\n\n******PROCESS TARGET STARTED******\n\n\n')
process_target_log.write("---> Start Time " + start_time)
# Logging all Standard In/Output
sys.stdout = process_target_log
sys.stderr = process_target_log
base_path = db_model.select_from_table("projectobsno", column_keys,
where_con, 0)
base_path = base_path[0]
uvfits_full_path = base_path + "/PRECALIB/" + calibrated_fits_file
# uvfits_full_path = base_path+"/PRECALIB/"+calibrated_fits_file
print "Copying " + uvfits_full_path + " to " + fits_dir
copying_fits = os.system("cp " + uvfits_full_path + " " + fits_dir)
uvfits_file = calibrated_fits_file
# Starting spam.process_target(SPLIT_FITS_FILE)
try:
spam.process_target(uvfits_file,
allow_selfcal_skip=True,
add_freq_to_name=True)
# If this process_target is success call
# GADPU API setSuccessStatus for the current fits_id
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
success_update_data = {
"set": {
"status": "checking",
"end_time": current_date_timestamp,
"comments": "processing done, checking"
},
"where": {
"imaging_id": imaging_id
}
}
db_model.update_table(success_update_data, "imaginginput")
except Exception, e:
process_target_log.write("Error: " + str(e))
# If this process_target is a failure call
# GADPU API setFailedStatus for the current fits_id
current_date_timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
success_update_data = {
"set": {
"status": "failed",
"end_time": current_date_timestamp,
},
"where": {
"imaging_id": imaging_id
}
}
db_model.update_table(success_update_data, "imaginginput")
print("Error: spam.process_tagret Failed " + uvfits_file)
def pre_calibration_targets(self):
print("Started Stage3: ")
spam.set_aips_userid(33)
dbutils = DBUtils()
fileutils = FileUtils()
cycle_id = self.pipeline_configuration()["cycle_id"]
# while True:
# columnKeys = {"calibration_id"}
# whereData = {"comments": "c15", "status": "copying"}
# uncalibrated_uvfits = dbutils.select_from_table("calibrationinput", columnKeys, whereData, 0)
# if not uncalibrated_uvfits:
# break
# print("Waiting for bandwidth ... ")
# time.sleep(50)
columnKeys = {"calibration_id", "project_id", "uvfits_file"}
whereData = {"status": str(cycle_id)}
uncalibrated_uvfits = dbutils.select_from_table(
"calibrationinput", columnKeys, whereData, 0)
if not uncalibrated_uvfits:
print(
"All for the data is processed ... please check the DB for pre_calib"
)
spam.exit()
calibration_id = uncalibrated_uvfits[0]
project_id = uncalibrated_uvfits[1]
uvfits_file = uncalibrated_uvfits[2]
columnKeys = {"base_path", "observation_no"}
whereData = {"project_id": project_id, "cycle_id": int(cycle_id)}
project_details = dbutils.select_from_table("projectobsno", columnKeys,
whereData, 0)
base_path = project_details[1]
observation_no = project_details[0]
current_date_timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
projectobsno_update_data = {
"set": {
"status":
"processing",
"comments":
"running precalibrate_target, calibration_id = " +
str(calibration_id),
},
"where": {
"project_id": project_id
}
}
calibration_update_data = {
"set": {
"status": "copying",
"start_time": current_date_timestamp
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
dbutils.update_table(calibration_update_data, "calibrationinput")
UVFITS_FILE_NAME = uvfits_file
UVFITS_BASE_DIR = base_path
is_fits_dir = os.getcwd().split('/')
print(is_fits_dir)
SPAM_WORKING_DIR = os.getcwd()
print(SPAM_WORKING_DIR)
# for num in range(1, 3):
# SPAM_THREAD_DIR += "/" + is_fits_dir[num]
# if 'fits' not in is_fits_dir:
# SPAM_THREAD_DIR = os.getcwd()
SPAM_WORKING_DIR = os.getcwd() + "/fits/"
print(SPAM_WORKING_DIR, UVFITS_BASE_DIR, UVFITS_FILE_NAME)
UVFITS_FILE_PATH = UVFITS_BASE_DIR + "/" + UVFITS_FILE_NAME
print(UVFITS_FILE_PATH)
print(SPAM_WORKING_DIR)
fileutils.copy_files(UVFITS_FILE_PATH, SPAM_WORKING_DIR)
print("Copying done ==> Moving to pre_cal_target")
current_date_timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
calibration_update_data = {
"set": {
"status": "processing",
"start_time": current_date_timestamp
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(calibration_update_data, "calibrationinput")
fileutils.run_spam_precalibration_stage(UVFITS_BASE_DIR,
SPAM_WORKING_DIR,
UVFITS_FILE_NAME,
observation_no)
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
check_status_file = glob.glob(base_path + "/PRECALIB/failed_log.txt")
comments = "failed"
if check_status_file:
status = "failed"
comments = str(open(check_status_file[0], 'r').read())
else:
status = "success"
comments = "precalibrate_target done, calibration_id = " + str(
calibration_id)
projectobsno_update_data = {
"set": {
"status": status,
"comments": comments
},
"where": {
"project_id": project_id
}
}
calibration_update_data = {
"set": {
"status": status,
"end_time": current_date_timestamp,
"comments": comments
},
"where": {
"calibration_id": calibration_id
}
}
dbutils.update_table(projectobsno_update_data, "projectobsno")
dbutils.update_table(calibration_update_data, "calibrationinput")
def __main__():
spam.set_aips_userid(11)
hostname = socket.gethostname()
# Setting the Process Start Date Time
start_time = str(datetime.datetime.now())
# Taking system's in/out to backup variable
original_stdout = sys.stdout
original_stderr = sys.stderr
thread_dir = os.getcwd()
# Changing directory to fits/
os.chdir("fits/")
process_target_log = open('process_target.log', 'a+')
process_target_log.write('\n\n\n******PROCESS TARGET STARTED******\n\n\n')
process_target_log.write("---> Start Time " + start_time)
# Logging Compute Node Hostname
process_target_log.write("\n Node name" + hostname + "\n")
# Logging all Standard In/Output
sys.stdout = process_target_log
sys.stderr = process_target_log
datfil_dir = thread_dir + "/datfil/"
fits_dir = thread_dir + "/fits/"
curr_dir = thread_dir + "/fits/"
process_status = False
db_model = project_model.ProjectModel()
# Get random imaging_id & project_id
column_keys = [
tableSchema.imaginginputId, tableSchema.projectobsnoId,
"calibrated_fits_file"
]
where_con = {"status": "unprocessed", "comments": "cycle 19"}
to_be_processed = db_model.select_from_table("imaginginput", column_keys,
where_con, None)
imaginginput_details = random.choice(to_be_processed)
imaging_id = imaginginput_details["imaging_id"]
# Update status for imaginginput for selected imaging_id
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
update_data = {
"set": {
"status": "processing",
"start_time": current_date_timestamp,
"comments": "",
"end_time": current_date_timestamp
},
"where": {
"imaging_id": imaging_id,
}
}
db_model.update_table(update_data, "imaginginput")
project_id = imaginginput_details["project_id"]
calibrated_fits_file = imaginginput_details["calibrated_fits_file"]
# Using the above project_id, fetch base_path
column_keys = ["file_path"]
where_con = {"project_id": project_id}
base_path = db_model.select_from_table("projectobsno", column_keys,
where_con, 0)
base_path = base_path[0]
# Insert new thread to computethread
column_keys = [tableSchema.computenodeId, "threads_count"]
where_condition = {"node_name": hostname}
node_details = db_model.select_from_table("computenode", column_keys,
where_condition, 1)
node_id = node_details[0]
threads_count = node_details[1]
computethread_data = {
'pipeline_id':
1,
'node_id':
node_id,
'thread_dir':
thread_dir,
'status':
'processing',
'file_name':
calibrated_fits_file,
'comments':
"{'imaging_id': " + str(imaging_id) + ", 'project_id': " +
str(project_id) + "}"
}
thread_id = db_model.insert_into_table("computethread", computethread_data,
tableSchema.computethreadId)
# Update computenode with the above generated node_id & increment threads_count
node_update_data = {
"set": {
"threads_count": threads_count + 1,
"status": "processing"
},
"where": {
"node_id": node_id
}
}
db_model.update_table(node_update_data, "computenode")
uvfits_full_path = base_path + "/PRECALIB/" + calibrated_fits_file
#uvfits_full_path = base_path+"/PRECALIB/"+calibrated_fits_file
print "Copying " + uvfits_full_path + " to " + fits_dir
copying_fits = os.system("cp " + uvfits_full_path + " " + fits_dir)
uvfits_file = calibrated_fits_file
# Starting spam.process_target(SPLIT_FITS_FILE)
try:
spam.process_target(uvfits_file,
allow_selfcal_skip=True,
add_freq_to_name=True)
# If this process_target is success call
# GADPU API setSuccessStatus for the current fits_id
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
success_update_data = {
"set": {
"status": "checking",
"end_time": current_date_timestamp,
"comments": "processing done, checking"
},
"where": {
"imaging_id": imaging_id
}
}
db_model.update_table(success_update_data, "imaginginput")
except Exception, e:
process_target_log.write("Error: " + str(e))
# If this process_target is a failure call
# GADPU API setFailedStatus for the current fits_id
current_time_in_sec = time.time()
current_date_timestamp = datetime.datetime.fromtimestamp(
current_time_in_sec).strftime('%Y-%m-%d %H:%M:%S')
success_update_data = {
"set": {
"status": "failed",
"end_time": current_date_timestamp,
},
"where": {
"imaging_id": imaging_id
}
}
db_model.update_table(success_update_data, "imaginginput")
db_model.update_table(
{
"set": {
"status": "failed"
},
"where": {
"thread_id": thread_id,
"comments": str(e)
}
}, "computethread")
print("Error: spam.process_tagret Failed " + uvfits_file)
import os
import spam
import glob
import socket
import sys
import commands
import subprocess
import time
import random
from random import shuffle
hostname = socket.gethostname()
spam.set_aips_userid(11)
UVFITS_DATA = '/GARUDATA/IMAGING18/CYCLE18/'
PRECAL_PROCESSING = 'precalibration.processing'
PRECAL_SUCCESS = 'precalibration.success'
PRECAL_FAILED = 'precalibration.failed'
def check_haslam_flag(project):
'''
# Definition: check_haslam_flag() - takes PROPOSAL_CODE_DATE
# Returns: Boolean
# Description: This function checks for the all obslog files and
gets the is ALC set to OFF.
If th ALC at 13. in obslog if FLAGGED as OFF, then we set apply_tsys to
spam.pre_calibrate_targets(apply_tsys=False), apply_tsys is True by default
# Code: spam.pre_calibrate_targets(UVFITS_FILE_NAME, apply_tsys=False)
-----------------------------------------------------------------------------------
import spam
import time
import os, re
import glob
import trace
import sys
import time, datetime
import socket
hostname = socket.gethostname()
spam.set_aips_userid(12)
failed_files = open('datfil/failed_files.log', 'a+')
succeeded_files = open('datfil/succeeded_files.log', 'a+')
succeeded_files.write('Successful final processing\n\n')
succeeded_files.flush()
failed_files.write('Failed files\n\n')
failed_files.flush()
directory_log_pre = open('directory_log_pre.txt', 'a+')
directory_log_post = open('directory_log_post.txt', 'a+')
os.system('pwd')
def precalibrator(CURRENT_DIR):
os.chdir('fits/')
original_stdout = sys.stdout
original_stderr = sys.stderr
l2u_precal_log = open('l2u_precal.log', 'a+')
l2u_precal_log.write('\n\n\n******PRECALIBRATION STARTED******\n\n\n')
sys.stdout = l2u_precal_log
import spam
import time
import os, re
import glob
import trace
import sys
import time, datetime
import socket
hostname = socket.gethostname()
spam.set_aips_userid(10)
failed_files = open('datfil/failed_files.log', 'a+')
succeeded_files = open('datfil/succeeded_files.log', 'a+')
succeeded_files.write('Successful final processing\n\n')
succeeded_files.flush()
failed_files.write('Failed files\n\n')
failed_files.flush()
directory_log_pre = open('directory_log_pre.txt', 'a+')
directory_log_post = open('directory_log_post.txt', 'a+')
os.system('pwd')
def precalibrator(CURRENT_DIR):
os.chdir('fits/')
original_stdout = sys.stdout
original_stderr = sys.stderr
l2u_precal_log = open('l2u_precal.log', 'a+')
l2u_precal_log.write('\n\n\n******PRECALIBRATION STARTED******\n\n\n')
sys.stdout = l2u_precal_log
sys.stderr = l2u_precal_log