def run_ai(ais): """ Run aspect pipeline 'flt_run_pipe' over the aspect intervals described in the list of dictionaries passed as an argument """ ascds_env = getenv('source /home/ascds/.ascrc -r release', shell='tcsh') tcsh_shell("punlearn asp_l1_std", env=ascds_env) ocat_env = getenv( 'source /proj/sot/ska/data/aspect_authorization/set_ascds_ocat_vars.csh', shell='tcsh') for var in ['ASCDS_OCAT_UNAME', 'ASCDS_OCAT_SERVER', 'ASCDS_OCAT_PWORD']: ascds_env[var] = ocat_env[var] if opt.param is not None and len(opt.param): for param in opt.param: cmd = 'pset asp_l1_std {}'.format(param) tcsh_shell(cmd, env=ascds_env) logger_fh = FilelikeLogger(logger) for ai in ais: pipe_cmd = 'flt_run_pipe -r {root} -i {indir} -o {outdir} \ -t {pipe_ped} \ -a "INTERVAL_START"={istart} \ -a "INTERVAL_STOP"={istop} \ -a obiroot={obiroot} \ -a revision=1 '.format(**ai) if 'pipe_start_at' in ai: pipe_cmd = pipe_cmd + " -s {}".format(ai['pipe_start_at']) if 'pipe_stop_before' in ai: pipe_cmd = pipe_cmd + " -S {}".format(ai['pipe_stop_before']) if 'skip_slot' in ai: try: tcsh_shell(pipe_cmd + " -S check_star_data", env=ascds_env, logfile=logger_fh) except ShellError as sherr: # if shell error, just check to see if get_star_data completed successfully loglines = open(logger_fh.filename).read() if not re.search("get_star_data completed successfully", loglines): raise ShellError(sherr) cut_stars(ai) tcsh_shell(pipe_cmd + " -s check_star_data", env=ascds_env, logfile=logger_fh) else: logger.info('Running pipe command {}'.format(pipe_cmd)) tcsh_shell(pipe_cmd, env=ascds_env, logfile=logger_fh)
def test_env(self): envs = getenv('setenv TEST_ENV_VAR2 "hello"', shell='tcsh') assert envs['TEST_ENV_VAR2'] == 'hello' outlines = tcsh('echo $TEST_ENV_VAR2', env=envs) assert outlines == ['hello']
import os import string import re import math import unittest import time import numpy import astropy.io.fits as pyfits from datetime import datetime # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv( 'source /home/ascds/.ascrc -r release; source /home/mta/bin/reset_param ', shell='tcsh') ciaoenv = getenv('source /soft/ciao/bin/ciao.sh') # #--- reading directory list # path = '/data/aschrc6/wilton/isobe/Project8/HZ43/Scripts/house_keeping/dir_list' f = open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split(':', ent) var = atemp[1].strip()
# last update: Mar 19, 2021 # # # ################################################################################################# import sys import os import string import re import fnmatch # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release; \ source /home/mta/bin/reset_param', shell='tcsh') # #--- reading directory list # path = '/data/aschrc6/wilton/isobe/Project11/Scripts/house_keeping/dir_list' with open(path, 'r') as f: data = [line.strip() for line in f.readlines()] for ent in data: atemp = re.split(':', ent) var = atemp[1].strip() line = atemp[0].strip() exec("%s = %s" % (var, line)) # #--- append path to a privte folder
import os import string import re import math import unittest import time import numpy import astropy.io.fits as pyfits from datetime import datetime # #--- from ska # from Ska.Shell import getenv, bash #ascdsenv = getenv('source /home/ascds/.ascrc -r release; source /home/mta/bin/reset_param ', shell='tcsh') ascdsenv = getenv('source /home/ascds/.ascrc -r release ', shell='tcsh') ciaoenv = getenv('source /soft/ciao/bin/ciao.sh') # #--- reading directory list # path = '/data/aschrc6/wilton/isobe/Project8/HZ43/Scripts/house_keeping/dir_list' f = open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split(':', ent) var = atemp[1].strip()
import os import string import re import copy import math import Cookie import unittest import time import random # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release; source /home/mta/bin/reset_param ', shell='tcsh') ascdsenv['IDL_PATH'] = '+/usr/local/exelis/user_contrib/astron_Oct17/pro:+/home/mta/IDL:/home/nadams/pros:+/data/swolk/idl_libs:/home/mta/IDL/tara:widget_tools:utilities:event_browser' ascdsenv2 = getenv('source /proj/sot/ska/bin/ska_envs.csh', shell='tcsh') ascdsenv2['IDL_PATH'] = '+/usr/local/exelis/user_contrib/astron_Oct17/pro:+/home/mta/IDL:/home/nadams/pros:+/data/swolk/idl_libs:/home/mta/IDL/tara:widget_tools:utilities:event_browser' # #--- reading directory list # path = '/data/mta/Script/Python_script2.7/dir_list_py' f = open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data:
import os import string import re import copy import math import Cookie import unittest import time import random # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release; source /home/mta/bin/reset_param ', shell='tcsh') ascdsenv['IDL_PATH'] = '+/usr/local/rsi/user_contrib/astron_Oct09/pro:+/home/mta/IDL:/home/nadams/pros:+/data/swolk/idl_libs:/home/mta/IDL/tara:widget_tools:utilities:event_browser' ascdsenv2 = getenv('source /proj/sot/ska/bin/ska_envs.csh', shell='tcsh') ascdsenv2['IDL_PATH'] = '+/usr/local/rsi/user_contrib/astron_Oct09/pro:+/home/mta/IDL:/home/nadams/pros:+/data/swolk/idl_libs:/home/mta/IDL/tara:widget_tools:utilities:event_browser' # #--- reading directory list # path = '/data/mta/Script/Python_script2.7/dir_list_py' f = open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split(':', ent)
def test_ciao(self): envs = getenv('. /soft/ciao/bin/ciao.sh') test_script = ['printenv {}'.format(name) for name in sorted(envs)] outlines = bash('\n'.join(test_script), env=envs) assert outlines == [envs[name] for name in sorted(envs)]
from astropy.io import fits from Ska.Numpy import smooth, interpolate from Ska.Shell import bash, tcsh_shell, getenv from Ska.DBI import DBI from Ska.engarchive import fetch import Quaternion import Ska.quatutil REDO = False MTIME = 1457707041.222744 sqlaca = DBI(dbi='sybase', user='******') #XRAY_DATA = '/data/aca/archive/xray_for_periscope' projdir = '/proj/sot/ska/analysis/periscope_tilt_2016' XRAY_DATA = os.path.join(projdir, 'auto') ciao_env = getenv("source /soft/ciao/bin/ciao.csh", shell='tcsh') def get_on_axis_bright(srctable, x_center, y_center, limit=180): """ Given source table from celldetect and the X/Y center for the observation return a reduced source list with just the brightest source within radius limit :param srctable: table of sources containing at least X, Y, and NET_COUNTS for each source :param x_center: rough aimpoint X center from evt2 file :param y_center: rough aimpoint Y center from evt2 file :param limit: radial limit in pixels :returns: reduced source table containing just brightest source within radius limit """ dist = np.sqrt( ((srctable['X'] - x_center)**2 + (srctable['Y'] - y_center)**2))
def test_ciao(self): envs = getenv('source /soft/ciao/bin/ciao.csh', shell='tcsh') test_script = ['printenv {}'.format(name) for name in sorted(envs)] outlines = tcsh('\n'.join(test_script), env=envs) assert outlines == [envs[name] for name in sorted(envs)]
def test_env(self): envs = getenv('export TEST_ENV_VARA="hello"') assert envs['TEST_ENV_VARA'] == 'hello' outlines = bash('echo $TEST_ENV_VARA', env=envs) assert outlines == ['hello']
def test_ascds(self): envs = getenv('source /home/ascds/.ascrc -r release', shell='tcsh') test_script = ['printenv {}'.format(name) for name in sorted(envs)] outlines = tcsh('\n'.join(test_script), env=envs) assert outlines == [envs[name] for name in sorted(envs)]
import Quaternion import Ska.quatutil REDO = False MTIME = 1457707041.222744 sqlaca = DBI(dbi='sybase', user='******') #XRAY_DATA = '/data/aca/archive/xray_for_periscope' projdir = '/proj/sot/ska/analysis/periscope_tilt_2016' XRAY_DATA = os.path.join(projdir, 'auto') ciao_env = getenv("source /soft/ciao/bin/ciao.csh", shell='tcsh') def get_on_axis_bright(srctable, x_center, y_center, limit=180): """ Given source table from celldetect and the X/Y center for the observation return a reduced source list with just the brightest source within radius limit :param srctable: table of sources containing at least X, Y, and NET_COUNTS for each source :param x_center: rough aimpoint X center from evt2 file :param y_center: rough aimpoint Y center from evt2 file :param limit: radial limit in pixels :returns: reduced source table containing just brightest source within radius limit """ dist = np.sqrt(((srctable['X'] - x_center)**2 + (srctable['Y'] - y_center)**2))
import os import string import re import numpy import getopt import time import Chandra.Time import Ska.engarchive.fetch as fetch import random import unittest # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release; setenv ACISTOOLSDIR /home/pgf', shell='tcsh') # #--- reading directory list # path = '/data/mta/Script/ACIS/Focal/Script/house_keeping/dir_list' with open(path, 'r') as f: data = [line.strip() for line in f.readlines()] for ent in data: atemp = re.split(':', ent) var = atemp[1].strip() line = atemp[0].strip() exec("%s = %s" %(var, line)) # #--- append path to a private folder
import os import string import re import math import time import Chandra.Time import astropy.io.fits as pyfits # #--- from ska # from Ska.Shell import getenv, bash # #--- set ciao environment # ciaoenv = getenv('source /soft/ciao/bin/ciao.csh; \ source /home/mta/bin/reset_param; setenv PFILES "${PDIRS}";\ set path=(/soft/ciao/bin/ $path);', shell='tcsh') # #--- reading directory list # path = '/data/aschrc6/wilton/isobe/Project9/Scripts/Hrc_S/house_keeping/dir_list' #path = '/data/aschrc6/wilton/isobe/Project9/Scripts/Hrc_I/house_keeping/dir_list' with open(path, 'r') as f: data = [line.strip() for line in f.readlines()] for ent in data: atemp = re.split(':', ent) var = atemp[1].strip() line = atemp[0].strip() exec("%s = %s" % (var, line))
# # ##################################################################################### import os import sys import re import string import time import numpy import astropy.io.fits as pyfits import Ska.engarchive.fetch as fetch import Chandra.Time import datetime from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release', shell='tcsh') # #--- reading directory list # path = '/data/mta/Script/MTA_limit_trends/Scripts/house_keeping/dir_list' f = open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split(':', ent) var = atemp[1].strip() line = atemp[0].strip() exec "%s = %s" % (var, line) # #--- append path to a private folder
def run_ai(ais): """ Run aspect pipeline 'flt_run_pipe' over the aspect intervals described in the list of dictionaries passed as an argument """ ascds_env = getenv('source /home/ascds/.ascrc -r release', shell='tcsh') ocat_env = getenv( 'source /proj/sot/ska/data/aspect_authorization/set_ascds_ocat_vars.csh', shell='tcsh') for var in ['ASCDS_OCAT_UNAME', 'ASCDS_OCAT_SERVER', 'ASCDS_OCAT_PWORD']: ascds_env[var] = ocat_env[var] logger_fh = FilelikeLogger(logger) loglines = tcsh_shell("punlearn asp_l1_std", env=ascds_env, logfile=logger_fh) if opt.fdc_file is not None: tcsh_shell("pset asp_l1_std fdc='{}'".format(opt.fdc_file), env=ascds_env, logfile=logger_fh) for ai in ais: pipe_cmd = 'flt_run_pipe -r {root} -i {indir} -o {outdir} \ -t {pipe_ped} \ -a "INTERVAL_START"={istart} \ -a "INTERVAL_STOP"={istop} \ -a obiroot={obiroot} \ -a revision=1 '.format(**ai) start_pipe = PIPES[0] stop_pipe = None if 'pipe_start_at' in ai: if ai['pipe_start_at'] not in PIPES: raise ValueError(f"{ai['pipe_start_at']} not in PIPES list") start_pipe = ai['pipe_start_at'] if 'pipe_stop_before' in ai: if ai['pipe_stop_before'] not in PIPES: raise ValueError(f"{ai['pipe_stop_before']} not in PIPES list") stop_pipe = ai['pipe_stop_before'] # if the options start after or end before the stage to remove stars, # just do what is asked if (PIPES.index(start_pipe) > PIPES.index('check_star_data') or ((stop_pipe is not None) and (PIPES.index(stop_pipe) < PIPES.index('check_star_data')))): pipe_cmd = pipe_cmd + f' -s {start_pipe} ' if stop_pipe is not None: pipe_cmd = pipe_cmd + f' -S {stop_pipe} ' logger.info('Running pipe command {}'.format(pipe_cmd)) tcsh_shell(pipe_cmd, env=ascds_env, logfile=logger_fh) else: first_pipe = pipe_cmd + \ f' -s {start_pipe} ' + " -S check_star_data" logger.info('Running pipe command {}'.format(first_pipe)) tcsh_shell(first_pipe, env=ascds_env, logfile=logger_fh) star_files = glob(os.path.join(ai['outdir'], "*stars.txt")) if not len(star_files) == 1: logger.info( "Missing stars.txt, mocking one up from mica starcheck database" ) mock_stars_file(opt, ai) if 'skip_slot' in ai: logger.info("Cutting star as requested") cut_stars(ai) second_pipe = pipe_cmd + f' -s check_star_data ' if stop_pipe is not None: second_pipe = second_pipe + f' -S {stop_pipe}' logger.info('Running pipe command {}'.format(second_pipe)) tcsh_shell(second_pipe, env=ascds_env, logfile=logger_fh)
import string import math import numpy import unittest import time import pyfits import unittest from datetime import datetime from time import gmtime, strftime, localtime import Chandra.Time import Ska.engarchive.fetch as fetch # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release; punlearn dataseeker ', shell='tcsh') # #--- plotting routine # import matplotlib as mpl if __name__ == '__main__': mpl.use('Agg') from pylab import * import matplotlib.pyplot as plt import matplotlib.font_manager as font_manager import matplotlib.lines as lines # #--- reading directory list
import sys import re import string import random import operator import math import numpy import smtplib from ftplib import FTP import unittest # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release', shell='tcsh') # #--- reading directory list # path = '/data/mta/Script/MTA_Rad/house_keeping/dir_list_py' f= open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() f= open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split(':', ent) var = atemp[1].strip()
import re import string import random import math import astropy.io.fits as pyfits import os.path import sqlite3 import unittest import time from datetime import datetime from time import gmtime, strftime, localtime # #--- from ska # from Ska.Shell import getenv, bash ascdsenv = getenv('source /home/ascds/.ascrc -r release; source /home/mta/bin/reset_param', shell='tcsh') # #--- reading directory list # path = '/data/mta/Script/Envelope_trending/house_keeping/dir_list' f = open(path, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split(':', ent) var = atemp[1].strip() line = atemp[0].strip() exec "%s = %s" %(var, line) # #--- append path to a private folder
import sys import os import string import re import math import time import astropy.io.fits as pyfits # #--- from ska # from Ska.Shell import getenv, bash # #--- set ciao environment # ciaoenv = getenv('source /soft/ciao/bin/ciao.csh -o', shell='tcsh') # #--- reading directory list # path = '/data/aschrc6/wilton/isobe/Project9/Scripts/LETG/house_keeping/dir_list' with open(path, 'r') as f: data = [line.strip() for line in f.readlines()] for ent in data: atemp = re.split(':', ent) var = atemp[1].strip() line = atemp[0].strip() exec("%s = %s" % (var, line)) # #--- append path to a private folders