def test_astrometric_singleton(self, dataset_names): """ Tests pipeline-style processing of a singleton exposure using runastrodriz. """ # Get sample data through astroquery flcfile = aqutils.retrieve_observation(dataset_names, suffix=['FLC'])[0] fltfile = aqutils.retrieve_observation(dataset_names, suffix=['FLT'])[0] rawfile = aqutils.retrieve_observation(dataset_names, suffix=['RAW'])[0] # Retrieve reference files for these as well self.get_input_file('', fltfile, docopy=False) # Insure environment variables are set for full processing os.environ['ASTROMETRY_STEP_CONTROL'] = 'on' os.environ['ASTROMETRY_COMPUTE_APOSTERIORI'] = 'on' os.environ['ASTROMETRY_APPLY_APRIORI'] = 'on' # Run pipeline processing using runastrodriz.process(rawfile, force=True, inmemory=True) # compare WCSNAMEs from flt and flc files flc_wcsname = fits.getval(flcfile, 'wcsname', ext=1) flt_wcsname = fits.getval(fltfile, 'wcsname', ext=1) # Perform comparisons: # - WCSNAME values should contain '-' from either a priori or a posteriori solution # - WCSNAME value should be the same for FLT and FLC images assert ('-' in flc_wcsname) assert ('-' in flt_wcsname) assert (flc_wcsname == flt_wcsname)
def test_alignpipe_randomlist(tmpdir, dataset): """ Tests which validate whether mosaics can be aligned to an astrometric standard. Characteristics of these tests: * A reference WCS is generated based upon all the input images for the field. * A source astrometric catalog is created using the Photutils package to detect explicitly sources in the images. * An astrometric catalog is created to extract astrometric positions for the found sources in the input images' field-of-view using GAIADR2 (preferred) or GAIADR1. * Cross matching/fitting is done between found sources and catalog coordinates with the Tweakwcs package. * The quality of the fit is evaluated against a minimum threshold and potentially another fit algorithm is invoked or an alternative catalog is used in an effort to obtain a better quality fit. * If the option is set, the WCS information is updated for the input exposures. The default is False. * No mosaic is generated. * An output table containing characterizations of the process and associated fit is generated. Success Criteria: * Success criterion hard-coded for this test represents whether a statistical sample (70%) of ACS and WFC3 datasets were able to be aligned to within 10mas RMS. * RMS values are extracted from the table output from `perform_align` * This criterion will need to be determined by the user after the test has been run based on how many datasets were run and skipped. The input master_list CSV file is output from a database and lists associations and singletons for ACS and WFC3 instruments randomly sorted. The actual data files are downloaded from MAST via astroquery. This test file can be executed in the following manner: $ pytest -n # -s --basetemp=/internal/hladata/yourUniqueDirectoryHere --bigdata --slow --master_list ACSWFC3ListDefault50.csv --start_row 0 --num_rows 50 test_pipe_randomlist.py >& test_pipe_random_output.txt & $ tail -f test_pipe_random_output.txt * The `-n #` option can be used to run tests in parallel if `pytest-xdist` has been installed where `#` is the number of cpus to use. * Note: When running this test, the `--basetemp` directory should be set to a unique existing directory to avoid deleting previous test output. * The default master list exists in the tests/hla directory and contains 50 datasets. The full master list of thousands of datasets resides in Artifactory as ACSWFC3List.csv (https://bytesalad.stsci.edu/artifactory/hst-hla-pipeline/dev/master_lists). """ # Start by resetting the logging for all the modules used rl = logging.getLogger('stwcs.wcsutil.headerlet') if len(rl.handlers) > 1: del rl.handlers[-1] print("TEST_RANDOM_ALIGN. Dataset: ", dataset) output_name = dataset + '.ecsv' current_dt = datetime.datetime.now() print(str(current_dt)) subdir = "" prevdir = os.getcwd() # create working directory specified for the test if not tmpdir.ensure(subdir, dir=True): curdir = tmpdir.mkdir(subdir).strpath else: curdir = tmpdir.join(subdir).strpath os.chdir(curdir) return_value = 1 try: # The dataset name is an ipppssoot only. dataset = dataset.lower() # Download the data necessary for processing - The dataset list contains only the ipppssoot of a # singleton or an ASN. Need to get the FLT/FLC files as runastrodriz expects them to be present. input_parameter = "RAW" filename_for_runastrodriz = dataset.lower() + "_raw.fits" if dataset.endswith("0"): input_parameter = "ASN" filename_for_runastrodriz = dataset.lower() + "_asn.fits" log.info("Input parameter: {}".format(input_parameter)) files_on_disk = check_disk_get_data([dataset], suffix=input_parameter, archive=False, clobber=False) log.info("Dataset: {}".format(dataset)) if os.path.exists('mastDownload'): shutil.rmtree('mastDownload') log.info("\nFiles: {}".format(files_on_disk)) # Insure environment variables are set for full processing os.environ['ASTROMETRY_STEP_CONTROL'] = 'on' os.environ['ASTROMETRY_COMPUTE_APOSTERIORI'] = 'on' os.environ['ASTROMETRY_APPLY_APRIORI'] = 'on' flts = sorted(glob.glob('*fl?.fits')) jref_dir = 'jref.old/' if '16r12191j_mdz' in fits.getval(flts[0], 'mdriztab') else 'jref/' os.environ['jref'] = os.path.join(os.environ['crrefer'], jref_dir) log.info("JREF: {}".format(os.environ['jref'])) # Run pipeline processing using # runastrodriz accepts *_raw.fits or *_asn.fits, but it assumes the *_fl[t|c].fits files # are also present runastrodriz.process(filename_for_runastrodriz, force=True) return_value = 0 # Catch anything that happens as this dataset will be considered a failure, but # the processing of datasets should continue. This is meant to catch # unexpected errors and generate sufficient output exception # information so algorithmic problems can be addressed. except Exception as except_details: traceback.print_exc() pytest.fail("TEST_RANDOM. Exception Dataset: {}\n", dataset) return_value = 1 """ finally: # Perform some clean up if os.path.isfile('ref_cat.ecsv'): os.remove('ref_cat.ecsv') if os.path.isfile('refcatalog.cat'): os.remove('refcatalog.cat') for filename in os.listdir(): if filename.endswith('flt.fits') or filename.endswith('flc.fits'): os.remove(filename) """ assert return_value == 0 # Return to original directory os.chdir(prevdir)