def test_log_to_file(tmpdir, level): local_path = tmpdir.join('test.log') log_file = local_path.open('wb') log_path = str(local_path.realpath()) orig_level = log.level try: if level is not None: log.setLevel(level) with log.log_to_file(log_path): log.error("Error message") log.warning("Warning message") log.info("Information message") log.debug("Debug message") log_file.close() finally: log.setLevel(orig_level) log_file = local_path.open('rb') log_entries = log_file.readlines() log_file.close() if level is None: # The log level *should* be set to whatever it was in the config level = conf.log_level # Check list length if level == 'DEBUG': assert len(log_entries) == 4 elif level == 'INFO': assert len(log_entries) == 3 elif level == 'WARN': assert len(log_entries) == 2 elif level == 'ERROR': assert len(log_entries) == 1 # Check list content assert eval(log_entries[0].strip())[-3:] == ('astropy.tests.test_logger', 'ERROR', 'Error message') if len(log_entries) >= 2: assert eval( log_entries[1].strip())[-3:] == ('astropy.tests.test_logger', 'WARNING', 'Warning message') if len(log_entries) >= 3: assert eval( log_entries[2].strip())[-3:] == ('astropy.tests.test_logger', 'INFO', 'Information message') if len(log_entries) >= 4: assert eval( log_entries[3].strip())[-3:] == ('astropy.tests.test_logger', 'DEBUG', 'Debug message')
def concatenate_one(strip, logfile = os.path.join(constants.LOGDIR, 'concatenation.log')): with log.log_to_file(logfile): # Strips are defined by the start longitude log.info('Concatenating L={0}'.format(strip)) for mode in ['light', 'full']: for part in ['a', 'b']: concat = Concatenator(strip, part, mode) concat.run() return strip
def test_log_to_file(tmpdir, level): local_path = tmpdir.join('test.log') log_file = local_path.open('wb') log_path = str(local_path.realpath()) orig_level = log.level try: if level is not None: log.setLevel(level) with log.log_to_file(log_path): log.error("Error message") log.warning("Warning message") log.info("Information message") log.debug("Debug message") log_file.close() finally: log.setLevel(orig_level) log_file = local_path.open('rb') log_entries = log_file.readlines() log_file.close() if level is None: # The log level *should* be set to whatever it was in the config level = conf.log_level # Check list length if level == 'DEBUG': assert len(log_entries) == 4 elif level == 'INFO': assert len(log_entries) == 3 elif level == 'WARN': assert len(log_entries) == 2 elif level == 'ERROR': assert len(log_entries) == 1 # Check list content assert eval(log_entries[0].strip())[-3:] == ( 'astropy.tests.test_logger', 'ERROR', 'Error message') if len(log_entries) >= 2: assert eval(log_entries[1].strip())[-3:] == ( 'astropy.tests.test_logger', 'WARNING', 'Warning message') if len(log_entries) >= 3: assert eval(log_entries[2].strip())[-3:] == ( 'astropy.tests.test_logger', 'INFO', 'Information message') if len(log_entries) >= 4: assert eval(log_entries[3].strip())[-3:] == ( 'astropy.tests.test_logger', 'DEBUG', 'Debug message')
def calibrate_one(filename): """Applies the re-calibration to a single bandmerged field catalogue.""" with log.log_to_file(os.path.join(constants.LOGDIR, 'apply_calibration.log')): try: ca = CalibrationApplicator() ca.run(filename) except Exception, e: log.error('%s: *UNEXPECTED EXCEPTION*: calibrate_one: %s' % (filename, e)) return filename
def prepare_one(run): with log.log_to_file(os.path.join(constants.LOGDIR, 'images.log')): result = [] for ccd in constants.EXTENSIONS: try: img = SurveyImage(run, ccd) img.save() result.append(img.get_metadata()) img.fits_orig.close() # avoid memory leak except Exception, e: log.error(str(run)+': '+util.get_pid()+': '+str(e)) return result
def seam_one(strip, logfile = os.path.join(constants.LOGDIR, 'seaming.log')): with log.log_to_file(logfile): """Seams the fields in a given longitude strip.""" # Strips are defined by the start longitude of a 10 deg-wide strip assert(strip in np.arange(25, 215+1, constants.STRIPWIDTH)) log.info('{0}: strip{1}: START'.format(str(datetime.datetime.now())[0:19], strip)) # Intialize caching dictionary CACHE[strip] = {} # Which are our boundaries? # Note: we must allow FIELD_MAXDIST for border overlaps! lon1 = strip - constants.FIELD_MAXDIST lon2 = strip + constants.STRIPWIDTH + constants.FIELD_MAXDIST cond_strip = (constants.IPHASQC_COND_RELEASE & (IPHASQC['l'] >= lon1) & (IPHASQC['l'] < lon2)) n_fields = cond_strip.sum() # How many fields are in our strip? n_processed = 0 # Seam fields; do the best-seeing fields first! for idx in np.argsort(IPHASQC['seeing_max']): if cond_strip[idx]: n_processed += 1 log.info('{0}: strip{1}: {2}/{3}: seaming {4}'.format( str(datetime.datetime.now())[0:19], strip, n_processed, n_fields, IPHASQC['id'][idx])) log.info('{0}: strip{1}: cached {2:.2f} GB'.format( str(datetime.datetime.now())[0:19], strip, sys.getsizeof(CACHE[strip])/(1024**3))) try: s = SeamMachine(IPHASQC['id'][idx], IPHASQC['ra'][idx], IPHASQC['dec'][idx], strip) s.run() except SeamingException, e: log.error(str(e)) except Exception, e: pid = socket.gethostname()+'/'+str(os.getpid()) log.error('%s strip %s: %s: *UNEXPECTED EXCEPTION*: %s' % ( pid, strip, IPHASQC['id'][idx], e))
def test_log_to_file_origin2(tmpdir): local_path = tmpdir.join('test.log') log_file = local_path.open('wb') log_path = str(local_path.realpath()) with log.log_to_file(log_path, filter_origin='astropy.wcs'): log.error("Error message") log.warning("Warning message") log_file.close() log_file = local_path.open('rb') log_entries = log_file.readlines() log_file.close() assert len(log_entries) == 0
def test_log_to_file_level(tmpdir): local_path = tmpdir.join('test.log') log_file = local_path.open('wb') log_path = str(local_path.realpath()) with log.log_to_file(log_path, filter_level='ERROR'): log.error("Error message") log.warning("Warning message") log_file.close() log_file = local_path.open('rb') log_entries = log_file.readlines() log_file.close() assert len(log_entries) == 1 assert eval(log_entries[0].strip())[-2:] == ('ERROR', 'Error message')
def test_log_to_file_encoding(tmpdir, encoding): local_path = tmpdir.join('test.log') log_path = str(local_path.realpath()) orig_encoding = conf.log_file_encoding conf.log_file_encoding = encoding with log.log_to_file(log_path): for handler in log.handlers: if isinstance(handler, logging.FileHandler): if encoding: assert handler.stream.encoding == encoding else: assert handler.stream.encoding == locale.getpreferredencoding() conf.log_file_encoding = orig_encoding
def test_log_to_file_level(tmpdir): local_path = tmpdir.join('test.log') log_file = local_path.open('wb') log_path = str(local_path.realpath()) with log.log_to_file(log_path, filter_level='ERROR'): log.error("Error message") log.warning("Warning message") log_file.close() log_file = local_path.open('rb') log_entries = log_file.readlines() log_file.close() assert len(log_entries) == 1 assert eval(log_entries[0].strip())[-2:] == ( 'ERROR', 'Error message')
def bandmerge_one(fieldid): """Band-merge a single field """ with log.log_to_file(os.path.join(constants.LOGDIR, 'bandmerging.log')): engine = socket.gethostname()+'/'+str(os.getpid()) log.info('Starting {0} on {1}'.format(fieldid, engine)) # Which index does the field have in the QC table? idx = np.where(IPHASQC.field('id') == fieldid)[0] if len(idx) < 1: raise IPHASException('{}: error identifying runs'.format(fieldid)) # Carry out the band-merging bm = BandMerge(fieldid, IPHASQC.field('qflag')[idx[0]], IPHASQC.field('run_ha')[idx[0]], IPHASQC.field('run_r')[idx[0]], IPHASQC.field('run_i')[idx[0]]) status = bm.run() log.info('Finished {0} on {1} (returned {2})'.format(fieldid, engine, status)) return status
def offsets_one(run): """Returns all offsets for a given reference exposure. Parameters ---------- run : integer or string Telescope run number. Returns ------- offsets : list of dictionaries A sequence of dictionaries for each overlapping run. Each dictionary contains the fields run1/run2/offset/std/n. """ with log.log_to_file(os.path.join(constants.LOGDIR, 'offsets.log')): try: log.info('{0}: Computing offsets for {1}'.format(util.get_pid(), run)) om = OffsetMachine(run) return om.relative_offsets() except Exception, e: log.error('{0}: UNEXPECTED EXCEPTION FOR RUN {1}: {2}'.format(util.get_pid(), run, e)) return [None]
def hub_script(timeout=0): """ This main function is executed by the ``samp_hub`` command line tool. """ parser = argparse.ArgumentParser(prog="samp_hub " + __version__) parser.add_argument("-k", "--secret", dest="secret", metavar="CODE", help="custom secret code.") parser.add_argument("-d", "--addr", dest="addr", metavar="ADDR", help="listening address (or IP).") parser.add_argument("-p", "--port", dest="port", metavar="PORT", type=int, help="listening port number.") parser.add_argument("-f", "--lockfile", dest="lockfile", metavar="FILE", help="custom lockfile.") parser.add_argument("-w", "--no-web-profile", dest="web_profile", action="store_false", help="run the Hub disabling the Web Profile.", default=True) parser.add_argument("-P", "--pool-size", dest="pool_size", metavar="SIZE", type=int, help="the socket connections pool size.", default=20) timeout_group = parser.add_argument_group("Timeout group", "Special options to setup hub and client timeouts." "It contains a set of special options that allows to set up the Hub and " "clients inactivity timeouts, that is the Hub or client inactivity time " "interval after which the Hub shuts down or unregisters the client. " "Notification of samp.hub.disconnect MType is sent to the clients " "forcibly unregistered for timeout expiration.") timeout_group.add_argument("-t", "--timeout", dest="timeout", metavar="SECONDS", help="set the Hub inactivity timeout in SECONDS. By default it " "is set to 0, that is the Hub never expires.", type=int, default=0) timeout_group.add_argument("-c", "--client-timeout", dest="client_timeout", metavar="SECONDS", help="set the client inactivity timeout in SECONDS. By default it " "is set to 0, that is the client never expires.", type=int, default=0) parser.add_argument_group(timeout_group) log_group = parser.add_argument_group("Logging options", "Additional options which allow to customize the logging output. By " "default the SAMP Hub uses the standard output and standard error " "devices to print out INFO level logging messages. Using the options " "here below it is possible to modify the logging level and also " "specify the output files where redirect the logging messages.") log_group.add_argument("-L", "--log-level", dest="loglevel", metavar="LEVEL", help="set the Hub instance log level (OFF, ERROR, WARNING, INFO, DEBUG).", type=str, choices=["OFF", "ERROR", "WARNING", "INFO", "DEBUG"], default='INFO') log_group.add_argument("-O", "--log-output", dest="logout", metavar="FILE", help="set the output file for the log messages.", default="") parser.add_argument_group(log_group) adv_group = parser.add_argument_group("Advanced group", "Advanced options addressed to facilitate administrative tasks and " "allow new non-standard Hub behaviors. In particular the --label " "options is used to assign a value to hub.label token and is used to " "assign a name to the Hub instance. " "The very special --multi option allows to start a Hub in multi-instance mode. " "Multi-instance mode is a non-standard Hub behavior that enables " "multiple contemporaneous running Hubs. Multi-instance hubs place " "their non-standard lock-files within the <home directory>/.samp-1 " "directory naming them making use of the format: " "samp-hub-<PID>-<ID>, where PID is the Hub process ID while ID is an " "internal ID (integer).") adv_group.add_argument("-l", "--label", dest="label", metavar="LABEL", help="assign a LABEL to the Hub.", default="") adv_group.add_argument("-m", "--multi", dest="mode", help="run the Hub in multi-instance mode generating a custom " "lockfile with a random name.", action="store_const", const='multiple', default='single') parser.add_argument_group(adv_group) options = parser.parse_args() try: if options.loglevel in ("OFF", "ERROR", "WARNING", "DEBUG", "INFO"): log.setLevel(options.loglevel) if options.logout != "": context = log.log_to_file(options.logout) else: class dummy_context: def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): pass context = dummy_context() with context: args = copy.deepcopy(options.__dict__) del(args["loglevel"]) del(args["logout"]) hub = SAMPHubServer(**args) hub.start(False) if not timeout: while hub.is_running: time.sleep(0.01) else: time.sleep(timeout) hub.stop() except KeyboardInterrupt: try: hub.stop() except NameError: pass except OSError as e: print("[SAMP] Error: I/O error({0}): {1}".format(e.errno, e.strerror)) sys.exit(1) except SystemExit: pass
from astropy import log from datetime import datetime import time log.setLevel('INFO') t0 = time.time() with log.log_to_file("all_analysis_{0}.log".format(datetime.now().isoformat())): log.info("Starting make_apex_cubes postprocessing") execfile("make_apex_cubes.py") do_postprocessing() extract_co_subcubes(mergepath=mergepath) log.info("Creating pyradex grid. dt={0}".format(time.time()-t0)) execfile("pyradex_h2comm_grid.py") log.info("Redo dendro. dt={0}".format(time.time()-t0)) execfile("redo_dendro.py") # execfile("dendro_mask.py") # make_dend_303() # execfile("dendro_temperature.py") # do_dendro_temperatures_both() # execfile("dendrotem_plots.py") # execfile("make_piecewise_temcube.py") log.info("Make ratiotem cubesims. dt={0}".format(time.time()-t0)) execfile("make_ratiotem_cubesims.py") log.info("Make ratiotem integ. dt={0}".format(time.time()-t0)) execfile("make_ratio_integ.py") make_ratio_integ() make_ratio_max()
from meteorflux import db ####### # MAIN ####### if __name__ == '__main__': # Speed-ups autocommit = False remove_old = True # Check if we have an argument if len(sys.argv) < 2: raise Exception("Please supply the name of a ZIP file.") path = sys.argv[1] #log.setLevel('DEBUG') with log.log_to_file('ingestion.log'): mydb = db.FluxDB(autocommit=autocommit) if os.path.isdir(path): log.info("%s is a directory, will ingest all *.zip files inside." % path) metrec.ingest_dir(path, mydb, remove_old) else: myzip = metrec.ingest_zip(path, mydb, remove_old) mydb.commit()
def main(args=None): global ang_size global image_directory global main_reference_image global fwhm_input global do_conversion global do_registration global do_convolution global do_resampling global do_seds global do_cleanup global kernel_directory global im_pixsc global rot_angle ang_size = '' image_directory = '' main_reference_image = '' fwhm_input = '' do_conversion = False do_registration = False do_convolution = False do_resampling = False do_seds = False do_cleanup = False kernel_directory = '' im_pixsc = '' # note start time for log start_time = datetime.now() # parse arguments if args !=None: arglist = string.split(args) else: arglist = sys.argv[1:] parse_status = parse_command_line(arglist) if parse_status > 0: if __name__ == '__main__': sys.exit() else: return if (do_cleanup): # cleanup and exit cleanup_output_files() if __name__ == '__main__': sys.exit() else: return # Lists to store information global image_data global converted_data global registered_data global convolved_data global resampled_data global headers global filenames image_data = [] converted_data = [] registered_data = [] convolved_data = [] resampled_data = [] headers = [] filenames = [] # if not just cleaning up, make a log file which records input parameters logfile_name = 'imagecube_'+ start_time.strftime('%Y-%m-%d_%H%M%S') + '.log' with log.log_to_file(logfile_name,filter_origin='imagecube.imagecube'): log.info('imagecube started at %s' % start_time.strftime('%Y-%m-%d_%H%M%S')) log.info('imagecube called with arguments %s' % arglist) # Grab all of the .fits and .fit files in the specified directory all_files = glob.glob(image_directory + "/*.fit*") # no use doing anything if there aren't any files! if len(all_files) == 0: warnings.warn('No fits files found in directory %s' % image_directory, AstropyUserWarning ) if __name__ == '__main__': sys.exit() else: return # get images for (i,fitsfile) in enumerate(all_files): hdulist = fits.open(fitsfile) img_extens = find_image_planes(hdulist) # NOTETOSELF: right now we are just using the *first* image extension in a file # which is not what we want to do, ultimately. header = hdulist[img_extens[0]].header image = hdulist[img_extens[0]].data # Strip the .fit or .fits extension from the filename so we can append # things to it later on filename = os.path.splitext(hdulist.filename())[0] hdulist.close() # check to see if image has reasonable scale & orientation # NOTETOSELF: should this really be here? It's not relevant for just flux conversion. # want separate loop over image planes, after finishing file loop pixelscale = get_pixel_scale(header) fov = pixelscale * float(header['NAXIS1']) log.info("Checking %s: is pixel scale (%.2f\") < ang_size (%.2f\") < FOV (%.2f\") ?"% (fitsfile,pixelscale, ang_size,fov)) if (pixelscale < ang_size < fov): try: wavelength = header['WAVELNTH'] header['WAVELNTH'] = (wavelength, 'micron') # add the unit if it's not already there image_data.append(image) headers.append(header) filenames.append(filename) except KeyError: warnings.warn('Image %s has no WAVELNTH keyword, will not be used' % filename, AstropyUserWarning) else: warnings.warn("Image %s does not meet the above criteria." % filename, AstropyUserWarning) # end of loop over files # Sort the lists by their WAVELNTH value images_with_headers_unsorted = zip(image_data, headers, filenames) images_with_headers = sorted(images_with_headers_unsorted, key=lambda header: header[1]['WAVELNTH']) if (do_conversion): convert_images(images_with_headers) if (do_registration): register_images(images_with_headers) if (do_convolution): convolve_images(images_with_headers) if (do_resampling): resample_images(images_with_headers, logfile_name) if (do_seds): output_seds(images_with_headers) # all done! log.info('All tasks completed.') if __name__ == '__main__': sys.exit() else: return
def hub_script(timeout=0): """ This main function is executed by the ``samp_hub`` command line tool. """ parser = argparse.ArgumentParser(prog="samp_hub " + __version__) parser.add_argument("-k", "--secret", dest="secret", metavar="CODE", help="custom secret code.") parser.add_argument("-d", "--addr", dest="addr", metavar="ADDR", help="listening address (or IP).") parser.add_argument("-p", "--port", dest="port", metavar="PORT", type=int, help="listening port number.") parser.add_argument("-f", "--lockfile", dest="lockfile", metavar="FILE", help="custom lockfile.") parser.add_argument("-w", "--no-web-profile", dest="web_profile", action="store_false", help="run the Hub disabling the Web Profile.", default=True) parser.add_argument("-P", "--pool-size", dest="pool_size", metavar="SIZE", type=int, help="the socket connections pool size.", default=20) timeout_group = parser.add_argument_group( "Timeout group", "Special options to setup hub and client timeouts." "It contains a set of special options that allows to set up the Hub and " "clients inactivity timeouts, that is the Hub or client inactivity time " "interval after which the Hub shuts down or unregisters the client. " "Notification of samp.hub.disconnect MType is sent to the clients " "forcibly unregistered for timeout expiration.") timeout_group.add_argument( "-t", "--timeout", dest="timeout", metavar="SECONDS", help="set the Hub inactivity timeout in SECONDS. By default it " "is set to 0, that is the Hub never expires.", type=int, default=0) timeout_group.add_argument( "-c", "--client-timeout", dest="client_timeout", metavar="SECONDS", help="set the client inactivity timeout in SECONDS. By default it " "is set to 0, that is the client never expires.", type=int, default=0) parser.add_argument_group(timeout_group) log_group = parser.add_argument_group( "Logging options", "Additional options which allow to customize the logging output. By " "default the SAMP Hub uses the standard output and standard error " "devices to print out INFO level logging messages. Using the options " "here below it is possible to modify the logging level and also " "specify the output files where redirect the logging messages.") log_group.add_argument( "-L", "--log-level", dest="loglevel", metavar="LEVEL", help= "set the Hub instance log level (OFF, ERROR, WARNING, INFO, DEBUG).", type=str, choices=["OFF", "ERROR", "WARNING", "INFO", "DEBUG"], default='INFO') log_group.add_argument("-O", "--log-output", dest="logout", metavar="FILE", help="set the output file for the log messages.", default="") parser.add_argument_group(log_group) adv_group = parser.add_argument_group( "Advanced group", "Advanced options addressed to facilitate administrative tasks and " "allow new non-standard Hub behaviors. In particular the --label " "options is used to assign a value to hub.label token and is used to " "assign a name to the Hub instance. " "The very special --multi option allows to start a Hub in multi-instance mode. " "Multi-instance mode is a non-standard Hub behavior that enables " "multiple contemporaneous running Hubs. Multi-instance hubs place " "their non-standard lock-files within the <home directory>/.samp-1 " "directory naming them making use of the format: " "samp-hub-<PID>-<ID>, where PID is the Hub process ID while ID is an " "internal ID (integer).") adv_group.add_argument("-l", "--label", dest="label", metavar="LABEL", help="assign a LABEL to the Hub.", default="") adv_group.add_argument( "-m", "--multi", dest="mode", help="run the Hub in multi-instance mode generating a custom " "lockfile with a random name.", action="store_const", const='multiple', default='single') parser.add_argument_group(adv_group) options = parser.parse_args() try: if options.loglevel in ("OFF", "ERROR", "WARNING", "DEBUG", "INFO"): log.setLevel(options.loglevel) if options.logout != "": context = log.log_to_file(options.logout) else: class dummy_context: def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): pass context = dummy_context() with context: args = copy.deepcopy(options.__dict__) del (args["loglevel"]) del (args["logout"]) hub = SAMPHubServer(**args) hub.start(False) if not timeout: while hub.is_running: time.sleep(0.01) else: time.sleep(timeout) hub.stop() except KeyboardInterrupt: try: hub.stop() except NameError: pass except OSError as e: print(f"[SAMP] Error: I/O error({e.errno}): {e.strerror}") sys.exit(1) except SystemExit: pass
from meteorflux import metrec from meteorflux import db ####### # MAIN ####### if __name__ == '__main__': # Speed-ups autocommit = False remove_old = True # Check if we have an argument if len(sys.argv) < 2: raise Exception("Please supply the name of a ZIP file.") path = sys.argv[1] #log.setLevel('DEBUG') with log.log_to_file('ingestion.log'): mydb = db.FluxDB(autocommit=autocommit) if os.path.isdir(path): log.info("%s is a directory, will ingest all *.zip files inside." % path) metrec.ingest_dir(path, mydb, remove_old) else: myzip = metrec.ingest_zip(path, mydb, remove_old) mydb.commit()
try: result = do_subject(subject_id) agg_res = result.pop('aggregation_result', None) pd.to_pickle(agg_res, 'aggregation_results/{}.pickle.gz'.format(subject_id)) pd.to_pickle(result, 'results/{}.pickle.gz'.format(subject_id)) except Exception as e: log.warn((subject_id, e)) q.put(subject_id) bar.update(1) # for subject_id in tqdm.tqdm(subjects.index, desc='iterating_over_subjects'): # if not overwrite and os.path.exists('results/{}.pickle.gz'.format(subject_id)): # sleep(0.1) # continue # result = do_subject(subject_id) # if result is not None: # agg_res = result.pop('aggregation_result', None) # pd.to_pickle(agg_res, 'aggregation_results/{}.pickle.gz'.format(subject_id)) # pd.to_pickle(result, 'results/{}.pickle.gz'.format(subject_id)) if __name__ == '__main__': # suppress warnings to keep our beautiful progress bar working warnings.simplefilter('ignore', UserWarning) log.setLevel('WARN') np.seterr(divide='ignore', invalid='ignore') os.makedirs('logs', exist_ok=True) with log.log_to_file('logs/{}.log'.format(str(datetime.now()).replace(' ', '_'))): main()
def main(args=None): global ang_size global image_directory global main_reference_image global fwhm_input global do_conversion global do_registration global do_convolution global do_resampling global do_seds global do_cleanup global kernel_directory global im_pixsc # change variable name global rot_angle global make_2D ang_size = '' image_directory = '' main_reference_image = '' fwhm_input = '' do_conversion = False do_registration = False do_convolution = False do_resampling = False do_seds = False do_cleanup = False kernel_directory = '' im_pixsc = '' make_2D = False # note start time for log start_time = datetime.now() # parse arguments if args != None: arglist = args.split(' ') else: arglist = sys.argv[1:] parse_status = parse_command_line(arglist) if parse_status > 0: if __name__ == '__main__': sys.exit() else: return if (do_cleanup): # cleanup and exit cleanup_output_files() if __name__ == '__main__': sys.exit() else: return # NOTE_FROM_RK : A lot of these contants seem redundant and unused, need to figure out exactly # which ones are used and remove the rest # Lists to store information global image_data global converted_data global registered_data global convolved_data global resampled_data global headers global filenames converted_data = [] registered_data = [] convolved_data = [] resampled_data = [] filenames = [] image_data = [] headers = [] # append all the images before creating the stack hdus = [] # First HDU in the stack, just to store some information about the stack hdr = fits.Header() hdr['COMMENT'] = "Image stack created to form the data cube" #this is just to allow for a later sort on the HDU list, can be fixed later hdr['WAVELNTH'] = 0 primary_hdu = fits.PrimaryHDU(header=hdr) hdus.append(primary_hdu) # if not just cleaning up, make a log file which records input parameters logfile_name = 'imagecube_' + start_time.strftime( '%Y-%m-%d_%H%M%S') + '.log' with log.log_to_file(logfile_name, filter_origin='imagecube.imagecube'): log.info('imagecube started at %s' % start_time.strftime('%Y-%m-%d_%H%M%S')) log.info('imagecube called with arguments %s' % arglist) # Grab all of the .fits and .fit files in the specified directory all_files = glob.glob(image_directory + "/*.fit*") # no use doing anything if there aren't any files! if len(all_files) == 0: warnings.warn( 'No fits files found in directory %s' % image_directory, AstropyUserWarning) if __name__ == '__main__': sys.exit() else: return # get images for (i, fitsfile) in enumerate(all_files): hdulist = fits.open(fitsfile) img_extens = find_image_planes(hdulist) # NOTETOSELF: right now we are just using the *first* image extension in a file # which is not what we want to do, ultimately. header = hdulist[img_extens[0]].header image = hdulist[img_extens[0]].data # Strip the .fit or .fits extension from the filename so we can append # things to it later on filename = os.path.splitext(hdulist.filename())[0] hdulist.close() # check to see if image has reasonable scale & orientation # NOTETOSELF: should this really be here? It's not relevant for just flux conversion. # want separate loop over image planes, after finishing file loop pixelscale = get_pixel_scale(header) fov = pixelscale * float(header['NAXIS1']) log.info( "Checking %s: is pixel scale (%.2f\") < ang_size (%.2f\") < FOV (%.2f\") ?" % (fitsfile, pixelscale, ang_size, fov)) if (pixelscale < ang_size < fov): try: # there seems to be a different name for wavelength in some images, look into it wavelength = header['WAVELNTH'] header['WAVELNTH'] = ( wavelength, 'micron' ) # add the unit if it's not already there header['FILENAME'] = fitsfile a = fits.ImageHDU(header=header, data=image) hdus.append(a) except KeyError: warnings.warn( 'Image %s has no WAVELNTH keyword, will not be used' % filename, AstropyUserWarning) else: warnings.warn( "Image %s does not meet the above criteria." % filename, AstropyUserWarning) # end of loop over files # Sort the lists by their WAVELNTH value hdus.sort(key=lambda x: x.header['WAVELNTH']) # this is the image stack, the data structure stores the images in the following format : # Primary HDU : the first HDU contains some information on the stack created # Image HDU : the next 'n' image HDUs contain the headers and the data of the image files that # need to be processed by IMAGECUBE image_stack = fits.HDUList(hdus) # At this step, create a kernel stack as well. # It should consist of the 5 kernels that need to be used to convolve. # Generate the kernel filename by picking up the instruments for each image and the wavelength # Further, before convolving each image from this kernel_stack with images from the image_stack # Resample them so that the pixel scale match -- DOUBT # Pixel scale of kernel should match with that of the image pixel scale kernels = [] kernels.append([]) # this is the url from where the kernels will be downloaded url0 = "https://www.astro.princeton.edu/~ganiano/Kernels/Ker_2012/Kernels_fits_Files/Low_Resolution/Kernel_LoRes_" # all the images will be transformed to the PSF of the largest wavelength to_hdu = image_stack[-1] to_instr = str(to_hdu.header['INSTRUME']) to_wavelnth = to_hdu.header['WAVELNTH'] # small hack since MIPS channels sometimes have wavelengths of different levels of precision if (to_instr == "MIPS"): to_wavelnth = math.ceil(to_wavelnth) # For every image in our stack, we first look if there's a corresponding # kernel file in the dataset provided. If we dont find one, we look for one on the URL # mentioned and generated using the instrument name and wavlenegth. If the website does # not seem to have the corresponding kernels, we generate a Gaussian kernel using the # FWHM input and the corresponding pixel_scale for i in range(1, len(image_stack)): original_filename = os.path.basename( image_stack[i].header['FILENAME']) original_directory = os.path.dirname( image_stack[i].header['FILENAME']) kernel_filename = (original_directory + "/" + kernel_directory + "/" + original_filename + "_kernel.fits") log.info("Looking for " + kernel_filename) if os.path.exists(kernel_filename): log.info("Found a kernel; will convolve with it shortly.") # reading the kernel kernel_hdulist = fits.open(kernel_filename) kernel_image = kernel_hdulist[0].data kernel_hdulist.close() kernels.append(kernel_image) else: fr_instr = str(image_stack[i].header['INSTRUME']) fr_wavelnth = image_stack[i].header['WAVELNTH'] if (fr_instr == 'MIPS'): fr_wavelnth = math.ceil(fr_wavelnth) # This is the URL generated, from where we will donwload files. url = url0 + str(fr_instr) + "_" + str( fr_wavelnth) + "_to_" + str(to_instr) + "_" + str( to_wavelnth) + ".fits.gz" filename = url.split("/")[-1] # TODO : Look for these files if they're already downloaded so that these downloads do not need to # happen multiple times if the same kernel files are required. Ideally, make a kernels folder to handle this with open(filename, "wb") as f: r = requests.get(url) if not r.status_code == 404: f.write(r.content) with gzip.open(filename, 'rb') as f_in: with open(filename.split('.gz')[0], 'wb') as f_out: shutil.copyfileobj(f_in, f_out) log.info("File unzipped : ", filename.split('.gz')[0]) # resampling of the kernel, so that the file can be used for convolution resampled_kernel = resample_kernel( filename.split('.gz')[0], image_stack[i].header['FILENAME']) kernels.append(resampled_kernel) else: log.info( "This file doesn't seem to exist on the website : ", filename) native_pixelscale = get_pixel_scale( image_stack[i].header) sigma_input = (fwhm_input / (2 * math.sqrt(2 * math.log(2)) * native_pixelscale)) kernels.append(Gaussian2DKernel(sigma_input).array) kernel_stack = kernels if (do_conversion): image_stack = convert_images(image_stack) if (do_registration): image_stack = register_images(image_stack) if (do_convolution): image_stack = convolve_images(image_stack, kernel_stack) if (do_resampling): image_stack = resample_images(image_stack, logfile_name) if (do_seds): output_seds(image_stack) # all done! log.info('All tasks completed.') if __name__ == '__main__': sys.exit() else: return
from astropy import log from datetime import datetime import time log.setLevel('INFO') t0 = time.time() with log.log_to_file("all_analysis_{0}.log".format( datetime.now().isoformat())): log.info("Starting make_apex_cubes postprocessing") execfile("make_apex_cubes.py") do_postprocessing() extract_co_subcubes(mergepath=mergepath) log.info("Creating pyradex grid. dt={0}".format(time.time() - t0)) execfile("pyradex_h2comm_grid.py") log.info("Redo dendro. dt={0}".format(time.time() - t0)) execfile("redo_dendro.py") # execfile("dendro_mask.py") # make_dend_303() # execfile("dendro_temperature.py") # do_dendro_temperatures_both() # execfile("dendrotem_plots.py") # execfile("make_piecewise_temcube.py") log.info("Make ratiotem cubesims. dt={0}".format(time.time() - t0)) execfile("make_ratiotem_cubesims.py") log.info("Make ratiotem integ. dt={0}".format(time.time() - t0)) execfile("make_ratio_integ.py")