def satellite_angular_offset(index, clock_offset): # Fetch observed position of object at this time point pt_utc = path_x_y[index][3] pt_alt = path_alt_az[index][0] pt_az = path_alt_az[index][1] # Project position of this satellite in space at this time point t = ts.tai_jd(jd=jd_from_unix(utc=pt_utc + tai_utc_offset + clock_offset)) # Project position of this satellite in the observer's sky sight_line = sat - observer topocentric = sight_line.at(t) sat_alt, sat_az, sat_distance = topocentric.altaz() # Work out offset of satellite's position from observed moving object ang_mismatch = ang_dist(ra0=pt_az * pi / 180, dec0=pt_alt * pi / 180, ra1=sat_az.radians, dec1=sat_alt.radians) * 180 / pi return ang_mismatch, sat_distance
def shower_determination(utc_min, utc_max): """ Estimate the parent showers of all meteors observed between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should determine the parent showers of meteors (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should determine the parent showers of meteors (unix time). :type utc_max: float :return: None """ # Load list of meteor showers shower_list = read_shower_list() # Open connection to image archive db = obsarchive_db.ObservationDatabase(file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting meteor shower identification.") # Count how many images we manage to successfully fit outcomes = { 'successful_fits': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Status update logging.info("Searching for meteors within period {} to {}".format(date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for meteors within this time period conn.execute(""" SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid INNER JOIN archive_metadata am2 ON ao.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s AND am2.stringValue = "Meteor" ORDER BY ao.obsTime; """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the images we are going to work on logging.info("Estimating the parent showers of {:d} meteors.".format(len(results))) # Count how many meteors we find in each shower meteor_count_by_shower = {} # Analyse each meteor in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if 'pigazing:path' not in all_metadata: logging.info("Cannot process <{}> due to inadequate metadata.".format(item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId'] ) # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection( db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix ) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list_this = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration'] ) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Check number of points in path path_len = len(path_x_y) # List of candidate showers this meteor might belong to candidate_showers = [] # Test for each candidate meteor shower in turn for shower in shower_list: # Work out celestial coordinates of shower radiant in RA/Dec in hours/degs of epoch radiant_ra_at_epoch, radiant_dec_at_epoch = ra_dec_from_j2000(ra0=shower['RA'], dec0=shower['Decl'], utc_new=item['obsTime']) # Work out alt-az of the shower's radiant using known location of camera. Fits returned in degrees. alt_az_pos = alt_az(ra=radiant_ra_at_epoch, dec=radiant_dec_at_epoch, utc=item['obsTime'], latitude=projector.obstory_info['latitude'], longitude=projector.obstory_info['longitude']) # Work out position of the Sun (J2000) sun_ra_j2000, sun_dec_j2000 = sun_pos(utc=item['obsTime']) # Work out position of the Sun (RA, Dec of epoch) sun_ra_at_epoch, sun_dec_at_epoch = ra_dec_from_j2000(ra0=sun_ra_j2000, dec0=sun_dec_j2000, utc_new=item['obsTime']) # Offset from peak of shower year = 365.2524 peak_offset = (sun_ra_at_epoch * 180 / 12. - shower['peak']) * year / 360 # days while peak_offset < -year / 2: peak_offset += year while peak_offset > year / 2: peak_offset -= year start_offset = peak_offset + shower['start'] - 4 end_offset = peak_offset + shower['end'] + 4 # Estimate ZHR of shower at the time the meteor was observed zhr = 0 if abs(peak_offset) < 2: zhr = shower['zhr'] # Shower is within 2 days of maximum; use quoted peak ZHR value if start_offset < 0 < end_offset: zhr = max(zhr, 5) # Shower is not at peak, but is active; assume ZHR=5 # Correct hourly rate for the altitude of the shower radiant hourly_rate = zhr * sin(alt_az_pos[0] * pi / 180) # If hourly rate is zero, this shower is not active if hourly_rate <= 0: # logging.info("Meteor shower <{}> has zero rate".format(shower['name'])) continue # Work out angular distance of meteor from radiant (radians) path_radiant_sep = [ang_dist(ra0=pt[0], dec0=pt[1], ra1=radiant_ra_at_epoch * pi / 12, dec1=radiant_dec_at_epoch * pi / 180) for pt in path_ra_dec_at_epoch] change_in_radiant_dist = path_radiant_sep[-1] - path_radiant_sep[0] # radians # Reject meteors that travel *towards* the radiant if change_in_radiant_dist < 0: continue # Convert path to Cartesian coordinates on a unit sphere path_cartesian = [Vector.from_ra_dec(ra=ra * 12 / pi, dec=dec * 180 / pi) for ra, dec in path_ra_dec_at_epoch] # Work out cross product of first and last point, which is normal to path of meteors first = path_cartesian[0] last = path_cartesian[-1] path_normal = first.cross_product(last) # Work out angle of path normal to meteor shower radiant radiant_cartesian = Vector.from_ra_dec(ra=radiant_ra_at_epoch, dec=radiant_dec_at_epoch) theta = path_normal.angle_with(radiant_cartesian) # degrees if theta > 90: theta = 180 - theta # What is the angular separation of the meteor's path's closest approach to the shower radiant? radiant_angle = 90 - theta # Work out likelihood metric that this meteor belongs to this shower radiant_angle_std_dev = 2 # Allow 2 degree mismatch in radiant pos likelihood = hourly_rate * scipy.stats.norm(loc=0, scale=radiant_angle_std_dev).pdf(radiant_angle) # Store information about the likelihood this meteor belongs to this shower candidate_showers.append({ 'name': shower['name'], 'likelihood': likelihood, 'offset': radiant_angle, 'change_radiant_dist': change_in_radiant_dist, 'shower_rate': hourly_rate }) # Add model possibility for sporadic meteor hourly_rate = 5 likelihood = hourly_rate * (1. / 90.) # Mean value of Gaussian in range 0-90 degs candidate_showers.append({ 'name': "Sporadic", 'likelihood': likelihood, 'offset': 0, 'shower_rate': hourly_rate }) # Renormalise likelihoods to sum to unity sum_likelihood = sum(shower['likelihood'] for shower in candidate_showers) for shower in candidate_showers: shower['likelihood'] *= 100 / sum_likelihood # Sort candidates by likelihood candidate_showers.sort(key=itemgetter('likelihood'), reverse=True) # Report possible meteor shower identifications logging.info("{date} [{obs}] -- {showers}".format( date=date_string(utc=item['obsTime']), obs=item['observationId'], showers=", ".join([ "{} {:.1f}% ({:.1f} deg offset)".format(shower['name'], shower['likelihood'], shower['offset']) for shower in candidate_showers ]) )) # Identify most likely shower most_likely_shower = candidate_showers[0]['name'] # Update tally of meteors if most_likely_shower not in meteor_count_by_shower: meteor_count_by_shower[most_likely_shower] = 0 meteor_count_by_shower[most_likely_shower] += 1 # Store meteor identification user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:name", value=most_likely_shower)) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:radiant_offset", value=candidate_showers[0]['offset'])) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:path_length", value=ang_dist(ra0=path_ra_dec_at_epoch[0][0], dec0=path_ra_dec_at_epoch[0][1], ra1=path_ra_dec_at_epoch[-1][0], dec1=path_ra_dec_at_epoch[-1][1] ) * 180 / pi )) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:path_ra_dec", value="[[{:.3f},{:.3f}],[{:.3f},{:.3f}],[{:.3f},{:.3f}]]".format( path_ra_dec_at_epoch[0][0] * 12 / pi, path_ra_dec_at_epoch[0][1] * 180 / pi, path_ra_dec_at_epoch[int(path_len / 2)][0] * 12 / pi, path_ra_dec_at_epoch[int(path_len / 2)][1] * 180 / pi, path_ra_dec_at_epoch[-1][0] * 12 / pi, path_ra_dec_at_epoch[-1][1] * 180 / pi, ) )) # Meteor successfully identified outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} meteors successfully identified.".format(outcomes['successful_fits'])) logging.info("{:d} malformed database records.".format(outcomes['error_records'])) logging.info("{:d} rescued database records.".format(outcomes['rescued_records'])) logging.info("{:d} meteors with incomplete data.".format(outcomes['insufficient_information'])) # Report tally of meteors logging.info("Tally of meteors by shower:") for shower in sorted(meteor_count_by_shower.keys()): logging.info(" * {:32s}: {:6d}".format(shower, meteor_count_by_shower[shower])) # Clean up and exit db.commit() db.close_db() return
def plane_determination(utc_min, utc_max, source): """ Estimate the identity of aircraft observed between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should determine the identity of aircraft (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should determine the identity of aircraft (unix time). :type utc_max: float :param source: The source we should use for plane trajectories. Either 'adsb' or 'fr24'. :type source: str :return: None """ # Open connection to image archive db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting aircraft identification.") # Count how many images we manage to successfully fit outcomes = { 'successful_fits': 0, 'unsuccessful_fits': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Status update logging.info("Searching for aircraft within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for planes and satellites within this time period conn.execute( """ SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid INNER JOIN archive_metadata am2 ON ao.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s AND (am2.stringValue='Plane' OR am2.stringValue='Satellite' OR am2.stringValue='Junk') ORDER BY ao.obsTime """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the images we are going to work on logging.info("Estimating the identity of {:d} aircraft.".format( len(results))) # Analyse each aircraft in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if 'pigazing:path' not in all_metadata: logging.info( "Cannot process <{}> due to inadequate metadata.".format( item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId']) # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection(db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration']) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Check number of points in path path_len = len(path_x_y) # Look up list of aircraft tracks at the time of this sighting if source == 'adsb': aircraft_list = fetch_planes_from_adsb(utc=item['obsTime']) elif source == 'fr24': aircraft_list = fetch_planes_from_fr24(utc=item['obsTime']) else: raise ValueError("Unknown source <{}>".format(source)) # List of aircraft this moving object might be candidate_aircraft = [] # Check that we found a list of aircraft if aircraft_list is None: logging.info("{date} [{obs}] -- No aircraft records found.".format( date=date_string(utc=item['obsTime']), obs=item['observationId'])) outcomes['insufficient_information'] += 1 continue # Logging message about how many aircraft we're testing # logging.info("{date} [{obs}] -- Matching against {count:7d} aircraft.".format( # date=date_string(utc=item['obsTime']), # obs=item['observationId'], # count=len(aircraft_list) # )) # Test for each candidate aircraft in turn for aircraft in aircraft_list: # Fetch aircraft position at each time point along trajectory ang_mismatch_list = [] distance_list = [] altitude_list = [] def aircraft_angular_offset(index, clock_offset): # Fetch observed position of object at this time point pt_utc = sight_line_list[index]['utc'] observatory_position = sight_line_list[index]['obs_position'] observed_sight_line = sight_line_list[index]['line'].direction # Project position of this aircraft in space at this time point aircraft_position = path_interpolate(aircraft=aircraft, utc=pt_utc + clock_offset) if aircraft_position is None: return np.nan, np.nan, np.nan # Convert position to Cartesian coordinates aircraft_point = Point.from_lat_lng( lat=aircraft_position['lat'], lng=aircraft_position['lon'], alt=aircraft_position['altitude'] * feet, utc=None) # Work out offset of plane's position from observed moving object aircraft_sight_line = aircraft_point.to_vector( ) - observatory_position.to_vector() angular_offset = aircraft_sight_line.angle_with( other=observed_sight_line) # degrees distance = abs(aircraft_sight_line) altitude = aircraft_position['altitude'] * feet return angular_offset, distance, altitude def time_offset_objective(p): """ Objective function that we minimise in order to find the best fit clock offset between the observed and model paths. :param p: Vector with a single component: the clock offset :return: Metric to minimise """ # Turn input parameters into a time offset clock_offset = p[0] # Look up angular offset ang_mismatch, distance, altitude = aircraft_angular_offset( index=0, clock_offset=clock_offset) # Return metric to minimise return ang_mismatch * exp(clock_offset / 8) # Work out the optimum time offset between the plane's path and the observed path # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> # for more information about how this works parameters_initial = [0] parameters_optimised = scipy.optimize.minimize( time_offset_objective, np.asarray(parameters_initial), options={ 'disp': False, 'maxiter': 100 }).x # Construct best-fit linear trajectory for best-fitting parameters clock_offset = float(parameters_optimised[0]) # Check clock offset is reasonable if abs(clock_offset) > global_settings['max_clock_offset']: continue # Measure the offset between the plane's position and the observed position at each time point for index in range(path_len): # Look up angular mismatch at this time point ang_mismatch, distance, altitude = aircraft_angular_offset( index=index, clock_offset=clock_offset) # Keep list of the offsets at each recorded time point along the trajectory ang_mismatch_list.append(ang_mismatch) distance_list.append(distance) altitude_list.append(altitude) # Consider adding this plane to list of candidates mean_ang_mismatch = np.mean( np.asarray(ang_mismatch_list)) # degrees distance_mean = np.mean(np.asarray(distance_list)) # metres altitude_mean = np.mean(np.asarray(altitude_list)) # metres if mean_ang_mismatch < global_settings['max_mean_angular_mismatch']: start_time = sight_line_list[0]['utc'] end_time = sight_line_list[-1]['utc'] start_point = path_interpolate(aircraft=aircraft, utc=start_time + clock_offset) end_point = path_interpolate(aircraft=aircraft, utc=end_time + clock_offset) candidate_aircraft.append({ 'call_sign': aircraft['call_sign'], # string 'hex_ident': aircraft['hex_ident'], # string 'distance': distance_mean / 1e3, # km 'altitude': altitude_mean / 1e3, # km 'clock_offset': clock_offset, # seconds 'offset': mean_ang_mismatch, # degrees 'start_point': start_point, 'end_point': end_point }) # Add model possibility for null aircraft if len(candidate_aircraft) == 0: candidate_aircraft.append({ 'call_sign': "Unidentified", 'hex_ident': "Unidentified", 'distance': 0, 'altitude': 0, 'clock_offset': 0, 'offset': 0, 'start_point': None, 'end_point': None }) # Sort candidates by score for candidate in candidate_aircraft: candidate['score'] = hypot( candidate['offset'], candidate['clock_offset'], ) candidate_aircraft.sort(key=itemgetter('score')) # Report possible satellite identifications logging.info("{prefix} -- {aircraft}".format( prefix=logging_prefix, aircraft=", ".join([ "{} ({:.1f} deg offset; clock offset {:.1f} sec; distance {:.1f} km)" .format(aircraft['call_sign'], aircraft['offset'], aircraft['clock_offset'], aircraft['distance']) for aircraft in candidate_aircraft ]))) # Identify most likely aircraft most_likely_aircraft = candidate_aircraft[0] # Fetch extra information about plane plane_info = fetch_aircraft_data( hex_ident=most_likely_aircraft['hex_ident']) # Store aircraft identification user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:call_sign", value=most_likely_aircraft['call_sign'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:hex_ident", value=most_likely_aircraft['hex_ident'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:clock_offset", value=most_likely_aircraft['clock_offset'])) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="plane:angular_offset", value=most_likely_aircraft['offset'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:distance", value=most_likely_aircraft['distance'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:mean_altitude", value=most_likely_aircraft['altitude'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:path", value=json.dumps([ most_likely_aircraft['start_point'], most_likely_aircraft['end_point'] ]))) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:path_length", value=ang_dist(ra0=path_ra_dec_at_epoch[0][0], dec0=path_ra_dec_at_epoch[0][1], ra1=path_ra_dec_at_epoch[-1][0], dec1=path_ra_dec_at_epoch[-1][1]) * 180 / pi)) aircraft_operator = "" if 'operator' in plane_info and plane_info['operator']: aircraft_operator = plane_info['operator'] elif 'owner' in plane_info and plane_info['owner']: aircraft_operator = plane_info['owner'] db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:operator", value=aircraft_operator)) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:model", value=plane_info.get( 'model', ''))) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:manufacturer", value=plane_info.get( 'manufacturername', ''))) # Aircraft successfully identified if most_likely_aircraft['call_sign'] == "Unidentified": outcomes['unsuccessful_fits'] += 1 else: outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} aircraft successfully identified.".format( outcomes['successful_fits'])) logging.info("{:d} aircraft not identified.".format( outcomes['unsuccessful_fits'])) logging.info("{:d} malformed database records.".format( outcomes['error_records'])) logging.info("{:d} rescued database records.".format( outcomes['rescued_records'])) logging.info("{:d} aircraft with incomplete data.".format( outcomes['insufficient_information'])) # Clean up and exit db.commit() db.close_db() return
def calibrate_lens(filenames: list, verbose: bool = True, diagnostics_run_id=None): """ Calibrate the radial distortion and sky orientation of one image, or a group of images, taken from a particular lens / camera setup. If multiple images are passed, they may have different orientations on the sky, but are assumed to share the same radial distortions. The best fit parameters are displayed to stdout, and also stored in the global variable <parameters_final>. :param filenames: List of the JSON files describing the images we are to calibrate. :type filenames: List<str> :param verbose: Boolean switch indicating whether we produce diagnostic messages listing the quality of fit of each star listed. :type verbose: bool :param diagnostics_run_id: If set, we produce a series of PNG graphs in </tmp> illustrating the quality of the fit achieved in terms of the radial offset of the position of each star from its theoretically predicted position. If this shows systematic trends away from zero, it suggests that the radial distortion has been poorly fitted. The string value supplied here is appended as a suffix to all the image files, to distinguish them from other runs of this tool. :type diagnostics_run_id: str|int :return: None """ # We store the details of the list of parameters we are currently fitting in global variables (yuck!) as they # need to be accessed by the objective function which does the function fitting. global parameters_radial, parameters_image global fitting_parameters, fitting_parameter_indices, fitting_parameter_names global fitting_star_list, fitting_filename_list global parameters_final # Details of all the images we are currently fitting img_size_x = [] # Horizontal pixel width of each image we are fitting img_size_y = [] # Vertical pixel height of each image we are fitting input_config = [ ] # The full JSON structures describing each image we are fitting star_list = [] # Lists of the stars in each image we are fitting # Start creating list of parameters we are to fit fitting_parameters = { } # fitting_parameters[parameter_name] = dictionary of info about the parameter (see above) fitting_parameter_indices = { } # fitting_parameter_indices[parameter_name] = index fitting_parameter_names = [ ] # fitting_parameter_names[index] = parameter_name fitting_star_list = None # global copy of <star_list> # Compile a list of all the free parameters in this fitting run # All fitting runs require a single set of radial distortion parameters # We create descriptors for these parameters in <fitting_parameters> now for item in parameters_radial: parameter_index = len(fitting_parameters) fitting_parameters[item['name']] = item.copy() fitting_parameters[item['name']][ 'image'] = None # These parameters do not refer to any single image fitting_parameter_indices[item['name']] = parameter_index fitting_parameter_names.append(item['name']) # For every image we are fitting, we need a separate set of parameters describing the image's orientation on the sky # We create descriptors for these parameters now for filename in filenames: # Read the JSON description for this image image_info = read_input_data(filename=filename, show_warnings=verbose) # Add this image to the list of images we are fitting img_size_x.append(image_info[0]) img_size_y.append(image_info[1]) input_config.append(image_info[2]) star_list.append(image_info[3]) # Create all the parameters in <fitting_parameters> which describe the orientation of this image on the sky for item in parameters_image: # Calculate the index that this parameter will take in the vector of trial values in the optimiser parameter_index = len(fitting_parameters) # Create a unique name for this parameter. We may have multiple images, so we prepend the image's filename # to the parameter name, to distinguish it from other images. key = "{}_{}".format(filename, item['name']) # Create parameter fitting_parameters[key] = item.copy() fitting_parameters[key]['image'] = filename fitting_parameter_indices[key] = parameter_index fitting_parameter_names.append(key) # Create global list of all the images we are fitting, and the stars within each image fitting_star_list = star_list fitting_filename_list = filenames # Set a sensible default RA / Dec for the centre of each image for index, filename in enumerate(filenames): # Use the first reference star in each image as a guesstimate of celestial coordinates of the centre ra0 = star_list[index][0]['ra'] dec0 = star_list[index][0]['dec'] key_ra = "{}_{}".format(filename, 'ra') fitting_parameters[key_ra]['default'] = ra0 key_dec = "{}_{}".format(filename, 'dec') fitting_parameters[key_dec]['default'] = dec0 # Create vectors of initial values, and step sizes, for each parameter we are to fit parameters_scale = [ fitting_parameters[key]['step_size'] for key in fitting_parameter_names ] parameters_initial = [ fitting_parameters[key]['default'] / fitting_parameters[key]['step_size'] for key in fitting_parameter_names ] # Solve the system of equations # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> for more information about how this works parameters_optimised = scipy.optimize.minimize( mismatch, numpy.asarray(parameters_initial), options={ 'disp': verbose, 'maxiter': 1e8 }).x # Extract the best-fitting set of parameter values, in physical units (not in units of optimiser step size) parameters_final = [ parameters_optimised[i] * parameters_scale[i] for i in range(len(fitting_parameter_names)) ] # Display best fit parameter values if verbose: logging.info("Observatory: {}".format( [item['observatory'] for item in input_config])) logging.info("Lens: {}".format([item['lens'] for item in input_config])) logging.info("Best fit parameters were:") for index, key in enumerate(fitting_parameter_names): logging.info("{:30s} : {:.8f}".format( fitting_parameters[key]['title'], parameters_final[index] * fitting_parameters[key]['display_scale'])) # Print JSON string describing the radial distortion model for index, filename in enumerate(filenames): scale_x = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'width')]] scale_y = scale_x * parameters_final[ fitting_parameter_indices['aspect']] k1 = parameters_final[fitting_parameter_indices['k1']] k2 = parameters_final[fitting_parameter_indices['k2']] k3 = parameters_final[fitting_parameter_indices['k3']] logging.info( "Barrel parameters: [{:12.8f}, {:12.8f}, {:14.10f}, {:14.10f}, {:14.10f}]" .format(scale_x * 180 / pi, scale_y * 180 / pi, k1, k2, k3)) # In verbose mode, print detailed information about how well each star was fitted if verbose: # Loop over all of the images we are fitting for index, filename in enumerate(filenames): # Extract the parameter values relevant to this image ra0 = parameters_final[fitting_parameter_indices["{}_{}".format( filename, 'ra')]] dec0 = parameters_final[fitting_parameter_indices["{}_{}".format( filename, 'dec')]] scale_x = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'width')]] scale_y = scale_x * parameters_final[ fitting_parameter_indices['aspect']] pos_ang = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'pa')]] k1 = parameters_final[fitting_parameter_indices['k1']] k2 = parameters_final[fitting_parameter_indices['k2']] k3 = parameters_final[fitting_parameter_indices['k3']] logging.info("Image <{}>".format(filename)) logging.info("Stars used in fitting process:") # Print the quality of fit to each star individually for star in star_list[index]: # Calculate the theoretically predicted position of this star pos = gnomonic_project(ra=star['ra'], dec=star['dec'], ra0=ra0, dec0=dec0, size_x=1, size_y=1, scale_x=scale_x, scale_y=scale_y, pos_ang=pos_ang, barrel_k1=k1, barrel_k2=k2, barrel_k3=k3) # Calculate the offset between the theoretically predicted and observed positions of this star distance = hypot((star['xpos'] - pos[0]) * img_size_x[index], (star['ypos'] - pos[1]) * img_size_y[index]) # Print diagnostic details for this star logging.info(""" User-supplied position ({:4.0f},{:4.0f}). Model position ({:4.0f},{:4.0f}). Mismatch {:5.1f} pixels. """.format(star['xpos'] * img_size_x[index], star['ypos'] * img_size_y[index], pos[0] * img_size_x[index], pos[1] * img_size_y[index], distance).strip()) # Debugging: output a data file listing the observed and theoretical positions of each star if diagnostics_run_id is not None: # Create diagnostic data file in /tmp output_filename = "/tmp/point_offsets_{}.dat".format( diagnostics_run_id) with open(output_filename, "w") as output: # Column headings output.write("# x_user_input y_user_input x_model y_model\n") # Loop over all the images we are simultaneously fitting for index, filename in enumerate(filenames): # Leave a blank line between data points from different images output.write("\n\n") # Extract the parameter values relevant to this image ra0 = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'ra')]] dec0 = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'dec')]] scale_x = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'width')]] scale_y = scale_x * parameters_final[ fitting_parameter_indices['aspect']] pos_ang = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'pa')]] k1 = parameters_final[fitting_parameter_indices['k1']] k2 = parameters_final[fitting_parameter_indices['k2']] k3 = parameters_final[fitting_parameter_indices['k3']] # Loop over all the stars in this image for star in star_list[index]: # Calculate the theoretically predicted position of this star pos = gnomonic_project(ra=star['ra'], dec=star['dec'], ra0=ra0, dec0=dec0, size_x=1, size_y=1, scale_x=scale_x, scale_y=scale_y, pos_ang=pos_ang, barrel_k1=k1, barrel_k2=k2, barrel_k3=k3) # Output the theoretically predicted and observed positions of this star output.write("{:4.0f} {:4.0f} {:4.0f} {:4.0f}\n".format( star['xpos'] * img_size_x[index], star['ypos'] * img_size_y[index], pos[0] * img_size_x[index], pos[1] * img_size_y[index])) # Debugging: output a data file listing the theoretical and observed radial positions of each star if diagnostics_run_id is not None: # Create diagnostic data file in /tmp logging.info( "Producing diagnostic file <{}>".format(diagnostics_run_id)) output_filename = "/tmp/radial_distortion_{}.dat".format( diagnostics_run_id) with open(output_filename, "w") as output: # Column headings output.write( "# x/pixel, y/pixel, offset/pixel, radius/pixel , Angular distance/rad , " "Tangent-space distance , Barrel-corrected tan-space dist\n") # Loop over all the images we are simultaneously fitting for index, filename in enumerate(filenames): # Leave a blank line between data points from different images output.write("\n\n") # Extract the parameter values relevant to this image ra0 = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'ra')]] dec0 = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'dec')]] scale_x = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'width')]] scale_y = scale_x * parameters_final[ fitting_parameter_indices['aspect']] pos_ang = parameters_final[fitting_parameter_indices[ "{}_{}".format(filename, 'pa')]] k1 = parameters_final[fitting_parameter_indices['k1']] k2 = parameters_final[fitting_parameter_indices['k2']] k3 = parameters_final[fitting_parameter_indices['k3']] # Loop over all the stars in this image for star in star_list[index]: # Calculate the theoretically predicted position of this star pos = gnomonic_project(ra=star['ra'], dec=star['dec'], ra0=ra0, dec0=dec0, size_x=1, size_y=1, scale_x=scale_x, scale_y=scale_y, pos_ang=pos_ang, barrel_k1=k1, barrel_k2=k2, barrel_k3=k3) # Calculate the offset in the projected position of this star (pixels) offset = hypot((star['xpos'] - pos[0]) * img_size_x[index], (star['ypos'] - pos[1]) * img_size_y[index]) # Calculate the angular distance of this star from the centre of the field (rad) angular_distance = ang_dist(ra1=star['ra'], dec1=star['dec'], ra0=ra0, dec0=dec0) # Calculate the pixel distance of this star from the centre of the field # (horizontal pixels; after radial distortion) pixel_distance = hypot( (star['xpos'] - 0.5) * img_size_x[index], (star['ypos'] - 0.5) * img_size_x[index] * tan(scale_y / 2.) / tan(scale_x / 2.)) # Calculate the distance of this star from the centre of the field # (tangent space; before radial distortion) tan_distance = tan(angular_distance) # Apply radial correction to the position this star in tangent space r = tan_distance / tan(scale_x / 2) bc_kn = 1. - k1 - k2 - k3 r2 = r * (bc_kn + k1 * (r**2) + k2 * (r**4) + k3 * (r**6)) # Calculate the distance of this star from the centre of the field # (tangent space; after radial distortion) barrel_corrected_tan_dist = r2 * tan(scale_x / 2) # Display diagnostic table of calculated values output.write( "{:4.0f} {:4.0f} {:8.4f} {:8.4f} {:8.4f} {:8.4f} {:8.4f}\n" .format( star['xpos'] * img_size_x[index], star['ypos'] * img_size_y[index], offset, pixel_distance, angular_distance * 180 / pi, tan_distance / tan(scale_x / 2) * img_size_x[index] / 2, barrel_corrected_tan_dist / tan(scale_x / 2) * img_size_x[index] / 2)) # Write pyxplot script to plot quality of fit output_filename = "/tmp/radial_distortion_{}.ppl".format( diagnostics_run_id) with open(output_filename, "w") as output: output.write(""" set width 30 ; set term png dpi 100 set xlabel 'Distance from centre of field / pixels' set ylabel 'Observed pixel distance from centre (after distortion) - Tan space distance (before distortion)' set output '/tmp/radial_distortion_{0}_a.png' """.format(diagnostics_run_id)) for index, filename in enumerate(filenames): output.write(""" f_{1}(x) = a + b * x ** 2 # fit f_{1}() withouterrors '/tmp/radial_distortion_{0}.dat' using 4:$6/$4 index {1} via a, b """.format(diagnostics_run_id, index)) output.write("plot ") for index, filename in enumerate(filenames): output.write( "'/tmp/radial_distortion_{0}.dat' using 4:$6-$4 index {1}, " .format(diagnostics_run_id, index)) # output.write("f_{1}(x), ".format(diagnostics_run_id, index)) output.write( "'/tmp/radial_distortion_{0}.dat' using 4:$6-$7 index {1}, " .format(diagnostics_run_id, index)) output.write( "-1 notitle w col green lt 2, 0 notitle w col green lt 2, 1 notitle w col green lt 2\n" ) output.write(""" set ylabel 'Observed pixel distance from centre (after distortion) - Tan space distance (after distortion)' set output '/tmp/radial_distortion_{0}_b.png' """.format(diagnostics_run_id)) output.write("plot ") for index, filename in enumerate(filenames): output.write( "'/tmp/radial_distortion_{0}.dat' using 4:$7-$4 index {1}, " .format(diagnostics_run_id, index)) # output.write("'/tmp/radial_distortion_{0}.dat' using 4:$6/f_{1}($4)-$4 index {1}, ".format(diagnostics_run_id, index)) output.write( "-1 notitle w col green lt 2, 0 notitle w col green lt 2, 1 notitle w col green lt 2\n" ) # Run pyxplot os.system( "pyxplot /tmp/radial_distortion_{}.ppl".format(diagnostics_run_id))
def satellite_determination(utc_min, utc_max): """ Estimate the identity of spacecraft observed between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should determine the identity of spacecraft (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should determine the identity of spacecraft (unix time). :type utc_max: float :return: None """ # Open connection to image archive db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting satellite identification.") # Count how many images we manage to successfully fit outcomes = { 'successful_fits': 0, 'unsuccessful_fits': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Status update logging.info("Searching for satellites within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for satellites within this time period conn.execute( """ SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid INNER JOIN archive_metadata am2 ON ao.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s AND (am2.stringValue='Plane' OR am2.stringValue='Satellite' OR am2.stringValue='Junk') ORDER BY ao.obsTime """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the images we are going to work on logging.info("Estimating the identity of {:d} spacecraft.".format( len(results))) # Analyse each spacecraft in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if 'pigazing:path' not in all_metadata: logging.info( "Cannot process <{}> due to inadequate metadata.".format( item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId']) # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection(db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list_this = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration']) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Check number of points in path path_len = len(path_x_y) # Look up list of satellite orbital elements at the time of this sighting spacecraft_list = fetch_satellites(utc=item['obsTime']) # List of candidate satellites this object might be candidate_satellites = [] # Check that we found a list of spacecraft if spacecraft_list is None: logging.info( "{date} [{obs}] -- No spacecraft records found.".format( date=date_string(utc=item['obsTime']), obs=item['observationId'])) outcomes['insufficient_information'] += 1 continue # Logging message about how many spacecraft we're testing # logging.info("{date} [{obs}] -- Matching against {count:7d} spacecraft.".format( # date=date_string(utc=item['obsTime']), # obs=item['observationId'], # count=len(spacecraft_list) # )) # Test for each candidate satellite in turn for spacecraft in spacecraft_list: # Unit scaling deg2rad = pi / 180.0 # 0.0174532925199433 xpdotp = 1440.0 / (2.0 * pi) # 229.1831180523293 # Model the path of this spacecraft model = Satrec() model.sgp4init( # whichconst: gravity model WGS72, # opsmode: 'a' = old AFSPC mode, 'i' = improved mode 'i', # satnum: Satellite number spacecraft['noradId'], # epoch: days since 1949 December 31 00:00 UT jd_from_unix(spacecraft['epoch']) - 2433281.5, # bstar: drag coefficient (/earth radii) spacecraft['bStar'], # ndot (NOT USED): ballistic coefficient (revs/day) spacecraft['meanMotionDot'] / (xpdotp * 1440.0), # nddot (NOT USED): mean motion 2nd derivative (revs/day^3) spacecraft['meanMotionDotDot'] / (xpdotp * 1440.0 * 1440), # ecco: eccentricity spacecraft['ecc'], # argpo: argument of perigee (radians) spacecraft['argPeri'] * deg2rad, # inclo: inclination (radians) spacecraft['incl'] * deg2rad, # mo: mean anomaly (radians) spacecraft['meanAnom'] * deg2rad, # no_kozai: mean motion (radians/minute) spacecraft['meanMotion'] / xpdotp, # nodeo: right ascension of ascending node (radians) spacecraft['RAasc'] * deg2rad) # Wrap within skyfield to convert to topocentric coordinates ts = load.timescale() sat = EarthSatellite.from_satrec(model, ts) # Fetch spacecraft position at each time point along trajectory ang_mismatch_list = [] distance_list = [] # e, r, v = model.sgp4(jd_from_unix(utc=item['obsTime']), 0) # logging.info("{} {} {}".format(str(e), str(r), str(v))) tai_utc_offset = 39 # seconds def satellite_angular_offset(index, clock_offset): # Fetch observed position of object at this time point pt_utc = path_x_y[index][3] pt_alt = path_alt_az[index][0] pt_az = path_alt_az[index][1] # Project position of this satellite in space at this time point t = ts.tai_jd(jd=jd_from_unix(utc=pt_utc + tai_utc_offset + clock_offset)) # Project position of this satellite in the observer's sky sight_line = sat - observer topocentric = sight_line.at(t) sat_alt, sat_az, sat_distance = topocentric.altaz() # Work out offset of satellite's position from observed moving object ang_mismatch = ang_dist(ra0=pt_az * pi / 180, dec0=pt_alt * pi / 180, ra1=sat_az.radians, dec1=sat_alt.radians) * 180 / pi return ang_mismatch, sat_distance def time_offset_objective(p): """ Objective function that we minimise in order to find the best fit clock offset between the observed and model paths. :param p: Vector with a single component: the clock offset :return: Metric to minimise """ # Turn input parameters into a time offset clock_offset = p[0] # Look up angular offset ang_mismatch, sat_distance = satellite_angular_offset( index=0, clock_offset=clock_offset) # Return metric to minimise return ang_mismatch * exp(clock_offset / 8) # First, chuck out satellites with large angular offsets observer = wgs84.latlon( latitude_degrees=projector.obstory_info['latitude'], longitude_degrees=projector.obstory_info['longitude'], elevation_m=0) ang_mismatch, sat_distance = satellite_angular_offset( index=0, clock_offset=0) # Check angular offset is reasonable if ang_mismatch > global_settings['max_angular_mismatch']: continue # Work out the optimum time offset between the satellite's path and the observed path # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> # for more information about how this works parameters_initial = [0] parameters_optimised = scipy.optimize.minimize( time_offset_objective, np.asarray(parameters_initial), options={ 'disp': False, 'maxiter': 100 }).x # Construct best-fit linear trajectory for best-fitting parameters clock_offset = float(parameters_optimised[0]) # Check clock offset is reasonable if abs(clock_offset) > global_settings['max_clock_offset']: continue # Measure the offset between the satellite's position and the observed position at each time point for index in range(path_len): # Look up angular mismatch at this time point ang_mismatch, sat_distance = satellite_angular_offset( index=index, clock_offset=clock_offset) # Keep list of the offsets at each recorded time point along the trajectory ang_mismatch_list.append(ang_mismatch) distance_list.append(sat_distance.km) # Consider adding this satellite to list of candidates mean_ang_mismatch = np.mean(np.asarray(ang_mismatch_list)) distance_mean = np.mean(np.asarray(distance_list)) if mean_ang_mismatch < global_settings['max_mean_angular_mismatch']: candidate_satellites.append({ 'name': spacecraft['name'], # string 'noradId': spacecraft['noradId'], # int 'distance': distance_mean, # km 'clock_offset': clock_offset, # seconds 'offset': mean_ang_mismatch, # degrees 'absolute_magnitude': spacecraft['mag'] }) # Add model possibility for null satellite candidate_satellites.append({ 'name': "Unidentified", 'noradId': 0, 'distance': 35.7e3 * 0.25, # Nothing is visible beyond 25% of geostationary orbit distance 'clock_offset': 0, 'offset': 0, 'absolute_magnitude': None }) # Sort candidates by score - use absolute mag = 20 for satellites with no mag for candidate in candidate_satellites: candidate['score'] = hypot( candidate['distance'] / 1e3, candidate['clock_offset'], (20 if candidate['absolute_magnitude'] is None else candidate['absolute_magnitude']), ) candidate_satellites.sort(key=itemgetter('score')) # Report possible satellite identifications logging.info("{prefix} -- {satellites}".format( prefix=logging_prefix, satellites=", ".join([ "{} ({:.1f} deg offset; clock offset {:.1f} sec)".format( satellite['name'], satellite['offset'], satellite['clock_offset']) for satellite in candidate_satellites ]))) # Identify most likely satellite most_likely_satellite = candidate_satellites[0] # Store satellite identification user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="satellite:name", value=most_likely_satellite['name'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="satellite:norad_id", value=most_likely_satellite['noradId'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="satellite:clock_offset", value=most_likely_satellite['clock_offset'])) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="satellite:angular_offset", value=most_likely_satellite['offset'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="satellite:path_length", value=ang_dist(ra0=path_ra_dec_at_epoch[0][0], dec0=path_ra_dec_at_epoch[0][1], ra1=path_ra_dec_at_epoch[-1][0], dec1=path_ra_dec_at_epoch[-1][1]) * 180 / pi)) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="satellite:path_ra_dec", value="[[{:.3f},{:.3f}],[{:.3f},{:.3f}],[{:.3f},{:.3f}]]". format( path_ra_dec_at_epoch[0][0] * 12 / pi, path_ra_dec_at_epoch[0][1] * 180 / pi, path_ra_dec_at_epoch[int(path_len / 2)][0] * 12 / pi, path_ra_dec_at_epoch[int(path_len / 2)][1] * 180 / pi, path_ra_dec_at_epoch[-1][0] * 12 / pi, path_ra_dec_at_epoch[-1][1] * 180 / pi, ))) # Satellite successfully identified if most_likely_satellite['name'] == "Unidentified": outcomes['unsuccessful_fits'] += 1 else: outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} satellites successfully identified.".format( outcomes['successful_fits'])) logging.info("{:d} satellites not identified.".format( outcomes['unsuccessful_fits'])) logging.info("{:d} malformed database records.".format( outcomes['error_records'])) logging.info("{:d} rescued database records.".format( outcomes['rescued_records'])) logging.info("{:d} satellites with incomplete data.".format( outcomes['insufficient_information'])) # Clean up and exit db.commit() db.close_db() return
def average_daily_fits(conn, db, obstory_id, utc_max, utc_min): """ Average all of the orientation fixes within a given time period, excluding extreme fits. Update the observatory's status with a altitude and azimuth of the average fit, if it has a suitably small error bar. :param conn: Database connection object. :param db: Database object. :param obstory_id: Observatory publicId. :param utc_max: Unix time of the end of the time period. :param utc_min: Unix time of the beginning of the time period. :return: None """ # Divide up the time period in which we are working into individual nights, and then work on each night individually logging.info("Averaging daily fits within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Each night is a 86400-second period daily_block_size = 86400 # Make sure that blocks start at noon utc_min = (floor(utc_min / daily_block_size + 0.5) - 0.5) * daily_block_size time_blocks = list( np.arange(start=utc_min, stop=utc_max + daily_block_size, step=daily_block_size)) # Start new block whenever we have a hardware refresh, even if it's in the middle of the night! conn.execute( """ SELECT time FROM archive_metadata WHERE observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) AND fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey='refresh') AND time BETWEEN %s AND %s """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: time_blocks.append(item['time']) # Make sure that start points for time blocks are in order time_blocks.sort() # Work on each time block (i.e. night) in turn for block_index, utc_block_min in enumerate(time_blocks[:-1]): # End point for this time block utc_block_max = time_blocks[block_index + 1] # Search for observations with orientation fits conn.execute( """ SELECT am1.floatValue AS altitude, am2.floatValue AS azimuth, am3.floatValue AS pa, am4.floatValue AS tilt, am5.floatValue AS width_x_field, am6.floatValue AS width_y_field, am7.stringValue AS fit_quality FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:altitude") INNER JOIN archive_metadata am2 ON o.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:azimuth") INNER JOIN archive_metadata am3 ON o.uid = am3.observationId AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:pa") INNER JOIN archive_metadata am4 ON o.uid = am4.observationId AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:tilt") INNER JOIN archive_metadata am5 ON o.uid = am5.observationId AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_x_field") INNER JOIN archive_metadata am6 ON o.uid = am6.observationId AND am6.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_y_field") INNER JOIN archive_metadata am7 ON o.uid = am7.observationId AND am7.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:fit_quality") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_block_min, utc_block_max)) results = conn.fetchall() # Remove results with poor fit results_filtered = [] fit_threshold = 2 # pixels for item in results: fit_quality = float(json.loads(item['fit_quality'])[0]) if fit_quality > fit_threshold: continue item['weight'] = 1 / (fit_quality + 0.1) results_filtered.append(item) results = results_filtered # Report how many images we found logging.info( "Averaging fits within period {} to {}: Found {} fits.".format( date_string(utc_block_min), date_string(utc_block_max), len(results))) # Average the fits we found if len(results) < 4: logging.info("Insufficient images to reliably average.") continue # What fraction of the worst fits do we reject? rejection_fraction = 0.25 # Reject the 25% of fits which are further from the average rejection_count = int(len(results) * rejection_fraction) # Convert alt-az fits into radians and average # Iteratively remove the point furthest from the mean results_filtered = results # Iteratively take the average of the fits, reject the furthest outlier, and then take a new average for iteration in range(rejection_count): # Average the (alt, az) measurements for this observatory by finding their centroid on a sphere alt_az_list = [[i['altitude'] * deg, i['azimuth'] * deg] for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] alt_az_best = mean_angle_2d(pos_list=alt_az_list, weights=weights_list)[0] # Work out the offset of each fit from the average fit_offsets = [ ang_dist(ra0=alt_az_best[1], dec0=alt_az_best[0], ra1=fitted_alt_az[1], dec1=fitted_alt_az[0]) for fitted_alt_az in alt_az_list ] # Reject the worst fit which is further from the average fits_with_weights = list(zip(fit_offsets, results_filtered)) fits_with_weights.sort(key=operator.itemgetter(0)) fits_with_weights.reverse() # Create a new list of orientation fits, with the worst outlier excluded results_filtered = [item[1] for item in fits_with_weights[1:]] # Convert alt-az fits into radians and average by finding their centroid on a sphere alt_az_list = [[i['altitude'] * deg, i['azimuth'] * deg] for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] [alt_az_best, alt_az_error] = mean_angle_2d(pos_list=alt_az_list, weights=weights_list) # Average other angles by finding their centroid on a circle output_values = {} for quantity in ['tilt', 'pa', 'width_x_field', 'width_y_field']: # Iteratively remove the point furthest from the mean results_filtered = results # Iteratively take the average of the values for each parameter, reject the furthest outlier, # and then take a new average for iteration in range(rejection_count): # Average quantity measurements quantity_values = [i[quantity] * deg for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] quantity_mean = mean_angle(angle_list=quantity_values, weights=weights_list)[0] # Work out the offset of each fit from the average fit_offsets = [] for index, quantity_value in enumerate(quantity_values): offset = quantity_value - quantity_mean if offset < -pi: offset += 2 * pi if offset > pi: offset -= 2 * pi fit_offsets.append(abs(offset)) # Reject the worst fit which is furthest from the average fits_with_weights = list(zip(fit_offsets, results_filtered)) fits_with_weights.sort(key=operator.itemgetter(0)) fits_with_weights.reverse() results_filtered = [item[1] for item in fits_with_weights[1:]] # Filtering finished; now convert each fit into radians and average values_filtered = [i[quantity] * deg for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] value_best = mean_angle(angle_list=values_filtered, weights=weights_list)[0] output_values[quantity] = value_best * rad # Print fit information success = ( alt_az_error * rad < 0.1 ) # Only accept determinations with better precision than 0.1 deg adjective = "SUCCESSFUL" if success else "REJECTED" logging.info("""\ {} ORIENTATION FIT from {:2d} images: Alt: {:.2f} deg. Az: {:.2f} deg. PA: {:.2f} deg. \ ScaleX: {:.2f} deg. ScaleY: {:.2f} deg. Uncertainty: {:.2f} deg.\ """.format(adjective, len(results_filtered), alt_az_best[0] * rad, alt_az_best[1] * rad, output_values['tilt'], output_values['width_x_field'], output_values['width_y_field'], alt_az_error * rad)) # Update observatory status if success: # Flush any previous observation status flush_orientation(obstory_id=obstory_id, utc_min=utc_block_min - 1, utc_max=utc_block_min + 1) user = settings['pigazingUser'] timestamp = time.time() db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:altitude", value=alt_az_best[0] * rad, time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:azimuth", value=alt_az_best[1] * rad, time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:pa", value=output_values['pa'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:tilt", value=output_values['tilt'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:width_x_field", value=output_values['width_x_field'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:width_y_field", value=output_values['width_y_field'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:uncertainty", value=alt_az_error * rad, time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:image_count", value=len(results), time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.commit()