def list_triangulations(utc_min=None, utc_max=None): """ Display a list of all the trajectories of moving objects registered in the database. :param utc_min: Only show observations made after the specified time stamp. :type utc_min: float :param utc_max: Only show observations made before the specified time stamp. :type utc_max: float :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Compile search criteria for observation groups where = ["g.semanticType = (SELECT uid FROM archive_semanticTypes WHERE name=\"{}\")". format(simultaneous_event_type) ] args = [] if utc_min is not None: where.append("g.time>=%s") args.append(utc_min) if utc_max is not None: where.append("g.time<=%s") args.append(utc_max) # Search for observation groups containing groups of simultaneous detections conn.execute(""" SELECT g.publicId AS groupId, g.time AS time, am.stringValue AS objectType, am2.floatValue AS speed, am3.floatValue AS mean_altitude, am4.floatValue AS max_angular_offset, am5.floatValue AS max_baseline, am6.stringValue AS radiant_direction, am7.floatValue AS sight_line_count, am8.stringValue AS path FROM archive_obs_groups g INNER JOIN archive_metadata am ON g.uid = am.groupId AND am.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") INNER JOIN archive_metadata am2 ON g.uid = am2.groupId AND am2.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:speed") INNER JOIN archive_metadata am3 ON g.uid = am3.groupId AND am3.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:mean_altitude") INNER JOIN archive_metadata am4 ON g.uid = am4.groupId AND am4.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:max_angular_offset") INNER JOIN archive_metadata am5 ON g.uid = am5.groupId AND am5.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:max_baseline") INNER JOIN archive_metadata am6 ON g.uid = am6.groupId AND am6.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:radiant_direction") INNER JOIN archive_metadata am7 ON g.uid = am7.groupId AND am7.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:sight_line_count") INNER JOIN archive_metadata am8 ON g.uid = am8.groupId AND am8.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="triangulation:path") WHERE """ + " AND ".join(where) + """ ORDER BY g.time; """, args) results = conn.fetchall() # Count how many simultaneous detections we find by type detections_by_type = {} # Compile tally by type for item in results: # Add this triangulation to tally if item['objectType'] not in detections_by_type: detections_by_type[item['objectType']] = 0 detections_by_type[item['objectType']] += 1 # List information about each observation in turn print("{:16s} {:20s} {:20s} {:8s} {:10s}".format("GroupId", "Time", "Object type", "Speed", "Altitude")) for item in results: # Print triangulation information print("{:16s} {:20s} {:20s} {:8.0f} {:10.0f}".format(item['groupId'], date_string(item['time']), item['objectType'], item['speed'], item['mean_altitude'] )) # Report tally of events print("\nTally of events by type:") for event_type in sorted(detections_by_type.keys()): print(" * {:26s}: {:6d}".format(event_type, detections_by_type[event_type]))
def do_triangulation(utc_min, utc_max, utc_must_stop): # We need to share the list of sight lines to each moving object with the objective function that we minimise global sight_line_list, time_span, seed_position # Start triangulation process logging.info( "Triangulating simultaneous object detections between <{}> and <{}>.". format(date_string(utc_min), date_string(utc_max))) db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) # Count how many objects we manage to successfully fit outcomes = { 'successful_fits': 0, 'failed_fits': 0, 'inadequate_baseline': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Compile search criteria for observation groups where = [ "g.semanticType = (SELECT uid FROM archive_semanticTypes WHERE name=\"{}\")" .format(simultaneous_event_type) ] args = [] if utc_min is not None: where.append("o.obsTime>=%s") args.append(utc_min) if utc_max is not None: where.append("o.obsTime<=%s") args.append(utc_max) # Open direct connection to database conn = db.con # Search for observation groups containing groups of simultaneous detections conn.execute( """ SELECT g.publicId AS groupId, o.publicId AS observationId, o.obsTime, f.repositoryFname, am.stringValue AS objectType, l.publicId AS observatory FROM archive_obs_groups g INNER JOIN archive_obs_group_members m on g.uid = m.groupId INNER JOIN archive_observations o ON m.childObservation = o.uid INNER JOIN archive_observatories l ON o.observatory = l.uid LEFT OUTER JOIN archive_files f on o.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video") INNER JOIN archive_metadata am ON g.uid = am.groupId AND am.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE """ + " AND ".join(where) + """ ORDER BY o.obsTime; """, args) results = conn.fetchall() # Compile list of events into list of groups obs_groups = {} obs_group_ids = [] for item in results: key = item['groupId'] if key not in obs_groups: obs_groups[key] = [] obs_group_ids.append({ 'groupId': key, 'time': item['obsTime'], 'type': item['objectType'] }) obs_groups[key].append(item) # Loop over list of simultaneous event detections for group_info in obs_group_ids: # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}/{type:16s}]".format( date=date_string(utc=group_info['time']), obs=group_info['groupId'], type=group_info['type']) # If we've run out of time, stop now time_now = time.time() if utc_must_stop is not None and time_now > utc_must_stop: break # Make a list of all our sight-lines to this object, from all observatories sight_line_list = [] observatory_list = {} # Fetch information about each observation in turn for item in obs_groups[group_info['groupId']]: # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file( repository_fname=item['repositoryFname']) file_metadata = { item.key: item.value for item in file_obj.meta } else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection(db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list_this = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration']) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Add to observatory_list, now that we've checked this observatory has all necessary information if item['observatory'] not in observatory_list: observatory_list[item['observatory']] = projector.obstory_info # Add sight lines from this observatory to list which combines all observatories sight_line_list.extend(sight_line_list_this) # If we have fewer than four sight lines, don't bother trying to triangulate if len(sight_line_list) < 4: logging.info( "{prefix} -- Giving up triangulation as we only have {x:d} sight lines to object." .format(prefix=logging_prefix, x=len(sight_line_list))) continue # Initialise maximum baseline between the stations which saw this objects maximum_baseline = 0 # Check the distances between all pairs of observatories obstory_info_list = [ Point.from_lat_lng(lat=obstory['latitude'], lng=obstory['longitude'], alt=0, utc=None) for obstory in observatory_list.values() ] pairs = [[obstory_info_list[i], obstory_info_list[j]] for i in range(len(obstory_info_list)) for j in range(i + 1, len(obstory_info_list))] # Work out maximum baseline between the stations which saw this objects for pair in pairs: maximum_baseline = max( maximum_baseline, abs(pair[0].displacement_vector_from(pair[1]))) # If we have no baselines of over 1 km, don't bother trying to triangulate if maximum_baseline < 1000: logging.info( "{prefix} -- Giving up triangulation as longest baseline is only {x:.0f} m." .format(prefix=logging_prefix, x=maximum_baseline)) outcomes['inadequate_baseline'] += 1 continue # Set time range of sight lines time_span = [ min(item['utc'] for item in sight_line_list), max(item['utc'] for item in sight_line_list) ] # Create a seed point to start search for object path. We pick a point above the centroid of the observatories # that saw the object centroid_v = sum(item['obs_position'].to_vector() for item in sight_line_list) / len(sight_line_list) centroid_p = Point(x=centroid_v.x, y=centroid_v.y, z=centroid_v.z) centroid_lat_lng = centroid_p.to_lat_lng(utc=None) seed_position = Point.from_lat_lng(lat=centroid_lat_lng['lat'], lng=centroid_lat_lng['lng'], alt=centroid_lat_lng['alt'] * 2e4, utc=None) # Attempt to fit a linear trajectory through all of the sight lines that we have collected parameters_initial = [0, 0, 0, 0, 0, 0] # Solve the system of equations # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> # for more information about how this works parameters_optimised = scipy.optimize.minimize( angular_mismatch_objective, numpy.asarray(parameters_initial), options={ 'disp': False, 'maxiter': 1e8 }).x # Construct best-fit linear trajectory for best-fitting parameters best_triangulation = line_from_parameters(parameters_optimised) # logging.info("Best fit path of object is <{}>.".format(best_triangulation)) # logging.info("Mismatch of observed sight lines from trajectory are {} deg.".format( # ["{:.1f}".format(best_triangulation.find_closest_approach(s['line'])['angular_distance']) # for s in sight_line_list] # )) # Find sight line with the worst match mismatch_list = sight_line_mismatch_list(trajectory=best_triangulation) maximum_mismatch = max(mismatch_list) # Reject trajectory if it deviates by more than 8 degrees from any observation if maximum_mismatch > 8: logging.info( "{prefix} -- Trajectory mismatch is too great ({x:.1f} deg).". format(prefix=logging_prefix, x=maximum_mismatch)) outcomes['failed_fits'] += 1 continue # Convert start and end points of path into (lat, lng, alt) start_point = best_triangulation.point(0).to_lat_lng(utc=None) start_point['utc'] = time_span[0] end_point = best_triangulation.point(1).to_lat_lng(utc=None) end_point['utc'] = time_span[1] # Calculate linear speed of object speed = abs(best_triangulation.direction) / ( time_span[1] - time_span[0]) # m/s # Calculate radiant direction for this object radiant_direction_vector = best_triangulation.direction * -1 radiant_direction_coordinates = radiant_direction_vector.to_ra_dec( ) # hours, degrees radiant_greenwich_hour_angle = radiant_direction_coordinates['ra'] radiant_dec = radiant_direction_coordinates['dec'] instantaneous_sidereal_time = sidereal_time(utc=(utc_min + utc_max) / 2) # hours radiant_ra = radiant_greenwich_hour_angle + instantaneous_sidereal_time # hours radiant_direction = [radiant_ra, radiant_dec] # Store triangulated information in database user = settings['pigazingUser'] timestamp = time.time() triangulation_metadata = { "triangulation:speed": speed, "triangulation:mean_altitude": (start_point['alt'] + end_point['alt']) / 2 / 1e3, # km "triangulation:max_angular_offset": maximum_mismatch, "triangulation:max_baseline": maximum_baseline, "triangulation:radiant_direction": json.dumps(radiant_direction), "triangulation:sight_line_count": len(sight_line_list), "triangulation:path": json.dumps([start_point, end_point]) } # Set metadata on the observation group for metadata_key, metadata_value in triangulation_metadata.items(): db.set_obsgroup_metadata(user_id=user, group_id=group_info['groupId'], utc=timestamp, meta=mp.Meta(key=metadata_key, value=metadata_value)) # Set metadata on each observation individually for item in obs_groups[group_info['groupId']]: for metadata_key, metadata_value in triangulation_metadata.items(): db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key=metadata_key, value=metadata_value)) # Commit metadata to database db.commit() # Report outcome logging.info( "{prefix} -- Success -- {path}; speed {mph:11.1f} mph; {sight_lines:6d} detections." .format( prefix=logging_prefix, path="{:5.1f} {:5.1f} {:10.1f} -> {:5.1f} {:5.1f} {:10.1f}". format( start_point['lat'], start_point['lng'], start_point['alt'] / 1e3, # deg deg km end_point['lat'], end_point['lng'], end_point['alt'] / 1e3), mph=speed / 0.44704, # mph sight_lines=len(sight_line_list))) # Triangulation successful outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} objects successfully triangulated.".format( outcomes['successful_fits'])) logging.info("{:d} objects could not be triangulated.".format( outcomes['failed_fits'])) logging.info("{:d} objects had an inadequate baseline.".format( outcomes['inadequate_baseline'])) logging.info("{:d} malformed database records.".format( outcomes['error_records'])) logging.info("{:d} rescued database records.".format( outcomes['rescued_records'])) logging.info("{:d} objects with incomplete data.".format( outcomes['insufficient_information'])) # Commit changes db.commit() db.close_db()
def list_images(utc_min=None, utc_max=None, username=None, obstory=None, img_type=None, obs_type=None, stride=1): """ Display a list of all the images registered in the database. :param utc_min: Only show observations made after the specified time stamp. :type utc_min: float :param utc_max: Only show observations made before the specified time stamp. :type utc_max: float :param username: Optionally specify a username, to filter only images by a particular user :type username: str :param obstory: The public id of the observatory we are to show observations from :type obstory: str :param img_type: Only show images with this semantic type :type img_type: str :param obs_type: Only show observations with this semantic type :type obs_type: str :param stride: Only show every nth observation matching the search criteria :type stride: int :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() where = ["1"] args = [] if utc_min is not None: where.append("o.obsTime>=%s") args.append(utc_min) if utc_max is not None: where.append("o.obsTime<=%s") args.append(utc_max) if username is not None: where.append("o.userId=%s") args.append(username) if obstory is not None: where.append("l.publicId=%s") args.append(obstory) if obs_type is not None: where.append("ast.name=%s") args.append(obs_type) conn.execute( """ SELECT o.uid, o.userId, l.name AS place, o.obsTime FROM archive_observations o INNER JOIN archive_observatories l ON o.observatory = l.uid INNER JOIN archive_semanticTypes ast ON o.obsType = ast.uid WHERE """ + " AND ".join(where) + """ ORDER BY obsTime DESC LIMIT 200; """, args) results = conn.fetchall() # List information about each observation in turn sys.stdout.write("{:6s} {:10s} {:32s} {:17s} {:20s}\n".format( "obsId", "Username", "Observatory", "Time", "Images")) for counter, obs in enumerate(results): # Only show every nth hit if counter % stride != 0: continue # Print observation information sys.stdout.write("{:6d} {:10s} {:32s} {:17s} ".format( obs['uid'], obs['userId'], obs['place'], date_string(obs['obsTime']))) where = ["f.observationId=%s"] args = [obs['uid']] if img_type is not None: where.append("ast.name=%s") args.append(img_type) # Fetch list of files in this observation conn.execute( """ SELECT ast.name AS semanticType, repositoryFname, am.floatValue AS skyClarity FROM archive_files f INNER JOIN archive_semanticTypes ast ON f.semanticType = ast.uid LEFT OUTER JOIN archive_metadata am ON f.uid = am.fileId AND am.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="pigazing:skyClarity") WHERE """ + " AND ".join(where) + """; """, args) files = conn.fetchall() for count, item in enumerate(files): if count > 0: sys.stdout.write("\n{:69s}".format("")) if item['skyClarity'] is None: item['skyClarity'] = 0 sys.stdout.write("{:40s} {:32s} {:10.1f}".format( item['semanticType'], item['repositoryFname'], item['skyClarity'])) sys.stdout.write("\n")
def list_orientation_fixes(obstory_id, utc_min, utc_max): """ List all the orientation fixes for a particular observatory. :param obstory_id: The ID of the observatory we want to list orientation fixes for. :param utc_min: The start of the time period in which we should list orientation fixes (unix time). :param utc_max: The end of the time period in which we should list orientation fixes (unix time). :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Start compiling list of orientation fixes orientation_fixes = [] # Select observations with orientation fits conn.execute( """ SELECT am1.floatValue AS altitude, am2.floatValue AS azimuth, am3.floatValue AS tilt, am4.floatValue AS width_x_field, am5.floatValue AS width_y_field, o.obsTime AS time FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:altitude") INNER JOIN archive_metadata am2 ON o.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:azimuth") INNER JOIN archive_metadata am3 ON o.uid = am3.observationId AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:tilt") INNER JOIN archive_metadata am4 ON o.uid = am4.observationId AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_x_field") INNER JOIN archive_metadata am5 ON o.uid = am5.observationId AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_y_field") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: orientation_fixes.append({ 'time': item['time'], 'average': False, 'fit': item }) # Select observatory orientation fits conn.execute( """ SELECT am1.floatValue AS altitude, am2.floatValue AS azimuth, am3.floatValue AS tilt, am4.floatValue AS width_x_field, am5.floatValue AS width_y_field, am1.time AS time FROM archive_observatories o INNER JOIN archive_metadata am1 ON o.uid = am1.observatory AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:altitude") INNER JOIN archive_metadata am2 ON o.uid = am2.observatory AND am2.time=am1.time AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:azimuth") INNER JOIN archive_metadata am3 ON o.uid = am3.observatory AND am3.time=am1.time AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:tilt") INNER JOIN archive_metadata am4 ON o.uid = am4.observatory AND am4.time=am1.time AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_x_field") INNER JOIN archive_metadata am5 ON o.uid = am5.observatory AND am5.time=am1.time AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_y_field") WHERE o.publicId=%s AND am1.time BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: orientation_fixes.append({ 'time': item['time'], 'average': True, 'fit': item }) # Sort fixes by time orientation_fixes.sort(key=itemgetter('time')) # Display column headings print("""\ {:1s} {:16s} {:9s} {:9s} {:9s} {:8s} {:8s}\ """.format("", "Time", "Alt", "Az", "Tilt", "FoV X", "FoV Y")) # Display fixes for item in orientation_fixes: print("""\ {:s} {:16s} {:9.4f} {:9.4f} {:9.4f} {:8.3f} {:8.3f} {:s}\ """.format("\n>" if item['average'] else " ", date_string(item['time']), item['fit']['altitude'], item['fit']['azimuth'], item['fit']['tilt'], item['fit']['width_x_field'], item['fit']['width_y_field'], "\n" if item['average'] else "")) # Clean up and exit return
def frame_drop_detection(utc_min, utc_max): """ Detect video frame drop events between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should search for video frame drop (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should search for video frame drop (unix time). :type utc_max: float :return: None """ # Open connection to image archive db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting video frame drop detection.") # Count how many images we manage to successfully fit outcomes = { 'frame_drop_events': 0, 'non_frame_drop_events': 0, 'error_records': 0, 'rescued_records': 0 } # Status update logging.info("Searching for frame drops within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for meteors within this time period conn.execute( """ SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory, am6.stringValue AS type FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid LEFT OUTER JOIN archive_metadata am6 ON ao.uid = am6.observationId AND am6.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s ORDER BY ao.obsTime """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the videos we are going to work on logging.info("Searching for dropped frames within {:d} videos.".format( len(results))) # Analyse each video in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if ('pigazing:path' not in all_metadata) or ('pigazing:videoStart' not in all_metadata): logging.info( "Cannot process <{}> due to inadequate metadata.".format( item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}/{type:16s}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId'], type=item['type'] if item['type'] is not None else '') # Read path of the moving object in pixel coordinates path_json = all_metadata['pigazing:path'] try: path_x_y = json.loads(path_json) except json.decoder.JSONDecodeError: # Attempt JSON repair; sometimes JSON content gets truncated original_json = path_json fixed_json = "],[".join(original_json.split("],[")[:-1]) + "]]" try: path_x_y = json.loads(fixed_json) # logging.info("{prefix} -- RESCUE: In: {detections:.0f} / {duration:.1f} sec; " # "Rescued: {count:d} / {json_span:.1f} sec".format( # prefix=logging_prefix, # detections=all_metadata['pigazing:detections'], # duration=all_metadata['pigazing:duration'], # count=len(path_x_y), # json_span=path_x_y[-1][3] - path_x_y[0][3] # )) outcomes['rescued_records'] += 1 except json.decoder.JSONDecodeError: logging.info( "{prefix} -- !!! JSON error".format(prefix=logging_prefix)) outcomes['error_records'] += 1 continue # Check number of points in path path_len = len(path_x_y) # Make list of object speed at each point path_speed = [] # pixels/sec path_distance = [] for i in range(path_len - 1): pixel_distance = hypot(path_x_y[i + 1][0] - path_x_y[i][0], path_x_y[i + 1][1] - path_x_y[i][1]) time_interval = (path_x_y[i + 1][3] - path_x_y[i][3]) + 1e-8 speed = pixel_distance / time_interval path_speed.append(speed) path_distance.append(pixel_distance) # Start making a list of frame-drop events frame_drop_points = [] # Scan through for points with anomalously high speed scan_half_window = 4 for i in range(len(path_speed)): scan_min = max(0, i - scan_half_window) scan_max = min(scan_min + 2 * scan_half_window, len(path_speed) - 1) median_speed = max(np.median(path_speed[scan_min:scan_max]), 1) if (path_distance[i] > 16) and (path_speed[i] > 4 * median_speed): break_time = np.mean([path_x_y[i + 1][3], path_x_y[i][3]]) video_time = break_time - all_metadata['pigazing:videoStart'] break_distance = path_distance[i] # significance = path_speed[i]/median_speed frame_drop_points.append([ i + 1, float("%.4f" % break_time), float("%.1f" % video_time), round(break_distance) ]) # Report result if len(frame_drop_points) > 0: logging.info("{prefix} -- {x}".format(prefix=logging_prefix, x=frame_drop_points)) # Store frame-drop list user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="frame_drop:list", value=json.dumps(frame_drop_points))) # Video successfully analysed if len(frame_drop_points) == 0: outcomes['non_frame_drop_events'] += 1 else: outcomes['frame_drop_events'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} videos with frame-drop.".format( outcomes['frame_drop_events'])) logging.info("{:d} videos without frame-drop.".format( outcomes['non_frame_drop_events'])) logging.info("{:d} malformed database records.".format( outcomes['error_records'])) logging.info("{:d} rescued database records.".format( outcomes['rescued_records'])) # Clean up and exit db.commit() db.close_db() return
def satellite_determination(utc_min, utc_max): """ Estimate the identity of spacecraft observed between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should determine the identity of spacecraft (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should determine the identity of spacecraft (unix time). :type utc_max: float :return: None """ # Open connection to image archive db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting satellite identification.") # Count how many images we manage to successfully fit outcomes = { 'successful_fits': 0, 'unsuccessful_fits': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Status update logging.info("Searching for satellites within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for satellites within this time period conn.execute( """ SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid INNER JOIN archive_metadata am2 ON ao.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s AND (am2.stringValue='Plane' OR am2.stringValue='Satellite' OR am2.stringValue='Junk') ORDER BY ao.obsTime """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the images we are going to work on logging.info("Estimating the identity of {:d} spacecraft.".format( len(results))) # Analyse each spacecraft in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if 'pigazing:path' not in all_metadata: logging.info( "Cannot process <{}> due to inadequate metadata.".format( item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId']) # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection(db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list_this = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration']) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Check number of points in path path_len = len(path_x_y) # Look up list of satellite orbital elements at the time of this sighting spacecraft_list = fetch_satellites(utc=item['obsTime']) # List of candidate satellites this object might be candidate_satellites = [] # Check that we found a list of spacecraft if spacecraft_list is None: logging.info( "{date} [{obs}] -- No spacecraft records found.".format( date=date_string(utc=item['obsTime']), obs=item['observationId'])) outcomes['insufficient_information'] += 1 continue # Logging message about how many spacecraft we're testing # logging.info("{date} [{obs}] -- Matching against {count:7d} spacecraft.".format( # date=date_string(utc=item['obsTime']), # obs=item['observationId'], # count=len(spacecraft_list) # )) # Test for each candidate satellite in turn for spacecraft in spacecraft_list: # Unit scaling deg2rad = pi / 180.0 # 0.0174532925199433 xpdotp = 1440.0 / (2.0 * pi) # 229.1831180523293 # Model the path of this spacecraft model = Satrec() model.sgp4init( # whichconst: gravity model WGS72, # opsmode: 'a' = old AFSPC mode, 'i' = improved mode 'i', # satnum: Satellite number spacecraft['noradId'], # epoch: days since 1949 December 31 00:00 UT jd_from_unix(spacecraft['epoch']) - 2433281.5, # bstar: drag coefficient (/earth radii) spacecraft['bStar'], # ndot (NOT USED): ballistic coefficient (revs/day) spacecraft['meanMotionDot'] / (xpdotp * 1440.0), # nddot (NOT USED): mean motion 2nd derivative (revs/day^3) spacecraft['meanMotionDotDot'] / (xpdotp * 1440.0 * 1440), # ecco: eccentricity spacecraft['ecc'], # argpo: argument of perigee (radians) spacecraft['argPeri'] * deg2rad, # inclo: inclination (radians) spacecraft['incl'] * deg2rad, # mo: mean anomaly (radians) spacecraft['meanAnom'] * deg2rad, # no_kozai: mean motion (radians/minute) spacecraft['meanMotion'] / xpdotp, # nodeo: right ascension of ascending node (radians) spacecraft['RAasc'] * deg2rad) # Wrap within skyfield to convert to topocentric coordinates ts = load.timescale() sat = EarthSatellite.from_satrec(model, ts) # Fetch spacecraft position at each time point along trajectory ang_mismatch_list = [] distance_list = [] # e, r, v = model.sgp4(jd_from_unix(utc=item['obsTime']), 0) # logging.info("{} {} {}".format(str(e), str(r), str(v))) tai_utc_offset = 39 # seconds def satellite_angular_offset(index, clock_offset): # Fetch observed position of object at this time point pt_utc = path_x_y[index][3] pt_alt = path_alt_az[index][0] pt_az = path_alt_az[index][1] # Project position of this satellite in space at this time point t = ts.tai_jd(jd=jd_from_unix(utc=pt_utc + tai_utc_offset + clock_offset)) # Project position of this satellite in the observer's sky sight_line = sat - observer topocentric = sight_line.at(t) sat_alt, sat_az, sat_distance = topocentric.altaz() # Work out offset of satellite's position from observed moving object ang_mismatch = ang_dist(ra0=pt_az * pi / 180, dec0=pt_alt * pi / 180, ra1=sat_az.radians, dec1=sat_alt.radians) * 180 / pi return ang_mismatch, sat_distance def time_offset_objective(p): """ Objective function that we minimise in order to find the best fit clock offset between the observed and model paths. :param p: Vector with a single component: the clock offset :return: Metric to minimise """ # Turn input parameters into a time offset clock_offset = p[0] # Look up angular offset ang_mismatch, sat_distance = satellite_angular_offset( index=0, clock_offset=clock_offset) # Return metric to minimise return ang_mismatch * exp(clock_offset / 8) # First, chuck out satellites with large angular offsets observer = wgs84.latlon( latitude_degrees=projector.obstory_info['latitude'], longitude_degrees=projector.obstory_info['longitude'], elevation_m=0) ang_mismatch, sat_distance = satellite_angular_offset( index=0, clock_offset=0) # Check angular offset is reasonable if ang_mismatch > global_settings['max_angular_mismatch']: continue # Work out the optimum time offset between the satellite's path and the observed path # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> # for more information about how this works parameters_initial = [0] parameters_optimised = scipy.optimize.minimize( time_offset_objective, np.asarray(parameters_initial), options={ 'disp': False, 'maxiter': 100 }).x # Construct best-fit linear trajectory for best-fitting parameters clock_offset = float(parameters_optimised[0]) # Check clock offset is reasonable if abs(clock_offset) > global_settings['max_clock_offset']: continue # Measure the offset between the satellite's position and the observed position at each time point for index in range(path_len): # Look up angular mismatch at this time point ang_mismatch, sat_distance = satellite_angular_offset( index=index, clock_offset=clock_offset) # Keep list of the offsets at each recorded time point along the trajectory ang_mismatch_list.append(ang_mismatch) distance_list.append(sat_distance.km) # Consider adding this satellite to list of candidates mean_ang_mismatch = np.mean(np.asarray(ang_mismatch_list)) distance_mean = np.mean(np.asarray(distance_list)) if mean_ang_mismatch < global_settings['max_mean_angular_mismatch']: candidate_satellites.append({ 'name': spacecraft['name'], # string 'noradId': spacecraft['noradId'], # int 'distance': distance_mean, # km 'clock_offset': clock_offset, # seconds 'offset': mean_ang_mismatch, # degrees 'absolute_magnitude': spacecraft['mag'] }) # Add model possibility for null satellite candidate_satellites.append({ 'name': "Unidentified", 'noradId': 0, 'distance': 35.7e3 * 0.25, # Nothing is visible beyond 25% of geostationary orbit distance 'clock_offset': 0, 'offset': 0, 'absolute_magnitude': None }) # Sort candidates by score - use absolute mag = 20 for satellites with no mag for candidate in candidate_satellites: candidate['score'] = hypot( candidate['distance'] / 1e3, candidate['clock_offset'], (20 if candidate['absolute_magnitude'] is None else candidate['absolute_magnitude']), ) candidate_satellites.sort(key=itemgetter('score')) # Report possible satellite identifications logging.info("{prefix} -- {satellites}".format( prefix=logging_prefix, satellites=", ".join([ "{} ({:.1f} deg offset; clock offset {:.1f} sec)".format( satellite['name'], satellite['offset'], satellite['clock_offset']) for satellite in candidate_satellites ]))) # Identify most likely satellite most_likely_satellite = candidate_satellites[0] # Store satellite identification user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="satellite:name", value=most_likely_satellite['name'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="satellite:norad_id", value=most_likely_satellite['noradId'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="satellite:clock_offset", value=most_likely_satellite['clock_offset'])) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="satellite:angular_offset", value=most_likely_satellite['offset'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="satellite:path_length", value=ang_dist(ra0=path_ra_dec_at_epoch[0][0], dec0=path_ra_dec_at_epoch[0][1], ra1=path_ra_dec_at_epoch[-1][0], dec1=path_ra_dec_at_epoch[-1][1]) * 180 / pi)) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="satellite:path_ra_dec", value="[[{:.3f},{:.3f}],[{:.3f},{:.3f}],[{:.3f},{:.3f}]]". format( path_ra_dec_at_epoch[0][0] * 12 / pi, path_ra_dec_at_epoch[0][1] * 180 / pi, path_ra_dec_at_epoch[int(path_len / 2)][0] * 12 / pi, path_ra_dec_at_epoch[int(path_len / 2)][1] * 180 / pi, path_ra_dec_at_epoch[-1][0] * 12 / pi, path_ra_dec_at_epoch[-1][1] * 180 / pi, ))) # Satellite successfully identified if most_likely_satellite['name'] == "Unidentified": outcomes['unsuccessful_fits'] += 1 else: outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} satellites successfully identified.".format( outcomes['successful_fits'])) logging.info("{:d} satellites not identified.".format( outcomes['unsuccessful_fits'])) logging.info("{:d} malformed database records.".format( outcomes['error_records'])) logging.info("{:d} rescued database records.".format( outcomes['rescued_records'])) logging.info("{:d} satellites with incomplete data.".format( outcomes['insufficient_information'])) # Clean up and exit db.commit() db.close_db() return
def calibrate_lens(obstory_id, utc_min, utc_max, utc_must_stop=None): """ Use astrometry.net to determine the orientation of a particular observatory. :param obstory_id: The ID of the observatory we want to determine the orientation for. :param utc_min: The start of the time period in which we should determine the observatory's orientation. :param utc_max: The end of the time period in which we should determine the observatory's orientation. :param utc_must_stop: The time by which we must finish work :return: None """ global parameter_scales, fit_list # Open connection to database [db0, conn] = connect_db.connect_db() # Open connection to image archive db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info( "Starting estimation of lens calibration for <{}>".format(obstory_id)) # Mathematical constants deg = pi / 180 rad = 180 / pi # Count how many successful fits we achieve successful_fits = 0 # Read properties of known lenses hw = hardware_properties.HardwareProps( path=os.path.join(settings['pythonPath'], "..", "configuration_global", "camera_properties")) # Reduce time window to where observations are present conn.execute( """ SELECT obsTime FROM archive_observations WHERE obsTime BETWEEN %s AND %s AND observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) ORDER BY obsTime ASC LIMIT 1 """, (utc_min, utc_max, obstory_id)) results = conn.fetchall() if len(results) == 0: logging.warning("No observations within requested time window.") return utc_min = results[0]['obsTime'] - 1 conn.execute( """ SELECT obsTime FROM archive_observations WHERE obsTime BETWEEN %s AND %s AND observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) ORDER BY obsTime DESC LIMIT 1 """, (utc_min, utc_max, obstory_id)) results = conn.fetchall() utc_max = results[0]['obsTime'] + 1 # Divide up time interval into day-long blocks logging.info("Searching for images within period {} to {}".format( date_string(utc_min), date_string(utc_max))) block_size = 3600 minimum_sky_clarity = 1e6 + 1400 utc_min = (floor(utc_min / block_size + 0.5) - 0.5) * block_size # Make sure that blocks start at noon time_blocks = list( np.arange(start=utc_min, stop=utc_max + block_size, step=block_size)) # Start new block whenever we have a hardware refresh conn.execute( """ SELECT time FROM archive_metadata WHERE observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) AND fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey='refresh') AND time BETWEEN %s AND %s """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: time_blocks.append(item['time']) # Make sure that start points for time blocks are in order time_blocks.sort() # Build list of images we are to analyse images_for_analysis = [] for block_index, utc_block_min in enumerate(time_blocks[:-1]): utc_block_max = time_blocks[block_index + 1] logging.info("Calibrating lens within period {} to {}".format( date_string(utc_block_min), date_string(utc_block_max))) # Search for background-subtracted time lapse image with best sky clarity within this time period conn.execute( """ SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, am.floatValue AS skyClarity FROM archive_files f INNER JOIN archive_observations ao on f.observationId = ao.uid INNER JOIN archive_metadata am ON f.uid = am.fileId AND am.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="pigazing:skyClarity") LEFT OUTER JOIN archive_metadata am2 ON f.uid = am2.fileId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:lens_barrel_parameters") WHERE ao.obsTime BETWEEN %s AND %s AND ao.observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:timelapse/backgroundSubtracted") AND am.floatValue > %s AND am2.uid IS NULL AND ao.astrometryProcessed IS NULL ORDER BY am.floatValue DESC LIMIT 1 """, (utc_block_min, utc_block_max, obstory_id, minimum_sky_clarity)) results = conn.fetchall() if len(results) > 0: images_for_analysis.append({ 'utc': results[0]['obsTime'], 'skyClarity': results[0]['skyClarity'], 'repositoryFname': results[0]['repositoryFname'], 'observationId': results[0]['observationId'] }) # Sort images into order of sky clarity images_for_analysis.sort(key=itemgetter("skyClarity")) images_for_analysis.reverse() # Display logging list of the images we are going to work on logging.info("Estimating the calibration of {:d} images:".format( len(images_for_analysis))) for item in images_for_analysis: logging.info("{:17s} {:04.0f} {:32s}".format(date_string(item['utc']), item['skyClarity'], item['repositoryFname'])) # Analyse each image in turn for item_index, item in enumerate(images_for_analysis): logging.info("Working on image {:32s} ({:4d}/{:4d})".format( item['repositoryFname'], item_index + 1, len(images_for_analysis))) # Make a temporary directory to store files in. # This is necessary as astrometry.net spams the cwd with lots of temporary junk tmp0 = "/tmp/dcf21_calibrateLens_{}".format(item['repositoryFname']) # logging.info("Created temporary directory <{}>".format(tmp)) os.system("mkdir {}".format(tmp0)) # Fetch observatory status obstory_info = db.get_obstory_from_id(obstory_id) obstory_status = None if obstory_info and ('name' in obstory_info): obstory_status = db.get_obstory_status(obstory_id=obstory_id, time=item['utc']) if not obstory_status: logging.info("Aborting -- no observatory status available.") continue # Fetch observatory status lens_name = obstory_status['lens'] lens_props = hw.lens_data[lens_name] # This is an estimate of the *maximum* angular width we expect images to have. # It should be within a factor of two of correct! estimated_image_scale = lens_props.fov # Find image orientation orientation filename = os.path.join(settings['dbFilestore'], item['repositoryFname']) if not os.path.exists(filename): logging.info("Error: File <{}> is missing!".format( item['repositoryFname'])) continue # 1. Copy image into working directory # logging.info("Copying file") img_name = item['repositoryFname'] command = "cp {} {}/{}_tmp.png".format(filename, tmp0, img_name) # logging.info(command) os.system(command) # 2. We estimate the distortion of the image by passing a series of small portions of the image to # astrometry.net. We use this to construct a mapping between (x, y) pixel coordinates to (RA, Dec). # Define the size of each small portion we pass to astrometry.net fraction_x = 0.15 fraction_y = 0.15 # Create a list of the centres of the portions we send fit_list = [] portion_centres = [{'x': 0.5, 'y': 0.5}] # Points along the leading diagonal of the image for z in np.arange(0.1, 0.9, 0.1): if z != 0.5: portion_centres.append({'x': z, 'y': z}) portion_centres.append({'x': (z + 0.5) / 2, 'y': z}) portion_centres.append({'x': z, 'y': (z + 0.5) / 2}) # Points along the trailing diagonal of the image for z in np.arange(0.1, 0.9, 0.1): if z != 0.5: portion_centres.append({'x': z, 'y': 1 - z}) portion_centres.append({'x': (1.5 - z) / 2, 'y': z}) portion_centres.append({'x': z, 'y': (1.5 - z) / 2}) # Points down the vertical centre-line of the image for z in np.arange(0.15, 0.85, 0.1): portion_centres.append({'x': 0.5, 'y': z}) # Points along the horizontal centre-line of the image for z in np.arange(0.15, 0.85, 0.1): portion_centres.append({'x': z, 'y': 0.5}) # Fetch the pixel dimensions of the image we are working on d = image_dimensions("{}/{}_tmp.png".format(tmp0, img_name)) @dask.delayed def analyse_image_portion(image_portion): # Make a temporary directory to store files in. # This is necessary as astrometry.net spams the cwd with lots of temporary junk tmp = "/tmp/dcf21_calibrateLens_{}_{}".format( item['repositoryFname'], image_portion['index']) # logging.info("Created temporary directory <{}>".format(tmp)) os.system("mkdir {}".format(tmp)) # Use ImageMagick to crop out each small piece of the image command = """ cd {6} ; \ rm -f {5}_tmp3.png ; \ convert {0}_tmp.png -colorspace sRGB -define png:format=png24 -crop {1:d}x{2:d}+{3:d}+{4:d} +repage {5}_tmp3.png """.format(os.path.join(tmp0, img_name), int(fraction_x * d[0]), int(fraction_y * d[1]), int((image_portion['x'] - fraction_x / 2) * d[0]), int((image_portion['y'] - fraction_y / 2) * d[1]), img_name, tmp) # logging.info(command) os.system(command) # Check that we've not run out of time if utc_must_stop and (time.time() > utc_must_stop): logging.info("We have run out of time! Aborting.") os.system("rm -Rf {}".format(tmp)) return None # How long should we allow astrometry.net to run for? timeout = "40s" # Run astrometry.net. Insert --no-plots on the command line to speed things up. # logging.info("Running astrometry.net") estimated_width = 2 * math.atan( math.tan(estimated_image_scale / 2 * deg) * fraction_x) * rad astrometry_output = os.path.join(tmp, "txt") command = """ cd {5} ; \ timeout {0} solve-field --no-plots --crpix-center --scale-low {1:.1f} \ --scale-high {2:.1f} --overwrite {3}_tmp3.png > {4} 2> /dev/null \ """.format(timeout, estimated_width * 0.6, estimated_width * 1.2, img_name, astrometry_output, tmp) # logging.info(command) os.system(command) # Parse the output from astrometry.net assert os.path.exists( astrometry_output), "Path <{}> doesn't exist".format( astrometry_output) fit_text = open(astrometry_output).read() # logging.info(fit_text) # Clean up # logging.info("Removing temporary directory <{}>".format(tmp)) os.system("rm -Rf {}".format(tmp)) # Extract celestial coordinates of the centre of the frame from astrometry.net output test = re.search( r"\(RA H:M:S, Dec D:M:S\) = \(([\d-]*):(\d\d):([\d.]*), [+]?([\d-]*):(\d\d):([\d\.]*)\)", fit_text) if not test: logging.info("FAIL(POS): Point ({:.2f},{:.2f}).".format( image_portion['x'], image_portion['y'])) return None ra_sign = sgn(float(test.group(1))) ra = abs(float(test.group(1))) + float(test.group(2)) / 60 + float( test.group(3)) / 3600 if ra_sign < 0: ra *= -1 dec_sign = sgn(float(test.group(4))) dec = abs(float(test.group(4))) + float( test.group(5)) / 60 + float(test.group(6)) / 3600 if dec_sign < 0: dec *= -1 # If astrometry.net achieved a fit, then we report it to the user logging.info( "FIT: RA: {:7.2f}h. Dec {:7.2f} deg. Point ({:.2f},{:.2f}).". format(ra, dec, image_portion['x'], image_portion['y'])) # Also, populate <fit_list> with a list of the central points of the image fragments, and their (RA, Dec) # coordinates. return { 'ra': ra * pi / 12, 'dec': dec * pi / 180, 'x': image_portion['x'], 'y': image_portion['y'], 'radius': hypot(image_portion['x'] - 0.5, image_portion['y'] - 0.5) } # Analyse each small portion of the image in turn dask_tasks = [] for index, image_portion in enumerate(portion_centres): image_portion['index'] = index dask_tasks.append( analyse_image_portion(image_portion=image_portion)) fit_list = dask.compute(*dask_tasks) # Remove fits which returned None fit_list = [i for i in fit_list if i is not None] # Clean up os.system("rm -Rf {}".format(tmp0)) os.system("rm -Rf /tmp/tmp.*") # Make histogram of fits as a function of radius radius_histogram = [0] * 10 for fit in fit_list: radius_histogram[int(fit['radius'] * 10)] += 1 logging.info("Fit histogram vs radius: {}".format(radius_histogram)) # Reject this image if there are insufficient fits from astrometry.net if min(radius_histogram[:5]) < 2: logging.info("Insufficient fits to continue") continue # Fit a gnomonic projection to the image, with barrel correction, to fit all the celestial positions of the # image fragments. # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> for more information ra0 = fit_list[0]['ra'] dec0 = fit_list[0]['dec'] parameter_scales = [ pi / 4, pi / 4, pi / 4, pi / 4, pi / 4, pi / 4, 5e-2, 5e-6 ] parameters_default = [ ra0, dec0, pi / 4, pi / 4, 0, lens_props.barrel_parameters[2], 0 ] parameters_initial = [ parameters_default[i] / parameter_scales[i] for i in range(len(parameters_default)) ] fitting_result = scipy.optimize.minimize(mismatch, parameters_initial, method='nelder-mead', options={ 'xtol': 1e-8, 'disp': True, 'maxiter': 1e8, 'maxfev': 1e8 }) parameters_optimal = fitting_result.x parameters_final = [ parameters_optimal[i] * parameter_scales[i] for i in range(len(parameters_default)) ] # Display best fit numbers headings = [["Central RA / hr", 12 / pi], ["Central Decl / deg", 180 / pi], ["Image width / deg", 180 / pi], ["Image height / deg", 180 / pi], ["Position angle / deg", 180 / pi], ["barrel_k1", 1], ["barrel_k2", 1]] logging.info( "Fit achieved to {:d} points with offset of {:.5f}. Best fit parameters were:" .format(len(fit_list), fitting_result.fun)) for i in range(len(parameters_default)): logging.info("{0:30s} : {1}".format( headings[i][0], parameters_final[i] * headings[i][1])) # Reject fit if objective function too large if fitting_result.fun > 1e-4: logging.info("Rejecting fit as chi-squared too large.") continue # Reject fit if k1/k2 values are too extreme if (abs(parameters_final[5]) > 0.3) or (abs(parameters_final[6]) > 0.1): logging.info("Rejecting fit as parameters seem extreme.") continue # Update observation status successful_fits += 1 user = settings['pigazingUser'] timestamp = time.time() barrel_parameters = [ parameters_final[2] * 180 / pi, parameters_final[3] * 180 / pi, parameters_final[5], parameters_final[6], 0 ] db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="calibration:lens_barrel_parameters", value=json.dumps(barrel_parameters))) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="calibration:chi_squared", value=fitting_result.fun)) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="calibration:point_count", value=str(radius_histogram))) # Commit metadata changes db.commit() db0.commit() # Report how many fits we achieved logging.info( "Total of {:d} images successfully fitted.".format(successful_fits)) if successful_fits > 0: # Now determine mean lens calibration each day logging.info("Averaging daily fits within period {} to {}".format( date_string(utc_min), date_string(utc_max))) block_size = 86400 utc_min = (floor(utc_min / block_size + 0.5) - 0.5) * block_size # Make sure that blocks start at noon time_blocks = list( np.arange(start=utc_min, stop=utc_max + block_size, step=block_size)) # Start new block whenever we have a hardware refresh conn.execute( """ SELECT time FROM archive_metadata WHERE observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) AND fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey='refresh') AND time BETWEEN %s AND %s """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: time_blocks.append(item['time']) # Make sure that start points for time blocks are in order time_blocks.sort() for block_index, utc_block_min in enumerate(time_blocks[:-1]): utc_block_max = time_blocks[block_index + 1] # Select observations with calibration fits conn.execute( """ SELECT am1.stringValue AS barrel_parameters FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:lens_barrel_parameters") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_block_min, utc_block_max)) results = conn.fetchall() logging.info( "Averaging fits within period {} to {}: Found {} fits.".format( date_string(utc_block_min), date_string(utc_block_max), len(results))) # Average the fits we found if len(results) < 3: logging.info("Insufficient images to reliably average.") continue # Pick the median fit value_list = { 'scale_x': [], 'scale_y': [], 'barrel_k1': [], 'barrel_k2': [], 'barrel_k3': [] } for item in results: barrel_parameters = json.loads(item['barrel_parameters']) value_list['scale_x'].append(barrel_parameters[0]) value_list['scale_y'].append(barrel_parameters[1]) value_list['barrel_k1'].append(barrel_parameters[2]) value_list['barrel_k2'].append(barrel_parameters[3]) value_list['barrel_k3'].append(barrel_parameters[4]) median_values = {} for key, values in value_list.items(): values.sort() median_values[key] = values[len(values) // 2] # Print fit information logging.info("""\ CALIBRATION FIT from {:2d} images: %s. \ """.format( len(results), "; ".join([ "{}: {}".format(key, median) for key, median in median_values.items() ]))) # Flush any previous observation status flush_calibration(obstory_id=obstory_id, utc_min=utc_block_min - 1, utc_max=utc_block_min + 1) # Update observatory status user = settings['pigazingUser'] timestamp = time.time() barrel_parameters = [ median_values['scale_x'], median_values['scale_y'], median_values['barrel_k1'], median_values['barrel_k2'], median_values['barrel_k3'] ] db.register_obstory_metadata( obstory_id=obstory_id, key="calibration:lens_barrel_parameters", value=json.dumps(barrel_parameters), time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.commit() # Clean up and exit db.commit() db.close_db() db0.commit() conn.close() db0.close() return
def list_calibration_fixes(obstory_id, utc_min, utc_max): """ List all the calibration fixes for a particular observatory. :param obstory_id: The ID of the observatory we want to list calibration fixes for. :param utc_min: The start of the time period in which we should list calibration fixes (unix time). :param utc_max: The end of the time period in which we should list calibration fixes (unix time). :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Start compiling list of calibration fixes calibration_fixes = [] # Select observatory with calibration fits conn.execute( """ SELECT am1.stringValue AS barrel_parameters, am4.floatValue AS chi_squared, am5.stringValue AS point_count, o.obsTime AS time FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:lens_barrel_parameters") INNER JOIN archive_metadata am4 ON o.uid = am4.observationId AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:chi_squared") INNER JOIN archive_metadata am5 ON o.uid = am5.observationId AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:point_count") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: calibration_fixes.append({ 'time': item['time'], 'average': False, 'fit': item }) # Select observation calibration fits conn.execute( """ SELECT am1.stringValue AS barrel_parameters, am3.floatValue AS chi_squared, am4.stringValue AS point_count, am1.time AS time FROM archive_observatories o INNER JOIN archive_metadata am1 ON o.uid = am1.observatory AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:lens_barrel_parameters") LEFT OUTER JOIN archive_metadata am3 ON o.uid = am3.observatory AND am3.time=am1.time AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:chi_squared") LEFT OUTER JOIN archive_metadata am4 ON o.uid = am4.observatory AND am4.time=am1.time AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="calibration:point_count") WHERE o.publicId=%s AND am1.time BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: calibration_fixes.append({ 'time': item['time'], 'average': True, 'fit': item }) # Sort fixes by time calibration_fixes.sort(key=itemgetter('time')) # Display column headings print("""\ {:1s} {:16s} {:8s} {:8s} {:10s} {:12s} {:6s}\ """.format("", "Time", "barrelK1", "barrelK2", "barrelK3", "chi2", "points")) # Display fixes for item in calibration_fixes: # Deal with missing data if item['fit']['chi_squared'] is None: item['fit']['chi_squared'] = -1 if item['fit']['point_count'] is None: item['fit']['point_count'] = "-" # Display calibration fix barrel_parameters = json.loads(item['fit']['barrel_parameters']) print("""\ {:s} {:16s} {:8.4f} {:8.4f} {:10.7f} {:12.9f} {:s} {:s}\ """.format("\n>" if item['average'] else " ", date_string(item['time']), barrel_parameters[2], barrel_parameters[3], barrel_parameters[4], item['fit']['chi_squared'], item['fit']['point_count'], "\n" if item['average'] else "")) # Clean up and exit return
def list_simultaneous_detections(utc_min=None, utc_max=None): """ Display a list of all the simultaneous object detections registered in the database. :param utc_min: Only show observations made after the specified time stamp. :type utc_min: float :param utc_max: Only show observations made before the specified time stamp. :type utc_max: float :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Compile search criteria for observation groups where = [ "g.semanticType = (SELECT uid FROM archive_semanticTypes WHERE name=\"{}\")" .format(simultaneous_event_type) ] args = [] if utc_min is not None: where.append("o.obsTime>=%s") args.append(utc_min) if utc_max is not None: where.append("o.obsTime<=%s") args.append(utc_max) # Search for observation groups containing groups of simultaneous detections conn.execute( """ SELECT g.publicId AS groupId, o.publicId AS obsId, o.obsTime, am.stringValue AS objectType FROM archive_obs_groups g INNER JOIN archive_obs_group_members m on g.uid = m.groupId INNER JOIN archive_observations o ON m.childObservation = o.uid INNER JOIN archive_metadata am ON g.uid = am.groupId AND am.fieldId = (SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE """ + " AND ".join(where) + """ ORDER BY o.obsTime; """, args) results = conn.fetchall() # Count how many simultaneous detections we find by type detections_by_type = {} # Compile list of groups obs_groups = {} obs_group_ids = [] for item in results: key = item['groupId'] if key not in obs_groups: obs_groups[key] = [] obs_group_ids.append({ 'groupId': key, 'time': item['obsTime'], 'type': item['objectType'] }) # Add this simultaneous detection to tally if item['objectType'] not in detections_by_type: detections_by_type[item['objectType']] = 0 detections_by_type[item['objectType']] += 1 obs_groups[key].append(item['obsId']) # List information about each observation in turn print("{:16s} {:20s} {:20s} {:s}".format("Time", "groupId", "Object type", "Observations")) for group_info in obs_group_ids: # Print group information print("{:16s} {:20s} {:20s} {:s}".format( group_info['groupId'], date_string(group_info['time']), group_info['type'], " ".join(obs_groups[group_info['groupId']]))) # Report tally of events print("\nTally of events by type:") for event_type in sorted(detections_by_type.keys()): print(" * {:26s}: {:6d}".format(event_type, detections_by_type[event_type]))
def average_daily_fits(conn, db, obstory_id, utc_max, utc_min): """ Average all of the orientation fixes within a given time period, excluding extreme fits. Update the observatory's status with a altitude and azimuth of the average fit, if it has a suitably small error bar. :param conn: Database connection object. :param db: Database object. :param obstory_id: Observatory publicId. :param utc_max: Unix time of the end of the time period. :param utc_min: Unix time of the beginning of the time period. :return: None """ # Divide up the time period in which we are working into individual nights, and then work on each night individually logging.info("Averaging daily fits within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Each night is a 86400-second period daily_block_size = 86400 # Make sure that blocks start at noon utc_min = (floor(utc_min / daily_block_size + 0.5) - 0.5) * daily_block_size time_blocks = list( np.arange(start=utc_min, stop=utc_max + daily_block_size, step=daily_block_size)) # Start new block whenever we have a hardware refresh, even if it's in the middle of the night! conn.execute( """ SELECT time FROM archive_metadata WHERE observatory=(SELECT uid FROM archive_observatories WHERE publicId=%s) AND fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey='refresh') AND time BETWEEN %s AND %s """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: time_blocks.append(item['time']) # Make sure that start points for time blocks are in order time_blocks.sort() # Work on each time block (i.e. night) in turn for block_index, utc_block_min in enumerate(time_blocks[:-1]): # End point for this time block utc_block_max = time_blocks[block_index + 1] # Search for observations with orientation fits conn.execute( """ SELECT am1.floatValue AS altitude, am2.floatValue AS azimuth, am3.floatValue AS pa, am4.floatValue AS tilt, am5.floatValue AS width_x_field, am6.floatValue AS width_y_field, am7.stringValue AS fit_quality FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:altitude") INNER JOIN archive_metadata am2 ON o.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:azimuth") INNER JOIN archive_metadata am3 ON o.uid = am3.observationId AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:pa") INNER JOIN archive_metadata am4 ON o.uid = am4.observationId AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:tilt") INNER JOIN archive_metadata am5 ON o.uid = am5.observationId AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_x_field") INNER JOIN archive_metadata am6 ON o.uid = am6.observationId AND am6.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:width_y_field") INNER JOIN archive_metadata am7 ON o.uid = am7.observationId AND am7.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="orientation:fit_quality") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_block_min, utc_block_max)) results = conn.fetchall() # Remove results with poor fit results_filtered = [] fit_threshold = 2 # pixels for item in results: fit_quality = float(json.loads(item['fit_quality'])[0]) if fit_quality > fit_threshold: continue item['weight'] = 1 / (fit_quality + 0.1) results_filtered.append(item) results = results_filtered # Report how many images we found logging.info( "Averaging fits within period {} to {}: Found {} fits.".format( date_string(utc_block_min), date_string(utc_block_max), len(results))) # Average the fits we found if len(results) < 4: logging.info("Insufficient images to reliably average.") continue # What fraction of the worst fits do we reject? rejection_fraction = 0.25 # Reject the 25% of fits which are further from the average rejection_count = int(len(results) * rejection_fraction) # Convert alt-az fits into radians and average # Iteratively remove the point furthest from the mean results_filtered = results # Iteratively take the average of the fits, reject the furthest outlier, and then take a new average for iteration in range(rejection_count): # Average the (alt, az) measurements for this observatory by finding their centroid on a sphere alt_az_list = [[i['altitude'] * deg, i['azimuth'] * deg] for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] alt_az_best = mean_angle_2d(pos_list=alt_az_list, weights=weights_list)[0] # Work out the offset of each fit from the average fit_offsets = [ ang_dist(ra0=alt_az_best[1], dec0=alt_az_best[0], ra1=fitted_alt_az[1], dec1=fitted_alt_az[0]) for fitted_alt_az in alt_az_list ] # Reject the worst fit which is further from the average fits_with_weights = list(zip(fit_offsets, results_filtered)) fits_with_weights.sort(key=operator.itemgetter(0)) fits_with_weights.reverse() # Create a new list of orientation fits, with the worst outlier excluded results_filtered = [item[1] for item in fits_with_weights[1:]] # Convert alt-az fits into radians and average by finding their centroid on a sphere alt_az_list = [[i['altitude'] * deg, i['azimuth'] * deg] for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] [alt_az_best, alt_az_error] = mean_angle_2d(pos_list=alt_az_list, weights=weights_list) # Average other angles by finding their centroid on a circle output_values = {} for quantity in ['tilt', 'pa', 'width_x_field', 'width_y_field']: # Iteratively remove the point furthest from the mean results_filtered = results # Iteratively take the average of the values for each parameter, reject the furthest outlier, # and then take a new average for iteration in range(rejection_count): # Average quantity measurements quantity_values = [i[quantity] * deg for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] quantity_mean = mean_angle(angle_list=quantity_values, weights=weights_list)[0] # Work out the offset of each fit from the average fit_offsets = [] for index, quantity_value in enumerate(quantity_values): offset = quantity_value - quantity_mean if offset < -pi: offset += 2 * pi if offset > pi: offset -= 2 * pi fit_offsets.append(abs(offset)) # Reject the worst fit which is furthest from the average fits_with_weights = list(zip(fit_offsets, results_filtered)) fits_with_weights.sort(key=operator.itemgetter(0)) fits_with_weights.reverse() results_filtered = [item[1] for item in fits_with_weights[1:]] # Filtering finished; now convert each fit into radians and average values_filtered = [i[quantity] * deg for i in results_filtered] weights_list = [i['weight'] for i in results_filtered] value_best = mean_angle(angle_list=values_filtered, weights=weights_list)[0] output_values[quantity] = value_best * rad # Print fit information success = ( alt_az_error * rad < 0.1 ) # Only accept determinations with better precision than 0.1 deg adjective = "SUCCESSFUL" if success else "REJECTED" logging.info("""\ {} ORIENTATION FIT from {:2d} images: Alt: {:.2f} deg. Az: {:.2f} deg. PA: {:.2f} deg. \ ScaleX: {:.2f} deg. ScaleY: {:.2f} deg. Uncertainty: {:.2f} deg.\ """.format(adjective, len(results_filtered), alt_az_best[0] * rad, alt_az_best[1] * rad, output_values['tilt'], output_values['width_x_field'], output_values['width_y_field'], alt_az_error * rad)) # Update observatory status if success: # Flush any previous observation status flush_orientation(obstory_id=obstory_id, utc_min=utc_block_min - 1, utc_max=utc_block_min + 1) user = settings['pigazingUser'] timestamp = time.time() db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:altitude", value=alt_az_best[0] * rad, time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:azimuth", value=alt_az_best[1] * rad, time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:pa", value=output_values['pa'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:tilt", value=output_values['tilt'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:width_x_field", value=output_values['width_x_field'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:width_y_field", value=output_values['width_y_field'], time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:uncertainty", value=alt_az_error * rad, time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.register_obstory_metadata(obstory_id=obstory_id, key="orientation:image_count", value=len(results), time_created=timestamp, metadata_time=utc_block_min, user_created=user) db.commit()
def search_simultaneous_detections(utc_min, utc_max, utc_must_stop): # Count how many simultaneous detections we discover simultaneous_detections_by_type = {} db = obsarchive_db.ObservationDatabase(file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) # Search for moving objects within time span search = mp.ObservationSearch(observation_type="pigazing:movingObject/", time_min=utc_min, time_max=utc_max, limit=1000000) events_raw = db.search_observations(search) # Use only event descriptors, not other returned fields events = events_raw['obs'] # Make a list of which events are already members of groups events_used = [False] * len(events) # Look up the categorisation of each event for event in events: event.category = db.get_observation_metadata(event.id, "web:category") # Throw out junk events and unclassified events events = [x for x in events if x.category is not None and x.category not in ('Junk', 'Bin')] # Look up which pre-existing observation groups each event is in for index, event in enumerate(events): db.con.execute(""" SELECT COUNT(*) FROM archive_obs_groups grp WHERE grp.semanticType = (SELECT y.uid FROM archive_semanticTypes y WHERE y.name=%s) AND EXISTS (SELECT 1 FROM archive_obs_group_members x WHERE x.groupId=grp.uid AND x.childObservation=(SELECT z.uid FROM archive_observations z WHERE z.publicId=%s)); """, (simultaneous_event_type, event.id)) if db.con.fetchone()['COUNT(*)'] > 0: events_used[index] = True # Sort event descriptors into chronological order events.sort(key=lambda x: x.obs_time) # Look up the duration of each event, and calculate its end time for event in events: duration = 0 for meta in event.meta: if meta.key == "pigazing:duration": duration = meta.value event.duration = duration event.obs_time_end = event.obs_time + duration # Compile list of simultaneous object detections groups = [] # Search for simultaneous object detections for index in range(len(events)): # If we have already put this event in another simultaneous detection, don't add it to others if events_used[index]: continue # Look up time span of event event = events[index] obstory_id_list = [event.obstory_id] # List of all observatories which saw this event utc_min = event.obs_time # Earliest start time of any of the events in this group utc_max = event.obs_time_end # Latest end time of any of the events in this group events_used[index] = True prev_group_size = -1 group_members = [index] # Most events must be seen within a maximum offset of 1 second at different stations. # Planes are allowed an offset of up to 30 seconds due to their large parallax search_margin = 60 match_margin = 30 if event.category == "Plane" else 1 # Search for other events which fall within the same time span # Do this iteratively, as a preceding event can expand the end time of the group, and vice versa while len(group_members) > prev_group_size: prev_group_size = len(group_members) # Search for events at earlier times, and then at later times for search_direction in (-1, 1): # Start from the reference event candidate_index = index # Step through other events, providing they're within range while ((candidate_index >= 0) and (candidate_index < len(events))): # Fetch event record candidate = events[candidate_index] # Stop search if we've gone out of time range if ((candidate.obs_time_end < utc_min - search_margin) or (candidate.obs_time > utc_max + search_margin)): break # Check whether this is a simultaneous detection, with same categorisation if ((not events_used[candidate_index]) and (candidate.category == event.category) and (candidate.obs_time < utc_max + match_margin) and (candidate.obs_time_end > utc_min - match_margin)): # Add this event to the group, and update time span of event group_members.append(candidate_index) utc_min = min(utc_min, candidate.obs_time) utc_max = max(utc_max, candidate.obs_time_end) # Compile a list of all the observatories which saw this event if candidate.obstory_id not in obstory_id_list: obstory_id_list.append(candidate.obstory_id) # Record that we have added this event to a group events_used[candidate_index] = True # Step on to the next candidate event to add into group candidate_index += search_direction # We have found a coincident detection only if multiple observatories saw an event at the same time if len(obstory_id_list) < 2: continue # Update tally of events by type if event.category not in simultaneous_detections_by_type: simultaneous_detections_by_type[event.category] = 0 simultaneous_detections_by_type[event.category] += 1 # Initialise maximum baseline between the stations which saw this objects maximum_obstory_spacing = 0 # Work out locations of all observatories which saw this event obstory_locs = [] for obstory_id in obstory_id_list: obstory_info = db.get_obstory_from_id(obstory_id) obstory_loc = Point.from_lat_lng(lat=obstory_info['latitude'], lng=obstory_info['longitude'], alt=0, utc=(utc_min + utc_max) / 2 ) obstory_locs.append(obstory_loc) # Check the distances between all pairs of observatories pairs = [[obstory_locs[i], obstory_locs[j]] for i in range(len(obstory_id_list)) for j in range(i + 1, len(obstory_id_list)) ] # Work out maximum baseline between the stations which saw this objects for pair in pairs: maximum_obstory_spacing = max(maximum_obstory_spacing, abs(pair[0].displacement_vector_from(pair[1]))) # Create information about this simultaneous detection groups.append({'time': (utc_min + utc_max) / 2, 'obstory_list': obstory_id_list, 'time_spread': utc_max - utc_min, 'geographic_spacing': maximum_obstory_spacing, 'category': event.category, 'observations': [{'obs': events[x]} for x in group_members], 'ids': [events[x].id for x in group_members]}) # Report individual events we found for item in groups: logging.info(""" {time} -- {count:3d} stations; max baseline {baseline:5.0f} m; time spread {spread:4.1f} sec; type <{category}> """.format(time=dcf_ast.date_string(item['time']), count=len(item['obstory_list']), baseline=item['geographic_spacing'], spread=item['time_spread'], category=item['category']).strip()) # Report statistics on events we found logging.info("{:6d} moving objects seen within this time period". format(len(events_raw['obs']))) logging.info("{:6d} moving objects rejected because they were unclassified". format(len(events_raw['obs']) - len(events))) logging.info("{:6d} simultaneous detections found.". format(len(groups))) # Report statistics by event type logging.info("Tally of simultaneous detections by type:") for event_type in sorted(simultaneous_detections_by_type.keys()): logging.info(" * {:32s}: {:6d}".format(event_type, simultaneous_detections_by_type[event_type])) # Record simultaneous event detections into the database for item in groups: # Create new observation group group = db.register_obsgroup(title="Multi-station detection", user_id="system", semantic_type=simultaneous_event_type, obs_time=item['time'], set_time=time.time(), obs=item['ids']) # logging.info("Simultaneous detection at {time} by {count:3d} stations (time spread {spread:.1f} sec)". # format(time=dcf_ast.date_string(item['time']), # count=len(item['obstory_list']), # spread=item['time_spread'])) # logging.info("Observation IDs: %s" % item['ids']) # Register group metadata timestamp = time.time() db.set_obsgroup_metadata(user_id="system", group_id=group.id, utc=timestamp, meta=mp.Meta(key="web:category", value=item['category'])) db.set_obsgroup_metadata(user_id="system", group_id=group.id, utc=timestamp, meta=mp.Meta(key="simultaneous:time_spread", value=item['time_spread'])) db.set_obsgroup_metadata(user_id="system", group_id=group.id, utc=timestamp, meta=mp.Meta(key="simulataneous:geographic_spread", value=item['geographic_spacing'])) # Commit changes db.commit()
def observing_loop(): obstory_id = installation_info['observatoryId'] logging.info("Observatory controller launched") # Fetch observatory status, e.g. location, etc logging.info("Fetching observatory status") latitude = known_observatories[obstory_id]['latitude'] longitude = known_observatories[obstory_id]['longitude'] altitude = 0 latest_position_update = 0 flag_gps = 0 # Make sure that observatory exists in the database # Start main observing loop while True: # Get a new MySQL connection because old one may not be connected any longer db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) # Get a GPS fix on the current time and our location gps_fix = get_gps_fix() if gps_fix: latitude = gps_fix['latitude'] longitude = gps_fix['longitude'] altitude = gps_fix['altitude'] flag_gps = 1 # Decide whether we should observe, or do some day-time maintenance tasks logging.info("Observation controller considering what to do next.") time_now = time.time() # How far below the horizon do we require the Sun to be before we start observing? angle_below_horizon = settings['sunRequiredAngleBelowHorizon'] sun_times_yesterday = sunset_times.sun_times( unix_time=time_now - 3600 * 24, longitude=longitude, latitude=latitude, angle_below_horizon=angle_below_horizon) sun_times_today = sunset_times.sun_times( unix_time=time_now, longitude=longitude, latitude=latitude, angle_below_horizon=angle_below_horizon) sun_times_tomorrow = sunset_times.sun_times( unix_time=time_now + 3600 * 24, longitude=longitude, latitude=latitude, angle_below_horizon=angle_below_horizon) logging.info("Sunrise at {}".format( dcf_ast.date_string(sun_times_yesterday[0]))) logging.info("Sunset at {}".format( dcf_ast.date_string(sun_times_yesterday[2]))) logging.info("Sunrise at {}".format( dcf_ast.date_string(sun_times_today[0]))) logging.info("Sunset at {}".format( dcf_ast.date_string(sun_times_today[2]))) logging.info("Sunrise at {}".format( dcf_ast.date_string(sun_times_tomorrow[0]))) logging.info("Sunset at {}".format( dcf_ast.date_string(sun_times_tomorrow[2]))) sun_margin = settings['sunMargin'] # Calculate whether it's currently night time, and how long until the next sunrise is_night_time = False seconds_till_sunrise = 0 # Test whether it is night time is we are between yesterday's sunset and today's sunrise if (time_now > sun_times_yesterday[2] + sun_margin) and ( time_now < sun_times_today[0] - sun_margin): logging.info(""" It is night time. We are between yesterday's sunset and today's sunrise. """.strip()) is_night_time = True seconds_till_sunrise = sun_times_today[0] - time_now # Test whether it is between yesterday's sunset and today's sunrise elif (time_now > sun_times_yesterday[2]) and (time_now < sun_times_today[0]): next_observing_time = sun_times_yesterday[2] + sun_margin next_observing_wait = next_observing_time - time_now if next_observing_wait > 0: logging.info(""" We are between yesterday's sunset and today's sunrise, but sun has recently set. \ Waiting {:.0f} seconds (until {}) to start observing. """.format(next_observing_wait, dcf_ast.date_string(next_observing_time)).strip()) db.commit() db.close_db() del db time.sleep(next_observing_wait + 2) continue # Test whether it is night time, since we are between today's sunrise and tomorrow's sunset elif (time_now > sun_times_today[2] + sun_margin) and ( time_now < sun_times_tomorrow[0] - sun_margin): logging.info(""" It is night time. We are between today's sunset and tomorrow's sunrise. """.strip()) is_night_time = True seconds_till_sunrise = sun_times_tomorrow[0] - time_now # Test whether we between today's sunset and tomorrow's sunrise elif (time_now > sun_times_today[2]) and (time_now < sun_times_tomorrow[0]): next_observing_time = sun_times_today[2] + sun_margin next_observing_wait = next_observing_time - time_now if next_observing_time > 0: logging.info(""" We are between today's sunset and tomorrow's sunrise, but sun has recently set. \ Waiting {:.0f} seconds (until {}) to start observing. """.format(next_observing_wait, dcf_ast.date_string(next_observing_time)).strip()) db.commit() db.close_db() del db time.sleep(next_observing_wait + 2) continue # Calculate time until the next sunset seconds_till_sunset = sun_times_yesterday[2] - time_now if seconds_till_sunset < -sun_margin: seconds_till_sunset = sun_times_today[2] - time_now if seconds_till_sunset < -sun_margin: seconds_till_sunset = sun_times_tomorrow[2] - time_now # If sunset was well in the past, and sunrise is well in the future, we should observe! minimum_time_worth_observing = 600 if is_night_time and (seconds_till_sunrise > (sun_margin + minimum_time_worth_observing)): # Check that observatory exists check_observatory_exists(db_handle=db, obs_id=obstory_id, utc=time.time()) # Fetch updated observatory status obstory_status = db.get_obstory_status(obstory_id=obstory_id) # If we've not stored a GPS fix in the database within the past six hours, do so now if flag_gps and (time.time() > latest_position_update + 6 * 3600): latest_position_update = time.time() db.register_obstory_metadata( obstory_id=obstory_id, key="latitude_gps", value=latitude, metadata_time=time.time(), time_created=time.time(), user_created=settings['pigazingUser']) db.register_obstory_metadata( obstory_id=obstory_id, key="longitude_gps", value=longitude, metadata_time=time.time(), time_created=time.time(), user_created=settings['pigazingUser']) db.register_obstory_metadata( obstory_id=obstory_id, key="altitude_gps", value=altitude, metadata_time=time.time(), time_created=time.time(), user_created=settings['pigazingUser']) # Create clipping region mask file mask_file = "/tmp/triggermask_%d.txt" % os.getpid() open(mask_file, "w").write("\n\n".join([ "\n".join([("%d %d" % tuple(p)) for p in point_list]) for point_list in json.loads(obstory_status["clipping_region"]) ])) # Commit updates to the database db.commit() db.close_db() del db # Calculate how long to observe for observing_duration = seconds_till_sunrise - sun_margin # Do not record too much video in one file, as otherwise the file will be big if not settings['realTime']: observing_duration = min(observing_duration, settings['videoMaxRecordTime']) # Start observing run t_stop = time_now + observing_duration logging.info(""" Starting observing run until {} (running for {:.0f} seconds). """.format(dcf_ast.date_string(t_stop), observing_duration).strip()) # Flick the relay to turn the camera on relay_control.camera_on() time.sleep(5) logging.info("Camera has been turned on.") # Observe! We use different binaries depending whether we're using a webcam-like camera, # or a DSLR connected via gphoto2 time_key = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') # Work out which C binary we're using to do observing if settings['realTime']: output_argument = "" if obstory_status["camera_type"] == "gphoto2": binary = "realtimeObserve_dslr" else: binary = "realtimeObserve" else: output_argument = """ --output \"{}/raw_video/{}_{}\" """.format( settings['dataPath'], time_key, obstory_id) if settings['i_am_a_rpi']: binary = "recordH264_openmax" else: binary = "recordH264_libav" binary_full_path = "{path}{debug}/{binary}".format( path=settings['binaryPath'], debug="/debug" if settings['debug'] else "", binary=binary) cmd = """ timeout {timeout} \ {binary} --utc-stop {utc_stop:.1f} \ --obsid \"{obsid}\" \ --device \"{device}\" \ --fps {fps} \ --width {width:d} \ --height {height:d} \ --mask \"{mask_file}\" \ --latitude {latitude} \ --longitude {longitude} \ --flag-gps {flag_gps} \ --flag-upside-down {upside_down} \ {output_argument} """.format(timeout=float(observing_duration + 300), binary=binary_full_path, utc_stop=float(t_stop), obsid=obstory_id, device=settings['videoDev'], width=int(obstory_status['camera_width']), height=int(obstory_status['camera_height']), fps=float(obstory_status['camera_fps']), mask_file=mask_file, latitude=float(latitude), longitude=float(longitude), flag_gps=int(flag_gps), upside_down=int(obstory_status['camera_upside_down']), output_argument=output_argument).strip() logging.info("Running command: {}".format(cmd)) os.system(cmd) # Flick the relay to turn the camera off relay_control.camera_off() time.sleep(5) logging.info("Camera has been turned off.") # Snooze for up to 10 minutes; we may rerun observing tasks in a while if they ended prematurely if time.time() < t_stop: snooze_duration = float(min(t_stop - time.time(), 600)) logging.info( "Snoozing for {:.0f} seconds".format(snooze_duration)) time.sleep(snooze_duration) continue # It is day time, so consider running day time tasks # First, commit updates to the database db.commit() db.close_db() del db # Estimate roughly when we're next going to be able to observe (i.e. shortly after sunset) next_observing_wait = seconds_till_sunset + sun_margin # If we've got more than an hour, it's worth doing some day time tasks # Do daytime tasks on a RPi only if we are doing real-time observation if (next_observing_wait > 3600) and (settings['realTime'] or not settings['i_am_a_rpi']): t_stop = time_now + next_observing_wait logging.info(""" Starting daytime tasks until {} (running for {:.0f} seconds). """.format(dcf_ast.date_string(t_stop), next_observing_wait).strip()) os.system("cd {} ; ./daytimeTasks.py --stop-by {}".format( os.path.join(settings['pythonPath'], "observe"), t_stop)) # Snooze for up to 30 minutes; we may rerun daytime tasks in a while if they ended prematurely if time.time() < t_stop: snooze_duration = float(min(t_stop - time.time(), 1800)) logging.info( "Snoozing for {:.0f} seconds".format(snooze_duration)) time.sleep(snooze_duration) else: if next_observing_wait < 0: next_observing_wait = 0 next_observing_wait += 30 t_stop = time_now + next_observing_wait logging.info(""" Not time to start observing yet, so sleeping until {} ({:.0f} seconds away). """.format(dcf_ast.date_string(t_stop), next_observing_wait).strip()) time.sleep(next_observing_wait) # Little snooze to prevent spinning around the loop snooze_duration = float(10) logging.info("Snoozing for {:.0f} seconds".format(snooze_duration)) time.sleep(snooze_duration)
def list_planes(obstory_id, utc_min, utc_max): """ List all the plane identifications for a particular observatory. :param obstory_id: The ID of the observatory we want to list identifications for. :param utc_min: The start of the time period in which we should list identifications (unix time). :param utc_max: The end of the time period in which we should list identifications (unix time). :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Start compiling list of plane identifications plane_identifications = [] # Select moving objects with plane identifications conn.execute( """ SELECT am1.stringValue AS call_sign, am2.floatValue AS ang_offset, am3.floatValue AS clock_offset, am4.floatValue AS duration, am5.stringValue AS hex_ident, am6.floatValue AS distance, am7.stringValue AS operator, am8.stringValue AS model, am9.stringValue AS manufacturer, o.obsTime AS time, o.publicId AS obsId FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:call_sign") INNER JOIN archive_metadata am2 ON o.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:angular_offset") INNER JOIN archive_metadata am3 ON o.uid = am3.observationId AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:clock_offset") INNER JOIN archive_metadata am4 ON o.uid = am4.observationId AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="pigazing:duration") INNER JOIN archive_metadata am5 ON o.uid = am5.observationId AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:hex_ident") LEFT JOIN archive_metadata am6 ON o.uid = am6.observationId AND am6.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:distance") LEFT JOIN archive_metadata am7 ON o.uid = am7.observationId AND am7.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:operator") LEFT JOIN archive_metadata am8 ON o.uid = am8.observationId AND am8.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:model") LEFT JOIN archive_metadata am9 ON o.uid = am9.observationId AND am9.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="plane:manufacturer") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: plane_identifications.append({ 'id': item['obsId'], 'time': item['time'], 'call_sign': item['call_sign'], 'ang_offset': item['ang_offset'], 'clock_offset': item['clock_offset'], 'duration': item['duration'], 'hex_ident': item['hex_ident'], 'distance': item['distance'], 'operator': item['operator'], 'model': item['model'], 'manufacturer': item['manufacturer'] }) # Sort identifications by time plane_identifications.sort(key=itemgetter('time')) # Display column headings print("""\ {:16s} {:18s} {:18s} {:8s} {:10s} {:10s} {:10s} {:30s} {:30s} {:30s}\ """.format("Time", "Call sign", "Hex ident", "Duration", "Ang offset", "Clock off", "Distance", "Operator", "Model", "Manufacturer")) # Display list of meteors for item in plane_identifications: print("""\ {:16s} {:18s} {:18s} {:8.1f} {:10.1f} {:10.1f} {:10.1f} {:30s} {:30s} {:30s}\ """.format(date_string(item['time']), item['call_sign'], item['hex_ident'], item['duration'], item['ang_offset'], item['clock_offset'], item['distance'], item['operator'], item['model'], item['manufacturer'])) # Clean up and exit return
def plane_determination(utc_min, utc_max, source): """ Estimate the identity of aircraft observed between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should determine the identity of aircraft (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should determine the identity of aircraft (unix time). :type utc_max: float :param source: The source we should use for plane trajectories. Either 'adsb' or 'fr24'. :type source: str :return: None """ # Open connection to image archive db = obsarchive_db.ObservationDatabase( file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting aircraft identification.") # Count how many images we manage to successfully fit outcomes = { 'successful_fits': 0, 'unsuccessful_fits': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Status update logging.info("Searching for aircraft within period {} to {}".format( date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for planes and satellites within this time period conn.execute( """ SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid INNER JOIN archive_metadata am2 ON ao.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s AND (am2.stringValue='Plane' OR am2.stringValue='Satellite' OR am2.stringValue='Junk') ORDER BY ao.obsTime """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the images we are going to work on logging.info("Estimating the identity of {:d} aircraft.".format( len(results))) # Analyse each aircraft in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if 'pigazing:path' not in all_metadata: logging.info( "Cannot process <{}> due to inadequate metadata.".format( item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId']) # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection(db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration']) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Check number of points in path path_len = len(path_x_y) # Look up list of aircraft tracks at the time of this sighting if source == 'adsb': aircraft_list = fetch_planes_from_adsb(utc=item['obsTime']) elif source == 'fr24': aircraft_list = fetch_planes_from_fr24(utc=item['obsTime']) else: raise ValueError("Unknown source <{}>".format(source)) # List of aircraft this moving object might be candidate_aircraft = [] # Check that we found a list of aircraft if aircraft_list is None: logging.info("{date} [{obs}] -- No aircraft records found.".format( date=date_string(utc=item['obsTime']), obs=item['observationId'])) outcomes['insufficient_information'] += 1 continue # Logging message about how many aircraft we're testing # logging.info("{date} [{obs}] -- Matching against {count:7d} aircraft.".format( # date=date_string(utc=item['obsTime']), # obs=item['observationId'], # count=len(aircraft_list) # )) # Test for each candidate aircraft in turn for aircraft in aircraft_list: # Fetch aircraft position at each time point along trajectory ang_mismatch_list = [] distance_list = [] altitude_list = [] def aircraft_angular_offset(index, clock_offset): # Fetch observed position of object at this time point pt_utc = sight_line_list[index]['utc'] observatory_position = sight_line_list[index]['obs_position'] observed_sight_line = sight_line_list[index]['line'].direction # Project position of this aircraft in space at this time point aircraft_position = path_interpolate(aircraft=aircraft, utc=pt_utc + clock_offset) if aircraft_position is None: return np.nan, np.nan, np.nan # Convert position to Cartesian coordinates aircraft_point = Point.from_lat_lng( lat=aircraft_position['lat'], lng=aircraft_position['lon'], alt=aircraft_position['altitude'] * feet, utc=None) # Work out offset of plane's position from observed moving object aircraft_sight_line = aircraft_point.to_vector( ) - observatory_position.to_vector() angular_offset = aircraft_sight_line.angle_with( other=observed_sight_line) # degrees distance = abs(aircraft_sight_line) altitude = aircraft_position['altitude'] * feet return angular_offset, distance, altitude def time_offset_objective(p): """ Objective function that we minimise in order to find the best fit clock offset between the observed and model paths. :param p: Vector with a single component: the clock offset :return: Metric to minimise """ # Turn input parameters into a time offset clock_offset = p[0] # Look up angular offset ang_mismatch, distance, altitude = aircraft_angular_offset( index=0, clock_offset=clock_offset) # Return metric to minimise return ang_mismatch * exp(clock_offset / 8) # Work out the optimum time offset between the plane's path and the observed path # See <http://www.scipy-lectures.org/advanced/mathematical_optimization/> # for more information about how this works parameters_initial = [0] parameters_optimised = scipy.optimize.minimize( time_offset_objective, np.asarray(parameters_initial), options={ 'disp': False, 'maxiter': 100 }).x # Construct best-fit linear trajectory for best-fitting parameters clock_offset = float(parameters_optimised[0]) # Check clock offset is reasonable if abs(clock_offset) > global_settings['max_clock_offset']: continue # Measure the offset between the plane's position and the observed position at each time point for index in range(path_len): # Look up angular mismatch at this time point ang_mismatch, distance, altitude = aircraft_angular_offset( index=index, clock_offset=clock_offset) # Keep list of the offsets at each recorded time point along the trajectory ang_mismatch_list.append(ang_mismatch) distance_list.append(distance) altitude_list.append(altitude) # Consider adding this plane to list of candidates mean_ang_mismatch = np.mean( np.asarray(ang_mismatch_list)) # degrees distance_mean = np.mean(np.asarray(distance_list)) # metres altitude_mean = np.mean(np.asarray(altitude_list)) # metres if mean_ang_mismatch < global_settings['max_mean_angular_mismatch']: start_time = sight_line_list[0]['utc'] end_time = sight_line_list[-1]['utc'] start_point = path_interpolate(aircraft=aircraft, utc=start_time + clock_offset) end_point = path_interpolate(aircraft=aircraft, utc=end_time + clock_offset) candidate_aircraft.append({ 'call_sign': aircraft['call_sign'], # string 'hex_ident': aircraft['hex_ident'], # string 'distance': distance_mean / 1e3, # km 'altitude': altitude_mean / 1e3, # km 'clock_offset': clock_offset, # seconds 'offset': mean_ang_mismatch, # degrees 'start_point': start_point, 'end_point': end_point }) # Add model possibility for null aircraft if len(candidate_aircraft) == 0: candidate_aircraft.append({ 'call_sign': "Unidentified", 'hex_ident': "Unidentified", 'distance': 0, 'altitude': 0, 'clock_offset': 0, 'offset': 0, 'start_point': None, 'end_point': None }) # Sort candidates by score for candidate in candidate_aircraft: candidate['score'] = hypot( candidate['offset'], candidate['clock_offset'], ) candidate_aircraft.sort(key=itemgetter('score')) # Report possible satellite identifications logging.info("{prefix} -- {aircraft}".format( prefix=logging_prefix, aircraft=", ".join([ "{} ({:.1f} deg offset; clock offset {:.1f} sec; distance {:.1f} km)" .format(aircraft['call_sign'], aircraft['offset'], aircraft['clock_offset'], aircraft['distance']) for aircraft in candidate_aircraft ]))) # Identify most likely aircraft most_likely_aircraft = candidate_aircraft[0] # Fetch extra information about plane plane_info = fetch_aircraft_data( hex_ident=most_likely_aircraft['hex_ident']) # Store aircraft identification user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:call_sign", value=most_likely_aircraft['call_sign'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:hex_ident", value=most_likely_aircraft['hex_ident'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:clock_offset", value=most_likely_aircraft['clock_offset'])) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta( key="plane:angular_offset", value=most_likely_aircraft['offset'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:distance", value=most_likely_aircraft['distance'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:mean_altitude", value=most_likely_aircraft['altitude'])) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:path", value=json.dumps([ most_likely_aircraft['start_point'], most_likely_aircraft['end_point'] ]))) db.set_observation_metadata( user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:path_length", value=ang_dist(ra0=path_ra_dec_at_epoch[0][0], dec0=path_ra_dec_at_epoch[0][1], ra1=path_ra_dec_at_epoch[-1][0], dec1=path_ra_dec_at_epoch[-1][1]) * 180 / pi)) aircraft_operator = "" if 'operator' in plane_info and plane_info['operator']: aircraft_operator = plane_info['operator'] elif 'owner' in plane_info and plane_info['owner']: aircraft_operator = plane_info['owner'] db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:operator", value=aircraft_operator)) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:model", value=plane_info.get( 'model', ''))) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="plane:manufacturer", value=plane_info.get( 'manufacturername', ''))) # Aircraft successfully identified if most_likely_aircraft['call_sign'] == "Unidentified": outcomes['unsuccessful_fits'] += 1 else: outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} aircraft successfully identified.".format( outcomes['successful_fits'])) logging.info("{:d} aircraft not identified.".format( outcomes['unsuccessful_fits'])) logging.info("{:d} malformed database records.".format( outcomes['error_records'])) logging.info("{:d} rescued database records.".format( outcomes['rescued_records'])) logging.info("{:d} aircraft with incomplete data.".format( outcomes['insufficient_information'])) # Clean up and exit db.commit() db.close_db() return
def list_satellites(obstory_id, utc_min, utc_max): """ List all the satellite identifications for a particular observatory. :param obstory_id: The ID of the observatory we want to list identifications for. :param utc_min: The start of the time period in which we should list identifications (unix time). :param utc_max: The end of the time period in which we should list identifications (unix time). :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Start compiling list of satellite identifications satellite_identifications = [] # Select moving objects with satellite identifications conn.execute( """ SELECT am1.stringValue AS satellite_name, am2.floatValue AS ang_offset, am3.floatValue AS clock_offset, am4.floatValue AS duration, am5.floatValue AS norad_id, o.obsTime AS time, o.publicId AS obsId FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="satellite:name") INNER JOIN archive_metadata am2 ON o.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="satellite:angular_offset") INNER JOIN archive_metadata am3 ON o.uid = am3.observationId AND am3.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="satellite:clock_offset") INNER JOIN archive_metadata am4 ON o.uid = am4.observationId AND am4.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="pigazing:duration") INNER JOIN archive_metadata am5 ON o.uid = am5.observationId AND am5.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="satellite:norad_id") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: satellite_identifications.append({ 'id': item['obsId'], 'time': item['time'], 'satellite_name': item['satellite_name'], 'ang_offset': item['ang_offset'], 'clock_offset': item['clock_offset'], 'duration': item['duration'], 'norad_id': int(item['norad_id']) }) # Sort identifications by time satellite_identifications.sort(key=itemgetter('time')) # Display column headings print("""\ {:16s} {:7s} {:32s} {:26s} {:8s} {:10s} {:10s}\ """.format("Time", "NORAD", "ID", "Satellite", "Duration", "Ang offset", "Clock offset")) # Display list of meteors for item in satellite_identifications: print("""\ {:16s} {:7d} {:32s} {:26s} {:8.1f} {:10.1f} {:10.1f}\ """.format( date_string(item['time']), item['norad_id'], item['id'], item['satellite_name'], item['duration'], item['ang_offset'], item['clock_offset'], )) # Clean up and exit return
def timelapse_movie(utc_min, utc_max, obstory, img_types, stride, label): """ Make a time lapse video of images registered in the database using the command line tool ffmpeg. :param utc_min: Only return observations made after the specified time stamp. :type utc_min: float :param utc_max: Only return observations made before the specified time stamp. :type utc_max: float :param obstory: The public id of the observatory we are to fetch observations from :type obstory: str :param img_types: Only return images with these semantic types :type img_types: list[str] :param stride: Only return every nth observation matching the search criteria :type stride: int :return: None """ # Temporary directory to hold the images we are going to show pid = os.getpid() tmp = os.path.join("/tmp", "dcf_movie_images_{:d}".format(pid)) os.system("mkdir -p {}".format(tmp)) file_list = fetch_images(utc_min=utc_min, utc_max=utc_max, obstory=obstory, img_types=img_types, stride=stride) # Report how many files we found print("Observatory <{}>".format(obstory)) print(" * {:d} matching files in time range {} --> {}".format(len(file_list), dcf_ast.date_string(utc_min), dcf_ast.date_string(utc_max))) # Make list of the stitched files filename_list = [] filename_format = "frame_{:d}_%08d.jpg".format(pid) for counter, file_item in enumerate(file_list): # Look up the date of this file [year, month, day, h, m, s] = dcf_ast.inv_julian_day(dcf_ast.jd_from_unix( utc=file_item['observation']['obsTime'] )) # Filename for stitched image fn = filename_format % counter # Make list of input files input_files = [os.path.join(settings['dbFilestore'], file_item[semanticType]['repositoryFname']) for semanticType in img_types] command = "\ convert {inputs} +append -gravity SouthWest -fill Red -pointsize 26 -font Ubuntu-Bold \ -annotate +16+10 '{date} - {label1} - {label2}' {output} \ ".format(inputs=" ".join(input_files), date="{:02d}/{:02d}/{:04d} {:02d}:{:02d}".format(day, month, year, h, m), label1="Sky clarity: {}".format(" / ".join(["{:04.0f}".format(file_item[semanticType]['skyClarity']) for semanticType in img_types])), label2=label, output=os.path.join(tmp, fn)) # print(command) os.system(command) filename_list.append(fn) command_line = "cd {} ; ffmpeg -r 10 -i {} -codec:v libx264 {}".format(tmp , filename_format, "timelapse.mp4") print(command_line) os.system(command_line)
def list_meteors(obstory_id, utc_min, utc_max): """ List all the meteor identifications for a particular observatory. :param obstory_id: The ID of the observatory we want to list meteor identifications for. :param utc_min: The start of the time period in which we should list meteor identifications (unix time). :param utc_max: The end of the time period in which we should list meteor identifications (unix time). :return: None """ # Open connection to database [db0, conn] = connect_db.connect_db() # Start compiling list of meteor identifications meteor_identifications = [] # Count how many meteors we find in each shower meteor_count_by_shower = {} # Select observations with orientation fits conn.execute(""" SELECT am1.stringValue AS name, am2.floatValue AS radiant_offset, o.obsTime AS time, o.publicId AS obsId FROM archive_observations o INNER JOIN archive_metadata am1 ON o.uid = am1.observationId AND am1.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="shower:name") INNER JOIN archive_metadata am2 ON o.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="shower:radiant_offset") WHERE o.observatory = (SELECT uid FROM archive_observatories WHERE publicId=%s) AND o.obsTime BETWEEN %s AND %s; """, (obstory_id, utc_min, utc_max)) results = conn.fetchall() for item in results: meteor_identifications.append({ 'id': item['obsId'], 'time': item['time'], 'shower': item['name'], 'offset': item['radiant_offset'] }) # Update tally of meteors if item['name'] not in meteor_count_by_shower: meteor_count_by_shower[item['name']] = 0 meteor_count_by_shower[item['name']] += 1 # Sort meteors by time meteor_identifications.sort(key=itemgetter('time')) # Display column headings print("""\ {:16s} {:20s} {:20s} {:5s}\ """.format("Time", "ID", "Shower", "Offset")) # Display list of meteors for item in meteor_identifications: print("""\ {:16s} {:20s} {:26s} {:5.1f}\ """.format(date_string(item['time']), item['id'], item['shower'], item['offset'] )) # Report tally of meteors logging.info("Tally of meteors by shower:") for shower in sorted(meteor_count_by_shower.keys()): logging.info(" * {:26s}: {:6d}".format(shower, meteor_count_by_shower[shower])) # Clean up and exit return
def shower_determination(utc_min, utc_max): """ Estimate the parent showers of all meteors observed between the unix times <utc_min> and <utc_max>. :param utc_min: The start of the time period in which we should determine the parent showers of meteors (unix time). :type utc_min: float :param utc_max: The end of the time period in which we should determine the parent showers of meteors (unix time). :type utc_max: float :return: None """ # Load list of meteor showers shower_list = read_shower_list() # Open connection to image archive db = obsarchive_db.ObservationDatabase(file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) logging.info("Starting meteor shower identification.") # Count how many images we manage to successfully fit outcomes = { 'successful_fits': 0, 'error_records': 0, 'rescued_records': 0, 'insufficient_information': 0 } # Status update logging.info("Searching for meteors within period {} to {}".format(date_string(utc_min), date_string(utc_max))) # Open direct connection to database conn = db.con # Search for meteors within this time period conn.execute(""" SELECT ao.obsTime, ao.publicId AS observationId, f.repositoryFname, l.publicId AS observatory FROM archive_observations ao LEFT OUTER JOIN archive_files f ON (ao.uid = f.observationId AND f.semanticType=(SELECT uid FROM archive_semanticTypes WHERE name="pigazing:movingObject/video")) INNER JOIN archive_observatories l ON ao.observatory = l.uid INNER JOIN archive_metadata am2 ON ao.uid = am2.observationId AND am2.fieldId=(SELECT uid FROM archive_metadataFields WHERE metaKey="web:category") WHERE ao.obsType=(SELECT uid FROM archive_semanticTypes WHERE name='pigazing:movingObject/') AND ao.obsTime BETWEEN %s AND %s AND am2.stringValue = "Meteor" ORDER BY ao.obsTime; """, (utc_min, utc_max)) results = conn.fetchall() # Display logging list of the images we are going to work on logging.info("Estimating the parent showers of {:d} meteors.".format(len(results))) # Count how many meteors we find in each shower meteor_count_by_shower = {} # Analyse each meteor in turn for item_index, item in enumerate(results): # Fetch metadata about this object, some of which might be on the file, and some on the observation obs_obj = db.get_observation(observation_id=item['observationId']) obs_metadata = {item.key: item.value for item in obs_obj.meta} if item['repositoryFname']: file_obj = db.get_file(repository_fname=item['repositoryFname']) file_metadata = {item.key: item.value for item in file_obj.meta} else: file_metadata = {} all_metadata = {**obs_metadata, **file_metadata} # Check we have all required metadata if 'pigazing:path' not in all_metadata: logging.info("Cannot process <{}> due to inadequate metadata.".format(item['observationId'])) continue # Make ID string to prefix to all logging messages about this event logging_prefix = "{date} [{obs}]".format( date=date_string(utc=item['obsTime']), obs=item['observationId'] ) # Project path from (x,y) coordinates into (RA, Dec) projector = PathProjection( db=db, obstory_id=item['observatory'], time=item['obsTime'], logging_prefix=logging_prefix ) path_x_y, path_ra_dec_at_epoch, path_alt_az, sight_line_list_this = projector.ra_dec_from_x_y( path_json=all_metadata['pigazing:path'], path_bezier_json=all_metadata['pigazing:pathBezier'], detections=all_metadata['pigazing:detectionCount'], duration=all_metadata['pigazing:duration'] ) # Check for error if projector.error is not None: if projector.error in outcomes: outcomes[projector.error] += 1 continue # Check for notifications for notification in projector.notifications: if notification in outcomes: outcomes[notification] += 1 # Check number of points in path path_len = len(path_x_y) # List of candidate showers this meteor might belong to candidate_showers = [] # Test for each candidate meteor shower in turn for shower in shower_list: # Work out celestial coordinates of shower radiant in RA/Dec in hours/degs of epoch radiant_ra_at_epoch, radiant_dec_at_epoch = ra_dec_from_j2000(ra0=shower['RA'], dec0=shower['Decl'], utc_new=item['obsTime']) # Work out alt-az of the shower's radiant using known location of camera. Fits returned in degrees. alt_az_pos = alt_az(ra=radiant_ra_at_epoch, dec=radiant_dec_at_epoch, utc=item['obsTime'], latitude=projector.obstory_info['latitude'], longitude=projector.obstory_info['longitude']) # Work out position of the Sun (J2000) sun_ra_j2000, sun_dec_j2000 = sun_pos(utc=item['obsTime']) # Work out position of the Sun (RA, Dec of epoch) sun_ra_at_epoch, sun_dec_at_epoch = ra_dec_from_j2000(ra0=sun_ra_j2000, dec0=sun_dec_j2000, utc_new=item['obsTime']) # Offset from peak of shower year = 365.2524 peak_offset = (sun_ra_at_epoch * 180 / 12. - shower['peak']) * year / 360 # days while peak_offset < -year / 2: peak_offset += year while peak_offset > year / 2: peak_offset -= year start_offset = peak_offset + shower['start'] - 4 end_offset = peak_offset + shower['end'] + 4 # Estimate ZHR of shower at the time the meteor was observed zhr = 0 if abs(peak_offset) < 2: zhr = shower['zhr'] # Shower is within 2 days of maximum; use quoted peak ZHR value if start_offset < 0 < end_offset: zhr = max(zhr, 5) # Shower is not at peak, but is active; assume ZHR=5 # Correct hourly rate for the altitude of the shower radiant hourly_rate = zhr * sin(alt_az_pos[0] * pi / 180) # If hourly rate is zero, this shower is not active if hourly_rate <= 0: # logging.info("Meteor shower <{}> has zero rate".format(shower['name'])) continue # Work out angular distance of meteor from radiant (radians) path_radiant_sep = [ang_dist(ra0=pt[0], dec0=pt[1], ra1=radiant_ra_at_epoch * pi / 12, dec1=radiant_dec_at_epoch * pi / 180) for pt in path_ra_dec_at_epoch] change_in_radiant_dist = path_radiant_sep[-1] - path_radiant_sep[0] # radians # Reject meteors that travel *towards* the radiant if change_in_radiant_dist < 0: continue # Convert path to Cartesian coordinates on a unit sphere path_cartesian = [Vector.from_ra_dec(ra=ra * 12 / pi, dec=dec * 180 / pi) for ra, dec in path_ra_dec_at_epoch] # Work out cross product of first and last point, which is normal to path of meteors first = path_cartesian[0] last = path_cartesian[-1] path_normal = first.cross_product(last) # Work out angle of path normal to meteor shower radiant radiant_cartesian = Vector.from_ra_dec(ra=radiant_ra_at_epoch, dec=radiant_dec_at_epoch) theta = path_normal.angle_with(radiant_cartesian) # degrees if theta > 90: theta = 180 - theta # What is the angular separation of the meteor's path's closest approach to the shower radiant? radiant_angle = 90 - theta # Work out likelihood metric that this meteor belongs to this shower radiant_angle_std_dev = 2 # Allow 2 degree mismatch in radiant pos likelihood = hourly_rate * scipy.stats.norm(loc=0, scale=radiant_angle_std_dev).pdf(radiant_angle) # Store information about the likelihood this meteor belongs to this shower candidate_showers.append({ 'name': shower['name'], 'likelihood': likelihood, 'offset': radiant_angle, 'change_radiant_dist': change_in_radiant_dist, 'shower_rate': hourly_rate }) # Add model possibility for sporadic meteor hourly_rate = 5 likelihood = hourly_rate * (1. / 90.) # Mean value of Gaussian in range 0-90 degs candidate_showers.append({ 'name': "Sporadic", 'likelihood': likelihood, 'offset': 0, 'shower_rate': hourly_rate }) # Renormalise likelihoods to sum to unity sum_likelihood = sum(shower['likelihood'] for shower in candidate_showers) for shower in candidate_showers: shower['likelihood'] *= 100 / sum_likelihood # Sort candidates by likelihood candidate_showers.sort(key=itemgetter('likelihood'), reverse=True) # Report possible meteor shower identifications logging.info("{date} [{obs}] -- {showers}".format( date=date_string(utc=item['obsTime']), obs=item['observationId'], showers=", ".join([ "{} {:.1f}% ({:.1f} deg offset)".format(shower['name'], shower['likelihood'], shower['offset']) for shower in candidate_showers ]) )) # Identify most likely shower most_likely_shower = candidate_showers[0]['name'] # Update tally of meteors if most_likely_shower not in meteor_count_by_shower: meteor_count_by_shower[most_likely_shower] = 0 meteor_count_by_shower[most_likely_shower] += 1 # Store meteor identification user = settings['pigazingUser'] timestamp = time.time() db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:name", value=most_likely_shower)) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:radiant_offset", value=candidate_showers[0]['offset'])) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:path_length", value=ang_dist(ra0=path_ra_dec_at_epoch[0][0], dec0=path_ra_dec_at_epoch[0][1], ra1=path_ra_dec_at_epoch[-1][0], dec1=path_ra_dec_at_epoch[-1][1] ) * 180 / pi )) db.set_observation_metadata(user_id=user, observation_id=item['observationId'], utc=timestamp, meta=mp.Meta(key="shower:path_ra_dec", value="[[{:.3f},{:.3f}],[{:.3f},{:.3f}],[{:.3f},{:.3f}]]".format( path_ra_dec_at_epoch[0][0] * 12 / pi, path_ra_dec_at_epoch[0][1] * 180 / pi, path_ra_dec_at_epoch[int(path_len / 2)][0] * 12 / pi, path_ra_dec_at_epoch[int(path_len / 2)][1] * 180 / pi, path_ra_dec_at_epoch[-1][0] * 12 / pi, path_ra_dec_at_epoch[-1][1] * 180 / pi, ) )) # Meteor successfully identified outcomes['successful_fits'] += 1 # Update database db.commit() # Report how many fits we achieved logging.info("{:d} meteors successfully identified.".format(outcomes['successful_fits'])) logging.info("{:d} malformed database records.".format(outcomes['error_records'])) logging.info("{:d} rescued database records.".format(outcomes['rescued_records'])) logging.info("{:d} meteors with incomplete data.".format(outcomes['insufficient_information'])) # Report tally of meteors logging.info("Tally of meteors by shower:") for shower in sorted(meteor_count_by_shower.keys()): logging.info(" * {:32s}: {:6d}".format(shower, meteor_count_by_shower[shower])) # Clean up and exit db.commit() db.close_db() return
def list_observatory_status(utc_min, utc_max, obstory): """ List all the metadata updates posted by a particular observatory between two given unix times. :param utc_min: Only list metadata updates after the specified unix time :param utc_max: Only list metadata updates before the specified unix time :param obstory: ID of the observatory we are to list events from :return: None """ # Open connection to image archive db = obsarchive_db.ObservationDatabase(file_store_path=settings['dbFilestore'], db_host=installation_info['mysqlHost'], db_user=installation_info['mysqlUser'], db_password=installation_info['mysqlPassword'], db_name=installation_info['mysqlDatabase'], obstory_id=installation_info['observatoryId']) try: obstory_info = db.get_obstory_from_id(obstory_id=obstory) except ValueError: print("Unknown observatory <{}>. Run ./listObservatories.py to see a list of available observatories.". format(obstory)) sys.exit(0) title = "Observatory <{}>".format(obstory) print("\n\n{}\n{}".format(title, "-" * len(title))) search = mp.ObservatoryMetadataSearch(obstory_ids=[obstory], time_min=utc_min, time_max=utc_max) data = db.search_obstory_metadata(search) data = data['items'] data.sort(key=lambda x: x.time) print(" * {:d} matching metadata items in time range {} --> {}".format(len(data), dcf_ast.date_string(utc_min), dcf_ast.date_string(utc_max))) # Check which items remain current refreshed = False data.reverse() keys_seen = [] for item in data: # The magic metadata keyword "refresh" causes all older metadata to be superseded if item.key == "refresh" and not refreshed: item.still_current = True refreshed = True # If we don't have a later metadata update for the same keyword, then this metadata remains current elif item.key not in keys_seen and not refreshed: item.still_current = True keys_seen.append(item.key) # This metadata item has been superseded else: item.still_current = False data.reverse() # Display list of items for item in data: if item.still_current: current_flag = "+" else: current_flag = " " print(" * {} [ID {}] {} -- {:16s} = {}".format(current_flag, item.id, dcf_ast.date_string(item.time), item.key, item.value))