def fetch_tray_ids(): logger.info('Fetching entity assignment info to extract tray IDs') client = minimal_honeycomb.MinimalHoneycombClient() result = client.request( request_type="query", request_name='entityAssignments', arguments=None, return_object=[ {'data': [ 'entity_assignment_id', {'entity': [ {'... on Tray': [ 'tray_id' ]} ]} ]} ] ) df = pd.json_normalize(result.get('data')) df.rename( columns={ 'entity.tray_id': 'tray_id', }, inplace=True ) logger.info( 'Found {} entity assignments for trays'.format( df['tray_id'].notna().sum())) df.set_index('entity_assignment_id', inplace=True) return df
def search_datapoints(query_list, return_data, chunk_size=100, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None): logger.info('Searching for datapoints that match the specified parameters') if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret) result = client.bulk_query(request_name='searchDatapoints', arguments={ 'query': { 'type': 'QueryExpression!', 'value': { 'operator': 'AND', 'children': query_list } } }, return_data=return_data, id_field_name='data_id', chunk_size=chunk_size) logger.info('Fetched {} datapoints'.format(len(result))) return result
def write_ble_radio_pings( datapoints_df, source_id_column_name='tag_assignment_id', source_type='MEASURED', chunk_size=100, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None ): datapoints_df = datapoints_df.rename( columns={ 'tag_device_id': 'tag_device', 'anchor_device_id': 'anchor_device', 'rssi': 'signal_strength', source_id_column_name: 'source' } ) datapoints_df = datapoints_df.reindex(columns=[ 'timestamp', 'tag_device', 'anchor_device', 'signal_strength', 'source' ]) datapoints_df['timestamp'] = datapoints_df['timestamp'].apply(lambda x: minimal_honeycomb.to_honeycomb_datetime(x.to_pydatetime()) ) datapoints_df['source_type'] = source_type datapoints_list = datapoints_df.to_dict(orient='records') logger.info('Writing data for {} radio pings to Honeycomb'.format(len(datapoints_list))) if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret ) result = client.bulk_mutation( request_name='createRadioPing', arguments={ 'radioPing': { 'type': 'RadioPingInput', 'value': datapoints_list } }, return_object=[ 'radio_ping_id' ], chunk_size=chunk_size ) radio_ping_ids = [datum.get('radio_ping_id') for datum in result] return radio_ping_ids
def fetch_environment_id( environment_id=None, environment_name=None, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None ): if environment_id is not None: if environment_name is not None: raise ValueError('If environment ID is specified, environment name cannot be specified') return environment_id if environment_name is not None: logger.info('Fetching environment ID for specified environment name') if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret ) result = client.bulk_query( request_name='findEnvironments', arguments={ 'name': { 'type': 'String', 'value': environment_name } }, return_data=[ 'environment_id' ], id_field_name='environment_id' ) if len(result) == 0: raise ValueError('No environments match environment name {}'.format( environment_name )) if len(result) > 1: raise ValueError('Multiple environments match environment name {}'.format( environment_name )) environment_id = result[0].get('environment_id') logger.info('Found environment ID for specified environment name') return environment_id return None
def write_extrinsic_calibration_data(data, start_datetime, coordinate_space_id, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None): extrinsic_calibration_data_columns = [ 'device_id', 'rotation_vector', 'translation_vector' ] if not set(extrinsic_calibration_data_columns).issubset(set(data.columns)): raise ValueError('Data must contain the following columns: {}'.format( extrinsic_calibration_data_columns)) extrinsic_calibration_data_df = data.reset_index().reindex( columns=extrinsic_calibration_data_columns) extrinsic_calibration_data_df.rename(columns={'device_id': 'device'}, inplace=True) extrinsic_calibration_data_df[ 'start'] = honeycomb_io.utils.to_honeycomb_datetime(start_datetime) extrinsic_calibration_data_df['coordinate_space'] = coordinate_space_id extrinsic_calibration_data_df[ 'rotation_vector'] = extrinsic_calibration_data_df[ 'rotation_vector'].apply(lambda x: x.tolist()) extrinsic_calibration_data_df[ 'translation_vector'] = extrinsic_calibration_data_df[ 'translation_vector'].apply(lambda x: x.tolist()) records = extrinsic_calibration_data_df.to_dict(orient='records') if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret) result = client.bulk_mutation(request_name='createExtrinsicCalibration', arguments={ 'extrinsicCalibration': { 'type': 'ExtrinsicCalibrationInput', 'value': records } }, return_object=['extrinsic_calibration_id']) ids = None if len(result) > 0: ids = [datum.get('extrinsic_calibration_id') for datum in result] return ids
def fetch_camera_device_ids(environment_name, start_time, end_time, camera_device_types=DEFAULT_CAMERA_DEVICE_TYPES): client = minimal_honeycomb.MinimalHoneycombClient() result = client.request( request_type='query', request_name='findEnvironments', arguments={'name': { 'type': 'String', 'value': environment_name }}, return_object=[{ 'data': [{ 'assignments': [ 'start', 'end', 'assigned_type', { 'assigned': [ '__typename', { '... on Device': ['device_id', 'device_type'] } ] } ] }] }]) environments = result.get('data') if len(environments) == 0: raise ValueError('No environments match environment name {}'.format( environment_name)) if len(environments) > 1: raise ValueError('More than one environments matched name {}'.format( environment_name)) assignments = environments[0].get('assignments') camera_device_ids = list() for assignment in assignments: if assignment.get('start') is not None and (pd.to_datetime( assignment.get('start')).to_pydatetime() > end_time): continue if assignment.get('end') is not None and (pd.to_datetime( assignment.get('end')).to_pydatetime() < start_time): continue if assignment.get('assigned').get('__typename') != 'Device': continue if assignment.get('assigned').get( 'device_type') not in camera_device_types: continue camera_device_ids.append(assignment.get('assigned').get('device_id')) return camera_device_ids
def write_intrinsic_calibration_data(data, start_datetime, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None): intrinsic_calibration_data_columns = [ 'device_id', 'image_width', 'image_height', 'camera_matrix', 'distortion_coefficients' ] if not set(intrinsic_calibration_data_columns).issubset(set(data.columns)): raise ValueError('Data must contain the following columns: {}'.format( intrinsic_calibration_data_columns)) intrinsic_calibration_data_df = data.reset_index().reindex( columns=intrinsic_calibration_data_columns) intrinsic_calibration_data_df.rename(columns={'device_id': 'device'}, inplace=True) intrinsic_calibration_data_df[ 'start'] = honeycomb_io.utils.to_honeycomb_datetime(start_datetime) intrinsic_calibration_data_df[ 'camera_matrix'] = intrinsic_calibration_data_df[ 'camera_matrix'].apply(lambda x: x.tolist()) intrinsic_calibration_data_df[ 'distortion_coefficients'] = intrinsic_calibration_data_df[ 'distortion_coefficients'].apply(lambda x: x.tolist()) records = intrinsic_calibration_data_df.to_dict(orient='records') if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret) result = client.bulk_mutation(request_name='createIntrinsicCalibration', arguments={ 'intrinsicCalibration': { 'type': 'IntrinsicCalibrationInput', 'value': records } }, return_object=['intrinsic_calibration_id']) ids = None if len(result) > 0: ids = [datum.get('intrinsic_calibration_id') for datum in result] return ids
def generate_client( client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None ): if client is None: client=minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret ) return client
def write_position_data(data, start_datetime, coordinate_space_id, assigned_type='DEVICE', client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None): position_data_columns = ['device_id', 'position'] if not set(position_data_columns).issubset(set(data.columns)): raise ValueError('Data must contain the following columns: {}'.format( position_data_columns)) position_data_df = data.reset_index().reindex( columns=position_data_columns) position_data_df.rename(columns={'device_id': 'assigned'}, inplace=True) position_data_df.rename(columns={'position': 'coordinates'}, inplace=True) position_data_df['start'] = honeycomb_io.utils.to_honeycomb_datetime( start_datetime) position_data_df['assigned_type'] = assigned_type position_data_df['coordinate_space'] = coordinate_space_id position_data_df['coordinates'] = position_data_df['coordinates'].apply( lambda x: x.tolist()) records = position_data_df.to_dict(orient='records') if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret) result = client.bulk_mutation(request_name='assignToPosition', arguments={ 'positionAssignment': { 'type': 'PositionAssignmentInput!', 'value': records } }, return_object=['position_assignment_id']) ids = None if len(result) > 0: ids = [datum.get('position_assignment_id') for datum in result] return ids
def fetch_entity_info(): logger.info( 'Fetching entity assignment info to extract tray and person names') client = minimal_honeycomb.MinimalHoneycombClient() result = client.request( request_type="query", request_name='entityAssignments', arguments=None, return_object=[{ 'data': [ 'entity_assignment_id', { 'entity': [ 'entity_type: __typename', { '... on Tray': ['tray_id', 'tray_name: name'] }, { '... on Person': [ 'person_id', 'person_name: name', 'person_short_name: short_name' ] } ] } ] }]) df = pd.json_normalize(result.get('data')) df.rename(columns={ 'entity.entity_type': 'entity_type', 'entity.tray_id': 'tray_id', 'entity.tray_name': 'tray_name', 'entity.person_id': 'person_id', 'entity.person_name': 'person_name', 'entity.person_short_name': 'person_short_name', }, inplace=True) df.set_index('entity_assignment_id', inplace=True) logger.info( 'Found {} entity assignments for trays and {} entity assignments for people' .format(df['tray_id'].notna().sum(), df['person_id'].notna().sum())) return df
def fetch_environment_by_name(environment_name): logger.info('Fetching Environments data') client = minimal_honeycomb.MinimalHoneycombClient() result = client.request( request_type="query", request_name="environments", arguments=None, return_object=[ {'data': [ 'environment_id', 'name' ] } ] ) logger.info('Found environments data: {} records'.format( len(result.get('data')))) df = pd.DataFrame(result.get('data')) df = df[df['name'].str.lower().isin([environment_name.lower()])].reset_index(drop=True) if len(df) > 0: return df.loc[0] return None
def search_objects( request_name, query_list, return_data, id_field_name, chunk_size=100, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None ): if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret ) result = client.bulk_query( request_name=request_name, arguments={ 'query': { 'type': 'QueryExpression!', 'value': { 'operator': 'AND', 'children': query_list } } }, return_data=return_data, id_field_name=id_field_name, chunk_size=chunk_size ) return result
def fetch_material_names(): logger.info('Fetching material assignment info to extract material names') client = minimal_honeycomb.MinimalHoneycombClient() result = client.request(request_type="query", request_name='materialAssignments', arguments=None, return_object=[{ 'data': [ 'material_assignment_id', { 'material': ['material_id', 'material_name: name'] } ] }]) df = pd.json_normalize(result.get('data')) df.rename(columns={ 'material.material_id': 'material_id', 'material.material_name': 'material_name' }, inplace=True) df.set_index('material_assignment_id', inplace=True) logger.info('Found {} material assignments'.format( df['material_id'].notna().sum())) return df
def __init__(self, time_series_database=True, object_database=True, environment_name_honeycomb=None, object_type_honeycomb=None, object_id_field_name_honeycomb=None, write_chunk_size=20, read_chunk_size=1000, honeycomb_uri=None, honeycomb_token_uri=None, honeycomb_audience=None, honeycomb_client_id=None, honeycomb_client_secret=None): """ Constructor for DatabaseConnectionHoneycomb. If time_series_database and object_database are both True, database is an object time series database (e.g., a measurement database) and datapoints are identified by timestamp and object ID. If object_database is True and time_series_database is False, database is an object database (e.g., a device configuration database) and datapoints are identified by object ID. If time_series_database is True and object_database is False, behavior is not defined (for now). For an object time series database, Honeycomb environment, object type, and object ID field name must be specified. If Honeycomb access parameters (URI, token URI, audience, client ID, client secret) are not specified, method will attempt to read from corresponding environment variables (HONEYCOMB_URI, HONEYCOMB_TOKEN_URI, HONEYCOMB_AUDIENCE, HONEYCOMB_CLIENT_ID, HONEYCOMB_CLIENT_SECRET). Parameters: time_series_database (bool): Boolean indicating whether database is a time series database (default is True) object_database (bool): Boolean indicating whether database is an object database (default is True) environment_name_honeycomb (string): Name of the Honeycomb environment that the data is associated with object_type_honeycomb (string): Honeycomb object type that the data is associated with (e.g. DEVICE, PERSON) object_id_field_name_honeycomb (string): Honeycomb field name that holds the object ID (e.g., part_number) write_chunk_size (int): Number of datapoints to write in each request (default is 20) read_chunk_size (int): Number of datapoints to read in each request (default is 1000) honeycomb_uri (string): Honeycomb URI honeycomb_token_uri (string): Honeycomb token URI honeycomb_audience (string): Honeycomb audience honeycomb_client_id (string): Honeycomb client ID honeycomb_client_secret (string): Honeycomb client secret """ if not time_series_database and not object_database: raise ValueError( 'Database must be a time series database, an object database, or an object time series database' ) if time_series_database and object_database and environment_name_honeycomb is None: raise ValueError( 'Honeycomb environment name must be specified for object time series database' ) if time_series_database and object_database and object_type_honeycomb is None: raise ValueError( 'Honeycomb object type must be specified for object time series database' ) if time_series_database and object_database and object_id_field_name_honeycomb is None: raise ValueError( 'Honeycomb object ID field name must be specified for object time series database' ) self.time_series_database = time_series_database self.object_database = object_database self.environment_name_honeycomb = environment_name_honeycomb self.object_type_honeycomb = object_type_honeycomb self.object_id_field_name_honeycomb = object_id_field_name_honeycomb self.write_chunk_size = write_chunk_size self.read_chunk_size = read_chunk_size self.honeycomb_client = minimal_honeycomb.MinimalHoneycombClient( uri=honeycomb_uri, token_uri=honeycomb_token_uri, audience=honeycomb_audience, client_id=honeycomb_client_id, client_secret=honeycomb_client_secret) if self.environment_name_honeycomb is not None: findEnvironment_result = self.honeycomb_client.request( request_type='query', request_name='findEnvironment', arguments={ 'name': { 'type': 'String', 'value': self.environment_name_honeycomb } }, return_object=[{ 'data': ['environment_id'] }]) if len(findEnvironment_result.get('data')) == 0: raise ValueError( 'Environment name {} matched no environments'.format( self.environment_name_honeycomb)) if len(findEnvironment_result.get('data')) > 1: raise ValueError( 'Environment name {} matched more than one environment'. format(self.environment_name_honeycomb)) environment_id = findEnvironment_result.get('data')[0].get( 'environment_id') getEnvironment_result = self.honeycomb_client.request( request_type='query', request_name='getEnvironment', arguments={ 'environment_id': { 'type': 'ID!', 'value': environment_id } }, return_object=[ 'name', { 'assignments': [ 'assignment_id', 'start', 'end', 'assigned_type', { 'assigned': [{ '... on Device': [ 'device_id', 'device_type', 'part_number', 'serial_number', 'name', 'mac_address', 'tag_id' ] }, { '... on Person': [ 'person_id', 'name', 'first_name', 'last_name', 'nickname', 'short_name', 'person_type', 'transparent_classroom_id' ] }, { '... on Material': [ 'material_id', 'name', 'transparent_classroom_id' ] }, { '... on Tray': [ 'tray_id', 'part_number', 'name', 'serial_number' ] }] } ] } ]) self.environment = getEnvironment_result
def fetch_material_assignments(): logger.info('Fetching material assignment IDs') client = minimal_honeycomb.MinimalHoneycombClient() result = client.request( request_type="query", request_name='materialAssignments', arguments=None, return_object=[{ 'data': ['material_assignment_id', { 'tray': ['tray_id'] }, 'start', 'end'] }]) if len(result.get('data')) == 0: raise ValueError('No material assignments found') logger.info('Found {} material assignments'.format(len( result.get('data')))) assignments_dict = dict() for material_assignment in result.get('data'): tray_id = material_assignment['tray']['tray_id'] assignment = { 'material_assignment_id': material_assignment['material_assignment_id'], 'start': material_assignment['start'], 'end': material_assignment['end'] } if tray_id in assignments_dict.keys(): assignments_dict[tray_id].append(assignment) else: assignments_dict[tray_id] = [assignment] for tray_id in assignments_dict.keys(): num_assignments = len(assignments_dict[tray_id]) # Convert timestamp strings to Pandas datetime objects for assignment_index in range(num_assignments): assignments_dict[tray_id][assignment_index][ 'start'] = pd.to_datetime( assignments_dict[tray_id][assignment_index]['start'], utc=True) assignments_dict[tray_id][assignment_index][ 'end'] = pd.to_datetime( assignments_dict[tray_id][assignment_index]['end'], utc=True) # Sort assignment list by start time assignments_dict[tray_id] = sorted( assignments_dict[tray_id], key=lambda assignment: assignment['start']) # Check integrity of assignment list if num_assignments > 1: for assignment_index in range(1, num_assignments): if pd.isna(assignments_dict[tray_id][assignment_index - 1]['end']): raise ValueError( 'Assignment {} starts at {} but previous assignment for this device {} starts at {} and has no end time' .format( assignments_dict[tray_id][assignment_index] ['material_assignment_id'], assignments_dict[tray_id][assignment_index] ['start'], assignments_dict[tray_id] [assignment_index - 1]['material_assignment_id'], assignments_dict[tray_id][assignment_index - 1]['start'])) if assignments_dict[tray_id][assignment_index][ 'start'] < assignments_dict[tray_id][assignment_index - 1]['end']: raise ValueError( 'Assignment {} starts at {} but previous assignment for this device {} starts at {} and ends at {}' .format( assignments_dict[tray_id][assignment_index] ['material_assignment_id'], assignments_dict[tray_id][assignment_index] ['start'], assignments_dict[tray_id] [assignment_index - 1]['material_assignment_id'], assignments_dict[tray_id][assignment_index - 1]['start'], assignments_dict[tray_id][assignment_index - 1]['end'])) return assignments_dict
def fetch_camera_assignment_ids_from_camera_properties( start=None, end=None, camera_device_ids=None, camera_part_numbers=None, camera_names=None, camera_serial_numbers=None, chunk_size=100, client=None, uri=None, token_uri=None, audience=None, client_id=None, client_secret=None): if camera_device_ids is None and camera_names is None and camera_part_numbers is None and camera_serial_numbers is None: return None query_list = list() if camera_device_ids is not None: query_list.append({ 'field': 'device_id', 'operator': 'IN', 'values': camera_device_ids }) if camera_part_numbers is not None: query_list.append({ 'field': 'part_number', 'operator': 'IN', 'values': camera_part_numbers }) if camera_names is not None: query_list.append({ 'field': 'name', 'operator': 'IN', 'values': camera_names }) if camera_serial_numbers is not None: query_list.append({ 'field': 'serial_number', 'operator': 'IN', 'values': camera_serial_numbers }) logger.info( 'Fetching camera assignments for cameras with specified properties') if client is None: client = minimal_honeycomb.MinimalHoneycombClient( uri=uri, token_uri=token_uri, audience=audience, client_id=client_id, client_secret=client_secret) result = client.bulk_query(request_name='searchDevices', arguments={ 'query': { 'type': 'QueryExpression!', 'value': { 'operator': 'AND', 'children': query_list } } }, return_data=[ 'device_id', { 'assignments': ['assignment_id', 'start', 'end'] } ], id_field_name='device_id', chunk_size=chunk_size) assignments = list() for datum in result: if datum.get('assignments') is not None and len( datum.get('assignments')) > 0: assignments.extend(datum.get('assignments')) filtered_assignments = minimal_honeycomb.filter_assignments( assignments=assignments, start_time=start, end_time=end) if len(filtered_assignments) == 0: raise ValueError( 'No camera assignments match specified camera device IDs/names/part numbers/serial numbers and time span' ) camera_assignment_ids = [ assignment.get('assignment_id') for assignment in filtered_assignments ] return camera_assignment_ids