def fetch_tray_ids():
logger.info('Fetching entity assignment info to extract tray IDs')
client = minimal_honeycomb.MinimalHoneycombClient()
result = client.request(
request_type="query",
request_name='entityAssignments',
arguments=None,
return_object=[
{'data': [
'entity_assignment_id',
{'entity': [
{'... on Tray': [
'tray_id'
]}
]}
]}
]
)
df = pd.json_normalize(result.get('data'))
df.rename(
columns={
'entity.tray_id': 'tray_id',
},
inplace=True
)
logger.info(
'Found {} entity assignments for trays'.format(
df['tray_id'].notna().sum()))
df.set_index('entity_assignment_id', inplace=True)
return df
def search_datapoints(query_list,
return_data,
chunk_size=100,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None):
logger.info('Searching for datapoints that match the specified parameters')
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret)
result = client.bulk_query(request_name='searchDatapoints',
arguments={
'query': {
'type': 'QueryExpression!',
'value': {
'operator': 'AND',
'children': query_list
}
}
},
return_data=return_data,
id_field_name='data_id',
chunk_size=chunk_size)
logger.info('Fetched {} datapoints'.format(len(result)))
return result
def write_ble_radio_pings(
datapoints_df,
source_id_column_name='tag_assignment_id',
source_type='MEASURED',
chunk_size=100,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None
):
datapoints_df = datapoints_df.rename(
columns={
'tag_device_id': 'tag_device',
'anchor_device_id': 'anchor_device',
'rssi': 'signal_strength',
source_id_column_name: 'source'
}
)
datapoints_df = datapoints_df.reindex(columns=[
'timestamp',
'tag_device',
'anchor_device',
'signal_strength',
'source'
])
datapoints_df['timestamp'] = datapoints_df['timestamp'].apply(lambda x:
minimal_honeycomb.to_honeycomb_datetime(x.to_pydatetime())
)
datapoints_df['source_type'] = source_type
datapoints_list = datapoints_df.to_dict(orient='records')
logger.info('Writing data for {} radio pings to Honeycomb'.format(len(datapoints_list)))
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret
)
result = client.bulk_mutation(
request_name='createRadioPing',
arguments={
'radioPing': {
'type': 'RadioPingInput',
'value': datapoints_list
}
},
return_object=[
'radio_ping_id'
],
chunk_size=chunk_size
)
radio_ping_ids = [datum.get('radio_ping_id') for datum in result]
return radio_ping_ids
def fetch_environment_id(
environment_id=None,
environment_name=None,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None
):
if environment_id is not None:
if environment_name is not None:
raise ValueError('If environment ID is specified, environment name cannot be specified')
return environment_id
if environment_name is not None:
logger.info('Fetching environment ID for specified environment name')
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret
)
result = client.bulk_query(
request_name='findEnvironments',
arguments={
'name': {
'type': 'String',
'value': environment_name
}
},
return_data=[
'environment_id'
],
id_field_name='environment_id'
)
if len(result) == 0:
raise ValueError('No environments match environment name {}'.format(
environment_name
))
if len(result) > 1:
raise ValueError('Multiple environments match environment name {}'.format(
environment_name
))
environment_id = result[0].get('environment_id')
logger.info('Found environment ID for specified environment name')
return environment_id
return None
def write_extrinsic_calibration_data(data,
start_datetime,
coordinate_space_id,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None):
extrinsic_calibration_data_columns = [
'device_id', 'rotation_vector', 'translation_vector'
]
if not set(extrinsic_calibration_data_columns).issubset(set(data.columns)):
raise ValueError('Data must contain the following columns: {}'.format(
extrinsic_calibration_data_columns))
extrinsic_calibration_data_df = data.reset_index().reindex(
columns=extrinsic_calibration_data_columns)
extrinsic_calibration_data_df.rename(columns={'device_id': 'device'},
inplace=True)
extrinsic_calibration_data_df[
'start'] = honeycomb_io.utils.to_honeycomb_datetime(start_datetime)
extrinsic_calibration_data_df['coordinate_space'] = coordinate_space_id
extrinsic_calibration_data_df[
'rotation_vector'] = extrinsic_calibration_data_df[
'rotation_vector'].apply(lambda x: x.tolist())
extrinsic_calibration_data_df[
'translation_vector'] = extrinsic_calibration_data_df[
'translation_vector'].apply(lambda x: x.tolist())
records = extrinsic_calibration_data_df.to_dict(orient='records')
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret)
result = client.bulk_mutation(request_name='createExtrinsicCalibration',
arguments={
'extrinsicCalibration': {
'type': 'ExtrinsicCalibrationInput',
'value': records
}
},
return_object=['extrinsic_calibration_id'])
ids = None
if len(result) > 0:
ids = [datum.get('extrinsic_calibration_id') for datum in result]
return ids
def fetch_camera_device_ids(environment_name,
start_time,
end_time,
camera_device_types=DEFAULT_CAMERA_DEVICE_TYPES):
client = minimal_honeycomb.MinimalHoneycombClient()
result = client.request(
request_type='query',
request_name='findEnvironments',
arguments={'name': {
'type': 'String',
'value': environment_name
}},
return_object=[{
'data': [{
'assignments': [
'start', 'end', 'assigned_type', {
'assigned': [
'__typename', {
'... on Device': ['device_id', 'device_type']
}
]
}
]
}]
}])
environments = result.get('data')
if len(environments) == 0:
raise ValueError('No environments match environment name {}'.format(
environment_name))
if len(environments) > 1:
raise ValueError('More than one environments matched name {}'.format(
environment_name))
assignments = environments[0].get('assignments')
camera_device_ids = list()
for assignment in assignments:
if assignment.get('start') is not None and (pd.to_datetime(
assignment.get('start')).to_pydatetime() > end_time):
continue
if assignment.get('end') is not None and (pd.to_datetime(
assignment.get('end')).to_pydatetime() < start_time):
continue
if assignment.get('assigned').get('__typename') != 'Device':
continue
if assignment.get('assigned').get(
'device_type') not in camera_device_types:
continue
camera_device_ids.append(assignment.get('assigned').get('device_id'))
return camera_device_ids
def write_intrinsic_calibration_data(data,
start_datetime,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None):
intrinsic_calibration_data_columns = [
'device_id', 'image_width', 'image_height', 'camera_matrix',
'distortion_coefficients'
]
if not set(intrinsic_calibration_data_columns).issubset(set(data.columns)):
raise ValueError('Data must contain the following columns: {}'.format(
intrinsic_calibration_data_columns))
intrinsic_calibration_data_df = data.reset_index().reindex(
columns=intrinsic_calibration_data_columns)
intrinsic_calibration_data_df.rename(columns={'device_id': 'device'},
inplace=True)
intrinsic_calibration_data_df[
'start'] = honeycomb_io.utils.to_honeycomb_datetime(start_datetime)
intrinsic_calibration_data_df[
'camera_matrix'] = intrinsic_calibration_data_df[
'camera_matrix'].apply(lambda x: x.tolist())
intrinsic_calibration_data_df[
'distortion_coefficients'] = intrinsic_calibration_data_df[
'distortion_coefficients'].apply(lambda x: x.tolist())
records = intrinsic_calibration_data_df.to_dict(orient='records')
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret)
result = client.bulk_mutation(request_name='createIntrinsicCalibration',
arguments={
'intrinsicCalibration': {
'type': 'IntrinsicCalibrationInput',
'value': records
}
},
return_object=['intrinsic_calibration_id'])
ids = None
if len(result) > 0:
ids = [datum.get('intrinsic_calibration_id') for datum in result]
return ids
def generate_client(
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None
):
if client is None:
client=minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret
)
return client
def write_position_data(data,
start_datetime,
coordinate_space_id,
assigned_type='DEVICE',
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None):
position_data_columns = ['device_id', 'position']
if not set(position_data_columns).issubset(set(data.columns)):
raise ValueError('Data must contain the following columns: {}'.format(
position_data_columns))
position_data_df = data.reset_index().reindex(
columns=position_data_columns)
position_data_df.rename(columns={'device_id': 'assigned'}, inplace=True)
position_data_df.rename(columns={'position': 'coordinates'}, inplace=True)
position_data_df['start'] = honeycomb_io.utils.to_honeycomb_datetime(
start_datetime)
position_data_df['assigned_type'] = assigned_type
position_data_df['coordinate_space'] = coordinate_space_id
position_data_df['coordinates'] = position_data_df['coordinates'].apply(
lambda x: x.tolist())
records = position_data_df.to_dict(orient='records')
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret)
result = client.bulk_mutation(request_name='assignToPosition',
arguments={
'positionAssignment': {
'type': 'PositionAssignmentInput!',
'value': records
}
},
return_object=['position_assignment_id'])
ids = None
if len(result) > 0:
ids = [datum.get('position_assignment_id') for datum in result]
return ids
def fetch_entity_info():
logger.info(
'Fetching entity assignment info to extract tray and person names')
client = minimal_honeycomb.MinimalHoneycombClient()
result = client.request(
request_type="query",
request_name='entityAssignments',
arguments=None,
return_object=[{
'data': [
'entity_assignment_id', {
'entity': [
'entity_type: __typename', {
'... on Tray': ['tray_id', 'tray_name: name']
}, {
'... on Person': [
'person_id', 'person_name: name',
'person_short_name: short_name'
]
}
]
}
]
}])
df = pd.json_normalize(result.get('data'))
df.rename(columns={
'entity.entity_type': 'entity_type',
'entity.tray_id': 'tray_id',
'entity.tray_name': 'tray_name',
'entity.person_id': 'person_id',
'entity.person_name': 'person_name',
'entity.person_short_name': 'person_short_name',
},
inplace=True)
df.set_index('entity_assignment_id', inplace=True)
logger.info(
'Found {} entity assignments for trays and {} entity assignments for people'
.format(df['tray_id'].notna().sum(), df['person_id'].notna().sum()))
return df
def fetch_environment_by_name(environment_name):
logger.info('Fetching Environments data')
client = minimal_honeycomb.MinimalHoneycombClient()
result = client.request(
request_type="query",
request_name="environments",
arguments=None,
return_object=[
{'data':
[
'environment_id',
'name'
]
}
]
)
logger.info('Found environments data: {} records'.format(
len(result.get('data'))))
df = pd.DataFrame(result.get('data'))
df = df[df['name'].str.lower().isin([environment_name.lower()])].reset_index(drop=True)
if len(df) > 0:
return df.loc[0]
return None
def search_objects(
request_name,
query_list,
return_data,
id_field_name,
chunk_size=100,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None
):
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret
)
result = client.bulk_query(
request_name=request_name,
arguments={
'query': {
'type': 'QueryExpression!',
'value': {
'operator': 'AND',
'children': query_list
}
}
},
return_data=return_data,
id_field_name=id_field_name,
chunk_size=chunk_size
)
return result
def fetch_material_names():
logger.info('Fetching material assignment info to extract material names')
client = minimal_honeycomb.MinimalHoneycombClient()
result = client.request(request_type="query",
request_name='materialAssignments',
arguments=None,
return_object=[{
'data': [
'material_assignment_id', {
'material':
['material_id', 'material_name: name']
}
]
}])
df = pd.json_normalize(result.get('data'))
df.rename(columns={
'material.material_id': 'material_id',
'material.material_name': 'material_name'
},
inplace=True)
df.set_index('material_assignment_id', inplace=True)
logger.info('Found {} material assignments'.format(
df['material_id'].notna().sum()))
return df
def __init__(self,
time_series_database=True,
object_database=True,
environment_name_honeycomb=None,
object_type_honeycomb=None,
object_id_field_name_honeycomb=None,
write_chunk_size=20,
read_chunk_size=1000,
honeycomb_uri=None,
honeycomb_token_uri=None,
honeycomb_audience=None,
honeycomb_client_id=None,
honeycomb_client_secret=None):
"""
Constructor for DatabaseConnectionHoneycomb.
If time_series_database and object_database are both True, database is
an object time series database (e.g., a measurement database) and
datapoints are identified by timestamp and object ID.
If object_database is True and time_series_database is False, database
is an object database (e.g., a device configuration database) and
datapoints are identified by object ID.
If time_series_database is True and object_database is False, behavior
is not defined (for now).
For an object time series database, Honeycomb environment, object type,
and object ID field name must be specified.
If Honeycomb access parameters (URI, token URI, audience, client ID,
client secret) are not specified, method will attempt to read from
corresponding environment variables (HONEYCOMB_URI, HONEYCOMB_TOKEN_URI,
HONEYCOMB_AUDIENCE, HONEYCOMB_CLIENT_ID, HONEYCOMB_CLIENT_SECRET).
Parameters:
time_series_database (bool): Boolean indicating whether database is a time series database (default is True)
object_database (bool): Boolean indicating whether database is an object database (default is True)
environment_name_honeycomb (string): Name of the Honeycomb environment that the data is associated with
object_type_honeycomb (string): Honeycomb object type that the data is associated with (e.g. DEVICE, PERSON)
object_id_field_name_honeycomb (string): Honeycomb field name that holds the object ID (e.g., part_number)
write_chunk_size (int): Number of datapoints to write in each request (default is 20)
read_chunk_size (int): Number of datapoints to read in each request (default is 1000)
honeycomb_uri (string): Honeycomb URI
honeycomb_token_uri (string): Honeycomb token URI
honeycomb_audience (string): Honeycomb audience
honeycomb_client_id (string): Honeycomb client ID
honeycomb_client_secret (string): Honeycomb client secret
"""
if not time_series_database and not object_database:
raise ValueError(
'Database must be a time series database, an object database, or an object time series database'
)
if time_series_database and object_database and environment_name_honeycomb is None:
raise ValueError(
'Honeycomb environment name must be specified for object time series database'
)
if time_series_database and object_database and object_type_honeycomb is None:
raise ValueError(
'Honeycomb object type must be specified for object time series database'
)
if time_series_database and object_database and object_id_field_name_honeycomb is None:
raise ValueError(
'Honeycomb object ID field name must be specified for object time series database'
)
self.time_series_database = time_series_database
self.object_database = object_database
self.environment_name_honeycomb = environment_name_honeycomb
self.object_type_honeycomb = object_type_honeycomb
self.object_id_field_name_honeycomb = object_id_field_name_honeycomb
self.write_chunk_size = write_chunk_size
self.read_chunk_size = read_chunk_size
self.honeycomb_client = minimal_honeycomb.MinimalHoneycombClient(
uri=honeycomb_uri,
token_uri=honeycomb_token_uri,
audience=honeycomb_audience,
client_id=honeycomb_client_id,
client_secret=honeycomb_client_secret)
if self.environment_name_honeycomb is not None:
findEnvironment_result = self.honeycomb_client.request(
request_type='query',
request_name='findEnvironment',
arguments={
'name': {
'type': 'String',
'value': self.environment_name_honeycomb
}
},
return_object=[{
'data': ['environment_id']
}])
if len(findEnvironment_result.get('data')) == 0:
raise ValueError(
'Environment name {} matched no environments'.format(
self.environment_name_honeycomb))
if len(findEnvironment_result.get('data')) > 1:
raise ValueError(
'Environment name {} matched more than one environment'.
format(self.environment_name_honeycomb))
environment_id = findEnvironment_result.get('data')[0].get(
'environment_id')
getEnvironment_result = self.honeycomb_client.request(
request_type='query',
request_name='getEnvironment',
arguments={
'environment_id': {
'type': 'ID!',
'value': environment_id
}
},
return_object=[
'name', {
'assignments': [
'assignment_id', 'start', 'end', 'assigned_type', {
'assigned': [{
'... on Device': [
'device_id', 'device_type',
'part_number', 'serial_number', 'name',
'mac_address', 'tag_id'
]
}, {
'... on Person': [
'person_id', 'name', 'first_name',
'last_name', 'nickname', 'short_name',
'person_type',
'transparent_classroom_id'
]
}, {
'... on Material': [
'material_id', 'name',
'transparent_classroom_id'
]
}, {
'... on Tray': [
'tray_id', 'part_number', 'name',
'serial_number'
]
}]
}
]
}
])
self.environment = getEnvironment_result
def fetch_material_assignments():
logger.info('Fetching material assignment IDs')
client = minimal_honeycomb.MinimalHoneycombClient()
result = client.request(
request_type="query",
request_name='materialAssignments',
arguments=None,
return_object=[{
'data':
['material_assignment_id', {
'tray': ['tray_id']
}, 'start', 'end']
}])
if len(result.get('data')) == 0:
raise ValueError('No material assignments found')
logger.info('Found {} material assignments'.format(len(
result.get('data'))))
assignments_dict = dict()
for material_assignment in result.get('data'):
tray_id = material_assignment['tray']['tray_id']
assignment = {
'material_assignment_id':
material_assignment['material_assignment_id'],
'start': material_assignment['start'],
'end': material_assignment['end']
}
if tray_id in assignments_dict.keys():
assignments_dict[tray_id].append(assignment)
else:
assignments_dict[tray_id] = [assignment]
for tray_id in assignments_dict.keys():
num_assignments = len(assignments_dict[tray_id])
# Convert timestamp strings to Pandas datetime objects
for assignment_index in range(num_assignments):
assignments_dict[tray_id][assignment_index][
'start'] = pd.to_datetime(
assignments_dict[tray_id][assignment_index]['start'],
utc=True)
assignments_dict[tray_id][assignment_index][
'end'] = pd.to_datetime(
assignments_dict[tray_id][assignment_index]['end'],
utc=True)
# Sort assignment list by start time
assignments_dict[tray_id] = sorted(
assignments_dict[tray_id],
key=lambda assignment: assignment['start'])
# Check integrity of assignment list
if num_assignments > 1:
for assignment_index in range(1, num_assignments):
if pd.isna(assignments_dict[tray_id][assignment_index -
1]['end']):
raise ValueError(
'Assignment {} starts at {} but previous assignment for this device {} starts at {} and has no end time'
.format(
assignments_dict[tray_id][assignment_index]
['material_assignment_id'],
assignments_dict[tray_id][assignment_index]
['start'], assignments_dict[tray_id]
[assignment_index - 1]['material_assignment_id'],
assignments_dict[tray_id][assignment_index -
1]['start']))
if assignments_dict[tray_id][assignment_index][
'start'] < assignments_dict[tray_id][assignment_index -
1]['end']:
raise ValueError(
'Assignment {} starts at {} but previous assignment for this device {} starts at {} and ends at {}'
.format(
assignments_dict[tray_id][assignment_index]
['material_assignment_id'],
assignments_dict[tray_id][assignment_index]
['start'], assignments_dict[tray_id]
[assignment_index - 1]['material_assignment_id'],
assignments_dict[tray_id][assignment_index -
1]['start'],
assignments_dict[tray_id][assignment_index -
1]['end']))
return assignments_dict
def fetch_camera_assignment_ids_from_camera_properties(
start=None,
end=None,
camera_device_ids=None,
camera_part_numbers=None,
camera_names=None,
camera_serial_numbers=None,
chunk_size=100,
client=None,
uri=None,
token_uri=None,
audience=None,
client_id=None,
client_secret=None):
if camera_device_ids is None and camera_names is None and camera_part_numbers is None and camera_serial_numbers is None:
return None
query_list = list()
if camera_device_ids is not None:
query_list.append({
'field': 'device_id',
'operator': 'IN',
'values': camera_device_ids
})
if camera_part_numbers is not None:
query_list.append({
'field': 'part_number',
'operator': 'IN',
'values': camera_part_numbers
})
if camera_names is not None:
query_list.append({
'field': 'name',
'operator': 'IN',
'values': camera_names
})
if camera_serial_numbers is not None:
query_list.append({
'field': 'serial_number',
'operator': 'IN',
'values': camera_serial_numbers
})
logger.info(
'Fetching camera assignments for cameras with specified properties')
if client is None:
client = minimal_honeycomb.MinimalHoneycombClient(
uri=uri,
token_uri=token_uri,
audience=audience,
client_id=client_id,
client_secret=client_secret)
result = client.bulk_query(request_name='searchDevices',
arguments={
'query': {
'type': 'QueryExpression!',
'value': {
'operator': 'AND',
'children': query_list
}
}
},
return_data=[
'device_id', {
'assignments':
['assignment_id', 'start', 'end']
}
],
id_field_name='device_id',
chunk_size=chunk_size)
assignments = list()
for datum in result:
if datum.get('assignments') is not None and len(
datum.get('assignments')) > 0:
assignments.extend(datum.get('assignments'))
filtered_assignments = minimal_honeycomb.filter_assignments(
assignments=assignments, start_time=start, end_time=end)
if len(filtered_assignments) == 0:
raise ValueError(
'No camera assignments match specified camera device IDs/names/part numbers/serial numbers and time span'
)
camera_assignment_ids = [
assignment.get('assignment_id') for assignment in filtered_assignments
]
return camera_assignment_ids