def testGetModelDetailsEmptyFields(self): client = Mock() model = Mock(project='project_id', dataset_id='dataset_id', model_id='model_id', description=None, labels={}, created=datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None), expires=datetime.datetime(2021, 7, 14, 13, 23, 45, 67, tzinfo=None), location='US', modified=datetime.datetime(2020, 7, 15, 15, 11, 23, 32, tzinfo=None), model_type=0, label_columns=[], feature_columns=[], training_runs=[Mock(start_time=Mock(seconds=1234567))]) client.get_model = Mock(return_value=model) bigquery = BigQueryService(client) expected = { 'details': { 'id': 'project_id.dataset_id.model_id', 'name': 'model_id', 'description': None, 'labels': None, 'date_created': '2020-07-14T13:23:45.000067', 'expires': '2021-07-14T13:23:45.000067', 'location': 'US', 'last_modified': '2020-07-15T15:11:23.000032', 'model_type': Model.ModelType(0).name, 'schema_labels': [], 'feature_columns': [], 'training_runs': ['1970-01-15T06:56:07+00:00'] } } result = bigquery.get_model_details('some_model_id') self.assertEqual(expected, result)
def testGetDatasetDetailsFull(self): client = Mock() dataset = Mock(project='project_id', dataset_id='dataset_id', description='description of dataset', labels={ 'label_0': 'value_0', 'label_1': 'value_1' }, created=datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None), default_table_expiration_ms=17280000000, location='US', modified=datetime.datetime(2020, 7, 15, 15, 11, 23, 32, tzinfo=None), self_link='https://bigquery.googleapis.com/bigquery' + \ '/v2/projects/project_id/datasets/dataset_id') client.get_dataset = Mock(return_value=dataset) bigquery = BigQueryService(client) expected = { 'details': { 'id': 'project_id.dataset_id', 'name': 'dataset_id', 'description': 'description of dataset', 'labels': ['label_0: value_0', 'label_1: value_1'], 'date_created': '2020-07-14T13:23:45.000067', 'default_expiration': 17280000000, 'location': 'US', 'last_modified': '2020-07-15T15:11:23.000032', 'project': 'project_id', 'link': 'https://bigquery.googleapis.com/bigquery' + \ '/v2/projects/project_id/datasets/dataset_id' } } result = bigquery.get_dataset_details('some_dataset_id') self.assertEqual(expected, result)
def testNestedTable(self): # ensures records schema names are properly displayed and items # within records are properly separated client = Mock() schema_field_0 = Mock(field_type='STRING', mode='NULLABLE') schema_field_0.name = 'field_name_0' schema_subfield = Mock(field_type='INTEGER', mode='NULLABLE') schema_subfield.name = 'subfield' schema_subfield_2 = Mock(field_type='DATETIME', mode='NULLABLE') schema_subfield_2.name = 'other_subfield' schema_field_1 = Mock(field_type='RECORD', mode='NULLABLE', fields=[schema_subfield, schema_subfield_2]) schema_field_1.name = 'field_name_1' rows = MagicMock(schema=[schema_field_0, schema_field_1]) row_0 = [ 'hello', { 'subfield': 1, 'other_subfield': datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None) } ] row_1 = ['goodbye', {'subfield': 2, 'other_subfield': None}] row_2 = [None, None] rows.__iter__.return_value = [row_0, row_1, row_2] client.list_rows = MagicMock(return_value=rows) bigquery = BigQueryService(client) expected = { 'fields': [ 'field_name_0', 'field_name_1.subfield', 'field_name_1.other_subfield' ], 'rows': [['hello', 1, '2020-07-14 13:23:45.000067 '], ['goodbye', 2, None], [None, None, None]] } result = bigquery.get_table_preview('some_table_id') self.assertEqual(expected, result)
def testRepeatedTable(self): # ensures that repeated fields are returned as strings, and that # repeated record field name headers are not expanded client = Mock() schema_field_0 = Mock(field_type='STRING', description='this field is a repeated string', mode='REPEATED') schema_field_0.name = 'field_name_0' schema_subfield = Mock(field_type='INTEGER', description='this field is an integer', mode='NULLABLE') schema_subfield.name = 'subfield' schema_field_1 = Mock(field_type='RECORD', description='this field is a repeated record', mode='REPEATED', fields=[schema_subfield]) schema_field_1.name = 'field_name_1' rows = MagicMock(schema=[schema_field_0, schema_field_1]) row_0 = [['hello', 'hi'], [{'subfield': 1}]] row_1 = [['goodbye', 'bye'], [{'subfield': 2}, {'subfield': 3}]] row_2 = [[], []] row_3 = [[], [{'subfield': None}]] rows.__iter__.return_value = [row_0, row_1, row_2, row_3] client.list_rows = MagicMock(return_value=rows) bigquery = BigQueryService(client) expected = { 'fields': ['field_name_0', 'field_name_1'], 'rows': [['["hello", "hi"]', '[{"subfield": 1}]'], ['["goodbye", "bye"]', '[{"subfield": 2}, {"subfield": 3}]'], ['[]', '[]'], ['[]', '[{"subfield": null}]']] } result = bigquery.get_table_preview('some_table_id') self.assertEqual(expected, result)
def testEmptyTable(self): # same as nested table, but ensures # that when all entries are None the header # still displays properly. client = Mock() schema_field_0 = Mock(field_type='STRING', mode='NULLABLE') schema_field_0.name = 'field_name_0' schema_subfield = Mock(field_type='INTEGER', mode='NULLABLE') schema_subfield.name = 'subfield' schema_subfield_2 = Mock(field_type='DATETIME', mode='NULLABLE') schema_subfield_2.name = 'other_subfield' schema_field_1 = Mock(field_type='RECORD', mode='NULLABLE', fields=[schema_subfield, schema_subfield_2]) schema_field_1.name = 'field_name_1' rows = MagicMock(schema=[schema_field_0, schema_field_1]) row_0 = [None, None] rows.__iter__.return_value = [row_0] client.list_rows = MagicMock(return_value=rows) bigquery = BigQueryService(client) expected = { 'fields': [ 'field_name_0', 'field_name_1.subfield', 'field_name_1.other_subfield' ], 'rows': [[None, None, None]] } result = bigquery.get_table_preview('some_table_id') self.assertEqual(expected, result)
def _get_training_run_details(args): return BigQueryService.get().get_training_run_details(args["modelId"], int(args["runIndex"]))
def _get_model_details(args): return BigQueryService.get().get_model_details(args["modelId"])
def _get_view_details(args): return BigQueryService.get().get_view_details(args["viewId"])
def _get_table_preview(args): return BigQueryService.get().get_table_preview(args["tableId"])
def _get_table_details(args): return BigQueryService.get().get_table_details(args["tableId"])
def _get_dataset_details(args): return BigQueryService.get().get_dataset_details(args['datasetId'])
def testGetTrainingRunDetails(self): client = Mock() options = [['data_split_column', 'column_name_1'], ['data_split_eval_fraction', 0.3], ['data_split_method', 1], ['distance_type', 1], ['early_stop', BoolValue(value=False)], ['initial_learn_rate', 0.1], ['input_label_columns', ['column1', 'column2']], ['kmeans_initialization_column', 'column_name_2'], ['kmeans_initialization_method', 1], ['l1_regularization', DoubleValue(value=0.5)], ['l2_regularization', DoubleValue(value=0.6)], ['label_class_weights', [0.2, 0.3]], ['learn_rate', 0.7], ['learn_rate_strategy', 1], ['loss_type', 1], ['max_iterations', 20], ['min_relative_progress', DoubleValue(value=0.4)], ['model_uri', 'model.uri.string'], ['num_clusters', 7], ['optimization_strategy', 1], ['warm_start', BoolValue(value=True)]] mocked_options = [] for option in options: option_mock = Mock() option_mock.name = option[0] mocked_options.append([option_mock, option[1]]) training_options = Mock() training_options.ListFields = Mock(return_value=mocked_options) training_run = Mock(training_options=training_options, results=['result', 'result', 'result']) model = Mock(training_runs=[training_run]) client.get_model = Mock(return_value=model) bigquery = BigQueryService(client) expected = { 'details': { 'actual_iterations': 3, 'data_split_column': 'column_name_1', 'data_split_eval_fraction': 0.3, 'data_split_method': Model.DataSplitMethod(1).name, 'distance_type': Model.DistanceType(1).name, 'early_stop': 'False', 'initial_learn_rate': 0.1, 'input_label_columns': "['column1', 'column2']", 'kmeans_initialization_column': 'column_name_2', 'kmeans_initialization_method': Model.KmeansEnums.KmeansInitializationMethod(1).name, 'l1_regularization': 0.5, 'l2_regularization': 0.6, 'label_class_weights': '[0.2, 0.3]', 'learn_rate': 0.7, 'learn_rate_strategy': Model.LearnRateStrategy(1).name, 'loss_type': Model.LossType(1).name, 'max_iterations': 20, 'min_relative_progress': 0.4, 'model_uri': 'model.uri.string', 'num_clusters': 7, 'optimization_strategy': Model.OptimizationStrategy(1).name, 'warm_start': 'True' } } result = bigquery.get_training_run_details('some_model_id', 0) self.assertEqual(expected, result)
def testGetModelDetailsFull(self): client = Mock() label_col_0 = Mock() label_col_0.name = 'schema_label_0' label_col_0.type = Mock(type_kind=7) label_col_1 = Mock() label_col_1.name = 'schema_label_1' label_col_1.type = Mock(type_kind=9) feature_col_0 = Mock() feature_col_0.name = 'feature_col_0' feature_col_0.type = Mock(type_kind=8) model = Mock(project='project_id', dataset_id='dataset_id', model_id='model_id', description='description of model', labels={ 'label_0': 'value_0', 'label_1': 'value_1' }, created=datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None), expires=datetime.datetime(2021, 7, 14, 13, 23, 45, 67, tzinfo=None), location='US', modified=datetime.datetime(2020, 7, 15, 15, 11, 23, 32, tzinfo=None), model_type=0, label_columns=[label_col_0, label_col_1], feature_columns=[feature_col_0], training_runs=[ Mock(start_time=Mock(seconds=1234567)), Mock(start_time=Mock(seconds=1234789)) ]) client.get_model = Mock(return_value=model) bigquery = BigQueryService(client) expected = { 'details': { 'id': 'project_id.dataset_id.model_id', 'name': 'model_id', 'description': 'description of model', 'labels': ['label_0: value_0', 'label_1: value_1'], 'date_created': '2020-07-14T13:23:45.000067', 'last_modified': '2020-07-15T15:11:23.000032', 'expires': '2021-07-14T13:23:45.000067', 'location': 'US', 'model_type': Model.ModelType(0).name, 'schema_labels': [{ 'name': 'schema_label_0', 'type': SqlTypeNames[StandardSqlDataTypes(7).name].name }, { 'name': 'schema_label_1', 'type': SqlTypeNames[StandardSqlDataTypes(9).name].name }], 'feature_columns': [{ 'name': 'feature_col_0', 'type': SqlTypeNames[StandardSqlDataTypes(8).name].name }], 'training_runs': ['1970-01-15T06:56:07+00:00', '1970-01-15T06:59:49+00:00'] } } result = bigquery.get_model_details('some_model_id') self.assertEqual(expected, result)
def testGetViewDetailsFull(self): client = Mock() schema_field_0 = Mock(field_type='STRING', description='this field is a string', mode='NULLABLE') schema_field_0.name = 'field_name_0' schema_field_1 = Mock(field_type='INTEGER', description='this field is an integer', mode='NULLABLE') schema_field_1.name = 'field_name_1' table = Mock( project='project_id', dataset_id='dataset_id', table_id='view_id', description='description of view', labels={ 'label_0': 'value_0', 'label_1': 'value_1' }, created=datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None), expires=datetime.datetime(2021, 7, 14, 13, 23, 45, 67, tzinfo=None), modified=datetime.datetime(2020, 7, 15, 15, 11, 23, 32, tzinfo=None), self_link='https://bigquery.googleapis.com/bigquery/v2/' + \ 'projects/project_id/datasets/dataset_id/view_id', schema=[schema_field_0, schema_field_1], view_query='SELECT * FROM `project_id.dataset_id.table_id LIMIT 200', view_use_legacy_sql=False) client.get_table = Mock(return_value=table) bigquery = BigQueryService(client) expected = { 'details': { 'id': 'project_id.dataset_id.view_id', 'name': 'view_id', 'description': 'description of view', 'labels': ['label_0: value_0', 'label_1: value_1'], 'date_created': '2020-07-14T13:23:45.000067', 'expires': '2021-07-14T13:23:45.000067', 'last_modified': '2020-07-15T15:11:23.000032', 'project': 'project_id', 'dataset': 'dataset_id', 'link': 'https://bigquery.googleapis.com/bigquery' + \ '/v2/projects/project_id/datasets/dataset_id/view_id', 'schema': [{ 'name': 'field_name_0', 'type': 'STRING', 'description': 'this field is a string', 'mode': 'NULLABLE' }, { 'name': 'field_name_1', 'type': 'INTEGER', 'description': 'this field is an integer', 'mode': 'NULLABLE' }], 'query': 'SELECT * FROM `project_id.dataset_id.table_id LIMIT 200', 'legacy_sql': False } } result = bigquery.get_view_details('some_view_id') self.assertEqual(expected, result)
def testGetTableDetailsNestedSchema(self): client = Mock() schema_field_0 = Mock(field_type='STRING', description='this field is a string and a child', mode='NULLABLE') schema_field_0.name = 'child' schema_field_1 = Mock(field_type='RECORD', description='this field is a record', mode='NULLABLE', fields=[schema_field_0]) schema_field_1.name = 'record' table = Mock(project='project_id', dataset_id='dataset_id', table_id='table_id', description='description of table', labels={ 'label_0': 'value_0', 'label_1': 'value_1' }, created=datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None), expires=datetime.datetime(2021, 7, 14, 13, 23, 45, 67, tzinfo=None), location='US', modified=datetime.datetime(2020, 7, 15, 15, 11, 23, 32, tzinfo=None), self_link='https://bigquery.googleapis.com/bigquery' + \ '/v2/projects/project_id/datasets/dataset_id', num_rows=123456, num_bytes=2000000, schema=[schema_field_1]) client.get_table = Mock(return_value=table) bigquery = BigQueryService(client) expected = { 'details': { 'id': 'project_id.dataset_id.table_id', 'name': 'table_id', 'description': 'description of table', 'labels': ['label_0: value_0', 'label_1: value_1'], 'date_created': '2020-07-14T13:23:45.000067', 'expires': '2021-07-14T13:23:45.000067', 'location': 'US', 'last_modified': '2020-07-15T15:11:23.000032', 'project': 'project_id', 'dataset': 'dataset_id', 'link': 'https://bigquery.googleapis.com/bigquery' + \ '/v2/projects/project_id/datasets/dataset_id', 'num_rows': 123456, 'num_bytes': 2000000, 'schema': [{ 'name': 'record', 'type': 'RECORD', 'description': 'this field is a record', 'mode': 'NULLABLE' }, { 'name': 'record.child', 'type': 'STRING', 'description': 'this field is a string and a child', 'mode': 'NULLABLE' }] } } result = bigquery.get_table_details('some_table_id') self.assertEqual(expected, result)
def testGetTableDetailsEmptyFields(self): client = Mock() table = Mock(project='project_id', dataset_id='dataset_id', table_id='table_id', description=None, labels={}, created=datetime.datetime(2020, 7, 14, 13, 23, 45, 67, tzinfo=None), expires=None, location='US', modified=datetime.datetime(2020, 7, 15, 15, 11, 23, 32, tzinfo=None), self_link='https://bigquery.googleapis.com/' + \ 'bigquery/v2/projects/project_id/datasets/dataset_id', num_rows=0, num_bytes=0, schema=[]) client.get_table = Mock(return_value=table) bigquery = BigQueryService(client) expected = { 'details': { 'id': 'project_id.dataset_id.table_id', 'name': 'table_id', 'description': None, 'labels': None, 'date_created': '2020-07-14T13:23:45.000067', 'expires': None, 'location': 'US', 'last_modified': '2020-07-15T15:11:23.000032', 'project': 'project_id', 'dataset': 'dataset_id', 'link': 'https://bigquery.googleapis.com/' + \ 'bigquery/v2/projects/project_id/datasets/dataset_id', 'num_rows': 0, 'num_bytes': 0, 'schema': [] } } result = bigquery.get_table_details('some_table_id') self.assertEqual(expected, result)