def test_partition_sensor(self): sys.argv = ['qds.py', 'partitionsensor', 'check', '-d', '{"schema" : "default", "table" : "nation_s3_rcfile_p", "columns" : [{"column" : "p", "values" : [1, 2]}]}'] print_command() Connection._api_call = Mock(return_value={'status': True}) qds.main() Connection._api_call.assert_called_with( "POST", "sensors/partition_sensor", {"schema" : "default", "table" : "nation_s3_rcfile_p", "columns" : [{"column" : "p", "values" : [1, 2]}]})
def test_list_actions_seq_id(self): sys.argv = ['qds.py', 'scheduler', 'list-actions', '123', '--sequence_id', '123'] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = list_actions_side_effect qds.main() Connection._api_call.assert_has_calls([call("GET", "scheduler/123",params=None), call("GET", "scheduler/123/actions/123", params={})])
def test_list(self): sys.argv = ['qds.py', 'group', 'list'] print_command() Connection._api_call = Mock(return_value={'groups': []}) qds.main() Connection._api_call.assert_called_with( "GET", "groups", params=None)
def test_list_users(self): sys.argv = ['qds.py', 'group', 'list-users', '123'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "GET", "groups/123/qbol_users", params=None)
def test_remove_roles(self): sys.argv = ['qds.py', 'group', 'remove-roles', '123', '456,789'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "PUT", "groups/123", {'removed_roles': '456,789'})
def test_remove_users(self): sys.argv = ['qds.py', 'group', 'remove-users', '123', '456,789'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( 'PUT', 'groups/123', {'removed_members': '456,789'})
def test_minimal(self): sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'update', '123'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('PUT', 'clusters/123', {})
def test_rerun(self): sys.argv = ['qds.py', 'action', 'rerun', '123'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with("POST", "actions/123/rerun", None)
def test_unassign_role(self): sys.argv = ['qds.py', 'role', 'unassign_role', '123', "--group_id", "456"] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_has_calls(call("PUT", "roles/123/groups/456/unassign", params=None))
def test_duplicate(self): sys.argv = ['qds.py', 'role', 'duplicate', '123'] print_command() Connection._api_call = Mock(return_value={'roles':[]}) qds.main() Connection._api_call.assert_has_calls(call("POST", "roles/123/duplicate", params=None))
def test_all(self): sys.argv = ['qds.py', 'account', 'create', '--name', 'new_account', '--location', 's3://bucket/path', '--storage-access-key', 'dummy', '--storage-secret-key', 'dummy', '--compute-access-key', 'dummy', '--compute-secret-key', 'dummy', '--previous-account-plan', 'true', '--aws-region', 'us-east-1'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with("POST", "account", {'account': { 'name': 'new_account', 'acc_key': 'dummy', 'level': 'free', 'compute_type': 'CUSTOMER_MANAGED', 'aws_region': 'us-east-1', 'storage_type': 'CUSTOMER_MANAGED', 'CacheQuotaSizeInGB': '25', 'secret': 'dummy', 'use_previous_account_plan': 'true', 'compute_secret_key': 'dummy', 'compute_access_key': 'dummy', 'defloc': 's3://bucket/path'}})
def test_list_groups(self): sys.argv = ['qds.py', 'role', 'list_groups', '123'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_has_calls(call("GET", "roles/123/groups", params=None))
def test_minimal(self): sys.argv = ['qds.py', 'report', 'canonical_hive_commands'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with("GET", "reports/canonical_hive_commands", params={})
def test_results(self): sys.argv = ['qds.py', 'action', 'results', '123'] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = common_side_effect qds.main() Connection._api_call.assert_has_calls([call("GET", "actions/123", params=None),call("GET", "commands/123/results", params={'inline': True})])
def test_duplicate(self): sys.argv = ['qds.py', 'group', 'duplicate', '123'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "POST", "groups/123/duplicate", {})
def test_update(self): sys.argv = ['qds.py', 'role', 'update', '123', '--name', 'test'] print_command() Connection._api_call = Mock(return_value={'roles':[]}) qds.main() Connection._api_call.assert_has_calls(call("PUT", "roles/123", params={'name':'test'}))
def test_limit(self): sys.argv = ['qds.py', 'report', 'all_commands', '--limit', '20'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "GET", "reports/all_commands", params={'limit': 20})
def test_by_user(self): sys.argv = ['qds.py', 'report', 'all_commands', '--by-user'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "GET", "reports/all_commands", params={'by_user': True})
def test_list_pages(self): sys.argv = ['qds.py', 'action', 'list', '--per-page', '2'] print_command() Connection._api_call = Mock(return_value={"actions": []}) qds.main() Connection._api_call.assert_called_with("GET", "actions", params={'per_page': '2'})
def test_list_actions_pages_fields(self): sys.argv = ['qds.py', 'scheduler', 'list-actions', '123', '--per-page', '2', '--fields', 'id', 'sequence_id'] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = list_actions_side_effect qds.main() Connection._api_call.assert_has_calls([call("GET","scheduler/123",params=None), call("GET", "scheduler/123/actions", params={'per_page':'2'})])
def test_sparkcmd(self): sys.argv = ['qds.py', 'sparkcmd', 'cancel', '123'] print_command() Connection._api_call = Mock(return_value={'kill_succeeded': True}) qds.main() Connection._api_call.assert_called_with("PUT", "commands/123", {'status': 'kill'})
def test_update_two_fields(self): sys.argv = ['qds.py', 'nezha', 'cubes', 'update', '123', '--cost', '100', '--query', 'select now()'] print_command() Connection._api_call = Mock(return_value={'nezha_cubes': []}) qds.main() Connection._api_call.assert_called_with( "PUT", "nezha_cubes/123", {'cost': '100', 'query': 'select now()'})
def test_minimal(self): sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('GET', 'clusters', params=None)
def test_file_sensor(self): sys.argv = ['qds.py', 'filesensor', 'check', '-d', '{"files":["s3://dev.canopydata.com/airflow"]}'] print_command() Connection._api_call = Mock(return_value={'status': True}) qds.main() Connection._api_call.assert_called_with( "POST", "sensors/file_sensor", {'files':['s3://dev.canopydata.com/airflow']})
def test_update_schema_name(self): sys.argv = ['qds.py', 'nezha', 'cubes', 'update', '123', '--schema_name', 'something'] print_command() Connection._api_call = Mock(return_value={'nezha_cubes': []}) qds.main() Connection._api_call.assert_called_with( "PUT", "nezha_cubes/123", {'schema_name': 'something'})
def test_delete_group(self): sys.argv = ['qds.py', 'group', 'delete', '123'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "DELETE", "groups/123", None)
def test_state_invalid(self): sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--state', 'invalid'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value=[{"cluster" : {"state" : "invalid"}}]) qds.main() Connection._api_call.assert_called_with('GET', 'clusters', params=None)
def test_sort_cpu(self): sys.argv = ['qds.py', 'report', 'all_commands', '--sort', 'cpu'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "GET", "reports/all_commands", params={'sort_column': 'cpu'})
def test_view(self): sys.argv = ['qds.py', 'action', 'view', '123'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with("GET", "actions/123", params=None)
def test_update_two_fields(self): sys.argv = ['qds.py', 'nezha', 'data_sources', 'update', '123', '--url', 'http://test', '--name', 'test'] print_command() Connection._api_call = Mock(return_value={'nezha_data_sources': []}) qds.main() Connection._api_call.assert_called_with( "PUT", "nezha_data_sources/123", {'name': 'test', 'url': 'http://test'})
def test_list_actions_pages(self): sys.argv = [ 'qds.py', 'scheduler', 'list-actions', '123', '--per-page', '2' ] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = list_actions_side_effect qds.main() Connection._api_call.assert_has_calls([ call("GET", "scheduler/123", params=None), call("GET", "scheduler/123/actions", params={'per_page': '2'}) ])
def test_submit_with_name(self): sys.argv = ['qds.py', 'dbtapquerycmd', 'submit', '--query', 'show tables', '--db_tap_id', 1, '--name', 'test_name'] print_command() Connection._api_call = Mock(return_value={'id': 1}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'db_tap_id': 1, 'query': 'show tables', 'tags': None, 'name': 'test_name', 'macros': None, 'command_type': 'DbTapQueryCommand', 'can_notify': False})
def test_submit_tags(self): sys.argv = ['qds.py', 'hadoopcmd', 'submit', '--name', 'test_name', '--tags', 'abc,def', 'jar', 's3://bucket/path-to-jar'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'sub_command': 'jar', 'sub_command_args': "'s3://bucket/path-to-jar'", 'name': 'test_name', 'tags': ['abc', 'def'], 'label': None, 'command_type': 'HadoopCommand', 'can_notify': False})
def test_submit_s3distcp(self): sys.argv = ['qds.py', 'hadoopcmd', 'submit', 's3distcp', '--src', 'source', '--dest', 'destincation'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'sub_command': 's3distcp', 'sub_command_args': "'--src' 'source' '--dest' 'destincation'", 'name': None, 'label': None, 'tags': None, 'command_type': 'HadoopCommand', 'can_notify': False})
def test_update_group(self): sys.argv = ['qds.py', 'group', 'update', '123', '--name', 'sdk-test', '--members', '7,8,9', '--roles', '10,11', '--remove-members', '5,6', '--remove-roles', '12,13'] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with("PUT", "groups/123", {'members': '7,8,9', 'removed_roles': '12,13', 'removed_members': '5,6', 'name': 'sdk-test', 'roles': '10,11'})
def test_update_two_fields(self): sys.argv = [ 'qds.py', 'nezha', 'update', 'partitions', '123', '--cost', '100', '--query', 'select now()' ] print_command() Connection._api_call = Mock(return_value={'nezha_partitions': []}) qds.main() Connection._api_call.assert_called_with("PUT", "nezha_partitions/123", { 'cost': '100', 'query': 'select now()' })
def test_update_two_fields(self): sys.argv = [ 'qds.py', 'nezha', 'update', 'data_sources', '123', '--url', 'http://test', '--name', 'test' ] print_command() Connection._api_call = Mock(return_value={'nezha_data_sources': []}) qds.main() Connection._api_call.assert_called_with("PUT", "nezha_data_sources/123", { 'name': 'test', 'url': 'http://test' })
def test_od_od_od(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-ondemand-percentage', '100'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}}, 'label': ['test_label']}})
def test_list_actions_seq_id(self): sys.argv = [ 'qds.py', 'scheduler', 'list-actions', '123', '--sequence_id', '123' ] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = list_actions_side_effect qds.main() Connection._api_call.assert_has_calls([ call("GET", "scheduler/123", params=None), call("GET", "scheduler/123/actions/123", params={}) ])
def test_state_down(self): sys.argv = [ 'qds.py', '--version', 'v2', 'cluster', 'list', '--state', 'down' ] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value=[{ "cluster": { "state": "down" } }]) qds.main() Connection._api_call.assert_called_with('GET', 'clusters', params=None)
def test_invite_without_group(self): sys.argv = [ 'qds.py', 'user', 'invite', '--email', '*****@*****.**', '--account-id', '34' ] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with("POST", "users/invite_new", { 'invitee_email': '*****@*****.**', 'account': '34', 'groups': None })
def test_no_aws_region(self): sys.argv = ['qds.py', 'account', 'create', '--name', 'new_account', '--location', 's3://bucket/path', '--storage-access-key', 'dummy', '--storage-secret-key', 'dummy', '--compute-access-key', 'dummy', '--compute-secret-key', 'dummy', '--previous-account-plan', 'true'] print_command() Connection._api_call = Mock(return_value={}) with self.assertRaises(SystemExit): qds.main()
def test_submit_template_with_inline_json(self): sys.argv = [ 'qds.py', 'template', 'submit', '--id', '14', '--j', '{"input_vars" : [{"table" : "accounts"}]}' ] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = submit_actions_side_effect qds.main() data = {'input_vars': [{'table': "'accounts'"}]} Connection._api_call.assert_called_with("POST", "command_templates/14/run", data)
def test_clone_template(self): file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'input_clone_template.json') sys.argv = [ 'qds.py', 'template', 'clone', '--id', '12', '--data', file_path ] print_command() Connection._api_call = Mock() qds.main() with open(file_path) as f: data = json.load(f) Connection._api_call.assert_called_with( "POST", "command_templates/12/duplicate", data)
def test_submit_with_tags(self): sys.argv = ['qds.py', 'dbtapquerycmd', 'submit', '--query', "select * from table_1 limit \$limit\$", '--db_tap_id', 1, '--tags', 'tag1,tag2'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'macros': None, 'db_tap_id': 1, 'query': "select * from table_1 limit \$limit\$", 'tags': ["tag1", "tag2"], 'name': None, 'command_type': 'DbTapQueryCommand', 'can_notify': False})
def test_submit_query(self): sys.argv = ['qds.py', 'prestocmd', 'submit', '--query', 'show tables'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with( 'POST', 'commands', { 'macros': None, 'label': None, 'query': 'show tables', 'command_type': 'PrestoCommand', 'can_notify': False, 'script_location': None })
def test_invite_with_multiple_groups(self): sys.argv = [ 'qds.py', 'user', 'invite', '--email', '*****@*****.**', '--account-id', '34', '--groups', "system-admin,system-user" ] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "POST", "users/invite_new", { 'invitee_email': '*****@*****.**', 'account': '34', 'groups': 'system-admin,system-user' })
def test_submit_script_location(self): sys.argv = ['qds.py', 'prestocmd', 'submit', '--script_location', 's3://bucket/path-to-script'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'macros': None, 'label': None, 'tags': None, 'name': None, 'query': None, 'command_type': 'PrestoCommand', 'can_notify': False, 'script_location': 's3://bucket/path-to-script'})
def test_minimal(self): sys.argv = [ 'qds.py', '--version', 'v2', 'cluster', 'clone', '1234', '--label', 'test_label1', 'test_label2' ] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( 'POST', 'clusters/1234/clone', {'cluster_info': { 'label': ['test_label1', 'test_label2'] }})
def test_view_with_fields(self): sys.argv = [ 'qds.py', 'nezha', 'view', 'cubes', '123', '--fields', 'id', 'table_name' ] print_command() Connection._api_call = Mock(return_value={ 'name': 'dummycubes', 'id': '1', 'table_name': '2' }) qds.main() Connection._api_call.assert_called_with("GET", "nezha_cubes/123", params=None)
def test_od_od_spot(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-spot-percentage', '100', '--autoscaling-maximum-bid-price-percentage', '50', '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { 'nodes': [{'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}})
def test_image_version_v22(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--flavour', 'hive', '--slave-instance-type', 'c1.xlarge', '--min-nodes', '3', '--image-version', '1.latest'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('POST', 'clusters', {'engine_config': {'flavour': 'hive'}, 'cluster_info': {'label': ['test_label'], 'min_nodes': 3, 'slave_instance_type': 'c1.xlarge', 'cluster_image_version': '1.latest', 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', 'maximum_bid_price_percentage': 100}]}}, 'label': ['test_label']}})
def test_submit_template(self): file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'input_run_template.json') sys.argv = [ 'qds.py', 'template', 'submit', '--id', '14', '--j', file_path ] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = submit_actions_side_effect qds.main() with open(file_path) as f: data = json.load(f) Connection._api_call.assert_called_with("POST", "command_templates/14/run", data)
def test_results(self): sys.argv = ['qds.py', 'action', 'results', '123'] print_command() Connection._api_call = Mock() Connection._api_call.side_effect = common_side_effect qds.main() Connection._api_call.assert_has_calls([ call("GET", "actions/123", params=None), call("GET", "commands/123/results", params={ 'inline': True, 'include_headers': 'false' }) ])
def test_logo(self): sys.argv = [ 'qds.py', 'account', 'branding', '--account-id', '4', '--logo-uri', 'https://www.xyz.com/image.jpg' ] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "PUT", "accounts/branding", { 'logo': { 'logo_uri': 'https://www.xyz.com/image.jpg' }, 'account_id': '4' })
def test_link(self): sys.argv = [ 'qds.py', 'account', 'branding', '--account-id', '4', '--link-url', 'https://www.xyz.com', '--link-label', 'Documentation' ] print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( "PUT", "accounts/branding", { 'link': { 'link_url': 'https://www.xyz.com', 'link_label': 'Documentation' }, 'account_id': '4' })
def test_submit_sample_size(self): sys.argv = ['qds.py', 'hivecmd', 'submit', '--query', 'show tables', '--sample_size', '1024'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'macros': None, 'label': None, 'tags': None, 'sample_size': '1024', 'name': None, 'query': 'show tables', 'command_type': 'HiveCommand', 'can_notify': False, 'script_location': None})
def test_submit_tags(self): sys.argv = ['qds.py', 'hivecmd', 'submit', '--script_location', 's3://bucket/path-to-script', '--tags', 'abc,def'] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with('POST', 'commands', {'macros': None, 'label': None, 'tags': ["abc", "def"], 'sample_size': None, 'name': None, 'query': None, 'command_type': 'HiveCommand', 'can_notify': False, 'script_location': 's3://bucket/path-to-script'})
def test_submit_jar_cluster_label(self): sys.argv = [ 'qds.py', 'hadoopcmd', 'submit', '--cluster-label', 'test_label', 'jar', 's3://bucket/path-to-jar' ] print_command() Connection._api_call = Mock(return_value={'id': 1234}) qds.main() Connection._api_call.assert_called_with( 'POST', 'commands', { 'sub_command': 'jar', 'sub_command_args': "'s3://bucket/path-to-jar'", 'label': 'test_label', 'command_type': 'HadoopCommand', 'can_notify': False })
def test_heterogeneous_config_v2(self): sys.argv = [ 'qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', '--heterogeneous-config', 'test' ] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with( 'POST', 'clusters', { 'cluster_info': { 'label': ['test_label'], 'heterogeneous_config': 'test' } })