def load_data(dataset_name, budget): """ Only works with categorical/ordinal columns as of now SQL scenario...? """ # Load dataset from service (dataset is pd.DataFrame) dataset_document = DATASET_CLIENT.read(dataset_name, budget) dataset = load_dataset(dataset_document) schema = load_metadata(dataset_document) # NOTE: As of right now, any data clipping per schema is not # required for the supported synthetic data scenarios # TODO: Support categorical, ordinal and continuous specification through schema categorical_columns = [] ordinal_columns = [] # range(0,len(data)) # TODO: Temporary support for dropping id column if 'pid' in dataset.columns: dataset.drop('pid', axis=1, inplace=True) if 'income' in dataset.columns: dataset.drop('income', axis=1, inplace=True) return dataset, dataset_document, { 'categorical_columns': categorical_columns, 'ordinal_columns': ordinal_columns }, schema
def test_read_release_no_penalty(dataset_client, dataset_name): """ READ (RELEASE) TEST Further readrelease calls do not incur budget """ dataset_client.custom_headers = {'client_guid': 'mock_user_guid'} dataset_document = dataset_client.read_released(dataset_name) df = load_dataset(dataset_document) assert isinstance(df, pd.pandas.core.frame.DataFrame)
def test_auth_register_release_csv(dataset): """ Tests registering with authorized guids, check successful reads for them + unsuccessful reads otherwise Check that subsequent reads don't incur budget """ # Generate some valid/invalid clients external_clients = [get_dataset_client() for _ in range(0, 9)] valid_clients = external_clients[5:9] invalid_clients = external_clients[0:5] valid_guids = [c._guid for c in valid_clients] service_client = get_dataset_client() dataset["authorized_users"] = valid_guids # Register dataset takes an optional list of valid ids 5-9 response = service_client.register(dataset) assert response.result == dataset['dataset_name'] # Fake DP Perturb, similar release will be done in module retrieved_dataset = service_client.read(dataset["dataset_name"], 1.0) retrieved_dataset.dataset_name = "release_authorize_csv" release_doc = { "dataset_name": "release_authorize_csv", "dataset_type": retrieved_dataset.dataset_type, retrieved_dataset.dataset_type: retrieved_dataset.csv_details, "release_cost": retrieved_dataset.release_cost, "budget": retrieved_dataset.budget, "authorized_users": retrieved_dataset.authorized_users } release_dataset = service_client.release(release_doc) assert release_dataset.dataset_name == "release_authorize_csv" # Should have same authorized users assert release_dataset.authorized_users == valid_guids # Attempt to read from released dataset with valid clients for c in valid_clients: dataset_document = c.read_released(release_dataset.dataset_name) df = load_dataset(dataset_document) assert isinstance(df, pd.pandas.core.frame.DataFrame) # Attempt to read from released dataset with invalid clients for c in invalid_clients: with pytest.raises(Exception) as error: c.read_released(release_dataset.dataset_name) assert error.typename == "HttpOperationError"
""" Sample Command line: python register_dataset.py private_csv csv_details 10.0 False local_path=serverside/path/to/example.csv """ if __name__ == "__main__": private_dataset_name = sys.argv[1] release_dataset_name = sys.argv[2] budget = sys.argv[3] with mlflow.start_run(): service_client = get_dataset_client() dataset_document = service_client.read(private_dataset_name, budget) input_dataframe = load_dataset(dataset_document) new_dataframe = pd.DataFrame( input_dataframe) # DP stuff here? "make_new_df"? prev_path = dataset_document['csv_details']['path'] new_path = os.path.join(os.path.dirname(prev_path), release_dataset_name + '.csv') prev_schema = dataset_document['csv_details']['schema'] new_schema = os.path.join(os.path.dirname(prev_schema), release_dataset_name + '.yaml') new_dataframe.to_csv(new_path, index=False) with open(prev_schema, 'w') as yaml_path: yaml.dump(prev_schema, yaml_path, default_flow_style=False)
dataset_name = sys.argv[1] # string budget = float(sys.argv[2]) # any number # We expect the next two inputs in the following format: json.dumps([col, names, here]) (e.g. '["a","b"]') x_features = json.loads(sys.argv[3]) # features y_targets = json.loads(sys.argv[4]) # class labels with mlflow.start_run(run_name="diffpriv_logreg"): # log attributes mlflow.log_param("dataset_name", dataset_name) mlflow.log_param("budget", budget) mlflow.log_param("x_features", x_features) mlflow.log_param("y_targets", y_targets) dataset_document = get_dataset_client().read(dataset_name, budget) dataset = load_dataset(dataset_document) schema = load_metadata(dataset_document) # use column names to get X and y from dataset to pass to LogisticRegression X = dataset[x_features] y = np.ravel( dataset[y_targets] ) # use ravel to convert the column vector to a 1d array to avoid issues later using fit # TODO Calculate max norm with schema norms = np.linalg.norm(dataset, axis=1) # norms for each column max_norm = np.amax(norms) logging.warning( 'Currently calculating the data norm instead of using schema-specified value. This is bad practice, and will eventually be changed' ) # provide a warning about bad practice
def test_load_dataverse_dataset_file(dataset_client, dataset_name, budget): dataset_document = dataset_client.read(dataset_name, budget) df = load_dataset(dataset_document) assert isinstance(df, pd.pandas.core.frame.DataFrame)