Esempio n. 1
0
def create(name):
    """
    Create a feature set
    """
    feast_client = Client(
        core_url=feast_config.get_config_property_or_fail("core_url")
    )  # type: Client

    feast_client.apply(FeatureSet(name=name))
Esempio n. 2
0
 def test_auth_success_with_insecure_channel_on_core_url(
     self, insecure_core_server_with_auth
 ):
     client = Client(
         core_url="localhost:50056",
         core_enable_auth=True,
         core_auth_token=_FAKE_JWT_TOKEN,
     )
     client.list_feature_sets()
Esempio n. 3
0
 def test_secure_channel_creation_with_secure_client(self, _mocked_obj):
     client = Client(core_url="localhost:50053",
                     serving_url="localhost:50054",
                     serving_secure=True,
                     core_secure=True)
     with mock.patch("grpc.secure_channel") as _grpc_mock, \
             mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value="test")) as _mocked_credentials:
         client._connect_core()
         _grpc_mock.assert_called_with(client.core_url,
                                       _mocked_credentials.return_value)
Esempio n. 4
0
File: cli.py Progetto: masonlr/feast
def start_stream_to_online(feature_table: str, jar: str):
    """
    Start stream to online sync job.
    """
    import feast.pyspark.aws.jobs

    client = Client()
    table = client.get_feature_table(feature_table)
    feast.pyspark.aws.jobs.start_stream_to_online(client, table,
                                                  [jar] if jar else [])
Esempio n. 5
0
 def secure_mock_client(self, mocker):
     client = Client(core_url=CORE_URL,
                     serving_url=SERVING_URL,
                     core_secure=True,
                     serving_secure=True)
     mocker.patch.object(client, "_connect_core")
     mocker.patch.object(client, "_connect_serving")
     client._core_url = CORE_URL
     client._serving_url = SERVING_URL
     return client
Esempio n. 6
0
def ingest(name, version, filename, file_type):
    """
    Ingest feature data into a feature set
    """

    feast_client = Client(core_url=feast_config.get_config_property_or_fail(
        "core_url"))  # type: Client

    feature_set = feast_client.get_feature_set(name=name, version=version)
    feature_set.ingest_file(file_path=filename)
Esempio n. 7
0
 def mock_client_with_auth(self):
     client = Client(
         core_url=CORE_URL,
         serving_url=SERVING_URL,
         enable_auth=True,
         auth_token=_FAKE_JWT_TOKEN,
     )
     client._core_url = CORE_URL
     client._serving_url = SERVING_URL
     return client
Esempio n. 8
0
File: cli.py Progetto: beoy/feast
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
    """
    Sync offline store to online.
    """
    import feast.pyspark.aws.jobs

    client = Client()
    table = client.get_feature_table(feature_table)
    feast.pyspark.aws.jobs.sync_offline_to_online(client, table, start_time,
                                                  end_time)
Esempio n. 9
0
 def secure_mock_client(self):
     client = Client(
         core_url=CORE_URL,
         serving_url=SERVING_URL,
         core_enable_ssl=True,
         serving_enable_ssl=True,
     )
     client._core_url = CORE_URL
     client._serving_url = SERVING_URL
     return client
Esempio n. 10
0
def entity_create(filename, project):
    """
    Create or update an entity
    """

    entities = [
        Entity.from_dict(entity_dict) for entity_dict in yaml_loader(filename)
    ]
    feast_client = Client()  # type: Client
    feast_client.apply_entity(entities, project)
Esempio n. 11
0
def feature_table_create(filename):
    """
    Create or update a feature table
    """

    feature_tables = [
        FeatureTable.from_dict(ft_dict) for ft_dict in yaml_loader(filename)
    ]
    feast_client = Client()  # type: Client
    feast_client.apply_feature_table(feature_tables)
Esempio n. 12
0
def feature_set_create(filename):
    """
    Create or update a feature set
    """

    feature_sets = [
        FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)
    ]
    feast_client = Client()  # type: Client
    feast_client.apply(feature_sets)
Esempio n. 13
0
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
    """
    Sync offline store data to online store
    """
    from datetime import datetime

    client = Client()
    table = client.get_feature_table(feature_table)
    client.start_offline_to_online_ingestion(
        table, datetime.fromisoformat(start_time),
        datetime.fromisoformat(end_time))
Esempio n. 14
0
File: cli.py Progetto: tleyden/feast
def feature_table_describe(name: str, project: str):
    """
    Describe a feature table
    """
    feast_client = Client()  # type: Client
    ft = feast_client.get_feature_table(name=name, project=project)

    if not ft:
        print(f'Feature table with name "{name}" could not be found')
        return

    print(yaml.dump(yaml.safe_load(str(ft)), default_flow_style=False, sort_keys=False))
Esempio n. 15
0
def client(pytestconfig):
    core_url = pytestconfig.getoption("core_url")
    serving_url = pytestconfig.getoption("serving_url")

    client = Client(
        core_url=core_url,
        serving_url=serving_url,
    )

    client.set_project(PROJECT_NAME)

    return client
Esempio n. 16
0
def client(core_url, serving_url, allow_dirty):
    # Get client for core and serving
    client = Client(core_url=core_url, serving_url=serving_url)

    # Ensure Feast core is active, but empty
    if not allow_dirty:
        feature_sets = client.list_feature_sets()
        if len(feature_sets) > 0:
            raise Exception(
                "Feast cannot have existing feature sets registered. Exiting tests."
            )

    return client
Esempio n. 17
0
def project_list():
    """
    List all projects
    """
    feast_client = Client()  # type: Client

    table = []
    for project in feast_client.list_projects():
        table.append([project])

    from tabulate import tabulate

    print(tabulate(table, headers=["NAME"], tablefmt="plain"))
Esempio n. 18
0
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
    """
    Sync offline store data to online store
    """
    from datetime import datetime

    client = Client()
    table = client.get_feature_table(feature_table)
    client.start_offline_to_online_ingestion(
        table,
        datetime.strptime(start_time, DATETIME_ISO),
        datetime.strptime(end_time, DATETIME_ISO),
    )
Esempio n. 19
0
def feature_set_describe(name: str, version: int):
    """
    Describe a feature set
    """
    feast_client = Client()  # type: Client
    fs = feast_client.get_feature_set(name=name, version=version)
    if not fs:
        print(
            f'Feature set with name "{name}" and version "{version}" could not be found'
        )
        return

    print(yaml.dump(yaml.safe_load(str(fs)), default_flow_style=False, sort_keys=False))
Esempio n. 20
0
def feature_set_list():
    """
    List all feature sets
    """
    feast_client = Client()  # type: Client

    table = []
    for fs in feast_client.list_feature_sets():
        table.append([fs.name, fs.version])

    from tabulate import tabulate

    print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain"))
Esempio n. 21
0
def feature_set_list():
    """
    List all feature sets
    """
    feast_client = Client()  # type: Client

    table = []
    for fs in feast_client.list_feature_sets(project="*", name="*"):
        table.append([fs.name, repr(fs)])

    from tabulate import tabulate

    print(tabulate(table, headers=["NAME", "REFERENCE"], tablefmt="plain"))
Esempio n. 22
0
def feature_set_create(filename):
    """
    Create or update a feature set
    """

    feature_sets = [
        FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)
    ]

    feast_client = Client(core_url=feast_config.get_config_property_or_fail(
        "core_url"))  # type: Client

    feast_client.apply(feature_sets)
Esempio n. 23
0
def list():
    """
    List all feature sets
    """
    feast_client = Client(core_url=feast_config.get_config_property_or_fail(
        "core_url"))  # type: Client

    table = []
    for fs in feast_client.list_feature_sets():
        table.append([fs.name, fs.version])

    from tabulate import tabulate

    print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain"))
Esempio n. 24
0
def apply(filename):
    """
    Apply a configuration to a resource by filename or stdin
    """

    resources = [
        ResourceFactory.get_resource(res_dict["kind"]).from_dict(res_dict)
        for res_dict in loaders.yaml_loader(filename)
    ]

    feast_client = Client(core_url=feast_config.get_config_property_or_fail(
        "core_url"))  # type: Client

    feast_client.apply(resources)
Esempio n. 25
0
def test_get_list_alltypes(client: Client, alltypes_entity: Entity,
                           alltypes_featuretable: FeatureTable):
    # ApplyEntity
    client.apply_entity(alltypes_entity)

    # GetEntity Check
    assert client.get_entity(name="alltypes_id") == alltypes_entity

    # ListEntities Check
    alltypes_filtering_labels = {"cat": "alltypes"}
    actual_alltypes_entities = client.list_entities(
        labels=alltypes_filtering_labels)
    assert len(actual_alltypes_entities) == 1

    # ApplyFeatureTable
    client.apply_feature_table(alltypes_featuretable)

    # GetFeatureTable Check
    actual_get_feature_table = client.get_feature_table(name="alltypes")
    assert actual_get_feature_table == alltypes_featuretable

    # ListFeatureTables Check
    actual_list_feature_table = [
        ft for ft in client.list_feature_tables() if ft.name == "alltypes"
    ][0]
    assert actual_list_feature_table == alltypes_featuretable
Esempio n. 26
0
def ingest_job_restart(job_id: str):
    """
    Restart job for id.
    Waits for the job to fully restart.
    """
    # find ingestion job for id
    feast_client = Client()
    jobs = feast_client.list_ingest_jobs(job_id=job_id)
    if len(jobs) < 1:
        print(f"Ingestion Job with id {job_id} could not be found")
        sys.exit(1)
    job = jobs[0]

    feast_client.restart_ingest_job(job)
Esempio n. 27
0
def test_ingest_into_bq(
    feast_client: Client,
    customer_entity: Entity,
    driver_entity: Entity,
    bq_dataframe: pd.DataFrame,
    bq_dataset: str,
    pytestconfig,
):
    bq_project = pytestconfig.getoption("bq_project")
    bq_table_id = f"bq_staging_{datetime.now():%Y%m%d%H%M%s}"
    ft = FeatureTable(
        name="basic_featuretable",
        entities=["driver_id", "customer_id"],
        features=[
            Feature(name="dev_feature_float", dtype=ValueType.FLOAT),
            Feature(name="dev_feature_string", dtype=ValueType.STRING),
        ],
        max_age=Duration(seconds=3600),
        batch_source=BigQuerySource(
            table_ref=f"{bq_project}:{bq_dataset}.{bq_table_id}",
            event_timestamp_column="datetime",
            created_timestamp_column="timestamp",
        ),
    )

    # ApplyEntity
    feast_client.apply(customer_entity)
    feast_client.apply(driver_entity)

    # ApplyFeatureTable
    feast_client.apply(ft)
    feast_client.ingest(ft, bq_dataframe, timeout=120)

    bq_client = bigquery.Client(project=bq_project)

    # Poll BQ for table until the table has been created
    def try_get_table():
        try:
            table = bq_client.get_table(
                bigquery.TableReference(
                    bigquery.DatasetReference(bq_project, bq_dataset), bq_table_id
                )
            )
        except NotFound:
            return None, False
        else:
            return table, True

    wait_retry_backoff(
        retry_fn=try_get_table,
        timeout_secs=30,
        timeout_msg="Timed out trying to get bigquery table",
    )

    query_string = f"SELECT * FROM `{bq_project}.{bq_dataset}.{bq_table_id}`"

    job = bq_client.query(query_string)
    query_df = job.to_dataframe()

    assert_frame_equal(query_df, bq_dataframe)
Esempio n. 28
0
def feature_set_list():
    """
    List all projects
    """
    feast_client = Client(core_url=feast_config.get_config_property_or_fail(
        "core_url"))  # type: Client

    table = []
    for project in feast_client.list_projects():
        table.append([project])

    from tabulate import tabulate

    print(tabulate(table, headers=["NAME"], tablefmt="plain"))
Esempio n. 29
0
def entity_describe(name: str, project: str):
    """
    Describe an entity
    """
    feast_client = Client()  # type: Client
    entity = feast_client.get_entity(name=name, project=project)

    if not entity:
        print(f'Entity with name "{name}" could not be found')
        return

    print(
        yaml.dump(yaml.safe_load(str(entity)),
                  default_flow_style=False,
                  sort_keys=False))
Esempio n. 30
0
File: cli.py Progetto: tleyden/feast
def feature_table_list(project: str, labels: str):
    """
    List all feature tables
    """
    feast_client = Client()  # type: Client

    labels_dict = _get_labels_dict(labels)

    table = []
    for ft in feast_client.list_feature_tables(project=project, labels=labels_dict):
        table.append([ft.name, ft.entities])

    from tabulate import tabulate

    print(tabulate(table, headers=["NAME", "ENTITIES"], tablefmt="plain"))