def search(
self,
query: Search,
all_results: bool = False,
records: List[Record] = None,
previous_after: int = 0) -> List[Record]:
if records is None:
records = []
response = self.client.search(query)
after = response.next_token
records.extend(response.records)
if after:
if all_results is False:
user_input = input(('There were more records than configured batch count, current record count is'
' {record_count}.\n\n'
'To retrieve another batch before storing type: "next"\n'
'To retrieve all batches before storing (this can cause very long load times) '
'type: "all"\n'
'To store all retrieved batches type: "store" (or any other response)\n'
'Current after index is {next_token}\n').format(next_token=after,
record_count=len(records)))
if user_input.lower() == "next":
print("get next")
query.next_token = after
self.search(query, records=records, previous_after=after, all_results=False)
elif user_input.lower() == "all":
query.next_token = after
self.search(query, records=records, previous_after=after, all_results=True)
else:
query.next_token = after
self.search(query, records=records, previous_after=after, all_results=True)
print("get all")
return records
def search_all_records():
if os.path.exists(credentials_path):
print("found", credentials_path)
client = e3db.Client(json.load(open(credentials_path)))
with open(output_path, 'w') as output_file:
print_and_write(output_file, "Opening file to write to: {0}".format(output_path))
print_and_write(output_file, "Running query for all records")
# Keep the end time the same.
# If you expect data coming in older than last year, increasing the days value in timedelta will query from an earlier time.
# Narrowing the start time will speed up the search if you expect the range is too large.
end_time = datetime.now() + timedelta(days=5)
start_time = end_time - timedelta(days=365)
# E3db Search by default does not provide data.
# More examples and information can be found here https://github.com/tozny/e3db-python#searching-records.
# If you want to include_data add in the flag include_data=True as shown below:
# query = Search(include_data=True, include_all_writers=True, next_token=0, count=1000)
query = Search(include_all_writers=True, next_token=0, count=1000)
query_narrow_time_range_and_write_to_file(output_file, client, start_time, end_time, query)
### Demonstration of storing records in memory instead of writing to file
# all_results = []
# query_narrow_time_range(client, start_time, end_time, query, all_results)
# for ar in all_results:
# write_full_records(output_file, ar)
else:
print("could not find credentials file at path: {0}".format(credentials_path))
def run():
# DO NOT SHARE THIS WORKSPACE WITH OTHERS IF YOUR CONFIG IS PRESENT
# DO NOT ADD YOUR CONFIG INTO SOURCE CONTROL
support = ClientSupport.from_config(
"/home/ec2-user/environment/tozstore-cloud9-integration/ta2resources/config.json")
# Construct search query here (The sample query retrieves all records your client can read with the replaced record type)
# For full examples on the search query see https://github.com/tozny/e3db-python
q = Search(count=50, include_data=True, include_all_writers=True).match(
record_type=["<Replace with a record type>"])
# The simplest approach is to use support.search_and_store(q)
support.search_and_store(q)
def main():
try:
client = e3db.Client(e3db.Config.load(client_name))
except:
register_and_write_credentials()
client = e3db.Client(e3db.Config.load(client_name))
write_data(client)
# Get all written records
results = client.search(Search())
print_results("Getting all records", results)
# Construct Wildcard Search for all Flora
wild_search = Search().match(strategy="WILDCARD", keys=["flora*"])
results = client.search(wild_search)
print_results("Getting all flora records", results)
# Construct Regex Search for all Fauna
regex_search = Search().match(strategy="REGEXP", keys=[".*fauna.*"])
results = client.search(regex_search)
print_results("Getting all fauna records", results)
# Construct Regex Search for all `test` tagged meta
regex_test_search = Search().match(strategy="REGEXP", keys=[".*test.*"])
results = client.search(regex_test_search)
print_results("Getting all test records", results)
# Exclude all `test` tagged meta from results
exclude_test_search = Search().exclude(strategy="WILDCARD",
keys=["*test*"])
results = client.search(exclude_test_search)
print_results("Excluding all test records", results)
# Fuzzy Queries look for close words
fuzzy_search = Search().match(strategy="FUZZY", record_types=["fauna"])
results = client.search(fuzzy_search)
print_results(
"Getting all records with record type close to 'fauna', includes record_type 'fluna'",
results)
# Get Flora meta records, but exclude test records with multiple strategies
multiple_strategy_search = Search().match(strategy="WILDCARD", plain={"*flora*":"*"})\
.exclude(strategy="REGEXP", keys=[".*test.*"])
results = client.search(multiple_strategy_search)
print_results(
"Gets records with 'flora' in the plain key with wildcards, but excludes all records with 'test' in the plain key",
results)
def insert_record(record: Record, query: Search, conn: connect, s3_bucket: str,
s3_location: str):
# this needs to use e3db fetched records
with conn:
with conn.cursor() as curs:
curs.execute(
("INSERT INTO fetched_records"
" (query_string, record_meta, file_data, record_id, record_id_updated_at, file_name,"
" s3_bucket, s3_location)"
"VALUES "
" (%(query_string)s, %(record_meta)s, %(file_data)s, %(record_id)s,"
" %(record_id_updated_at)s, %(file_name)s, %(s3_bucket)s, %(s3_location)s)"
), {
'query_string': dumps(query.to_json()),
'record_meta': dumps(record.meta.plain),
'file_data': dumps(record.meta.file_meta.to_json()),
'record_id': record.meta.record_id,
'record_id_updated_at': record.meta.last_modified,
'file_name': record.meta.file_meta._file_name,
's3_bucket': s3_bucket,
's3_location': s3_location
})
def delete_data(client):
results = client.search(Search())
for r in results:
client.delete(r.meta.record_id, r.meta.version)
if __name__ == '__main__':
# print(sys.argv)
roundNum = sys.argv[1].split('=')[1]
path = sys.argv[2].split('=')[1]
alicia_path = './alicia_creds.json'
bruce_path = './bruce_creds.json'
client = e3db.Client(json.load(open(path)))
alicia_client = e3db.Client(json.load(open(alicia_path)))
bruce_client = e3db.Client(json.load(open(bruce_path)))
# Query for the data from the players inputs for the given round
query = Search(include_data=True,
include_all_writers=True).match(condition="AND",
record_types=['move'],
keys=['round'],
values=[roundNum])
results = client.search(query)
player_move_dict = {}
for r in results:
move = r.to_json()['data']['moveType']
name = r.to_json()['data']['name']
player_move_dict[name] = move
# Fill in player move dict with the players moves for easier reference for the logic
bruce_move = player_move_dict['bruce']
alicia_move = player_move_dict['alicia']
def main():
try:
client = e3db.Client(e3db.Config.load(client_name))
except:
register_and_write_credentials()
client = e3db.Client(e3db.Config.load(client_name))
# comment me if using a custom client or data is already written
write_data(client)
# Get all written records
# By default Search(), the empty search, will match anything
results = client.search(Search())
print_results("Getting all records", results)
# The Search Method has two conditional options for constructing your queries: AND|OR
# OR is used by default and does not have to be explicitly stated as below.
server_1_or_server_2 = Search().match(condition="OR",
strategy="WILDCARD",
plain={
"*server_1*": "*",
"*server_2*": "*"
})
results = client.search(server_1_or_server_2)
print_results(
"getting records with server_1 or server_2 in meta with the plain field",
results)
server_1_and_server_2 = Search().match(condition="AND",
strategy="WILDCARD",
plain={
"*server_1*": "*",
"*server_2*": "*"
})
results = client.search(server_1_and_server_2)
print_results(
"getting records with server_1 and server_2 in meta with the plain field (returns nothing)",
results)
# without using the plain dictionary
server_1_or_server_2 = Search().match(condition="OR",
strategy="WILDCARD",
keys=["*server_1*", "*server_2*"])
results = client.search(server_1_or_server_2)
print_results(
"getting records with server_1 or server_2 using they keys field",
results)
# search fields can be intermingled
# AND is the logical operator and returns records that match all conditions within the match parameters.
record_and_plain_search = Search().match(condition="AND",
strategy="WILDCARD",
record_types=["flora"],
plain={"*test*": "*dand*"})
results = client.search(record_and_plain_search)
# The results we see here is all records of record_type `flora`, and key `*test*` and value `*dand*`
# We get the single record with meta "{'flora_test_12345':'dandelion'}"
print_results(
"get records by record type 'flora' AND plain containing '*test*':'*dand*' ",
results)
# Chained match clauses with match or exclude clauses with exclude are joined by the logical OR
chain_match = Search().match(condition="AND",
record_types=["flora"
]).match(condition="AND",
record_types=["fauna"])
# the above search is equivalent to below.
equivalent_to_chain_match = Search().match(condition="OR",
record_types=["flora", "fauna"])
# Chaining excludes example: It inheritely matches everything by default if no `match` parameter is provided,
# and then excludes fields in exclude.
# results = MATCH _ AND (EXCLUDE `flora` OR EXCLUDE `fauna`)
chain_exclude = Search().exclude(condition="AND",
record_types=["flora"]).exclude(
condition="AND",
record_types=["fauna"])
# the above search is equivalent to below.
equivalent_to_chain_exclude = Search().exclude(
condition="OR", record_types=["flora", "fauna"])
# Chaining match and exclude together will remove the exclude fields.
# This means records that equal the match clause AND do not equal the exclude clause are returned.
match_and_exclude = Search().match(record_types=["flora"]).exclude(
strategy="WILDCARD", keys=["*test*"])
results = client.search(match_and_exclude)
print_results(
"get records of record type 'flora' and do not have keys `*test*`",
results)
# Nested chaining allows you to specify varying strategies for different terms within a clause.
differing_strategies = Search().match(strategy="EXACT", record_types=["flora"])\
.match(strategy="WILDCARD", keys=["*12345"])\
.exclude(strategy="REGEXP", keys=[".*test.*"])\
.exclude(strategy="REGEXP", keys=[".*server_2.*"])
# results = (MATCH `flora` OR MATCH `*12345`) AND (EXCLUDE `.*test.*` OR EXCLUDE `.*server_2.*`)
results = client.search(differing_strategies)
print_results(
"Different matching strategies: this search will return an EXACT match to record_type `flora` OR WILDCARD match to keys `*12345`, and a REGEXP exclude to keys `.*test.*` OR REGEXP exclude to keys `.*server_2.*`",
results)
# Keep in mind that this chaining means your previous search object gets altered each time.
original_search = Search().exclude(record_types=["fauna"])
modified_search = original_search.exclude(record_types=["flora"])
results = client.search(modified_search)
print_results(
"modified_search and original_search will exclude both flora and fauna",
results)
import e3db
from e3db.types import Search
# For this example, instantiate a third party client
third_party_config = e3db.Config(
'a08ddc06-fake-fake-fake-e0589610a05b',
'fake6e9717e7ad5387c629553bf10db3055ed793376def7112ffa1624ec2fake',
'fake1467d1d79d078696d69e7d8082337d60a4aee13b55499f8fd9b9412efake',
'fakeLeDulPGJSddxY1GkvvZthoTZqsGmvS1AFszfake',
'fakemq3k9vfEaKPd1sRQubfZ2umWbWF0iB2buUSfake',
api_url = "https://api.e3db.com"
)
third_party_client = e3db.Client(third_party_config())
# Obtain records written by client from the third_party client
record_type = 'contact'
search_filter = Search(include_all_writers=True,include_data=True) \
.match(condition="AND", record_types=['contact'], writers=['tozny-client-id-that-shared-with-you'])
results = third_party_client.search(search_filter)
for record in results:
print(record.to_json())
import e3db
from e3db.types import Search
import os
import json
credentials_path = "credentials.json" # your e3db credentialss
if os.path.exists(credentials_path):
print("found", credentials_path)
client = e3db.Client(json.load(open(credentials_path)))
query = Search(include_all_writers=True, next_token=0, count=1000)
results = client.search(query)
print(
results.total_results
) # Total results available for searching within TozStore. If this exceeds 10k narrow your search, as we do not return results past than 10k.
if results.total_results > 0:
# Print the the first set of search results
for r in results:
print(r.to_json())
while results.next_token != 0: # If there are more results to page through grab the next page
results = client.search(query)
query.next_token = results.next_token
for r in results:
print(r.to_json())
else:
print("No results found in search")