def run(endpoint, database, path):
driver_config = ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
ensure_path_exists(driver, database, path)
create_tables(session, database)
describe_table(session, database, "series")
fill_tables_with_data(session, database)
select_simple(session, database)
upsert_simple(session, database)
select_prepared(session, database, 2, 3, 7)
select_prepared(session, database, 2, 3, 8)
explicit_tcl(session, database, 2, 6, 1)
select_prepared(session, database, 2, 6, 1)
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ())
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
with ydb.SessionPool(driver, size=10) as session_pool:
ensure_path_exists(driver, database, path)
full_path = os.path.join(database, path)
create_tables(session_pool, full_path)
describe_table(session_pool, full_path, "series")
fill_tables_with_data(session_pool, full_path)
select_simple(session_pool, full_path)
upsert_simple(session_pool, full_path)
select_prepared(session_pool, full_path, 2, 3, 7)
select_prepared(session_pool, full_path, 2, 3, 8)
explicit_tcl(session_pool, full_path, 2, 6, 1)
select_prepared(session_pool, full_path, 2, 6, 1)
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(endpoint, database, credentials=ydb.construct_credentials_from_environ())
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
with ydb.SessionPool(driver, size=10) as session_pool:
ensure_path_exists(driver, database, path)
create_tables(session_pool, path)
fill_data(session_pool, path)
peter_series = select_by_username(session_pool, path, "Peter Dinklage")
assert len(peter_series) == 2
emilia_series = select_by_username(session_pool, path, "Emilia Clarke")
assert len(emilia_series) == 3
def run(endpoint, database, path, auth_token):
driver_config = ydb.DriverConfig(endpoint,
database=database,
auth_token=auth_token)
try:
driver = ydb.Driver(driver_config)
driver.wait(timeout=5)
except TimeoutError:
raise RuntimeError("Connect failed to YDB")
try:
session = driver.table_client.session().create()
sample_data.prepare_data(driver, session, database, path)
db_path = os.path.join(database, path)
page_num = 0
for page in SchoolsPaginated(session, db_path, limit=3).pages():
page_num = page_num + 1
print("-- page: {} --".format(page_num))
for row in page.rows:
print(row.city.encode('utf-8'), row.number,
row.address.encode('utf-8'))
finally:
driver.stop()
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(endpoint,
database=database,
credentials=credentials_from_environ())
try:
driver = ydb.Driver(driver_config)
driver.wait(timeout=5)
session_pool = ydb.SessionPool(driver, size=10)
except TimeoutError:
raise RuntimeError("Connect failed to YDB")
try:
ensure_path_exists(driver, database, path)
create_tables(session_pool, database)
describe_table(session_pool, database, "series")
fill_tables_with_data(session_pool, database)
select_simple(session_pool, database)
upsert_simple(session_pool, database)
select_prepared(session_pool, database, 2, 3, 7)
select_prepared(session_pool, database, 2, 3, 8)
explicit_tcl(session_pool, database, 2, 6, 1)
select_prepared(session_pool, database, 2, 6, 1)
finally:
session_pool.stop()
driver.stop()
def create_task(event, context):
print(event)
task_id = str(uuid.uuid4())
session = boto3.Session(
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
region_name="ru-central1",
)
s3 = session.client("s3",
endpoint_url="https://storage.yandexcloud.net",
config=Config(signature_version="s3v4"))
presigned_url = s3.generate_presigned_url(
"put_object",
Params={
"Bucket": os.getenv('UPLOAD_BUCKET_NAME'),
"Key": task_id
},
ExpiresIn=500,
)
# write to DB
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
session.transaction().execute(
f"""
UPSERT INTO tasks (task_id, status) VALUES
("{task_id}", "NEW");
""",
commit_tx=True,
)
return {
'statusCode': 200,
'body': json.dumps({"presigned_url": presigned_url}),
}
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ())
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
raise RuntimeError("Connect failed to YDB")
_run(driver, database, path)
def get_task(event, context):
print(event)
task_id = event['params']['task_id']
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
result_sets = session.transaction(ydb.SerializableReadWrite()).execute(
f"""
SELECT
task_id,
status,
result_url
FROM tasks
WHERE task_id = "{task_id}";
""",
commit_tx=True,
)
body = None
if result_sets[0].rows[0].status.decode("utf-8") == "DONE":
body = {
"task_id": result_sets[0].rows[0].task_id.decode("utf-8"),
"status": result_sets[0].rows[0].status.decode("utf-8"),
"result_url": result_sets[0].rows[0].result_url.decode("utf-8")
}
else:
body = {
"task_id": result_sets[0].rows[0].task_id.decode("utf-8"),
"status": result_sets[0].rows[0].status.decode("utf-8")
}
return {
'statusCode': 200,
'body': json.dumps(body),
}
def _get_driver(self):
driver = ydb.Driver(
ydb.DriverConfig(
self._endpoint,
database=self._database,
auth_token=self._token,
)
)
try:
driver.wait(timeout=5)
except TimeoutError:
raise RuntimeError("YDB connect error")
return driver
def execute(config, query, params):
with ydb.Driver(config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
return None
session = driver.table_client.session().create()
prepared_query = session.prepare(query)
return session.transaction(ydb.SerializableReadWrite()).execute(
prepared_query, params, commit_tx=True)
def run(endpoint, database, path, auth_token):
driver_config = ydb.DriverConfig(endpoint,
database=database,
auth_token=auth_token)
try:
driver = ydb.Driver(driver_config)
driver.wait(timeout=5)
except TimeoutError:
raise RuntimeError("Connect failed to YDB")
try:
_run(driver, database, path)
finally:
driver.stop()
def _create_driver(endpoint, database):
driver_id = (endpoint, database, )
driver = _DRIVERS.get(driver_id)
if not driver:
driver_config = ydb.DriverConfig(
endpoint, database=database, credentials=Connection._create_credentials())
try:
driver = ydb.Driver(driver_config)
driver.wait(timeout=5)
except Exception:
raise DatabaseError('Failed to connect to YDB')
_DRIVERS[driver_id] = driver
return driver
def handler(event, context):
bucket_name = event['messages'][0]['details']['bucket_id']
object_name = event['messages'][0]['details']['object_id']
session = boto3.session.Session()
s3 = session.client(
service_name='s3',
endpoint_url='https://storage.yandexcloud.net',
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'))
presigned_url = s3.generate_presigned_url(
"get_object",
Params={
"Bucket": os.getenv('RESULTS_BUCKET_NAME'),
"Key": object_name
},
ExpiresIn=500,
)
# write to DB
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
session.transaction().execute(
f"""
UPSERT INTO tasks (task_id, status, result_url) VALUES
("{object_name}", "DONE", "{presigned_url}");
""",
commit_tx=True,
)
def list_tasks(event, context):
print(event)
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
result_sets = session.transaction(ydb.SerializableReadWrite()).execute(
f"""
SELECT
task_id,
status
FROM tasks;
""",
commit_tx=True,
)
result = {}
for row in result_sets[0].rows:
task_id = row.task_id.decode("utf-8")
status = row.status.decode("utf-8")
result[task_id] = status
return {
'statusCode': 200,
'body': json.dumps(result),
}
def session_pool_context(driver_config: ydb.DriverConfig,
size=1,
workers_threads_count=1):
with ydb.Driver(driver_config) as driver:
try:
logging.info("connecting to the database")
driver.wait(timeout=15)
except TimeoutError:
logging.critical(
f"connection failed\n"
f"last reported errors by discovery: {driver.discovery_debug_details()}"
)
raise
with ydb.SessionPool(
driver, size=size,
workers_threads_count=workers_threads_count) as session_pool:
try:
yield session_pool
except Exception as e:
logging.critical(
f"failed to create session pool due to {repr(e)}")
def handler(event, context):
bucket_name = event['messages'][0]['details']['bucket_id']
object_name = event['messages'][0]['details']['object_id']
# Create client
client = boto3.client(
service_name='sqs',
endpoint_url='https://message-queue.api.cloud.yandex.net',
region_name='ru-central1')
# Send message to queue
client.send_message(QueueUrl=QUEUE_URL, MessageBody=object_name)
# write to DB
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
session.transaction().execute(
f"""
UPSERT INTO tasks (task_id, status) VALUES
("{object_name}", "PROCESSING");
""",
commit_tx=True,
)
