def run(endpoint, database, path):
driver_config = ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
ensure_path_exists(driver, database, path)
create_tables(session, database)
describe_table(session, database, "series")
fill_tables_with_data(session, database)
select_simple(session, database)
upsert_simple(session, database)
select_prepared(session, database, 2, 3, 7)
select_prepared(session, database, 2, 3, 8)
explicit_tcl(session, database, 2, 6, 1)
select_prepared(session, database, 2, 6, 1)
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ())
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
with ydb.SessionPool(driver, size=10) as session_pool:
ensure_path_exists(driver, database, path)
full_path = os.path.join(database, path)
create_tables(session_pool, full_path)
describe_table(session_pool, full_path, "series")
fill_tables_with_data(session_pool, full_path)
select_simple(session_pool, full_path)
upsert_simple(session_pool, full_path)
select_prepared(session_pool, full_path, 2, 3, 7)
select_prepared(session_pool, full_path, 2, 3, 8)
explicit_tcl(session_pool, full_path, 2, 6, 1)
select_prepared(session_pool, full_path, 2, 6, 1)
def get_config():
endpoint = os.getenv("endpoint")
database = os.getenv("database")
if endpoint is None or database is None:
raise AssertionError("Нужно указать обе переменные окружения")
credentials = ydb.construct_credentials_from_environ()
return ydb.DriverConfig(endpoint, database, credentials=credentials)
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(endpoint, database, credentials=ydb.construct_credentials_from_environ())
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
with ydb.SessionPool(driver, size=10) as session_pool:
ensure_path_exists(driver, database, path)
create_tables(session_pool, path)
fill_data(session_pool, path)
peter_series = select_by_username(session_pool, path, "Peter Dinklage")
assert len(peter_series) == 2
emilia_series = select_by_username(session_pool, path, "Emilia Clarke")
assert len(emilia_series) == 3
def make_driver_config(endpoint, database, path):
return ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
def create_task(event, context):
print(event)
task_id = str(uuid.uuid4())
session = boto3.Session(
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
region_name="ru-central1",
)
s3 = session.client("s3",
endpoint_url="https://storage.yandexcloud.net",
config=Config(signature_version="s3v4"))
presigned_url = s3.generate_presigned_url(
"put_object",
Params={
"Bucket": os.getenv('UPLOAD_BUCKET_NAME'),
"Key": task_id
},
ExpiresIn=500,
)
# write to DB
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
session.transaction().execute(
f"""
UPSERT INTO tasks (task_id, status) VALUES
("{task_id}", "NEW");
""",
commit_tx=True,
)
return {
'statusCode': 200,
'body': json.dumps({"presigned_url": presigned_url}),
}
def run(endpoint, database, path):
driver_config = ydb.DriverConfig(
endpoint,
database,
credentials=ydb.construct_credentials_from_environ())
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
raise RuntimeError("Connect failed to YDB")
_run(driver, database, path)
def get_task(event, context):
print(event)
task_id = event['params']['task_id']
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
result_sets = session.transaction(ydb.SerializableReadWrite()).execute(
f"""
SELECT
task_id,
status,
result_url
FROM tasks
WHERE task_id = "{task_id}";
""",
commit_tx=True,
)
body = None
if result_sets[0].rows[0].status.decode("utf-8") == "DONE":
body = {
"task_id": result_sets[0].rows[0].task_id.decode("utf-8"),
"status": result_sets[0].rows[0].status.decode("utf-8"),
"result_url": result_sets[0].rows[0].result_url.decode("utf-8")
}
else:
body = {
"task_id": result_sets[0].rows[0].task_id.decode("utf-8"),
"status": result_sets[0].rows[0].status.decode("utf-8")
}
return {
'statusCode': 200,
'body': json.dumps(body),
}
def handler(event, context):
bucket_name = event['messages'][0]['details']['bucket_id']
object_name = event['messages'][0]['details']['object_id']
session = boto3.session.Session()
s3 = session.client(
service_name='s3',
endpoint_url='https://storage.yandexcloud.net',
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'))
presigned_url = s3.generate_presigned_url(
"get_object",
Params={
"Bucket": os.getenv('RESULTS_BUCKET_NAME'),
"Key": object_name
},
ExpiresIn=500,
)
# write to DB
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
session.transaction().execute(
f"""
UPSERT INTO tasks (task_id, status, result_url) VALUES
("{object_name}", "DONE", "{presigned_url}");
""",
commit_tx=True,
)
def list_tasks(event, context):
print(event)
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
result_sets = session.transaction(ydb.SerializableReadWrite()).execute(
f"""
SELECT
task_id,
status
FROM tasks;
""",
commit_tx=True,
)
result = {}
for row in result_sets[0].rows:
task_id = row.task_id.decode("utf-8")
status = row.status.decode("utf-8")
result[task_id] = status
return {
'statusCode': 200,
'body': json.dumps(result),
}
def handler(event, context):
bucket_name = event['messages'][0]['details']['bucket_id']
object_name = event['messages'][0]['details']['object_id']
# Create client
client = boto3.client(
service_name='sqs',
endpoint_url='https://message-queue.api.cloud.yandex.net',
region_name='ru-central1')
# Send message to queue
client.send_message(QueueUrl=QUEUE_URL, MessageBody=object_name)
# write to DB
driver_config = ydb.DriverConfig(
YDB_ENDPOINT,
DATABASE,
credentials=ydb.construct_credentials_from_environ(),
root_certificates=ydb.load_ydb_root_certificate(),
)
with ydb.Driver(driver_config) as driver:
try:
driver.wait(timeout=5)
except TimeoutError:
print("Connect failed to YDB")
print("Last reported errors by discovery:")
print(driver.discovery_debug_details())
exit(1)
session = driver.table_client.session().create()
session.transaction().execute(
f"""
UPSERT INTO tasks (task_id, status) VALUES
("{object_name}", "PROCESSING");
""",
commit_tx=True,
)