def run(endpoint, database, path): driver_config = ydb.DriverConfig( endpoint, database, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() ensure_path_exists(driver, database, path) create_tables(session, database) describe_table(session, database, "series") fill_tables_with_data(session, database) select_simple(session, database) upsert_simple(session, database) select_prepared(session, database, 2, 3, 7) select_prepared(session, database, 2, 3, 8) explicit_tcl(session, database, 2, 6, 1) select_prepared(session, database, 2, 6, 1)
def run(endpoint, database, path): driver_config = ydb.DriverConfig( endpoint, database, credentials=ydb.construct_credentials_from_environ()) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) with ydb.SessionPool(driver, size=10) as session_pool: ensure_path_exists(driver, database, path) full_path = os.path.join(database, path) create_tables(session_pool, full_path) describe_table(session_pool, full_path, "series") fill_tables_with_data(session_pool, full_path) select_simple(session_pool, full_path) upsert_simple(session_pool, full_path) select_prepared(session_pool, full_path, 2, 3, 7) select_prepared(session_pool, full_path, 2, 3, 8) explicit_tcl(session_pool, full_path, 2, 6, 1) select_prepared(session_pool, full_path, 2, 6, 1)
def run(endpoint, database, path): driver_config = ydb.DriverConfig(endpoint, database, credentials=ydb.construct_credentials_from_environ()) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) with ydb.SessionPool(driver, size=10) as session_pool: ensure_path_exists(driver, database, path) create_tables(session_pool, path) fill_data(session_pool, path) peter_series = select_by_username(session_pool, path, "Peter Dinklage") assert len(peter_series) == 2 emilia_series = select_by_username(session_pool, path, "Emilia Clarke") assert len(emilia_series) == 3
def run(endpoint, database, path, auth_token): driver_config = ydb.DriverConfig(endpoint, database=database, auth_token=auth_token) try: driver = ydb.Driver(driver_config) driver.wait(timeout=5) except TimeoutError: raise RuntimeError("Connect failed to YDB") try: session = driver.table_client.session().create() sample_data.prepare_data(driver, session, database, path) db_path = os.path.join(database, path) page_num = 0 for page in SchoolsPaginated(session, db_path, limit=3).pages(): page_num = page_num + 1 print("-- page: {} --".format(page_num)) for row in page.rows: print(row.city.encode('utf-8'), row.number, row.address.encode('utf-8')) finally: driver.stop()
def run(endpoint, database, path): driver_config = ydb.DriverConfig(endpoint, database=database, credentials=credentials_from_environ()) try: driver = ydb.Driver(driver_config) driver.wait(timeout=5) session_pool = ydb.SessionPool(driver, size=10) except TimeoutError: raise RuntimeError("Connect failed to YDB") try: ensure_path_exists(driver, database, path) create_tables(session_pool, database) describe_table(session_pool, database, "series") fill_tables_with_data(session_pool, database) select_simple(session_pool, database) upsert_simple(session_pool, database) select_prepared(session_pool, database, 2, 3, 7) select_prepared(session_pool, database, 2, 3, 8) explicit_tcl(session_pool, database, 2, 6, 1) select_prepared(session_pool, database, 2, 6, 1) finally: session_pool.stop() driver.stop()
def create_task(event, context): print(event) task_id = str(uuid.uuid4()) session = boto3.Session( aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'), region_name="ru-central1", ) s3 = session.client("s3", endpoint_url="https://storage.yandexcloud.net", config=Config(signature_version="s3v4")) presigned_url = s3.generate_presigned_url( "put_object", Params={ "Bucket": os.getenv('UPLOAD_BUCKET_NAME'), "Key": task_id }, ExpiresIn=500, ) # write to DB driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() session.transaction().execute( f""" UPSERT INTO tasks (task_id, status) VALUES ("{task_id}", "NEW"); """, commit_tx=True, ) return { 'statusCode': 200, 'body': json.dumps({"presigned_url": presigned_url}), }
def run(endpoint, database, path): driver_config = ydb.DriverConfig( endpoint, database, credentials=ydb.construct_credentials_from_environ()) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: raise RuntimeError("Connect failed to YDB") _run(driver, database, path)
def get_task(event, context): print(event) task_id = event['params']['task_id'] driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() result_sets = session.transaction(ydb.SerializableReadWrite()).execute( f""" SELECT task_id, status, result_url FROM tasks WHERE task_id = "{task_id}"; """, commit_tx=True, ) body = None if result_sets[0].rows[0].status.decode("utf-8") == "DONE": body = { "task_id": result_sets[0].rows[0].task_id.decode("utf-8"), "status": result_sets[0].rows[0].status.decode("utf-8"), "result_url": result_sets[0].rows[0].result_url.decode("utf-8") } else: body = { "task_id": result_sets[0].rows[0].task_id.decode("utf-8"), "status": result_sets[0].rows[0].status.decode("utf-8") } return { 'statusCode': 200, 'body': json.dumps(body), }
def _get_driver(self): driver = ydb.Driver( ydb.DriverConfig( self._endpoint, database=self._database, auth_token=self._token, ) ) try: driver.wait(timeout=5) except TimeoutError: raise RuntimeError("YDB connect error") return driver
def execute(config, query, params): with ydb.Driver(config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) return None session = driver.table_client.session().create() prepared_query = session.prepare(query) return session.transaction(ydb.SerializableReadWrite()).execute( prepared_query, params, commit_tx=True)
def run(endpoint, database, path, auth_token): driver_config = ydb.DriverConfig(endpoint, database=database, auth_token=auth_token) try: driver = ydb.Driver(driver_config) driver.wait(timeout=5) except TimeoutError: raise RuntimeError("Connect failed to YDB") try: _run(driver, database, path) finally: driver.stop()
def _create_driver(endpoint, database): driver_id = (endpoint, database, ) driver = _DRIVERS.get(driver_id) if not driver: driver_config = ydb.DriverConfig( endpoint, database=database, credentials=Connection._create_credentials()) try: driver = ydb.Driver(driver_config) driver.wait(timeout=5) except Exception: raise DatabaseError('Failed to connect to YDB') _DRIVERS[driver_id] = driver return driver
def handler(event, context): bucket_name = event['messages'][0]['details']['bucket_id'] object_name = event['messages'][0]['details']['object_id'] session = boto3.session.Session() s3 = session.client( service_name='s3', endpoint_url='https://storage.yandexcloud.net', aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY')) presigned_url = s3.generate_presigned_url( "get_object", Params={ "Bucket": os.getenv('RESULTS_BUCKET_NAME'), "Key": object_name }, ExpiresIn=500, ) # write to DB driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() session.transaction().execute( f""" UPSERT INTO tasks (task_id, status, result_url) VALUES ("{object_name}", "DONE", "{presigned_url}"); """, commit_tx=True, )
def list_tasks(event, context): print(event) driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() result_sets = session.transaction(ydb.SerializableReadWrite()).execute( f""" SELECT task_id, status FROM tasks; """, commit_tx=True, ) result = {} for row in result_sets[0].rows: task_id = row.task_id.decode("utf-8") status = row.status.decode("utf-8") result[task_id] = status return { 'statusCode': 200, 'body': json.dumps(result), }
def session_pool_context(driver_config: ydb.DriverConfig, size=1, workers_threads_count=1): with ydb.Driver(driver_config) as driver: try: logging.info("connecting to the database") driver.wait(timeout=15) except TimeoutError: logging.critical( f"connection failed\n" f"last reported errors by discovery: {driver.discovery_debug_details()}" ) raise with ydb.SessionPool( driver, size=size, workers_threads_count=workers_threads_count) as session_pool: try: yield session_pool except Exception as e: logging.critical( f"failed to create session pool due to {repr(e)}")
def handler(event, context): bucket_name = event['messages'][0]['details']['bucket_id'] object_name = event['messages'][0]['details']['object_id'] # Create client client = boto3.client( service_name='sqs', endpoint_url='https://message-queue.api.cloud.yandex.net', region_name='ru-central1') # Send message to queue client.send_message(QueueUrl=QUEUE_URL, MessageBody=object_name) # write to DB driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() session.transaction().execute( f""" UPSERT INTO tasks (task_id, status) VALUES ("{object_name}", "PROCESSING"); """, commit_tx=True, )