def run(endpoint, database, path): driver_config = ydb.DriverConfig( endpoint, database, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() ensure_path_exists(driver, database, path) create_tables(session, database) describe_table(session, database, "series") fill_tables_with_data(session, database) select_simple(session, database) upsert_simple(session, database) select_prepared(session, database, 2, 3, 7) select_prepared(session, database, 2, 3, 8) explicit_tcl(session, database, 2, 6, 1) select_prepared(session, database, 2, 6, 1)
def make_driver_config(endpoint, database, path): return ydb.DriverConfig( endpoint, database, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), )
def create_task(event, context): print(event) task_id = str(uuid.uuid4()) session = boto3.Session( aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'), region_name="ru-central1", ) s3 = session.client("s3", endpoint_url="https://storage.yandexcloud.net", config=Config(signature_version="s3v4")) presigned_url = s3.generate_presigned_url( "put_object", Params={ "Bucket": os.getenv('UPLOAD_BUCKET_NAME'), "Key": task_id }, ExpiresIn=500, ) # write to DB driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() session.transaction().execute( f""" UPSERT INTO tasks (task_id, status) VALUES ("{task_id}", "NEW"); """, commit_tx=True, ) return { 'statusCode': 200, 'body': json.dumps({"presigned_url": presigned_url}), }
def get_task(event, context): print(event) task_id = event['params']['task_id'] driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() result_sets = session.transaction(ydb.SerializableReadWrite()).execute( f""" SELECT task_id, status, result_url FROM tasks WHERE task_id = "{task_id}"; """, commit_tx=True, ) body = None if result_sets[0].rows[0].status.decode("utf-8") == "DONE": body = { "task_id": result_sets[0].rows[0].task_id.decode("utf-8"), "status": result_sets[0].rows[0].status.decode("utf-8"), "result_url": result_sets[0].rows[0].result_url.decode("utf-8") } else: body = { "task_id": result_sets[0].rows[0].task_id.decode("utf-8"), "status": result_sets[0].rows[0].status.decode("utf-8") } return { 'statusCode': 200, 'body': json.dumps(body), }
def handler(event, context): bucket_name = event['messages'][0]['details']['bucket_id'] object_name = event['messages'][0]['details']['object_id'] session = boto3.session.Session() s3 = session.client( service_name='s3', endpoint_url='https://storage.yandexcloud.net', aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY')) presigned_url = s3.generate_presigned_url( "get_object", Params={ "Bucket": os.getenv('RESULTS_BUCKET_NAME'), "Key": object_name }, ExpiresIn=500, ) # write to DB driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() session.transaction().execute( f""" UPSERT INTO tasks (task_id, status, result_url) VALUES ("{object_name}", "DONE", "{presigned_url}"); """, commit_tx=True, )
def list_tasks(event, context): print(event) driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() result_sets = session.transaction(ydb.SerializableReadWrite()).execute( f""" SELECT task_id, status FROM tasks; """, commit_tx=True, ) result = {} for row in result_sets[0].rows: task_id = row.task_id.decode("utf-8") status = row.status.decode("utf-8") result[task_id] = status return { 'statusCode': 200, 'body': json.dumps(result), }
def handler(event, context): bucket_name = event['messages'][0]['details']['bucket_id'] object_name = event['messages'][0]['details']['object_id'] # Create client client = boto3.client( service_name='sqs', endpoint_url='https://message-queue.api.cloud.yandex.net', region_name='ru-central1') # Send message to queue client.send_message(QueueUrl=QUEUE_URL, MessageBody=object_name) # write to DB driver_config = ydb.DriverConfig( YDB_ENDPOINT, DATABASE, credentials=ydb.construct_credentials_from_environ(), root_certificates=ydb.load_ydb_root_certificate(), ) with ydb.Driver(driver_config) as driver: try: driver.wait(timeout=5) except TimeoutError: print("Connect failed to YDB") print("Last reported errors by discovery:") print(driver.discovery_debug_details()) exit(1) session = driver.table_client.session().create() session.transaction().execute( f""" UPSERT INTO tasks (task_id, status) VALUES ("{object_name}", "PROCESSING"); """, commit_tx=True, )