def index(event, context):
evt = json.loads(event)['events'][0]
auth = oss2.StsAuth(context.credentials.access_key_id,
context.credentials.access_key_secret,
context.credentials.security_token)
key = evt['oss']['object']['key']
bucket = oss2.Bucket(auth, OSS_NET, 'sfive')
ots = tablestore.OTSClient(
OTS_NET,
context.credentials.access_key_id,
context.credentials.access_key_secret,
'YOUR-BUCKET',
sts_token=context.credentials.security_token,
)
def __init__(self, access_key_id, access_key_secret, end_point, instance_name, encoding=default.CODING,
retry_policy=OTSRetryPolicy(), socket_timeout=default.OTS_TIMEOUT, table_name=None, **kwargs):
"""
:param str access_key_id: Access Key ID
:param str access_key_secret: Access Key Secret
:param str end_point: Table store server domain started with https://
:param str instance_name: Instance name of table store
:param str or None encoding: Encoding
:param OTSRetryPolicy or None retry_policy: Retry policy
:param int or float or None socket_timeout: Socket timeout
:param str table_name: Default table name
"""
self.table_name = table_name
self.client = ts.OTSClient(
end_point,
access_key_id,
access_key_secret,
instance_name,
encoding=encoding,
retry_policy=retry_policy,
socket_timeout=socket_timeout,
**kwargs,
)
def Setup_Client(self, endpoint, access_key_id, access_key_secret,
instance_name, table_name):
self._tablename = table_name
self._Ali_client = tablestore.OTSClient(
endpoint,
access_key_id,
access_key_secret,
instance_name,
logger_name="table_store.log",
retry_policy=tablestore.WriteRetryPolicy())
self._client_setup = True
try:
describe_response = self._Ali_client.describe_table(
self._tablename)
print("Connected to the server, find the table.")
print(table_name)
print('TableName: %s' % describe_response.table_meta.table_name)
print('PrimaryKey: %s' %
describe_response.table_meta.schema_of_primary_key)
print('Reserved read throughput: %s' % describe_response.
reserved_throughput_details.capacity_unit.read)
print('Reserved write throughput: %s' % describe_response.
reserved_throughput_details.capacity_unit.write)
print('Last increase throughput time: %s' % describe_response.
reserved_throughput_details.last_increase_time)
print('Last decrease throughput time: %s' % describe_response.
reserved_throughput_details.last_decrease_time)
print('table options\'s time to live: %s' %
describe_response.table_options.time_to_live)
print('table options\'s max version: %s' %
describe_response.table_options.max_version)
print('table options\'s max_time_deviation: %s' %
describe_response.table_options.max_time_deviation)
except Exception as e:
print(e)
import warnings
warnings.filterwarnings('ignore')
sys.path.append(sys.argv[1])
# Please enter the information of the server here.
endpoint = "https://HCAd-Datasets3.cn-beijing.ots.aliyuncs.com"
access_key_id = sys.argv[3]
access_key_secret = sys.argv[4]
instance_name = "HCAd-Datasets3"
table_name = "HCA_d"
# Connect to the server
ots_client = tablestore.OTSClient(endpoint,
access_key_id,
access_key_secret,
instance_name,
logger_name="table_store.log",
retry_policy=tablestore.WriteRetryPolicy())
threads_num = int(sys.argv[5])
if threads_num < 0:
threads_num = multiprocessing.cpu_count() - 1
threads_num = min(multiprocessing.cpu_count() - 1, threads_num)
def Cell2Row(sample):
# build the attribute columns part
attribute_columns = []
for i in range(sample.shape[0]):
if sample.index[i] != "cid":
if sample.index[i] == 'user_id':
attribute_columns.append((sample.index[i], int(sample[i])))
def main():
parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.
RawTextHelpFormatter(prog, width=200),
description="""example:
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -a CreateTable -t yaconfig --meta '[
["Block", "STRING"],
["Section", "STRING"]
]'
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -a DescribeTable -t yaconfig
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -a ListTable
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t ImmTasks -a GetTask --owner 1023210024677934 --project imm-test-hl-doc-proj-shanghai --task formatconvert-00402572-d8e9-4750-ab88-ac3f388f63d1
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a GetRange > IMMConfig.json
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a GetRow --primary-keys '{"Block": "WebOfficeBilling", "Section": "SLS"}'
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a PutRow --rows '{"PrimaryKeys": {"Block": "WebOfficeBilling", "Section": "SLS"}, "Columns": {"Enable": true}}'
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a PutRow --primary-keys '{"Block": "WebOfficeBilling", "Section": "SLS"}' --column '{"Enable": false}'
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a UpdateRow --primary-keys '{"Block": "WebOfficeBilling", "Section": "SLS"}' --column '{"Enable": false}'
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a GetCanaryDeploy --primary-keys '{"Block": "Base", "Section": "Common"}' --column CanaryDeploy
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-regressi -t IMMConfig -a PutCanaryDeploy \\
--primary-keys '{"Block": "Base", "Section": "Common"}' --column CanaryDeploy --deploy '{
"ObjectType": "user",
"ObjectId": "default",
"Provider": "WPS",
"Function": "CONVERT",
"LatestVersion": {
"SyncAppName": "IMM_DEV_APP_CONVERT_WPS_default_20200520_regression",
"SyncClusterName": "IMM_DEV_CLUSTER_CONVERT_WPS_default_20200520_regression_sync",
"AsyncAppName": "IMM_DEV_APP_CONVERT_WPS_default_20200923_regression",
"AsyncClusterName": "IMM_DEV_CLUSTER_CONVERT_WPS_default_20200923_regression"
},
"CanaryDeployVersion": {
"SyncAppName": "IMM_DEV_APP_CONVERT_WPS_default_20200520_regression",
"SyncClusterName": "IMM_DEV_CLUSTER_CONVERT_WPS_default_20200520_regression_sync",
"AsyncAppName": "IMM_DEV_APP_CONVERT_WPS_default_20200923_regression",
"AsyncClusterName": "IMM_DEV_CLUSTER_CONVERT_WPS_default_20200923_regression"
},
"Percent": 100
}'
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a DelCanaryDeploy \\
--primary-keys '{"Block": "Base", "Section": "Common"}' --column CanaryDeploy --deploy '{
"ObjectType": "user",
"ObjectId": "default",
"Provider": "WPS",
"Function": "CONVERT"
}'
cat IMMConfig.json | python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d imm-dev-hl -t IMMConfig -a PutRow
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d configcenter -t config_imm_dev_hl -a GetConfigCenter
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d configcenter -t config_imm_dev_hl -a PutConfigCenter --config "$(cat 1.json)"
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d configcenter -t config_imm_dev_hl -a PutConfigCenter --config "$(cat 1.json)" --cmkey "xxx" --encrypt
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d configcenter -t config_imm_dev_hl -a GetConfigCenterByKey --key WebOffice_OSS_AK
python3 ots.py -c ~/.aksk/imm-dev -r cn-shanghai -d configcenter -t config_imm_dev_hl -a PutConfigCenterByKey --key WebOffice_OSS_AK --value xxx --cmkey "xxx" --encrypt
""")
parser.add_argument("-i", "--access-key-id", help="access key id")
parser.add_argument("-s", "--access-key-secret", help="access key secret")
parser.add_argument("-c", "--credential", help="credential file")
parser.add_argument("-e", "--endpoint", help="endpoint")
parser.add_argument("-r", "--region-id", help="region id")
parser.add_argument("-d", "--instance", help="instance")
parser.add_argument("-t", "--table", help="table")
parser.add_argument("--rows", help="row json format")
parser.add_argument("--primary-keys", help="primary keys json format")
parser.add_argument("--columns", help="columns json format")
parser.add_argument("--meta", help="table meta")
parser.add_argument("--change",
nargs="?",
const=True,
default=False,
type=str2bool,
help="change data")
parser.add_argument("--column", help="column name")
parser.add_argument("--deploy", help="canary deploy json format")
parser.add_argument("--owner", type=str, help="owner id")
parser.add_argument("--project", help="project")
parser.add_argument("--task", help="task id")
parser.add_argument("--key", help="configcenter key")
parser.add_argument("--value", help="configcenter value")
parser.add_argument("--cmkey", help="configcenter cmkey")
parser.add_argument("--config", help="configcenter config")
parser.add_argument("--encrypt",
nargs="?",
const=True,
default=False,
type=str2bool,
help="configcenter config")
parser.add_argument("-a",
"--action",
help="action",
choices=[
"ListTable", "GetRange", "PutRow", "UpdateRow",
"GetRow", "CreateTable", "DescribeTable",
"GetCanaryDeploy", "PutCanaryDeploy",
"DelCanaryDeploy", "GetTask", "GetConfigCenter",
"PutConfigCenter", "GetConfigCenterByKey",
"PutConfigCenterByKey"
])
args = parser.parse_args()
if args.credential:
args.access_key_id, args.access_key_secret = aksk.load_from_file(
args.credential)
if not args.endpoint:
args.endpoint = "https://{}.{}.ots.aliyuncs.com".format(
args.instance, args.region_id)
client = tablestore.OTSClient(args.endpoint, args.access_key_id,
args.access_key_secret, args.instance)
kms_cli = AcsClient(args.access_key_id, args.access_key_secret,
args.region_id)
if args.action == "ListTable":
print(json.dumps(list_table(client)))
elif args.action == "GetRange":
for row in get_range(client, args.table):
print(json.dumps(row))
elif args.action == "GetRow":
row = get_row(client, args.table, json.loads(args.primary_keys))
if row is not None:
print(json.dumps(row))
elif args.action == "UpdateRow":
if args.rows:
if args.rows[0] == "@":
rows = json.load((open(args.rows[1:])))
else:
rows = json.loads(args.rows)
if isinstance(rows, list):
for row in rows:
print(
json.dumps(
update_row(client, args.table, row, args.change)))
else:
print(
json.dumps(
update_row(client, args.table, rows, args.change)))
elif args.primary_keys and args.columns:
print(
json.dumps(
update_row(
client, args.table, {
"PrimaryKeys": json.loads(args.primary_keys),
"Columns": json.loads(args.columns)
}, args.change)))
else:
for row in sys.stdin:
print(
json.dumps(update_row(client, args.table,
json.loads(row))))
elif args.action == "PutRow":
if args.rows:
if args.rows[0] == "@":
rows = json.load((open(args.rows[1:])))
else:
rows = json.loads(args.rows)
if isinstance(rows, list):
for row in rows:
print(
json.dumps(
put_row(client, args.table, row, args.change)))
else:
print(
json.dumps(put_row(client, args.table, rows, args.change)))
elif args.primary_keys and args.columns:
print(
json.dumps(
put_row(
client, args.table, {
"PrimaryKeys": json.loads(args.primary_keys),
"Columns": json.loads(args.columns)
}, args.change)))
else:
for row in sys.stdin:
print(json.dumps(put_row(client, args.table, json.loads(row))))
elif args.action == "GetCanaryDeploy":
print(
json.dumps(
get_canary_deploy(client, args.table,
json.loads(args.primary_keys), args.column)))
elif args.action == "PutCanaryDeploy":
print(
json.dumps(
put_canary_deploy(client, args.table,
json.loads(args.primary_keys), args.column,
json.loads(args.deploy), args.change)))
elif args.action == "DelCanaryDeploy":
print(
json.dumps(
del_canary_deploy(client, args.table,
json.loads(args.primary_keys), args.column,
json.loads(args.deploy), args.change)))
elif args.action == "GetTask":
print(
json.dumps(
get_task(client, args.table, args.owner, args.project,
args.task)))
elif args.action == "CreateTable":
print(
json.dumps(create_table(client, args.table,
json.loads(args.meta))))
elif args.action == "DescribeTable":
print(json.dumps(describe_table(client, args.table)))
elif args.action == "GetConfigCenter":
print(json.dumps(get_configcenter(client, kms_cli, args.table)))
elif args.action == "PutConfigCenter":
print(
json.dumps(
put_configcenter(client, args.table, json.loads(args.config),
args.change, kms_cli, args.cmkey,
args.encrypt)))
elif args.action == "GetConfigCenterByKey":
print(
json.dumps(
get_configcenter_key(client, kms_cli, args.table, args.key)))
elif args.action == "PutConfigCenterByKey":
print(
json.dumps(
put_configcenter_key(client, args.table, args.key, args.value,
args.change, kms_cli, args.cmkey,
args.encrypt)))
else:
parser.print_help()