def update_row(client, table, row, change=False, view=True):
if not change and view:
res = get_row(client, table, row["PrimaryKeys"])
to_delete_keys = []
for col in res["Columns"]:
if col not in row["Columns"]:
to_delete_keys.append(col)
for key in to_delete_keys:
del (res["Columns"][key])
diff.color_diff(res, row)
return False
if not change:
return False
# 如果行不存在，插入行
# 如果行存在，修改或者新增 row 中的行，row 中不存在的列不变
meta = client.describe_table(table)
keys = [v[0] for v in meta.table_meta.schema_of_primary_key]
pks = []
for key in keys:
pks.append((key, row["PrimaryKeys"][key]))
cols = []
for col in row["Columns"]:
cols.append((col, row["Columns"][col]))
_, _ = client.update_row(
table, tablestore.Row(pks, {"PUT": cols}),
tablestore.Condition(tablestore.RowExistenceExpectation.IGNORE))
return True
def _Cell2Row(self, sample, meta):
# build the attribute columns part(metadata)
attribute_columns = []
attribute_columns.append(('organ', str(meta['organ'])))
attribute_columns.append(('region', str(meta['region'])))
attribute_columns.append(('subregion', str(meta['subregion'])))
attribute_columns.append(('seq_tech', str(meta['seq_tech'])))
attribute_columns.append(('sample_status', str(meta['sample_status'])))
attribute_columns.append(('donor_id', str(meta['donor_id'])))
attribute_columns.append(('donor_gender', str(meta['donor_gender'])))
attribute_columns.append(('donor_age', str(meta['donor_age'])))
attribute_columns.append(('cluster_name', str(meta['original_name'])))
attribute_columns.append(('cl_name', str(meta['cl_name'])))
attribute_columns.append(('hcad_name', str(meta['hcad_name'])))
# build the attribute columns part(gene)
for i in range(sample.shape[0]):
attribute_columns.append((sample.index[i], sample[i]))
# build the primary_key part
primary_key = [('study_id', str(meta['study_id'])),
('cell_id', str(meta['cell_id'])),
('user_id', int(meta['user_id']))]
# the maxium number of attribute columns in each writing operation is 1024, so we split a row into blocks
row_blocks = []
for i in range(
len(attribute_columns) // 1024 + 1
): # the maxium number of attribute columns in each writing operation is 1024
if i == 0:
row_blocks.append(
tablestore.Row(
primary_key,
attribute_columns[i *
1024:min(i * 1024 +
1024, len(attribute_columns
))]))
else:
row_blocks.append(
tablestore.Row(
primary_key, {
'PUT':
attribute_columns[i * 1024:min(
i * 1024 + 1024, len(attribute_columns))]
}))
return row_blocks
def Cell2Row(sample):
# build the attribute columns part
attribute_columns = []
for i in range(sample.shape[0]):
if sample.index[i] != "cid":
if sample.index[i] == 'user_id':
attribute_columns.append((sample.index[i], int(sample[i])))
elif isinstance(sample[i], np.int64):
attribute_columns.append((sample.index[i], float(sample[i])))
else:
attribute_columns.append((sample.index[i], sample[i]))
# build the primary_key part
primary_key = [('cid', int(sample['cid']))]
# the maxium number of attribute columns in each writing operation is 1024, so we split a row into blocks
row_blocks = []
for i in range(
len(attribute_columns) // 1024 + 1
): # the maxium number of attribute columns in each writing operation is 1024
if i == 0:
row_blocks.append(
tablestore.Row(
primary_key,
attribute_columns[i *
1024:min(i * 1024 +
1024, len(attribute_columns))]))
else:
row_blocks.append(
tablestore.Row(
primary_key, {
'PUT':
attribute_columns[i *
1024:min(i * 1024 +
1024, len(attribute_columns
))]
}))
return row_blocks
def put_row(client, table, row, change=False):
if not change:
res = get_row(client, table, row["PrimaryKeys"])
diff.color_diff(res, row)
return False
# 如果行不存在，插入新行
# 如果行存在，覆盖当前行，row 中不存在的列会被删除，等效于删除整行，再插入整行
meta = client.describe_table(table)
keys = [v[0] for v in meta.table_meta.schema_of_primary_key]
pks = []
for key in keys:
pks.append((key, row["PrimaryKeys"][key]))
cols = []
for col in row["Columns"]:
cols.append((col, row["Columns"][col]))
_, _ = client.put_row(table, tablestore.Row(pks, cols))
return True
def toTSRow(struct, data):
def getPK(k, v):
if k in data:
if v in (0, 3):
return int(data[k])
return data[k]
else:
if v == PK_INC:
return tablestore.PK_AUTO_INCR
else:
if k in struct['default']:
return struct['default'][k]
else:
raise ('PK %s except' % k)
pks = {k for k, v in struct['primary_key']}
pk = [(k, getPK(k, v)) for k, v in struct['primary_key']]
ac = [(k, v) for k, v in data.items() if not k in pks]
return tablestore.Row(pk, ac)
def insert(self, index, column=None, table_name=None, condition=None, return_type=None):
"""
:param list or tuple index: Primary key. Eg: [('ActID', 'abc')]
:param list or tuple column: Attribute column. Eg: [('StartTime', 0)]
:param str table_name: Table name
:param OTSRowCondition condition: Condition of existence of row
:param OTSReturnType return_type: API return content
:return:
"""
if table_name is None:
table_name = self.table_name
if condition is None:
condition = default.ROW_CONDITION
elif not isinstance(condition, OTSRowCondition):
raise InvalidParam(doc=OTSRowCondition)
if return_type is not None and not isinstance(return_type, OTSReturnType):
raise InvalidParam(doc=OTSReturnType)
row = ts.Row(index, column)
condition = ts.Condition(condition)
_, return_row = self.client.put_row(table_name, row, condition, return_type)
return StoreData(getattr(return_row, 'primary_key', None))
def format_update_row(index, column):
"""
:param list or tuple index:
:param list or tuple column:
:return:
"""
update_column = {
'PUT': list(),
'DELETE': list(),
'DELETE_ALL': list(),
}
if column is None:
column = tuple()
for item in column:
if item[1] is not None:
update_column['PUT'].append(tuple(item))
continue
if len(item) > 2:
update_column['DELETE'].append(tuple(item))
continue
update_column['DELETE_ALL'].append(tuple(item))
return ts.Row(index, update_column)
def update_row(self, primary_key, update_data):
# primary_key 是主键，如[('study_id','10.1038/s41467-019-10756-2'), ('cell_id','human_control-AAACCTGAGCTGAAAT'),('user_id',3)]
# updtae_data 是待更新的列的list，[(col1,val1),(col2,val2)]
try:
consumed, return_row, next_token = self._Ali_client.get_row(
self._tablename, primary_key, columns_to_get=['donor_gender'])
if return_row == None:
print("Error! This row doesn't exist in the table.")
else:
# convert update data to blockes
row = []
for i in range(
len(update_data) // 1024 + 1
): # the maxium number of attribute columns in each writing operation is 1024
row.append(
tablestore.Row(
primary_key, {
'PUT':
update_data[i *
1024:min(i * 1024 +
1024, len(update_data))]
}))
# try to update data
condition = tablestore.Condition(
tablestore.RowExistenceExpectation.IGNORE)
for i in range(len(row)):
try:
consumed, return_row = self._Ali_client.update_row(
self._tablename, row[i], condition)
except tablestore.OTSClientError as e:
print(e.get_error_message())
except tablestore.OTSServiceError as e:
print(e.get_error_message())
except tablestore.OTSClientError as e:
print(e.get_error_message())
except tablestore.OTSServiceError as e:
print(e.get_error_message())
def delete(self, data):
row = toTSRow(self.__model, data)
newrow = tablestore.Row(row.primary_key)
return self.__client.delete_row(self.__model['table_name'], newrow,
tablestore.Condition('IGNORE'))