async def create_all(self, module):
""" A coroutine that create all the models of a module
Args:
module: a module that defines several models
Return:
A list of created Model name
For example:
from tests import models
await db.create_all(models)
"""
self._checker()
succeed = []
for key, value in vars(module).items():
if hasattr(value, '__base__') and value.__base__ is self.Model:
if not await value.exist():
await value.create()
Logger.info(
"created Model '{table_name}' in db: '{db}'".format(
table_name=key, db=self.db_info.info.db.db
)
)
succeed.append(key)
else:
Logger.error("table '{}' already exists".format(key))
return succeed
async def drop_all(self, module):
""" A coroutine that drop all the models of a module
Args:
module: a module that defines several models
Return:
A list of droped Model name
For example:
from tests import models
await db.drop_all(models)
"""
self._checker()
succeed = []
for key, value in vars(module).items():
if hasattr(value, '__base__') and value.__base__ is self.Model:
if await value.exist():
await value.drop()
Logger.info(
"dropped Model '{model_name}' from db: '{db}'".format(
model_name=key, db=self.db_info.info.db.db
)
)
succeed.append(key)
else:
Logger.error(
message="drop table '{}' does not exist".format(key)
)
return succeed
async def _batch_add(cls, instances):
""" batch add instance """
if not isinstance(instances, (list, tuple)):
raise ValueError(f'Add illegal type {instances}')
cols = list(cls.__table__.field_dict.keys())
cols_copy = cols.copy()
for c in cols_copy:
c_type = cls.__table__.field_dict[c]
if hasattr(c_type, 'auto') and c_type.auto:
cols.remove(c)
values = []
for inst in instances:
values.append(cls._get_add_values(inst, cols))
insert_sql = cls._get_insert_sql(cols)
result = await RequestClient().execute(
insert_sql, values=values, is_batch=True
)
if result.affected != len(values):
Logger.error(
f'Failed to insert, affected rows: {result.affected}'
)
return result
def qiniu_upload_file(responce):
file_name = get_hash(responce.content)
store = TempFiles(file_name)
store.save(save_pic, responce)
try:
result_url = save_qiniu(file_name, os.path.dirname(store.filename))
except Exception as e:
Logger.error(error=e, task='qiniu_upload_file')
result_url = str(responce.url)
store.remove()
return result_url
def save_to_qiniu_by_url(url):
if not is_url(url):
return ''
new_url = transform_to_http(url)
try:
response = requests.get(new_url)
except ConnectionError as e:
Logger.error(error=e, task='save_to_qiniu_by_url')
if response.status_code != 200:
Logger.error('response status_code: {}'.format(response.status_code),
task='save_to_qiniu_by_url')
return str(url)
return qiniu_upload_file(response)
async def _do_add(cls, instance):
""" do add a instance data """
cols = list(instance.__dict__.keys())
values = cls._get_add_values(instance, cols)
insert_sql = cls._get_insert_sql(cols)
result = await RequestClient().execute(insert_sql, values=values)
if result.affected != 1:
Logger.error(
f'Failed to insert, affected rows: {result.affected}'
)
return None
if cls.__meta__.auto_pk is True:
instance._set_value(cls.__table__.pk, result.last_id, is_loader=True)
else:
result.last_id = instance.__dict__[cls.__table__.pk]
return result.last_id
async def batch_drop(self, *models):
""" A coroutine that batch drop same model.
Args:
models: one or more models
Return:
A list of droped table name
For example:
from tests.models import User, Order
await db.batch_drop(User, Order)
"""
self._checker()
succeed = []
if not models:
Logger.warning("parameter 'models' is empty, 'batch_drop' nothing to do")
return succeed
for model in models:
if not issubclass(model, self.Model):
raise ValueError(
'drop model type must be {}, get {}'.format(self.Model, model)
)
if await model.exist():
await model.drop()
Logger.info(
"dropped table '{table_name}' from db: '{db}'".format(
table_name=model.__meta__.table,
db=self.db_info.info.db.db
)
)
succeed.append(model.__meta__.table)
else:
Logger.error(
message="drop table '{}' does not exist".format(model.__meta__.table)
)
return succeed
def qiniu_fetch_file(purl):
max_retry = 5
if not is_url(purl):
Logger.warning(task='qiniu_fetch', message='input url:%s' % purl)
return ''
purl = transform_to_http(purl)
q_auth = qiniu.Auth(ACCESS_KEY, SECRET_KEY)
bucket_path = qiniu.BucketManager(q_auth)
for n in range(max_retry):
ret = bucket_path.fetch(purl, BUCKET_NAME)
if ret is None:
continue
elif isinstance(ret, tuple) and ret[0] is None:
continue
else:
key = ret[0]['key']
url = DOMAIN + str(key)
obj = urlparse.urlparse(url)
return obj.geturl()
else:
Logger.error(task='qiniu_fetch', message='max retry exceed')
return purl