async def create_all(self, module): """ A coroutine that create all the models of a module Args: module: a module that defines several models Return: A list of created Model name For example: from tests import models await db.create_all(models) """ self._checker() succeed = [] for key, value in vars(module).items(): if hasattr(value, '__base__') and value.__base__ is self.Model: if not await value.exist(): await value.create() Logger.info( "created Model '{table_name}' in db: '{db}'".format( table_name=key, db=self.db_info.info.db.db ) ) succeed.append(key) else: Logger.error("table '{}' already exists".format(key)) return succeed
async def drop_all(self, module): """ A coroutine that drop all the models of a module Args: module: a module that defines several models Return: A list of droped Model name For example: from tests import models await db.drop_all(models) """ self._checker() succeed = [] for key, value in vars(module).items(): if hasattr(value, '__base__') and value.__base__ is self.Model: if await value.exist(): await value.drop() Logger.info( "dropped Model '{model_name}' from db: '{db}'".format( model_name=key, db=self.db_info.info.db.db ) ) succeed.append(key) else: Logger.error( message="drop table '{}' does not exist".format(key) ) return succeed
async def _batch_add(cls, instances): """ batch add instance """ if not isinstance(instances, (list, tuple)): raise ValueError(f'Add illegal type {instances}') cols = list(cls.__table__.field_dict.keys()) cols_copy = cols.copy() for c in cols_copy: c_type = cls.__table__.field_dict[c] if hasattr(c_type, 'auto') and c_type.auto: cols.remove(c) values = [] for inst in instances: values.append(cls._get_add_values(inst, cols)) insert_sql = cls._get_insert_sql(cols) result = await RequestClient().execute( insert_sql, values=values, is_batch=True ) if result.affected != len(values): Logger.error( f'Failed to insert, affected rows: {result.affected}' ) return result
def qiniu_upload_file(responce): file_name = get_hash(responce.content) store = TempFiles(file_name) store.save(save_pic, responce) try: result_url = save_qiniu(file_name, os.path.dirname(store.filename)) except Exception as e: Logger.error(error=e, task='qiniu_upload_file') result_url = str(responce.url) store.remove() return result_url
def save_to_qiniu_by_url(url): if not is_url(url): return '' new_url = transform_to_http(url) try: response = requests.get(new_url) except ConnectionError as e: Logger.error(error=e, task='save_to_qiniu_by_url') if response.status_code != 200: Logger.error('response status_code: {}'.format(response.status_code), task='save_to_qiniu_by_url') return str(url) return qiniu_upload_file(response)
async def _do_add(cls, instance): """ do add a instance data """ cols = list(instance.__dict__.keys()) values = cls._get_add_values(instance, cols) insert_sql = cls._get_insert_sql(cols) result = await RequestClient().execute(insert_sql, values=values) if result.affected != 1: Logger.error( f'Failed to insert, affected rows: {result.affected}' ) return None if cls.__meta__.auto_pk is True: instance._set_value(cls.__table__.pk, result.last_id, is_loader=True) else: result.last_id = instance.__dict__[cls.__table__.pk] return result.last_id
async def batch_drop(self, *models): """ A coroutine that batch drop same model. Args: models: one or more models Return: A list of droped table name For example: from tests.models import User, Order await db.batch_drop(User, Order) """ self._checker() succeed = [] if not models: Logger.warning("parameter 'models' is empty, 'batch_drop' nothing to do") return succeed for model in models: if not issubclass(model, self.Model): raise ValueError( 'drop model type must be {}, get {}'.format(self.Model, model) ) if await model.exist(): await model.drop() Logger.info( "dropped table '{table_name}' from db: '{db}'".format( table_name=model.__meta__.table, db=self.db_info.info.db.db ) ) succeed.append(model.__meta__.table) else: Logger.error( message="drop table '{}' does not exist".format(model.__meta__.table) ) return succeed
def qiniu_fetch_file(purl): max_retry = 5 if not is_url(purl): Logger.warning(task='qiniu_fetch', message='input url:%s' % purl) return '' purl = transform_to_http(purl) q_auth = qiniu.Auth(ACCESS_KEY, SECRET_KEY) bucket_path = qiniu.BucketManager(q_auth) for n in range(max_retry): ret = bucket_path.fetch(purl, BUCKET_NAME) if ret is None: continue elif isinstance(ret, tuple) and ret[0] is None: continue else: key = ret[0]['key'] url = DOMAIN + str(key) obj = urlparse.urlparse(url) return obj.geturl() else: Logger.error(task='qiniu_fetch', message='max retry exceed') return purl