def data_and_condition_handler(obj, survey_id=None): if isinstance(obj, dict): for (k, v) in obj.items(): obj[k] = data_and_condition_handler(v, survey_id) elif isinstance(obj, list): for k, v in enumerate(obj): obj[k] = data_and_condition_handler(v, survey_id) elif isinstance(obj, ObjectId): obj = ObjectId(obj) elif isinstance(obj, str): if not obj.startswith(('date_', 'regex_')): return obj s = obj.split('_') if s[0] == 'date': d = s[1].split('-') obj = datetime(int(d[0]), int(d[1]), int(d[2])) elif s[0] == 'regex': obj = compile(s[1]) elif isinstance(obj, FileStorage): if not survey_id or not ObjectId.is_valid(survey_id): raise ValueError('invalid survey_id!') filename = '{}_{}_{}'.format(uuid1(), obj.name, obj.filename) file_path = join(FILE_FOLDER, survey_id, filename) obj.save(file_path) if obj.mimetype.startswith('image/') and obj.mimetype != 'image/gif': img = Image.open(file_path) if getsize(file_path) > 102400: img.thumbnail((img.width / 4, img.height / 4)) out = BytesIO() img.save(out, 'jpeg', quality=25) out.seek(0) data_url = b64encode(out.read()).decode() obj = 'img|{}|{}'.format(filename, data_url) else: obj = 'file|{}'.format(filename) return obj
def create_mongodb_index(db_alias, coll=None, drop_indexes=False, background=False, drop_dups=False): from model.mongo.index import MONGODB_INDEXES global MONGODB_INDEXES if db_alias is None and coll is not None: logger.error("db connection alias needed when specify coll") return False if db_alias is not None: MONGODB_INDEXES = {k: v for k, v in MONGODB_INDEXES.items() if k == db_alias} if coll is not None: MONGODB_INDEXES = {k: {k1: v1 for k1, v1 in v.items() if k1 == coll} for k, v in MONGODB_INDEXES.items()} for db_alias, colls in MONGODB_INDEXES.items(): db = get_mongo_database(db_alias) logger.info("begin process db {} ...".format(db.name)) for coll, indexes in colls.items(): coll = db[coll] logger.info( " begin process collection {} ...".format(coll.name)) if drop_indexes: logger.info(" begin drop indexes ...") coll.drop_indexes() logger.info(" end drop indexes") for index in indexes: keys = index[0] index_options = index[1] if len(index) >= 2 else {} index_options.update({ 'background': background }) while True: try: index_name = coll.create_index( keys, **index_options) except Exception as e: m = re.match( r"exception: E11000 duplicate key error collection: .+ (.+) dup key: { (.+) }", str(e)) if m and drop_dups: index_keys = m.group(1).rstrip('_1').rstrip( '_-1').replace('_1_', ' ').replace( '_-1_', ' ').split() index_values = [] for v in m.group(2).split(', '): v = v.lstrip(': ') if v.startswith('ObjectId'): v = ObjectId( re.match(r"ObjectId\('(.+)'\)", v).group(1)) elif v.startswith('"'): v = v.strip('"') elif re.match(r"\d+", v): v = int(v) index_values.append(v) query = { k: v for k, v in zip(index_keys, index_values)} logger.info( "duplicate docs: {}".format(list(coll.find(query)))) coll.delete_many(query) logger.info( "droped duplicate doc: {}".format((query))) continue logger.error("create index failed: {}".format(e)) break if index_name is not None: logger.info( " created index {}".format(index_name)) else: logger.info( " index {} already exists".format(keys)) break logger.info( " end process collection {}".format(coll.name)) logger.info("end process db {}".format(db.name))