def data_and_condition_handler(obj, survey_id=None):
if isinstance(obj, dict):
for (k, v) in obj.items():
obj[k] = data_and_condition_handler(v, survey_id)
elif isinstance(obj, list):
for k, v in enumerate(obj):
obj[k] = data_and_condition_handler(v, survey_id)
elif isinstance(obj, ObjectId):
obj = ObjectId(obj)
elif isinstance(obj, str):
if not obj.startswith(('date_', 'regex_')):
return obj
s = obj.split('_')
if s[0] == 'date':
d = s[1].split('-')
obj = datetime(int(d[0]), int(d[1]), int(d[2]))
elif s[0] == 'regex':
obj = compile(s[1])
elif isinstance(obj, FileStorage):
if not survey_id or not ObjectId.is_valid(survey_id):
raise ValueError('invalid survey_id!')
filename = '{}_{}_{}'.format(uuid1(), obj.name, obj.filename)
file_path = join(FILE_FOLDER, survey_id, filename)
obj.save(file_path)
if obj.mimetype.startswith('image/') and obj.mimetype != 'image/gif':
img = Image.open(file_path)
if getsize(file_path) > 102400:
img.thumbnail((img.width / 4, img.height / 4))
out = BytesIO()
img.save(out, 'jpeg', quality=25)
out.seek(0)
data_url = b64encode(out.read()).decode()
obj = 'img|{}|{}'.format(filename, data_url)
else:
obj = 'file|{}'.format(filename)
return obj
def create_mongodb_index(db_alias, coll=None, drop_indexes=False,
background=False, drop_dups=False):
from model.mongo.index import MONGODB_INDEXES
global MONGODB_INDEXES
if db_alias is None and coll is not None:
logger.error("db connection alias needed when specify coll")
return False
if db_alias is not None:
MONGODB_INDEXES = {k: v for k, v in MONGODB_INDEXES.items()
if k == db_alias}
if coll is not None:
MONGODB_INDEXES = {k: {k1: v1 for k1, v1 in v.items()
if k1 == coll}
for k, v in MONGODB_INDEXES.items()}
for db_alias, colls in MONGODB_INDEXES.items():
db = get_mongo_database(db_alias)
logger.info("begin process db {} ...".format(db.name))
for coll, indexes in colls.items():
coll = db[coll]
logger.info(
" begin process collection {} ...".format(coll.name))
if drop_indexes:
logger.info(" begin drop indexes ...")
coll.drop_indexes()
logger.info(" end drop indexes")
for index in indexes:
keys = index[0]
index_options = index[1] if len(index) >= 2 else {}
index_options.update({
'background': background
})
while True:
try:
index_name = coll.create_index(
keys, **index_options)
except Exception as e:
m = re.match(
r"exception: E11000 duplicate key error collection: .+ (.+) dup key: { (.+) }", str(e))
if m and drop_dups:
index_keys = m.group(1).rstrip('_1').rstrip(
'_-1').replace('_1_', ' ').replace(
'_-1_', ' ').split()
index_values = []
for v in m.group(2).split(', '):
v = v.lstrip(': ')
if v.startswith('ObjectId'):
v = ObjectId(
re.match(r"ObjectId\('(.+)'\)", v).group(1))
elif v.startswith('"'):
v = v.strip('"')
elif re.match(r"\d+", v):
v = int(v)
index_values.append(v)
query = {
k: v for k, v in zip(index_keys, index_values)}
logger.info(
"duplicate docs: {}".format(list(coll.find(query))))
coll.delete_many(query)
logger.info(
"droped duplicate doc: {}".format((query)))
continue
logger.error("create index failed: {}".format(e))
break
if index_name is not None:
logger.info(
" created index {}".format(index_name))
else:
logger.info(
" index {} already exists".format(keys))
break
logger.info(
" end process collection {}".format(coll.name))
logger.info("end process db {}".format(db.name))
