def sendmail_per_min(key, sub, text, min=300):
global last_time_sendmail
if key not in last_time_sendmail:
last_time_sendmail[key] = 0
if time.time() - last_time_sendmail[key] > min:
send_mail(sub, text)
last_time_sendmail[key] = time.time()
def _woker():
last_time_sendmail = 0
while True:
try:
namelists = data_util.get_undownloaded()
if len(namelists) > 0:
s = '\n'.join(['%s' % json.dumps(n, ensure_ascii=False) for n in namelists])
logger.info('download list:\n%s' % s)
for nl in namelists:
username = nl['username']
id = nl['id']
name = nl['name']
kw = nl['keyword']
logger.info('begin download %s(%s) for %s' % (name, kw, username))
save_to = data_util.prepare_download(username, id)
download(name, kw, save_to)
logger.info('finished download %s(%s) for %s' % (name, kw, username))
data_util.finish_download(username, id)
except Exception as e:
logger.error('downloader exception, raw: %s' % e, exc_info=True)
if time.time() - last_time_sendmail > 600:
send_mail('worker exception', str(e))
last_time_sendmail = time.time()
time.sleep(2)
def _transmitter():
last_time_sendmail = 0
while True:
id = ''
ch_name = ''
en_name = ''
username = ''
try:
ones = data_util.get_untransmitted(5)
random.shuffle(ones)
if len(ones) > 0:
s = '\n'.join(['%s %s %s %s'
% (o['username'], o['id'], o['chinese_name'], o['english_name']) for o in ones])
logger.info('transmit list:\n%s' % s)
for one in ones:
username = one['username']
id = one['id']
ch_name = one['chinese_name']
en_name = one['english_name']
directory = one['dir']
logger.info('begin transmit for %s, one: %s' % (username, json.dumps(one, ensure_ascii=False)))
transmit(username, id, directory)
data_util.finish_transmit(username, id)
logger.info('finished transmit for %s, one: %s' % (username, json.dumps(one, ensure_ascii=False)))
except Exception as e:
logger.error('transmitter exception when transmit %s/%s/%s for %s: %s'
% (id, ch_name, en_name, username, e), exc_info=True)
if time.time() - last_time_sendmail > 600:
net_util.send_mail('transmitter exception', str(e))
last_time_sendmail = time.time()
time.sleep(60)
def _woker():
last_time_sendmail = 0
while True:
try:
namelists = get_unsubmitted()
for nl in namelists:
logger.info('send downloader username: %s, names: %s' % (nl['username'], json.dumps(nl['namelist'], ensure_ascii=False)))
send_names(nl['username'], nl['namelist'])
finish_untransmitted(nl['username'], nl['namelist'])
if len(namelists) == 0:
test_downloader()
except Exception as e:
if time.time() - last_time_sendmail > 600:
send_mail('transmitter exception', str(e))
last_time_sendmail = time.time()
logger.error(e, exc_info=True)
time.sleep(60)
def get_list():
username = session.get('username')
get_type = request.args.get('type', None) # 'undownloaded', 'unannotated', 'annotated'
_from = request.args.get('from', 0)
_to = request.args.get('to', 100)
logger.info('get_list, username: %s, type: %s, from: %s, to: %s' % (username, get_type, _from, _to))
try:
_from = int(_from)
except:
_from = 0
try:
_to = int(_to)
except:
_to = 100
if username is None or get_type is None or get_type not in ['undownloaded', 'unannotated', 'annotated']:
return ''
try:
get_type = get_type.encode('utf8')
if get_type == 'undownloaded':
li = data_util.get_undownloaded_list(username)
elif get_type == 'unannotated':
li = data_util.get_unannotated_list(username)
elif get_type == 'annotated':
li = data_util.get_annotated_list(username)
else:
li = data_util.get_13000_list()
# li是列表, [{name: "人名(文件夹名)", thumbnail: "相对于TODO_DIR/FINISHED_DIR的人的第一张图片的路径"}]
if get_type != '13000':
for l in li:
if l['thumbnail'] != '':
l['thumbnail'] = url_for('data.static', filename='%s/%s/%s/%s' % (get_type, username, l['id'], l['thumbnail']))
return json.dumps({'length': len(li), 'data': li[_from: _to]})
except Exception as e:
logger.error('get_list exception, raw: %s' % e, exc_info=True)
send_mail('get_list exception', str(e))
return json.dumps({'msg': str(e)})
def _detector():
last_time_sendmail = 0
while True:
try:
namelists = data_util.get_undetected(10)
for nl in namelists:
username = nl['username']
id = nl['id']
directory = nl['dir']
with open(path.join(directory, '_options')) as f:
options = json.load(f)
skip = options['dont_detect_face']
with open(path.join(directory, '_name')) as f:
ch_name = f.readline().strip()
en_name = f.readline().strip()
try:
rects_path = path.join(directory, '_rects')
if skip is True:
logger.info('skip detect %s/%s/%s for %s' %
(id, ch_name, en_name, username))
for f in [
_ for _ in os.listdir(directory)
if _.endswith('.jpg')
]:
os.system('mv {}/{} {}/other_{}_0.jpg'.format(
directory, f, directory, f[:-4]))
if not os.path.exists(rects_path):
with open(rects_path, 'w+') as f1:
pass
else:
logger.info('detect %s/%s/%s for %s' %
(id, ch_name, en_name, username))
pic_num = 0
# 找到上次检测到达的id
for f in os.listdir(directory):
if f.startswith('face'):
this_num = int(f.split('_')[1])
if this_num >= pic_num:
pic_num = this_num + 1
for f in os.listdir(directory):
if f.startswith('face'): # 跳过之前完成的检测
continue
img_path = path.join(directory, f)
if f.endswith('.jpg'):
result = do_detect(img_path)
#result = {"rects":[{"x":1,"y":2,"w":3,"h":4},{"x":4,"y":4,"w":3,"h":4},{"x":8,"y":10,"w":6,"h":6}]}
if len(result['rects']
) > TOO_MANY_FACES_THRESHOLD:
logger.info(
'too many faces, remove. %s, %d faces'
% (f, len(result['rects'])))
else:
face_num = 0
detected_rects = {}
for rect in result['rects']:
dest_file = os.path.join(
directory, 'face_%s_%d.jpg' %
(f[:-4], face_num))
file_name = os.path.join(
'face_%s_%d.jpg' %
(f[:-4], face_num))
extended_rect = _extend_rect(
rect['x'], rect['y'], rect['w'],
rect['h'])
detected_rects[file_name] = [
extended_rect['x'],
extended_rect['y'],
extended_rect['w'],
extended_rect['h']
]
cmd = 'convert "%s" -crop %dx%d+%d+%d "%s"' \
% (img_path, extended_rect['w'], extended_rect['h'], extended_rect['x'], extended_rect['y'], dest_file)
os.system(cmd)
# 太大的没必要
if extended_rect[
'w'] > 600 or extended_rect[
'h'] > 600:
os.system(
'convert -resize %dx%d "%s" "%s"'
% (600, 600, dest_file,
dest_file))
face_num += 1
pic_num += 1
if not os.path.exists(rects_path):
with open(rects_path, 'w+') as f1:
json.dump(detected_rects, f1)
else:
with open(rects_path, 'r+') as f1:
rects = json.load(f1)
merge_rects = dict(
rects, **detected_rects)
with open(rects_path, 'w+') as f2:
json.dump(merge_rects, f2)
if not f.startswith('_'):
os.remove(img_path)
logger.info(
'finished detect %s/%s/%s for %s. %d pictures' %
(id, ch_name, en_name, username, pic_num))
data_util.finish_detect(username, id)
except Exception as e:
raise Exception({
'msg':
'error detecting %s/%s/%s for %s' %
(id, ch_name, en_name, username),
'raw':
e
})
except Exception as e:
if time.time() - last_time_sendmail > 600:
send_mail('detector exception', str(e))
last_time_sendmail = time.time()
logger.error('detector exception, raw: %s' % e, exc_info=True)
time.sleep(2)