def delete_redis(tag_id): try: data_list = [] key = f'newslive:tag_id:{str(tag_id)}' conn = redis.Redis(host=redis_setting['host'], port=redis_setting['port'], password=redis_setting['password'], db=redis_setting['db'], decode_responses=redis_setting['decode_responses']) for value in list(conn.smembers(key)): value_obj = eval(value) data_list.append(value_obj) # 获取今昨时间 today = datetime.date.today() # 当前日期 yesterday = datetime.date.today() - datetime.timedelta(days=1) time_list = [str(today), str(yesterday)] if data_list != []: for data in data_list: time = data['theme_time'].split(' ')[0] theme_id = data['theme_id'] # 时间时间是否在list中 if time not in time_list: if conn.sismember(key, str(data)): conn.srem(key, str(data)) # print(str(data)) # 删除图片 delete_img(theme_id) conn.close() except Exception as e: print(e) logger.error(f'delete_redis: {e}, time:{now_time}')
def delete_img(theme_id): try: path = 'Newslive/static/img_live/%s.jpg' % theme_id # path = '../../static/img_live/%s.jpg' % theme_id os.remove(path) # print('已删除:%s' % path) except Exception as e: print(e) logger.error(f'delete_img: {e}, time:{now_time}')
def exception_handler(exc, context): response = drf_exception_handler(exc, context) if response is None: logger.error('%s - %s - %s' % (context['view'], context['request'].method, exc)) return APIResponse(3, '异常', results={'detail': '服务器错误'}, http_status=status.HTTP_500_INTERNAL_SERVER_ERROR, exception=True ) return APIResponse(3, '异常', results=response.data, http_status=status.HTTP_401_UNAUTHORIZED)
def save_redis(key, value): try: conn = redis.Redis(host=redis_setting['host'], port=redis_setting['port'], password=redis_setting['password'], db=redis_setting['db'], decode_responses=redis_setting['decode_responses']) # for str_value in list(conn.smembers(key)): # value_obj = eval(str_value) # if value_obj['rich_text'] == value['rich_text']: # return conn.sadd(key, str(value)) # 172800 conn.close() except Exception as e: print(e) logger.error(f'save_redis: {e}, time:{now_time}')
def request_main(): while True: # print(now_time) for tag_id in tag_obj.keys(): # 清除缓存 if str(datetime.datetime.now()).split(' ')[1].split('.')[0].split(':')[0] + ':' + \ str(datetime.datetime.now()).split(' ')[1].split('.')[0].split(':')[1] == '23:00': try: detele_redis_task(tag_id) except Exception as e: print(e) logger.error(f'detele_redis_task: {e}, time:{now_time}') try: requests_live_data(tag_id) except Exception as e: print(e) logger.error(f'requests_live_data: {e}, time:{now_time}') time.sleep(60)
def requests_live_data(tag_id): try: url = base_url + str(tag_id) s = requests.session() s.keep_alive = False response = s.get(url=url, timeout=5) # 获取接口对象 obj_data_list = response.json()['result']['data']['feed']['list'] # print(obj_data_list) for obj_data in obj_data_list: theme_id = obj_data['id'] theme_time = obj_data['create_time'] rich_text = obj_data['rich_text'] # 如果为新浪广告 if del_theme(rich_text): continue img_text = obj_data['multimedia'] # 判断是否有图片 if img_text: # 有图片,请求图片数据 img = True img_url = img_text['img_url'][0].replace('\\', '') img_data = get_img_data(img_url, theme_id, 'save') else: img_data = '' img = False context = {} context['theme_id'] = str(theme_id) context['tag_id'] = str(tag_id) context['theme_time'] = theme_time context['rich_text'] = rich_text context['img'] = img context['img_data'] = img_data # print(context) data_key = f'newslive:tag_id:{str(tag_id)}' # 放入缓存 save_redis(data_key, context) except Exception as e: print(e) logger.error(f'requests_live_data: {e}, time:{now_time}')
def read_redis(tag_id, theme_id, type_method): try: data = {"list": []} key = f'newslive:tag_id:{str(tag_id)}' conn = redis.Redis(host=redis_setting['host'], port=redis_setting['port'], password=redis_setting['password'], db=redis_setting['db'], decode_responses=redis_setting['decode_responses']) for value in list(conn.smembers(key)): value_obj = eval(value) # 暂时用 图片路径,咱不用 B 64 # if value_obj['img']: # img_data = get_img_data('', value_obj['theme_id'], 'read') # value_obj['img_data'] = img_data # else: # value_obj['img_data'] = '' data['list'].append(value_obj) sort_data = sorted(data['list'], key=lambda i: i['theme_time'], reverse=True) # 清除新增 if theme_id != None and type_method == 'all': for i in range(0, len(sort_data)): if sort_data[i]['theme_id'] != theme_id: sort_data.pop(i) else: break if theme_id != None and type_method == 'update': list_1 = [] for i in range(0, len(sort_data)): if int(sort_data[i]['theme_id']) > int(theme_id): list_1.append(sort_data[i]) else: break sort_data = sorted(list_1, key=lambda i: i['theme_time'], reverse=True) data = {'list': sort_data} # print(data) conn.close() return data except Exception as e: print(e) logger.error(f'read_redis: {e}, time:{now_time}')
def get_img_data(url, id, method): try: if method == 'save': response = requests.get(url=url, timeout=5) file = f'/static/img_live/{id}.jpg' f = open(r'Newslive/static/img_live/%s.jpg' % id, 'ab') # 存储图片,多媒体文件需要参数b(二进制文件) # f = open(r'../../static/img_live/%s.jpg' % id, 'ab') # 存储图片,多媒体文件需要参数b(二进制文件) f.write(response.content) # 多媒体存储content f.close() return file else: # b64 f = open(r'Newslive/static/img_live/%s.jpg' % id, 'rb') # 图片,多媒体文件需要参数b(二进制文件) # f = open(r'../../static/img_live/%s.jpg' % id, 'rb') # 存储图片,多媒体文件需要参数b(二进制文件) f_data = f.read() # 多媒体存储content img_str = b64encode(f_data).decode('utf-8') f.close() return img_str except Exception as e: print(e) logger.error(f'get_img_data: {e}, time:{now_time}')