def start_thumb():
if not thumb_index_job.is_alive():
thumb_index_job.start()
return JSONEncoder().encode({"result": "started"})
def channels():
channel = request.args.get('channel')
keyword = request.args.get('keyword')
type = request.args.get('type')
page_num = 1 if not request.args.get('pageNum') or not is_number(
request.args.get('pageNum')) or int(
request.args.get('pageNum')) < 1 else int(
request.args.get('pageNum'))
page_size = 20 if not request.args.get('pageSize') or not is_number(
request.args.get('pageSize')) else int(request.args.get('pageSize'))
skip = (page_num - 1) * page_size
query = []
if channel:
channel_query = {"_id": {"$eq": channel}}
query.append(channel_query)
if keyword:
keyword_query = {
"name": {
"$regex": ".*{}.*".format(keyword),
"$options": "i"
}
}
query.append(keyword_query)
if type:
type_query = {"type": {"$eq": type}}
query.append(type_query)
result = []
if len(query) > 0:
result = mongo.db.channels.find({"$and": query})
else:
result = mongo.db.channels.find()
count = result.count()
# print(count)
result = result.skip(skip).limit(page_size)
output = []
for s in result:
output.append(s)
return JSONEncoder().encode({
"pagination": {
"current_page": page_num,
"total_count": count,
"page_size": page_size
},
"data": output
})
def playitems():
channel = request.args.get('channel')
keyword = request.args.get('keyword')
page_num = 1 if not request.args.get('pageNum') or not is_number(
request.args.get('pageNum')) or int(
request.args.get('pageNum')) < 1 else int(
request.args.get('pageNum'))
page_size = 20 if not request.args.get('pageSize') or not is_number(
request.args.get('pageSize')) else int(request.args.get('pageSize'))
skip = (page_num - 1) * page_size
query = []
if channel:
channel_query = {"channel": {"$eq": channel}}
query.append(channel_query)
if keyword:
keyword_query = {
"tags": {
"$elemMatch": {
"$regex": ".*{}.*".format(keyword),
"$options": "i"
}
}
}
query.append(keyword_query)
if len(query) > 0:
result = mongo.db.playitems.find({
"$and": query
}).sort([("thumb_success", -1), ("thumb_time", -1)])
else:
result = mongo.db.playitems.find().sort([("thumb_success", -1),
("thumb_time", -1)])
count = result.count()
# print(count)
result = result.skip(skip).limit(page_size)
output = []
for s in result:
output.append(s)
return JSONEncoder().encode({
"pagination": {
"current_page": page_num,
"total_count": count,
"page_size": page_size
},
"data": output
})
def start_thumb():
# for obj in mongo.db.playitems.find():
# try:
#
# obj['thumb_time'] = datetime.fromtimestamp(obj['thumb_time'])
# mongo.db.playitems.save(obj)
# except KeyError:
# pass
if not thumb_index_job.is_alive():
thumb_index_job.start()
return JSONEncoder().encode({
"result": "started"
})
def run(self):
log_handler = start_thread_logging()
if not os.path.exists(self.thumb_path):
os.makedirs(self.thumb_path)
for i in range(0, Config.THUMB_WORKDER_COUNT):
worker = ThumbDownloadWorker(self.queue, self.thumb_path,
self.mongo)
worker.setName("THUMB_WORKER-{}".format(i + 1))
worker.start()
logging.info(' {} started...............'.format(worker.getName()))
full_count = 0
while self.running_job:
if self.queue.full():
logging.info('queue full:{}, {}- {}'.format(
self.queue.qsize(), self.queue.maxsize, full_count))
time.sleep(3)
full_count += 1
continue
full_count = 0
try:
interval_time = time.mktime(
datetime.datetime.now().timetuple()) - 60 * 60 * 2
logging.info("query by time < {}".format(interval_time))
result = self.mongo.db.playitems.find({
"$or": [{
"thumb_time": {
"$exists": False
}
}, {
"thumb_time": {
"$lt": interval_time
}
}]
}).sort([("thumb_success", -1),
("thumb_time", -1)]).limit(Config.THUMB_WORKDER_COUNT)
logging.info('{} url to thumb: {}'.format(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
result.count()))
if result.count() == 0:
time.sleep(60)
else:
for s in result:
logging.info(JSONEncoder().encode(s))
# output.append(s)
self.queue.put([s['_id'], s['url']],
block=True,
timeout=None)
logging.info('url in queue:{}'.format(
self.queue.qsize()))
except Exception as ex:
traceback.print_exc()
logging.error('queue except:{}, {}'.format(
self.queue.qsize(), self.queue.maxsize))
pass
stop_thread_logging(log_handler)