def mineChan(self, board, thread):
try:
db = ChanDB(celery.conf.DATABASE)
url = "https://a.4cdn.org/"+str(board)+"/thread/"+str(thread)+".json"
response = urlopen(url).read().decode('utf8')
data = json.loads(response)
for pp in data["posts"]:
pp["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
pp["board_name"] = board
pp["thread_id"] = thread
pp["post_id"] = pp["no"]
logger.info(pp)
db.insert_post(pp)
logger.info("{0} : Attempting to remove: {1} from MQ on mongo".format(datetime.now().strftime("%c"), str(thread)+":"+str(board)))
db.remove_from_mq(str(thread)+":"+str(board))
except Exception as e:
# self.retry(exc=e)
logger.error(e)
raise e
class ChanMinner:
def __init__(self):
self.run = True
self.db = ChanDB(celeryconfig.DATABASE)
self.logger = logging.getLogger('4 Chan Minner')
self.logger.info('Starting 4Chan manner')
def start(self):
try:
url = "https://a.4cdn.org/boards.json"
response = urlopen(url).read().decode('utf8')
data = json.loads(response)
data["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.db.insert_home(data)
while self.run:
for b in data["boards"]:
time.sleep(10)
self.logger.info(b["board"])
board = basc_py4chan.Board(b["board"])
response = urlopen("https://a.4cdn.org/"+b["board"]+"/threads.json").read().decode('utf8')
boarddata = {"threads":json.loads(response)}
boarddata["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
boarddata["board_name"] = b["board"]
self.db.insert_board(boarddata)
for tid in board.get_all_thread_ids():
if self.db.is_in_mq(str(tid)+":"+str(b["board"])) == False:
self.db.add_to_mq(str(tid)+":"+str(b["board"]))
mineChan.delay(b["board"], tid)
time.sleep(10)
except Exception as e:
self.logger.error(e)
def stop(self):
self.run = False
def __init__(self):
self.run = True
self.db = ChanDB(celeryconfig.DATABASE)
self.logger = logging.getLogger('4 Chan Minner')
self.logger.info('Starting 4Chan manner')
