class MusicPipeline(object):
def __init__(self):
self.sock = Sock()
self.db = DB('music', 'postgres', 'dj')
def process_item(self, item, spider):
if len(self.db.query("""SELECT * FROM songs WHERE title=%s""", (item['title'], ))) == 0:
self.db.transaction(("""INSERT INTO songs (title, artist) VALUES (%s, %s)""",), ((item['title'],item['artist']),))
msg = item.type + '[*]' + item['title'] + '[*]' + item['download_link']
self.sock.send_msg(msg)
class StopDownloaders(object):
def __init__(self):
self.sock = Sock()
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
def spider_opened(self, spider):
#log.msg("opened spider %s" % spider.name)
# print '[START TO CRAWL SONGS]'
colorful.printc('[START TO CRAWL SONGS]', 'WARNING')
def spider_closed(self, spider):
#log.msg("closed spider %s" % spider.name)
# print '[INFO GATHERING COMPLETE]'
colorful.printc('[INFO GATHERING COMPLETE]', 'WARNING')
self.sock.send_msg('TERMINATE[*]--[*]--')
