def __init__(self, market):
self.market = market
resource = Resource(market)
rs0 = resource.get_mongo_rs0()
self.col_seed_table = rs0['crawl']['seedTable']
logger.info('invalid type {} '.format(tp))
return
metadata = {
'type': tp,
'handle': handle,
'category': category
}
self.create(url, 'youtubev2_social_seed', 'youtubev2.{}.{}'.format(tp, handle), 'schedule', **metadata)
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s, %(name)s %(levelname)s %(message)s',
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
market = 'en-us'
resource = Resource(market)
rs0 = resource.get_mongo_rs0()
col = rs0['crawl']['categories']
lder = Loader(market)
import json
with open(sys.argv[1]) as f:
for line in f:
if line.startswith('#'):
continue
items = line.strip().split(',', 2)
tp, handle, ch = items
if not ch:
ch = handle
lder.inject_youtube(tp, handle, ch)
