if log:
output['log'] = log
fn = writeFile(url, output)
# Data sent back to server
request = {
"links": data['links'] if data else list(),
"url": url,
"error": error,
"data": {
"time": t,
"path": os.path.abspath(fn)
}
}
else:
# Error sent back to server
request = {
'links': list(),
'url': url,
'error': error,
'data': None
}
url = getUrl(request)
except ConnectionRefusedError:
logging.info("Server unreachable, quitting.")
finally:
scraper.close()
s_nl = QuoraScraper()
s = QuoraScraper(True, args.EMAIL, args.PASS)
f = open(args.read)
o = open(args.output, 'a')
with open(args.seen) as seenF:
data = set(seenF.read().split('\n'))
seen = open(args.seen, 'a')
c = 0
try:
for line in f.readlines():
fn, url, hasData, hasTime = json.loads(line)
if fn in data or not hasData:
continue
if not hasTime:
hasTime = s_nl.processLog(url)
topics = s.getRealTopics(url)
o.write(json.dumps((fn, url, hasTime, topics)) + '\n')
seen.write(fn + '\n')
print(c)
c += 1
except KeyboardInterrupt:
pass
finally:
s_nl.close()
s.close()
f.close()
o.close()
seen.close()