def graph(query):
if MIXPANEL_TOKEN:
mp.track(request.remote_addr, 'rendered graph')
return Response(renderGraph(query.replace(".png","").replace(" ","")),mimetype='image/svg+xml')
def scrape(self):
today = getToday()
if self.args.date:
today = self.args.date
self.args.download = False # can't set another date and download
todayClean = re.sub(r'(\d{4}\-\d{2}\-\d{2}).*', r'\1', today)
htmlFile = self.settings['cacheDir'] + today + '.html'
listFile = self.settings['namesDir'] + today + '.lst'
slackChannel = self.settings['slack'].get('channel', None)
print('Starting %s' % (today, ))
print('HTML file %s' % (htmlFile, ))
print('List file %s' % (listFile, ))
if (self.args.download):
writeData(
Downloader().download(self.settings['teamUrl'], self.args.verbose),
htmlFile
)
allFiles = listFiles(self.settings['namesDir'])
prevListFile = getFilenameBefore(
[f for f in allFiles if f[-4:] == '.lst'],
listFile
)
print('Prev List file %s' % (prevListFile, ))
newNameList = HTMLPage(htmlFile).getPeopleNames()
print('Found %s new names (incl dogs)' % (len(newNameList), ))
if prevListFile:
oldNameList = readLines(prevListFile)
print('Found %s prev names' % (len(oldNameList), ))
ghosts = getMissingNames(oldNameList, newNameList)
freshies = getMissingNames(newNameList, oldNameList)
print('Found %s ghosts' % (len(ghosts), ))
print('Found %s new Freshies' % (len(freshies), ))
if self.args.verbose:
print('%s is a FreshGhost' % (ghosts, ))
print('%s is fresh' % (freshies, ))
if self.args.slack:
if len(ghosts) or (self.args.newbies and len(newbies)):
print('posting', today, freshies, ghosts, slackChannel)
postFacesToSlack(
self.settings['slack']['endpoint'],
today,
freshies,
ghosts,
slackChannel)
print('Posted ghosts & freshies to slack channel %s' % (slackChannel, ))
else:
print('No prev name list found. Is this the start of time?')
ghosts = []
freshies = []
if self.args.save:
print('Saving new names list to %s' % (listFile, ))
writeLines(listFile, newNameList)
jsonFileReader = open(self.settings['summaryFile'], 'r')
summaryData = json.load(jsonFileReader)
summaryData[today] = dict(
date=todayClean,
count=len(newNameList),
additions=len(freshies),
removals=len(ghosts),
)
jsonFileReader.close()
print('Adding to summary json %s' % (self.settings['summaryFile'], ))
jsonFileWriter = open(self.settings['summaryFile'], 'w')
json.dump(summaryData, jsonFileWriter)
jsonFileWriter.close()
if self.args.graph:
print('Building graph')
jsonFileReader = open(self.settings['summaryFile'], 'r')
renderGraph(
self.settings['graphDir'] + todayClean + '.png',
prepareGraphData(json.load(jsonFileReader), todayClean),
)
if self.args.slack:
postGraphToSlack(
self.settings['slack']['endpoint'],
todayClean,
self.settings['graphURLRoot'] + todayClean + '.png',
dict(
date=todayClean,
count=len(newNameList),
additions=len(freshies),
removals=len(ghosts),
),
slackChannel)
print('Posted graph image to slack channel %s' % (slackChannel, ))
print('Done')
def graph(query):
return Response(renderGraph(query.replace(".png", "").replace(" ", "")),
mimetype='image/png')
def graph(query):
if MIXPANEL_TOKEN:
mp.track(request.remote_addr, 'rendered graph')
return Response(renderGraph(query.replace(".png","").replace(" ","")),mimetype='image/svg+xml')
def graph(query):
return Response(renderGraph(query.replace(".png","").replace(" ","")),mimetype='image/png')