def crawlthemedia():
start = default_timer()
masterUpdates =[]
# #Let us begin
# #First we should call mr twitter from twitter_caller
name = comptwitterList()
masterUpdates.append(twitter_caller.competitor(name))
# then lets give LinkedIn a nudge
#comp,ids = complinkList()
#masterUpdates.append(linkedin.namedComp(comp,ids))
#then lets pull all the RSS feeds.
#masterUpdates.append(rss_puller.main())
#Send outputlist/dict to HTML generator for inclusion in Email
print len(masterUpdates)
if len(masterUpdates) > 1:
print "Emailing . . . . . "
#send somebody an email with updates
emailsender.emailGmailSender("robert.hamiltonsmith","kate-peter1", "*****@*****.**",
['*****@*****.**'], "Social Miner Updates ", masterUpdates)
print default_timer() - start
def crawlthemedia():
start = default_timer()
twitterUpdates =[]
linkedUpdates=[]
rssUpdates=[]
# #Let us begin
# #First we should call mr twitter from twitter_caller
twitterUpdates.append(twitter_caller.competitors())
# then lets give LinkedIn a nudge
linkedUpdates.append(linkedin.main())
#then lets pull all the RSS feeds.
rssUpdates.append(rss_puller.main())
# "title":a.title, "summary":a.summary_detail, "updated": a.updated,
# "links":a.links,"title_detail": a.title_detail,"link":a.link,
# "author":a.author,"id": a.id, "companyname":arse}
z = PrettyTable(["Competitor", "Title", "Links"])
z.align["Competitor"] = "l"
z.align["Title"] = 'c'
z.align["Text"] = 'c'
z.align["Links"] = 'c'
z.valign['Competitor'] = 't'
z.valign['Title'] = 't'
z.valign['Links'] = 't'
z.padding_width = 5
if len(rssUpdates) > 0:
for a in range(len(rssUpdates)):
for rss in rssUpdates[a]:
print rss.keys()
print rss['data'].keys()
tempTitle=""
tempComment=""
tempDetail=""
tempUrls=""
tempDesc=""
if rss['data'].has_key('title'):
print rss['data']['title']
chunksOut = chunks(rss['data']['title'], 40)
for ny in chunksOut:
tempTitle = tempTitle + "\n" + ny
if rss['data'].has_key('links'):
print rss['data']['links']
for lk in range(len(rss['data']['links'])):
tempUrls = tempUrls + "\n" + rss['data']['links'][lk]['href']
z.add_row([rss['competitor'],tempTitle,tempUrls])
#Send outputlist/dict to HTML generator for inclusion in Email
#loop through the updates and get simplistic data for simplistic people
x = PrettyTable(["Competitor", "Text", "Links"])
x.align["Competitor"] = "l"
x.align["Text"] = 'c'
x.align["Links"] = 'r'
x.border = True
x.header = True
x.padding_width = 5
if len(twitterUpdates) > 0:
for a in range(len(twitterUpdates[0])):
for update in twitterUpdates[0][a]:
#take text and add /n every 30 chars:
tempText=""
tempUrls=""
chunksOut = chunks(update['data']['texts'], 55)
#print chunksOut
for a in chunksOut:
tempText = tempText + "\n" + a
for b in update['data']['urls']:
#add \n
tempUrls = tempUrls + "\n" + b
#print tempText
#add a row to the table
x.add_row([update['competitor'],tempText, tempUrls])
y = PrettyTable(["Competitor","Type", "Comment", "Content", "Desc", "Location", "Position"])
y.align["Competitor"] = "l"
y.align["Type"] = 'r'
y.padding_width = 5
if len(linkedUpdates) > 0:
for b in range(len(linkedUpdates)):
for links in linkedUpdates[b]:
tempText=""
tempComment=""
tempUrls=""
tempDesc=""
tempPosition=""
tempLocation=""
print links['data'].keys()
print links['data']['action']
if links['data'].has_key('comment'):
print links['data']['comment']
chunksOut = chunks(links['data']['comment'], 55)
for ny in chunksOut:
tempComment = tempComment + "\n" + ny
if links['data'].has_key('content description'):
print links['data']['content description']
chunksOut = chunks(links['data']['content description'], 40)
for nz in chunksOut:
tempText = tempText + "\n" + nz
if links['data'].has_key('description'):
print links['data']['description']
chunksOut = chunks(links['data']['description'], 40)
for nj in chunksOut:
tempDesc = tempDesc + "\n" + nj
if links['data'].has_key('position'):
print links['data']['position']
tempPosition = links['data']['position']
if links['data'].has_key('siteJobRequest'):
print links['data']['siteJobRequest']
tempUrls = links['data']['siteJobRequest']
if links['data'].has_key('location'):
print links['data']['location']
tempLocation = links['data']['location']
y.add_row([links['competitor'],links['data']['action'],tempComment,tempText,
tempDesc,tempLocation,tempPosition + "\n" + tempUrls])
body = []
body.append(x)
body.append(y)
body.append(z)
print z
#unpack email credentials
user,pw = connections.email_connections()
#send somebody an email with updates
emailsender.emailGmailSender(user,pw, user,
connections.emailreciepients(), "Social Miner Updates ", body)
print default_timer() - start
