def pop_from_queue(external_id=None, account_id=None):
if account_id == None or external_id == None:
return json.dumps([]);
sql = "SELECT * FROM peepbuzz.twitter_queue WHERE external_id=" + str(external_id)
tweetsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
tweets = []
while True:
tweet = tweetsQ.fetch_row(1,1)
if tweet == ():
break
urls = []
urls = tweet[0]['urls'].split(',')
hashtags = []
hashtags = tweet[0]['hashtags'].split(',')
#format time
formattedTime=tweet[0]['created'].replace(' ', 'T')
formattedTime=formattedTime+'0000'
tweets.append({
"stream_name" : 'twitter',
"external_id" : tweet[0]['status_id'],
"urls" : urls,
"created" : formattedTime,
"promoter_id" : tweet[0]['promoter_id'],
"promoter" : tweet[0]['promoter'],
"thumbnail" : tweet[0]['thumbnail'],
"title" : None,
"summary" : tweet[0]['summary'],
"hashtags" : hashtags,
"discussion" : [],
"account_id" : account_id,
})
sql = "DELETE FROM peepbuzz.twitter_queue WHERE external_id=" + str(external_id)
tweetsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return json.dumps(tweets)
def on_status(self, status):
global follow_list
#format time into 2011-02-23T16:42:40+0000 format ala facebook
#Twitter Format: Wed Mar 23 22:51:50 +0000 2011
formattedTime = self.formatTime(status['created_at'])
hashtags = []
if len(status['entities']['hashtags']):
for val in status['entities']['hashtags']:
hashtags.append(val['text'].replace("'", "\\'"))
hashtag = ','.join(hashtags)
urls = []
if len(status['entities']['urls']):
for val in status['entities']['urls']:
urls.append(val['url'].replace("'", "\\'"))
url = ','.join(urls)
#print status['text']
text = status['text'].replace("'", "\\'")
if text[-1] == '\\':
text = text + " "
if str(status['user']['id']) in follow_list:
file_put_contents(str(status['user']['screen_name']) + " posted something")
infoModule.info.site['dblink'] = mysql_tools.db_connect()
sql = u"INSERT INTO `peepbuzz`.`twitter_queue` SET `status_id` = '" + str(status['id']) + "', `created` = '" + formattedTime + "', `promoter_id` = '" + str(status['user']['id']) + "', `promoter` = '" + status['user']['screen_name'] + "', `thumbnail` = '" + str(status['user']['profile_image_url']) + "', `summary` = '" + text + "', `external_id` = '" + str(status['user']['id']) + "', `hashtags` = '" + hashtag + "', `urls` = '" + url + "'";
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
infoModule.info.site['dblink'].close()
else:
pass
def tearDown(self):
if infoModule.info.site['dblink'] == None:
#need this code to recover from db disconnect test
infoModule.info.site['dblink'] = self.dblink
sql = "delete from peepbuzz.blocked_accounts where ba_id=" + str(self.ba_id)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
def cleanup(days):
link = mysql_tools.db_connect()
query = 'SELECT filament_id, story_id FROM peepbuzz.filaments WHERE created <= DATE_SUB(NOW(), INTERVAL '+str(days)+' DAY)'
result = mysql_tools.mysqlQuery(query, link)
while (1):
row = result.fetch_row(1,1)
if row == ():
break
query = 'DELETE from peepbuzz.filaments WHERE filament_id = "'+str(row[0]['filament_id'])+'"'
try:
result2 = mysql_tools.mysqlQuery(query, link)
except:
pprint.pprint(query)
sys.exit(1)
if row[0]['story_id'] != None:
query = 'SELECT count(*) from peepbuzz.filaments WHERE story_id = "'+str(row[0]['story_id'])+'"'
try:
result2 = mysql_tools.mysqlQuery(query, link)
except:
pprint.pprint(query)
sys.exit(1)
row = result2.fetch_row(1,1)
if row == None:
break
if row[0] == 0:
query = 'DELETE FROM peepbuzz.stories WHERE story_id = "'+str(row[0]['story_id'])+'"'
try:
result2 = mysql_tools.mysqlQuery(query, link)
except:
pprint.pprint(query)
sys.exit(1)
return True
def tearDown(self):
if infoModule.info.site['dblink'] == None:
#need this code to recover from db disconnect test
infoModule.info.site['dblink'] = self.dblink
sql = "delete from peepbuzz.blocked_accounts where ba_id=" + str(
self.ba_id)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
def createVideoStory(embed_code, v_celeb_list, source): #embed code, list of dicts, dict
recognized = False
v_source = {}
v_page_info = []
videoSource = []
newVideoSub = []
embed = re.search('<embed.*?src=\"http:\/\/www.youtube.com\/\w+\/([\w-]+).*?<\/embed>', embed_code, re.I)
if embed:
id = re.search('\/v\/[\w-]+', embed.group(0))
url = 'http://www.youtube.com/watch?v=' + id.group(0).replace('/v/', '')
youtube_page = urllib.urlopen(url).read()
youtube_page = youtube_page.replace('\n', '').replace('\t', '')
titleMatch = re.search('<title>Youtube ?- (.*?)</title>', youtube_page, re.I).group(0)
titleMatch = titleMatch.replace('<title>', '').replace('</title>', '').replace('YouTube- ', '').replace('YouTube - ', '')
v_page_info = { "title" : titleMatch, "url" : url, "embed_code": embed_code }
bodyMatch = re.search('<meta name="description" content="(.*?)">', youtube_page, re.I).group(0)
bodyMatch = bodyMatch.replace('<meta name="description" content="', '').replace('">', '')
sql = "select * from db_celebrifi.sources where title='youtube' and parent=0"
vSourceQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while True:
v_s = vSourceQ.fetch_row(1,1)
if v_s == ():
break
videoSource.append(v_s[0]['source_id'])
recognized = True
v_source['source_id'] = videoSource[0]
v_source['publish_immediately'] = True
if recognized :
sql = 'select * from db_celebrifi.subs where url like "' + v_page_info['url'] + '"'
urlCheckQuery = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if urlCheckQuery != False:
if urlCheckQuery.num_rows() > 0:
urlRows = urlCheckQuery.num_rows()
plog("found video url. incrementing score using: update subs set score=score + 1 where url='" + v_page_info['url']+"'", 2)
while True:
n_v = urlCheckQuery.fetch_row(1,1)
if n_v == ():
break
newVideoSub.append(n_v[0]['sub_id'])
sql = "insert into db_celebrifi.linkHistory set sub_id=" + newVideoSub[0] + ", feedIdx="+source['source_id'] +", linker_id=-1, linkerURL=" + embed_code + ", linkedOn=now(), userVote=false"
if debugMode == False:
linkHistory = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "update subs set score=score+1 where url='"+v_page_info['url']+"'"
if debugMode == False:
updateSubs = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
else:
plog("adding standalone video story", 2)
#addSub(bodyMatch, v_celeb_list, "", 0, v_source, v_page_info)
return 0
def getNewEntities():
''' find latest id from story_entites to see when the last entity was ID'd'''
query = 'select max(story_id) as max_story from peepbuzz.story_entities'
try:
storyIDQ = mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
except:
return False
storyIDRow = storyIDQ.fetch_row(1,1)
if storyIDRow == ():
return False
storyID = storyIDRow[0]['max_story']
if storyID == None:
storyID = 0
else:
storyID = 0
query = 'select story_id, title, body from peepbuzz.stories where story_id > '+str(storyID)
print query
try:
storiesQ = mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
except:
return False
while(1):
row = storiesQ.fetch_row(1,1)
if row==():
break
story_id = row[0]['story_id']
title = row[0]['title']
body = row[0]['body']
if body == None or title == None:
continue
url = 'http://informifi.com/enhancer/APIGateway.php'
request = {'command' : 'entities',
'byID' : 'true',
'title': title,
'searchText' : body}
data = urllib.urlencode(request)
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
res = response.read()
try:
entities = json.loads(res)
except ValueError:
continue
entityCount = len(entities)
print "found " + str(entityCount) + " entities"
for ents in entities:
primo = ents['primo']
if(primo== "Y"):
primo = 1
elif(primo =="N"):
primo = 10
q = 'insert into peepbuzz.story_entities (story_id, entity_id, primo) values ("'+str(story_id)+'","'+ents['id']+'","'+str(primo)+'")'
try:
insertQ = mysql_tools.mysqlQuery(q, infoModule.info.site['dblink'])
except:
return False
return True
def insertHash(hashtag, type, type_short, block):
query = 'insert into peepbuzz.hashtag_'+type+'_stats (hashtag, '+type_short+'_block, score) values ("'+hashtag+'","'+block+'","1")'
hashQ = mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
if(hashQ==False):
return False
else:
return True
def testUTF():
dblink = infoModule.info.site["dblink"]
sql = u"insert into peepbuzz.stories set title='foo faa \u2026 fum'"
# mysql_tools.mysqlQuery(sql, dblink)
sql = "select title from peepbuzz.stories"
accountsQ = mysql_tools.mysqlQuery(sql, dblink)
accounts = accountsQ.fetch_row(0, 1)
print accounts[0]["title"]
def setUp(self):
infoModule.info.site['dblink'] = mysql_tools.db_connect()
dblink = infoModule.info.site['dblink']
#set up accounts for test
# get valid account
sql = "SELECT user_id from peepbuzz.users limit 1"
userQ = mysql_tools.mysqlQuery(sql, dblink)
user = userQ.fetch_row(1,1)
sql = "SELECT account_id from peepbuzz.accounts limit 1"
accountQ = mysql_tools.mysqlQuery(sql, dblink)
account = accountQ.fetch_row(1,1)
self.account_id = account[0]['account_id']
self.user_id = user[0]['user_id']
sql = "insert into peepbuzz.blocked_accounts set user_id=" + self.user_id + ", unknown_account_id=" + self.unknown_account_id + ", account_id=" + self.account_id
testQ = mysql_tools.mysqlQuery(sql, dblink)
self.ba_id = dblink.insert_id()
def store_discussion(discussion_json, filament_id):
discussions = json.loads(discussion_json)
# get stream_id
sql = "SELECT stream_id FROM peepbuzz.filaments WHERE filament_id = " + str(filament_id)
stream_idQ = mysql_tools.mysqlQuery(sql, infoModule.info.site["dblink"])
row = stream_idQ.fetch_row(1, 1)
if row == ():
infoModule.info.errorList.append("ERROR: no such filament (" + str(filament_id) + ")")
return False
stream_id = row[0]["stream_id"]
discussion_ids = []
for discussion in discussions:
if "count" in discussion:
continue
else:
account_id, table = accountFinder(
stream_id, discussion["user_id"], discussion["user_name"], discussion["thumbnail"]
)
created = discussion["comment_created"]
body = discussion["body"]
body = body.replace("'", "\\'")
if table == "accounts":
field = "account_id"
mysql_tools.mysqlQuery(
u"INSERT INTO peepbuzz.discussions SET filament_id="
+ str(filament_id)
+ ", "
+ field
+ "="
+ str(account_id)
+ ', created="'
+ created
+ '", body="'
+ body
+ '"',
infoModule.info.site["dblink"],
)
discussion_id = infoModule.info.site["dblink"].insert_id()
discussion_ids.append(int(discussion_id))
return discussion_ids
def getIds(sub_id):
sql = 'SELECT celeb_id FROM ' + infoModule.info.site['database'] + '.subs_celebs WHERE sub_id = ' + str(sub_id)
entityIdsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
entityIds = entityIdsQ.fetch_row(0,1)
entityRows = []
for row in entityIds:
infoModule.info.entityList[row['celeb_id']] = {'position': None, 'frequency': 0, 'primo' : 'N'}
def testUTF():
dblink = infoModule.info.site['dblink']
sql = u'insert into peepbuzz.stories set title=\'foo faa \u2026 fum\''
#mysql_tools.mysqlQuery(sql, dblink)
sql = 'select title from peepbuzz.stories'
accountsQ = mysql_tools.mysqlQuery(sql, dblink)
accounts = accountsQ.fetch_row(0,1)
print accounts[0]['title']
def setUp(self):
infoModule.info.site['dblink'] = mysql_tools.db_connect()
dblink = infoModule.info.site['dblink']
#set up accounts for test
# get valid account
sql = "SELECT user_id from peepbuzz.users limit 1"
userQ = mysql_tools.mysqlQuery(sql, dblink)
user = userQ.fetch_row(1, 1)
sql = "SELECT account_id from peepbuzz.accounts limit 1"
accountQ = mysql_tools.mysqlQuery(sql, dblink)
account = accountQ.fetch_row(1, 1)
self.account_id = account[0]['account_id']
self.user_id = user[0]['user_id']
sql = "insert into peepbuzz.blocked_accounts set user_id=" + self.user_id + ", unknown_account_id=" + self.unknown_account_id + ", account_id=" + self.account_id
testQ = mysql_tools.mysqlQuery(sql, dblink)
self.ba_id = dblink.insert_id()
def URLInStories(URL):
dblink = infoModule.info.site["dblink"]
sql = "select story_id from peepbuzz.stories where url='" + URL + "' or original_url='" + URL + "'"
storyQ = mysql_tools.mysqlQuery(sql, dblink)
story = storyQ.fetch_row(1, 1)
if story == ():
return False
else:
return int(story[0]["story_id"])
def updateHash(hashtag, type, type_short, score, block):
score = int(score)
score = score + 1
query = 'update peepbuzz.hashtag_'+type+'_stats set score="'+str(score)+'" where hashtag="'+hashtag+'" and '+type_short+'_block="'+block+'"'
hashQ = mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
if(hashQ==False):
return False
else:
return True
def URLInStories(URL):
dblink = infoModule.info.site['dblink']
sql = "select story_id from peepbuzz.stories where url='" + URL + "' or original_url='" + URL + "'"
storyQ = mysql_tools.mysqlQuery(sql, dblink)
story = storyQ.fetch_row(1,1)
if story == ():
return False
else:
return int(story[0]['story_id'])
def getBlockedImages():
blockedImages = []
sql = "select * from " + infoModule.info.site['database'] + ".blocked_images"
esr = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while True:
blocked = esr.fetch_row(1,1)
if blocked == ():
break
blockedImages.append(blocked[0]['regex'])
return blockedImages
def addFollowing(user_id = None, account_id = None):
# Make sure we have a user to check against
if not user_id or not account_id:
return False
#Check to see if the follower exists yet
check = mysql_tools.mysqlQuery("SELECT * FROM `peepbuzz`.`following` WHERE `user_id` = '" + str(user_id) + "' AND `account_id` = '" + str(account_id) + "'", infoModule.info.site['dblink'])
if check.num_rows() > 0:
return True
# Since they did not exist let's add them
sql = "INSERT INTO `peepbuzz`.`following` SET `user_id` = '" + str(user_id) + "', `account_id` = '" + str(account_id) + "'"
print sql
add = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if not infoModule.info.site['dblink'].insert_id():
return False
return True
def on_status(self, status):
global follow_list
#format time into 2011-02-23T16:42:40+0000 format ala facebook
#Twitter Format: Wed Mar 23 22:51:50 +0000 2011
formattedTime = self.formatTime(status['created_at'])
hashtags = []
if len(status['entities']['hashtags']):
for val in status['entities']['hashtags']:
hashtags.append(val['text'].replace("'", "\\'"))
hashtag = ','.join(hashtags)
urls = []
if len(status['entities']['urls']):
for val in status['entities']['urls']:
urls.append(val['url'].replace("'", "\\'"))
url = ','.join(urls)
#print status['text']
text = status['text'].replace("'", "\\'")
if text[-1] == '\\':
text = text + " "
if str(status['user']['id']) in follow_list:
file_put_contents(
str(status['user']['screen_name']) + " posted something")
infoModule.info.site['dblink'] = mysql_tools.db_connect()
sql = u"INSERT INTO `peepbuzz`.`twitter_queue` SET `status_id` = '" + str(
status['id']
) + "', `created` = '" + formattedTime + "', `promoter_id` = '" + str(
status['user']['id']
) + "', `promoter` = '" + status['user'][
'screen_name'] + "', `thumbnail` = '" + str(
status['user']['profile_image_url']
) + "', `summary` = '" + text + "', `external_id` = '" + str(
status['user']['id']
) + "', `hashtags` = '" + hashtag + "', `urls` = '" + url + "'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
infoModule.info.site['dblink'].close()
else:
pass
def entityLibraryByUrl(lookupUrl, field):
celeb_idQ = mysql_tools.mysqlQuery('select celeb_id from db_topics.celebs where lookupUrl = "'+lookupUrl+'"', infoModule.info.site['dblink'])
celeb_id=celeb_idQ.fetch_row(1,1)
if celeb_id == ():
log.plog('no celeb_id found for ' + lookupUrl, 5)
return False
else:
cid = celeb_id[0]['celeb_id']
liveEntities[cid] = liveEntity(cid)
return liveEntities[cid].getData(field)
def getBlockedImages():
blockedImages = []
sql = "select * from " + infoModule.info.site[
'database'] + ".blocked_images"
esr = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while True:
blocked = esr.fetch_row(1, 1)
if blocked == ():
break
blockedImages.append(blocked[0]['regex'])
return blockedImages
def getEntityTotals(entity_id):
entDict = {}
sql = 'select celeb_id, stories_total, storiesWeighted_total, vertical from db_topics.celebStatsTotals where celeb_id =' + str(entity_id)
statsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while (1):
row = statsQ.fetch_row(1,1)
if row == ():
break;
idx = row[0]['vertical']
entDict[idx] = {"stories_total": int(row[0]['stories_total']), "storiesWeighted_total": int(row[0]['storiesWeighted_total'])}
return entDict
def accountFinder(stream_id, external_id, user_name, thumbnail):
''' Checking for external accounts in known accounts '''
sql = u"select account_id, thumbnail from peepbuzz.accounts where external_id='" + str(external_id) + "' and stream_id=" + str(stream_id) + " LIMIT 1"
known_account = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if known_account.num_rows() > 0:
''' return account_id and known_accounts table '''
row = known_account.fetch_row(1,1)
if row == ():
pass
else:
account_id = row[0]['account_id']
#check thumbnail against reported thumbnail and update if changed
if len(thumbnail) > 0 and thumbnail != row[0]['thumbnail']:
sql = "update peepbuzz.accounts set thumbnail='" + thumbnail + "' where account_id=" + str(account_id)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return account_id, 'accounts'
else:
''' no account, create it '''
new_account = mysql_tools.mysqlQuery(u"insert into peepbuzz.accounts set external_id='"+str(external_id)+"', stream_id='"+str(stream_id)+"', user_name='"+user_name+"', thumbnail='" + thumbnail + "'", infoModule.info.site['dblink'])
''' return insert id from account with uaccounts table '''
new_account_id = infoModule.info.site['dblink'].insert_id()
if len(thumbnail) > 0:
sql = "update peepbuzz.accounts set thumbnail='" + thumbnail + "' where account_id=" + str(new_account_id)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return str(new_account_id), 'accounts'
def twitterPic(picture, smsID, caption, celebID, date):
return
url = picture.group(0)
# Make sure that we have the http:// on the url
if url[:7] != 'http://':
url = 'http://' + url
#Create an image record for this
mysql_tools.mysqlQuery("INSERT INTO `db_celebrifi`.`images` SET `credit` = ''", link)
imageID = link.insert_id()
imageID = str(imageID)
print "http://angelina.celebrifi.com:81/imageFetcher.php?image_id=" + imageID + "&type=feed&url=" + url
imageFetch = "http://angelina.celebrifi.com:81/imageFetcher.php?image_id=" + imageID + "&type=feed&url=" + url
try: content = urllib2.urlopen(imageFetch).read()
except urllib2.URLError, e:
# Looks like we had a problem... So lets remove the image
mysql_tools.mysqlQuery("DELETE FROM `db_celebrifi`.`twitPics` WHERE `image_id` = '" + imageID + "'", link)
return False
def store_discussion(discussion_json, filament_id):
discussions = json.loads(discussion_json)
# get stream_id
sql = 'SELECT stream_id FROM peepbuzz.filaments WHERE filament_id = ' + str(
filament_id)
stream_idQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
row = stream_idQ.fetch_row(1, 1)
if row == ():
infoModule.info.errorList.append("ERROR: no such filament (" +
str(filament_id) + ")")
return False
stream_id = row[0]['stream_id']
discussion_ids = []
for discussion in discussions:
if 'count' in discussion:
continue
else:
account_id, table = accountFinder(stream_id, discussion['user_id'],
discussion['user_name'],
discussion['thumbnail'])
created = discussion['comment_created']
body = discussion['body']
body = body.replace("'", "\\'")
if table == 'accounts':
field = 'account_id'
mysql_tools.mysqlQuery(
u'INSERT INTO peepbuzz.discussions SET filament_id=' +
str(filament_id) + ', ' + field + '=' + str(account_id) +
', created="' + created + '", body="' + body + '"',
infoModule.info.site['dblink'])
discussion_id = infoModule.info.site['dblink'].insert_id()
discussion_ids.append(int(discussion_id))
return discussion_ids
def getIds(sub_id):
sql = 'SELECT celeb_id FROM ' + infoModule.info.site[
'database'] + '.subs_celebs WHERE sub_id = ' + str(sub_id)
entityIdsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
entityIds = entityIdsQ.fetch_row(0, 1)
entityRows = []
for row in entityIds:
infoModule.info.entityList[row['celeb_id']] = {
'position': None,
'frequency': 0,
'primo': 'N'
}
def checkForHash(hashtag, type, type_short, block):
# type: {daily, hourly}
# block : time/secs
query = 'select score from peepbuzz.hashtag_'+type+'_stats where '+type_short+'_block="'+block+'" and hashtag="'+hashtag+'"'
hashQ = mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
if(hashQ.num_rows()>0):
while (1):
row = hashQ.fetch_row(1,1)
if row == ():
break
score = row[0]['score']
return score
else:
return False
def addFollowing(user_id=None, account_id=None):
# Make sure we have a user to check against
if not user_id or not account_id:
return False
#Check to see if the follower exists yet
check = mysql_tools.mysqlQuery(
"SELECT * FROM `peepbuzz`.`following` WHERE `user_id` = '" +
str(user_id) + "' AND `account_id` = '" + str(account_id) + "'",
infoModule.info.site['dblink'])
if check.num_rows() > 0:
return True
# Since they did not exist let's add them
sql = "INSERT INTO `peepbuzz`.`following` SET `user_id` = '" + str(
user_id) + "', `account_id` = '" + str(account_id) + "'"
print sql
add = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if not infoModule.info.site['dblink'].insert_id():
return False
return True
def pop_from_queue(external_id=None, account_id=None):
if account_id == None or external_id == None:
return json.dumps([])
sql = "SELECT * FROM peepbuzz.twitter_queue WHERE external_id=" + str(
external_id)
tweetsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
tweets = []
while True:
tweet = tweetsQ.fetch_row(1, 1)
if tweet == ():
break
urls = []
urls = tweet[0]['urls'].split(',')
hashtags = []
hashtags = tweet[0]['hashtags'].split(',')
#format time
formattedTime = tweet[0]['created'].replace(' ', 'T')
formattedTime = formattedTime + '0000'
tweets.append({
"stream_name": 'twitter',
"external_id": tweet[0]['status_id'],
"urls": urls,
"created": formattedTime,
"promoter_id": tweet[0]['promoter_id'],
"promoter": tweet[0]['promoter'],
"thumbnail": tweet[0]['thumbnail'],
"title": None,
"summary": tweet[0]['summary'],
"hashtags": hashtags,
"discussion": [],
"account_id": account_id,
})
sql = "DELETE FROM peepbuzz.twitter_queue WHERE external_id=" + str(
external_id)
tweetsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return json.dumps(tweets)
def cleanup(days):
link = mysql_tools.db_connect()
query = 'SELECT filament_id, story_id FROM peepbuzz.filaments WHERE created <= DATE_SUB(NOW(), INTERVAL ' + str(
days) + ' DAY)'
result = mysql_tools.mysqlQuery(query, link)
while (1):
row = result.fetch_row(1, 1)
if row == ():
break
query = 'DELETE from peepbuzz.filaments WHERE filament_id = "' + str(
row[0]['filament_id']) + '"'
try:
result2 = mysql_tools.mysqlQuery(query, link)
except:
pprint.pprint(query)
sys.exit(1)
if row[0]['story_id'] != None:
query = 'SELECT count(*) from peepbuzz.filaments WHERE story_id = "' + str(
row[0]['story_id']) + '"'
try:
result2 = mysql_tools.mysqlQuery(query, link)
except:
pprint.pprint(query)
sys.exit(1)
row = result2.fetch_row(1, 1)
if row == None:
break
if row[0] == 0:
query = 'DELETE FROM peepbuzz.stories WHERE story_id = "' + str(
row[0]['story_id']) + '"'
try:
result2 = mysql_tools.mysqlQuery(query, link)
except:
pprint.pprint(query)
sys.exit(1)
return True
def addEntity(nameIdx, mptype):
phraseQ = mysql_tools.mysqlQuery("select phrase from db_topics.unknownNames where idx=" + str(nameIdx), infoModule.info.site['dblink'])
if phraseQ != None:
phrase = phraseQ.fetch_row(1,1)
if phrase == ():
log.plog("unknownNames query returned nothing", 4)
return
else:
log.plog("unknownNames query failed", 4)
return
nameParts = re.split("\s+", phrase[0]['phrase'])
if len(nameParts) == 2:
fname = nameParts[0]
mname = ""
lname = nameParts[1]
fname = fname.capitalize()
mname = mname.capitalize()
lname = lname.capitalize()
lookupUrl = fname + "-" + lname
elif len(nameParts) == 3:
fname = nameParts[0]
mname = nameParts[1]
lname = nameParts[2]
fname = fname.capitalize()
mname = mname.capitalize()
lname = lname.capitalize()
lookupUrl = fname + "-" + mname + "-" + lname
#final double check against notCelebs
sql = "SELECT * from db_topics.notCelebs WHERE phrase='" + phrase[0]['phrase'] + "'"
log.plog(sql, 2)
notCelebsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
notCelebs = notCelebsQ.fetch_row(1,1)
if notCelebs != ():
log.plog("attempted to add new entity that is in notCelebs table: " + phrase[0]['phrase'], 4)
return
#to block against this celeb being added later
mysql_tools.mysqlQuery("insert into db_topics.notCelebs set phrase='" + phrase[0]['phrase'] + "'", infoModule.info.site['dblink'])
searchName = phrase[0]['phrase']
checkCelebQ = mysql_tools.mysqlQuery("select * from db_topics.celebs where fullName='" + searchName + "'", infoModule.info.site['dblink'])
foundCelebs = checkCelebQ.num_rows()
log.plog("found " + str(foundCelebs) + " matching rows for '" + searchName + "'", 2)
if foundCelebs == 0:
log.plog("adding hidden entity '" + searchName + "'", 2)
sql = "insert into db_topics.celebs set status='A', fname='" + fname + "', mname='" + mname + "', lname='" + lname + "', fullName='" + searchName + "', searchName='" + searchName + "', lookupUrl='" + lookupUrl + "', bio='No bio available', mptype_id=" + mptype[0]['mptype_id']+ ", created=now()"
mptypeQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
# new entities should be added to the entities list for the page
newCid = infoModule.info.site['dblink'].insert_id()
if newCid > 0:
infoModule.info.entityList[newCid] = {'position': None, 'frequency': 1, 'primo' : 'N'}
log.plog('new entity id is ' + str(newCid) + ' added to list for this story', 3)
def nicknameFinder(workingText, stats, showNames=False):
log.plog('NICKNAMES', 2)
#args = ['/usr/bin/perl', 'nicknames.pl', workingText]
#nn = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
#load balance nickname connection
serverPicker = random.randint(0,1)
dollyHosts = ['angelina.celebrifi.com', 'britney.celebrifi.com', 'hedwig.celebrifi.com', 'fergie.celebrifi.com', 'catherinez.celebrifi.com', 'dolly.celebrifi.com']
machineName = os.uname()
#go to one server if in production, other if not
#for some cases, we need the nickname server to tell us WHY it found a nickname.
#that's what the showNames field is for
if showNames:
params = urllib.urlencode({'searchText': workingText, 'showNames':'true'})
else:
params = urllib.urlencode({'searchText': workingText})
if machineName[1] in dollyHosts:
socket = urllib.urlopen('http://hedwig.informifi.com:81/sourceReader/nicknames.pl', params)
else:
socket = urllib.urlopen('http://dev.informifi.com/sourceReader/nicknames.pl', params)
nn = socket.read()
nicknames = nn[:-1]
if nicknames == '':
return
log.plog('nickname Server returned ' + nicknames, 2)
nicknames = nicknames.split(",")
for i in range(len(nicknames)):
vals = nicknames[i].split(':')
eid = vals[0]
firstFound = vals[1]
if showNames:
nameUsed = vals[2]
if str(i) not in infoModule.info.entityList:
#test against db. If can't find entity in db, delete nickname
#test entity list against db temporarily
verifyQ = mysql_tools.mysqlQuery("select lookupUrl from db_topics.celebs where celeb_id=" + str(eid), infoModule.info.site['dblink'])
if verifyQ.num_rows() == 0:
log.plog("entity in list does not exist! %s." % str(eid), 5)
else:
log.plog("verified that %s is a real entity" % str(eid), 2)
#different returns based on showNames setting
if showNames:
infoModule.info.entityList[str(eid)] = {'position': None, 'frequency': 0, 'primo' : 'N', 'nameUsed' : nameUsed}
else:
infoModule.info.entityList[str(eid)] = {'position': None, 'frequency': 0, 'primo' : 'N'}
log.plog("found celeb: %s " % str(eid), 2)
if stats and (infoModule.info.entityList[str(eid)] == None or int(firstFound) < infoModule.info.entityList[str(eid)]['position']):
infoModule.info.entityList[str(eid)]['position'] = firstFound
def getEntityTotals(entity_id):
entDict = {}
sql = 'select celeb_id, stories_total, storiesWeighted_total, vertical from db_topics.celebStatsTotals where celeb_id =' + str(
entity_id)
statsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while (1):
row = statsQ.fetch_row(1, 1)
if row == ():
break
idx = row[0]['vertical']
entDict[idx] = {
"stories_total": int(row[0]['stories_total']),
"storiesWeighted_total": int(row[0]['storiesWeighted_total'])
}
return entDict
def insertHash(hashtag, type, type_short, block):
query = (
"insert into peepbuzz.hashtag_"
+ type
+ "_stats (hashtag, "
+ type_short
+ '_block, score) values ("'
+ hashtag
+ '","'
+ block
+ '","1")'
)
hashQ = mysql_tools.mysqlQuery(query, infoModule.info.site["dblink"])
if hashQ == False:
return False
else:
return True
def loadStreams():
query = mysql_tools.mysqlQuery("SELECT * FROM `peepbuzz`.`streams`", infoModule.info.site['dblink'])
streams = {}
if query.num_rows() > 0:
while True:
stream = query.fetch_row(1,1)
if stream == ():
break
if stream[0]['name'] in streams:
streams[stream[0]['name']] = stream[0]['stream_id']
else:
streams[stream[0]['name']] = {}
streams[stream[0]['name']] = stream[0]['stream_id']
else:
return False
return streams
def loadStreams():
query = mysql_tools.mysqlQuery("SELECT * FROM `peepbuzz`.`streams`",
infoModule.info.site['dblink'])
streams = {}
if query.num_rows() > 0:
while True:
stream = query.fetch_row(1, 1)
if stream == ():
break
if stream[0]['name'] in streams:
streams[stream[0]['name']] = stream[0]['stream_id']
else:
streams[stream[0]['name']] = {}
streams[stream[0]['name']] = stream[0]['stream_id']
else:
return False
return streams
def checkBlacklist(user_id = None, account_id = None):
# Make sure we have a user to check against
if not user_id or not account_id:
return False
if infoModule.info.site['dblink'] == None:
return False
# Find which field we are checking
account_where = "`account_id` = '" + str(account_id) + "'"
# Assemble and run the query
checkAccountQ = mysql_tools.mysqlQuery("SELECT * FROM `peepbuzz`.`blocked_accounts` WHERE " + account_where + " AND `user_id` = '" + str(user_id) + "'", infoModule.info.site['dblink'])
# Now check to see if a record exists
if checkAccountQ.num_rows() > 0:
# Looks like they blocked this account
return True
else:
# No blacklist record was found
return False
def updateHash(hashtag, type, type_short, score, block):
score = int(score)
score = score + 1
query = (
"update peepbuzz.hashtag_"
+ type
+ '_stats set score="'
+ str(score)
+ '" where hashtag="'
+ hashtag
+ '" and '
+ type_short
+ '_block="'
+ block
+ '"'
)
hashQ = mysql_tools.mysqlQuery(query, infoModule.info.site["dblink"])
if hashQ == False:
return False
else:
return True
def getUserList(demo=False):
if demo:
users = "133880286,149198901,101695592,77779709,145125358,135421739,24417275,50393960,131805814,8161232,31348594,128155589,132385468,38403110,123287206,22184352,109275593,59808841,24929621,18839785,97169575,87118217,87170183,101311381,113419517,66606024,56631494,23047158,813286,70179948,24705126,23088711,17842366,81634880,73571482,55593594,80648576,65076023,67227697,83683860,82550972,71281224,31711374,67003520,80932205,58270815,78589660,84770446,43047397,66974773,65560119,51807659,71785951,36736716,81037355,85037143,38323168,17109111,25764613,15989467,23406744,21801881,21424347,22765820,21043020,25966862,17386471,21575516,27606345,30632568,39157919,25477690,23728664,29498919,26911579,20752370,24110202,24326028,25574567,20061192,16354940,19429373,21990842,29625563,30204765,23328321,18320874,30313925,21370475,16434558,23233419,27488712,22543979,23210425,18346856,244655353,27474384,14171957,27583520,131144091,31732799,23551895,27554803,17494118,8316102,42132842,28682172,32613916,25768462,29971169,24721128,19885070,29297677,18287787,15764644,44621097,18278092,6250582,17672407,21807738,20314162,28637856,16627822,18501269,17104244,21225694,28353149,14959344,17825734,17429421,18824182,25914362,4358521,23532875,25154750,14142269,15090987,19415715,10552522,24914623,18444963,30341243,14715999,18663330,23686579,19099133,24333126,29103898,27982454,15690193,23237369,16672734,33087265,31580676,31108370,21113062,158769098,16031927,20714308,6017542,1880781,78945513,202022019,21313855,24618431,20073874,41634245,11134252,3135241,147725246,56413858,20545835,14345062,40116885,10774652,16180004,114655637,60680268,39308549,69773058,73402979,196168350,37764422,103060329,19416225,16555200,18451200,20094138,28195764,4898091,16311797,65146567,19658614,1917731,90725320,19659031,86364947,15450996,7228682,18956073,2704951,46002396,18513241,19611483,66369181,21186125,10126672,16244647,8775672,31310158,28576135,36681590,5988062,25395727,16869718,23725283,63150856,15988710,108351678,20024413,2467791,15918353,63395170,30411537,87103895,15985455,14669951,14844867,57428429,45190383,9300262,10168082,30864583,19417492,16467567,17546958,19553409,15918708,18229117,18216752,26533227,7713202,5357812,9508922,18916432,18217624,18906561,1367531,15675138,3108351,16032925,18646108,23970102,25320005,17454769,22701818,19899088,17844611,156783914,21721528,43008595,26429932,41073372,14553905,71028123,16305330,39803397,159875784,17855061,34346559,16643805,256865137,21987613,13601372,23388896,28706024,51893943,19475309,88663606,138135579,19317307,20532520,55656355,39880788,54579254,15416739,6604072,195223934,65629866,74231747,15883275,47956412,13783772,17998573,189181305,36775887,14619211,37360911,11640472,14089195,9695312,21052372,126707999,6836602,18185605,21466569,113436270,234814458,14085798,23023236,70858251,93819415,153830702,9636632,15430198,134537602,61621735,29899836,15394688,46241448,52786408,73060532,26765003,39588173,78377692,4517721,240892859,22696335,14253479,214677862,10122292,262246353,18510860,180505807,146993421,38883394,33696409,16476000,213796133,243361655,17667607,7995512,44366931,19374728,246913104,141335663,15066958,224300799,150723226,45564482,19195914,140882350,134758540,15117711,18990065,14411731,176936320,225847395,82149912,121836937,26520417,10876852,8383592,1319181,40519997,32518531,35677276,140496824,14079576,755756,2363571,811303,1586501,12798822,23838203,14981380,211891109,59477295,164725958,48499629,14514804,12101862,8237482,67358777,199097888,196352451,19658826,41187603,145753059,34147744,28520903,18051038,15846407,124245139,15591578,67140324,14109167,1284921,29631095,15891023,115230067,16204491,114870386,14179105,15719490,15645093,11336782,150990112,1652541,18164420,34228779,115699970,158865339,18956903,26642006,2195241,22569599,30815404,765564,40283581,19289684,14019652,16361348,19548547,14138514,18875991,138594660,60025762,16460799,15155313,13027572,71051839,6129732,11880802,18130749,18201016,20689408,11348282,16376711,17904776,15205798,46249360,26921818,132811099,52848021,5360012,14281466,22802398,43194755,13058232,12925072,104528362,16302090,135962109,9855132,45577446,24223641,121812725,65987618,21414576,17439893,22737278,11178672,14150661,56792882,24807616,16029780,30544925,33342154,972651,7581972,115710614,3939681,14173084,21875834,14266598,15639642,16012783,27911608,10235102,16661296,7215512,19621110,38218953,62591681,807095,14634720,27740227,19058681,24457744,16356190,15224867,10340482,89309995,17220934,12726792,15458181,54176768,56495963,23596644,15223265,18713552,13201312,37703408,15575625,33248363,107318424,133093395,19929890,56687786,142722336,53556894,52339151,45796568,57928790,106034413,171117515,21425125,137616033,104740065,44599982,70903729,67712707,53355887,108936259,95917731,18681139,97865628,15037033,16425333,56336341,57998991,124258971,171145913,86254626,43017766,99642673,77888423,148248527,42908132,88856792,87484569,111615736,122995784,40884854,36412963,56312411,56685329,56304605,27602673,155151279,37168231,177643755,73635568,24167314,6708952,116763916,19915362,75069067,169021842,14499829,140114710,36683668,15173291,17232815,17539739,68307227,34889448,19918047,39511166,95938895,71873770,18038892,94143715,5120691,12369372,19038395,16120265,18091904,24741685,19683378,39514461,18225966,16348549,20602394,43849753,37926315,4432431,27830610,97219875,33933259,808627,15234407,42226885,35593203,16190898,20214495,20032100,816653,749963,14511951,5763262,14293310,17074440,9460682,21457289,17899109,14874480,20280065,14761795,17004618,114423418,75050496,20224928,6799042,38087951,2184141,190929822,16303106,163866503,62770378,22484810,105675866,17940246,216004224,15485441,125463397,177168334,41814169,181936784,87429873,12044602,66883090,107544554,24671422,140055917,116290235,109740182,111186088,111499749,105122834,26530138,115474823,122474200,122062166,122553145,120641563,96400556,112065034,14435000,15677252,119557197,56115426,16684951,122405991,49989191,28573657,69570423,107139187,45553679,41355103,37632896,43275808,56378022,19337439,78262617,21235661,38143441,52437460,51096668,16350228,63681020,17536755,28114190,23775585,19734494,91083585,20626359,17872077,15378041,81690879,16454856,5172121,16489986,14866691,16232553,15745139,7744882,16591424,18302410,22927130,28333890,65276277,24447643,5768872,19655584,32561466,26208862,19076596,45567297,36021653,29525603,33235771,35846777,15364197,11396522,21695241,21466703,19529816,14322435,20598533,16694619,51376979,195703895,181809164,135456572,228240073,34131007,34197952,37034483,428333,5402612,16542390,112683235,94816809,28694130,70931004,90684808,85312583,90680573,18134734,80543346,17231318,14582285,16143542,14868835,40885516,97611573,115341081,132842874,132094623,96283286,121440641,132859239,94163409,123774113,48511486,83611972,18872373,76294950,198478747,24177024,98932084,42604232,61755650,16542626,22262999,50243043,53718666,113278275,138002210,146385708,130949035,33891322,29951362,50544680,20609518,159039098,154493261,24008967,73638336,145338127,68399705,119326030,114785416,119327640,25478634,102594253,18863815,63438962,117732057,43568964,15044593,56935361,92211391,96101258,78941611,113681700,4836211,92332298,93622144,65036099,79415553,51690631,42464353,32862653,41097723,71992917,15106168,38479920,30829649,16507391,41799998,23061619,54464822,31109504,47753886,65759352,55520719,28011481,18009267,16417648,14620392,7104672,14774255,33893248,45162347,127034693,194534748,110360770,54902524,129262111,97614907,18628249,18962189,18905875,18955181,21387965,12873942,20139348,18349474,19782758,18908190,17643126,12765022,19977028,14914525,17751959,10401102,15523506,16700571,7799412,16042788,16704472,18536826,8039622,15504971,97937152,137418466,155659213,19895282,117063289,92708272,181383752,48410382,106345557,34130969,15274585,92724677,121046433,69914008,196235064,70613646,215572137,121224923,155842625,222339038,64678911,106996530,149593459,104540706,166158304,211425214,74783784,206871837,154576522,99448420,139022142,108998148,26264871,77222698,129136581,16998020,103770785,119601029,30631766,87174678,13838562,97396580,21453086,20136728,66778288,35104773,56761423,48277760,19183953,233287824,18222378,245891639,291,13,20,12,261467278,266860992,28904686,259601739,82952848,269649865,57865199,177468546,29784580,22223740,254211714,54794774,258539150,156828337,34775386,21200792,96821243,24966698,25253084,21070607,112517482,227152801,258691740,37620532,266259739,32837358,263319842,36139712,265820934,247424639,237279585,175157275,23779324,51670525,23621049,159544785,150120631,20456522,31141686,239189534,26589987,79940688,232378553,216299334,21532770,95386169,246319243,253784164,40262755,262191109,191690176,221123236,183023359,258928180,123044254,42663256,97199688,34811069,122160076,47339690,231216037,28041769,97804036,25932078,35280333,259506017,39933869,259379883,258969640,15676578,253322314,253223778,257970633,45302570,55353207,160127025,22330908,237043022,257911756,5490392,191640178,25169275,112057008,136943034,173182035,170079806,70157977,23277580,122826599,99167992,44785212,236021571,32453930,45025729,250551703,95386370,14813320,243070104,17082958,156132825,153684144,23588838,211894963,239160456,81761893,131228928,110365072,44592558,243865694,188039706,242567007,22256645,126177871,20750019,245950301,23125912,23741531,127493777,16857864,21450669,63909970,20014300,180326389,152696429,215138502,112596025,219243547,90700803,187412435,202469077,194748700,205545055,91041719,175298380,124556528,176700507,122292598,175334528,116438547,155443008,15439395,174866331,130684475,181547288,182401865,138108708,154843423,85436197,29576445,76643486,153264772,97893963,135323671,150619089,142598575,21926199,98969077,29098641,33443003,142957691,105971448,59281659,33551876,16297426,30440025,25807119,42610327,21469205,33458339,84542880,28534252,21089219,112578012,85170052,63172203,47571031,26682924,106961801,92424604,80808661,16128694,9237112,47940349,161679647,19675870,112099774,116412043,188273028,127169087,66334209,56637669,52690690,80018213,14087270,39736606,56039856,21957826,100462369,69303631,51416299,70652594,141797364,178060134,177449553,105468025,94263219,58173133,120998613,58809331,144755081,20346956,95377752,111871312,44588485,57107167,100123541,17717614,53790896,103603559,101314413,89202391,24775528,21078156,79915337,64160613,20033989,92726549,22910295,90413046,90222460,82949914,44345790,75714954,10810102,27238177,17461978,48990944,53555106,41364508,66075817,44635677,61197829,44409004,116683144,146169170,67551899,40441273,17685964,25082055,23592970,75854417,114501238,17753033,67602060,58153957,57947582,66421178,14075928,107012281,252008932,220228167,47127694,643443,110843872,43598400,184095031,35927341,118201804,15093629,32613163,128260991,115313754,116586108,15588657,88065828,96827376,83529586,156888235,116677732,121131508,123255923,122928781,146066986,15941979,92753384,6509832,78526519,113586227,120786203,108725613,54387680,70565268,99693145,54829997,50374439,39730630,26565946,94743748,91855786,92732248,58166482,33801019,72857723,42581523,19397785,184966814,84554399,102261535,188161678,77324878,52369385,11695892,104864058,83200505,50745015,84276772,151554374,21380243,134805090,134805458,149186514,99735551,52171246,33465070,15639696,17774849,68663578,262625092,260839582,17482727,8933572,120965579,101779040,14407089,46073276,84352985,250495841,138822469,242253020,17293897,17160680,18745932,122994639,18549724,17342842,34351681,11589192,219617448,77739297,118757160,41094453,15355775,53712478,47685065,17152685,53256946,12716032,80330381,50973651,68164273,16237067,29173795,21014078,50944856,14582087,30486243,110384744,46704429,12757912,36916922,17381587,63129578,18185582,6463042,4047721,43533997,37629039,75896448,19329136,8805142,84620593,50026402,18639738,16832632,18267528,17289530,44582818,44317593,10807392,12954612,21213956,22179994,23326165,21324258,1468401,11347122,15234546,21131816,20067354,14094741,19538986,19923144,19637934,12687952,14862077,158132144,48100887,265084763,130894721,46611379,183764416,143244854,25521487,249308339,113425681,63796828,249492639,231653600,140223443,16913521,94618543,66237835,35080643,29461062,25798558,113101267,17874544,30464067,104249727,23018333,16667262,18220175,71876190,23511303,38002432,74021398,20495104,27041120,19359780,16410830,14302111,15235581,19343576,59542606,15185743,15846527,93122069,76396464,159959486,18432691,14668524,21724225,24868643,18458309,15774871,14872190,23713968,16037825,30915787,205558062,20691661,164912274,166679800,24148980,14841371,16364793,14456675,15573613,120194563,32588032,14702835,162501940,24072419,85339466,16909049,93306677,154511355,98929990,20789748,24647812,41822025,26095370,30006160,153407964,23314049,15907720,20155760,32661253,24019955,25705373,61903300,648,193402132,48115928,119547230,159190716,7885972,16007259,15924107,133238164,73057335,67024873,81731553,78727701,76439500,76089009,72358084,47938250,14402059,73585885,47859817,49548303,27216225,14909713,52612298,65380162,68141256,68909328,50648473,31693895,15354216,38088246,64666523,48588200,68619238,57591415,70175558,91935208,19394188,87021276,72506502,20536157,17728580,156347053,70300310,59019406,80629654,213988401,103061572,23006794,135109677,136594811,123822668,176295495,18735898,181041916,41552881,65659343,106310615,165109462,161548943,108252113,118893858,49657979,25365536,92284903,58547872,40255499,79708561,66306024,95385188,60011582,56913467,68098838,41790315,55778775,58387890,22868769,52603975,37750850,27205294,63982262,73303195,67836471,16423109,19662280,22563769,14506253,10350,24019308,23680913,9930742,989,20109973,22461427,18002296,62295319,14824849,16829827,22940219,15916582,244766451,97720704,157439536,86250341,29856819,71735224,83482914,76975035,20453602,122584206,92820744,81162841,118649107,49320694,88127502,118462618,115946251,119298031,18100055,100784944,77436150,80215266,58813207,60162223,62909470,80857366,80346657,66153834,75647231,75044398,189484942,69826593,183147986,166985108,158639282,149003180,150970545,18306168,131258077,140393642,102979395,101028230,89251621,85329365,59776155,23375688,16409683,14230524,85603854,3004231,65992743,57024049,159992786,21289311,16933337,67995848,21447363,50751556,70659694,114259282,16023803,69382909,101675562,60349064,57405030,71298473,71150249,67224001,65026150,33931356,45866936,36022940,43871900,45001048,46619479,21986512,18626887,116423810,22179561,41025510,15499660,16736293,58460325,100808813,45766975,70631526,39430669,39081659,53041067,76496142,66099575,76271580,198608682,43151567,18924866,17344361,18929773,15749983,14462907,16548435,18393089,15290123,14244769,188265738,14587429,212099989,97867221,27385146,45533787,145773163,87735340,48928735,40056109,61232019,62011530,39459292,26705065,62678144,69813431,20357019,79406971,40536441,13262442,20108560,46079391,55840387,15521075,54952792,52671787,43869144,18850433,18105473,52309424,51964156,783214,15825547,7108192,43542679,46103496,48059907,32825358,37224882,34284467,37382893,42825044,131442917,87942403,68217476,78659919,87994171,94523999,118649170,106309287,92945681,184861637,103913173,168606635,133690223,45022448,28344257,75226979,75196163,71837973,70023070,69044253,67902548,67091953,66981966,22913560,18174776,22125285,23476572,32479720,18175567,47174655,46085006,37352470,28290073,62552583,58136738,99925059,96751605,80285804,77445473,86896314,97186138,100778293,51475215,46392686,17800797,17006157,48410093,187507648,21551205,27781484,15076390,16578309,55422711,23524507,68359712,68071114,17732661,76986804,15766655,16324504,48907063,80864151,117333266,24409746,98816681,129724841,165174290,88287823,147293037,96500633,128124036,156003699,162913443,79453919,56143594,70307067,20584873,72756542,61703078,52655094,77447919,16382194,65054736,67799531,42826666,28824737,185122050,147001819,131607491,88718155,38740909,63509154,95943913,112508240,22534086,18393773,22548403,18131039,15573707,10798802,158645880,18286505,19399719,153051851,117426684,47715374,20326436,33402909,30055564,17600124,25053299,25166595,22536055,5920532,68509753,31194444,44462811,18450106,11018442,20040003,69615397,22171728,50687788,15162193,41338503,65104337,204491797,60054156,24047612,19628693,29271362,42712551,75014376,16147150,14620190,205789121,14275290,79320096,14365068,18735040,58582341,18639734,18665800,759251,19790261,91478624,98841303,113676911,42519612,21008866,18673446,18222770,18222599,15781432,14379501,18845207,22912596,208019751,20727198,18127783,5503832,268163148,29483675,109713729,42858171,23362483,37440098,53756003,249941736,18704659,184647530,122444371,21464713,90420314,106292326,111463209,25013763,100381915,62718650,22110491,93935921,94360081,95515846,96851347,111690277,212782910,22992332,23520739,21267612,32087193,58513684,46570247,115796187,28638191,110886577,110867752,48363171,32192094,26507551,27592116,43768449,56783491,32749378,91242884,55403897,76052943,17372853,22307857,46297163,19383317,27778893,24285686,19329393,21111883,17919972,36726043,28094035,192747213,44924709,257262765,263359642,262734281,90537810,17512682,128744229,22270337,22964152,21950364,47462001,102441113,45215683,247008365,14746298,83723250,1503121,22135992,102476260,247752114,225919331,16971999,131979670,14409497,15054622,170186151,14864693,21597063,242153407,248815930,20756490,157778204,15261298,260064723,21633857,20212370,254186356,17694756,37396006,17911421,124763921,13821862,21062471,24363945,153168449,14260037,21303400,44494276,5568292,77026540,18025954,16898332,12372812,126722715,116387194,22876454,178071411,110905570,17235724,22505100,239571244,136568901,15447566,249854076,17783446,23334601,142901066,57358233,65533134,41911874,112740986,232880095,17599795,17890304,157380172,87313007,258014885,17416700,14198654,259345913,1240681,33935084,21217402,32623377,27275087,15771056,92394644,16522352,1947301,91518617,188154135,241201438,197594920,13623532,36719281,188591875,234826866,138787319,211921304,9855382,153149305,115911638,166252256,19674502,86697288,135230178,153810519,106448460,48706233,13491312,104324860,82161632,117821354,72631996,47279525,18990471,66369206,67378554,18215973,110823581,113420831,105174965,63246339,54546430,40300027,61853389,21857760,76936279,78408666,76348185,62595437,49706919,54655632,61762853,43920155,44615672,48398586,78138151,33998183,76091017,21783950,59204932,44777959,24752484,39657463,14224719,14091091,17961886,16245822,16116713,14350393,17409240,6450322,14079839,9109712,14287409,10894562,14616022,14479720,14505838,15170346,16028241,15647676,16669075,16027546,15649433,18020120,17451742,8487622,15460572,15743810,19109172,18923032,18058609,20456198,19006707,20436059,20179628,20437286,17967675,17974455,17968875,16878156,266830495,14858000,176944763,128225158,16571326,123974480,20776147,16625082,51462013,48029363,65502117,27311044,43881089,243000634,136389258,238259855,160276009,248945322,29235589,20221159,23199697,63877097,36804447,132986633,130945778,17856665,15826822,222140591,27948604,166608401,164383046,244212645,13215382,21844854,24404039,11856892,143798379,180090338,9717322,20733972,33919343,71022729,17494010,18365936,16664681,12701412,26837501,18080708,3952661,52954222,76707735,57770539,234979566,15017893,174263663,106514937,61220477,115979444,43409523,66873399,46469610,189550099,14226495,123738314,6449282,16125224,57768217,14552725,21768766,49620730,15399512,40260612,146167871,196824385,48388417,7212562,16326882,48043084,16081831,23176276,134843187,97474887,55355654,15670644,52774637,173999134,16261445,190298721,4207961,21840737,30907284,136550204,16224666,21146211,258928816,241990563,28260687,259905595,18727438,15371215,196631065,41219840,95493980,15498429,102786512,180859549,14677751,20818801,16250929,93069110,18891923,19670455,8370082,16640598,93876666,47688440,29795115,14327933,34022952,17391507,224464606,32155953,22957302,5392522,46612327,114497182,109287731,17169320,33986037,14129299,222205782,224424419,225234560,15265641,21178616,22771961,18873711,14275426,24295482,21457205,15901113,20713061,16789970,73238146,69181624,22772264,79743108,47694175,21674448,138479037,18686907,24892146,49337217,21699092,17786474,69218324,49573859,48038030,171129408,14804435,28479007,142419647,198504506,46764631,68801460,16340501,14413466,4119741,82643763,89269664,85591466,8979992,42860402,39524002,46161313,21334893,40906518,255762643,96496611,76653027,14836128,22602881,22295473,77215251,28428838,121265704,47136500,95243532,59738746,18773422,50156221,37016784,234293544,32631520,252726343,24954899,118805793,15244981,38236362,31072439,43907446,17373943,20317201,31079712,70758813,33838795,27233809,51258903,37951729,50415064,37395827,30022594,17214648,88295976,20651632,16123781,119957997,24024655,33601431,23090525,37683259,27678902,43766620,17721514,15353981,18619979,31058793,39578080,22571950,20854518,19330213,36106105,47401289,22780988,25324156,22646228,29711814,31468242,41438347,29785315,31606227,46229563,31967467,33651080,39726700,24949726,15353815,25425425,23629827,22485793,20537454,14325369,19728030,22044289,24927543,28393637,29867516,20829172,15386108,22303692,21128035,21699429,17198500,15131310,5741722,5248441,237488632,20407539,17797448,30704824,153741488,15809249,29028408,19212009,56051834,14062236,65495877,112624301,28408675,59058446,33755676,84370522,264459407,86387057,17837853,24741917,118752616,223100019,93783780,66593978,53162803,192019265,17782766,249253283,22330646,199941365,102271048,43018238,20012204,108797271,31119337,153474021,235083242,109660695,49860231,2425151,190170608,49861057,169686021,17771860,49669034,122737411,66891455,188468721,50557402,29840295,229965178,222708016,244243477,15934926,184596818,65773523,15169907,110840720,68524064,129575180,101928415,25626894,246163327,207010069,40735690,230547432,141326970,28669612,39015718,128090469,49197437,163126581,107818778,16929701,50531062,53193168,39936193,44821096,46829683,46313918,49034233,30124262,33009321,28067175,35624959,34097876,26892157,30043495,21119636,79039503,17869798,21619519,40047532,32765534,14412533,14246001,50325797,14529929,12811952,7313362,14688489,245510831,14498484,53197137,16212685,47964412,41144996,816214,15008449,24784891,27589818,39008816,113880573,930061,87904422,1365841,1501471,14413715,1242511,14326840,20333461,657863,1835411,16219126,3829151,109275829,81890700,30345764,8442372,41269920,18840896,101844850,8953122,209792690,40168339,18915795,15664289,133578965,13213122,263767927,71294756,120943272,137466464,40553282,135897307,21348880,18227519,21111896,52772661,91888403,25355724,247033019,25527966,83590349,117928875,21251668,234629912,62915920,17095452,14245378,23778898,196272316,248427077,36554387,224119738,17291393,80638910,40892016,19107878,7380872,56700964,50073507,63004503,21268897,24431892,21307076,21982720,18791763,27901418,18172905,9532402,16037856,85131054,15210410,51263592,43556096,26624619,130617778,52070270,158414847,14268812,92079350,24816848,50323173,187005921,183665224,234521957,47260400,18678924,7651862,58601997,23544596,33792634,91181033,14603515,19071682,20064236,7768572,149217510,188975706,22169931,145492157,14677919,114589771,14693823,15519544,201870414,17781165,14702533,13784592,39050020,167181650,153051008,14861285,30261067,44177383,65647594,28143425,15194013,18959163,28144343,16228479,18901122,36958781,187319644,118022756,16129106,54885400,1068831,68539526,15339201,1344951,2890961,119525215,85634177,7152572,185644175,5561412,14434063,11270992,15024563,16256269,21157904,15572155,31182553,15309804,16160352,7334402,16129920,17912982,16598226,35373847,56705653,250831586,27529145,26618468,238073909,40626514,234132208,235120302,20402945,216431881,35989792,229686366,23539358,169608502,268123287,225278024,261835843,28376631,79293791,6446742,56518880,18915240,180188862,17200116,32948225,25398075,25677409,41737887,18987651,29952857,15537451,24322371,19877186,23967578,46252298,28471026,27406401,33549858,14817982,40141037,24472704,14860448,26933384,82688781,35133435,264938614,172562639,4170491,17292143,16347506,143664435,23065354,19942991,19410817,229129090,237620586,228296619,30959811,21656599,261555701,29365013,105916405,188639874,173581919,22831662,47218790,37367041,16302242,39288259,15126367,25482459,19867571,229390349,35586563,105029433,183089853,23184896,38127255,16966162,151159882,217231723,48488561,23065057,231389485,58294430,254179370,127750483,5162861,187279871,190064796,78575667,21077000,25174913,94216840,108788135,111635980,23593446,267854863,78445977,232992031,163570705,150669746,14342564,19658936,19418459,40959360,84388888,30343716,260845829,263815367,193732179,221792092,19716781,38254095,73181712,65707359,9624742,234489403,248735463,168673083,247486443,249410485,30912937,249348006,246088673,235373000,246357149,210926192,158398519,137823987,33530012,28602948,236511574,234002627,153944899,85396297,240812994,233783568,199325935,119250381,237770636,161791703,242892689,239949176,242426145,229873610,227341953,216503958,24721108,132201224,217547360,208692908,48086014,30216513,80612021,126017061,148453195,192957076,24745957,188019606,24232525,140519774,103922067,28582749,39249305,16583468,15022633,75124560,125445568,34972633,84119348,107545479,138770045,36948268,115111016,153865861,28599820,161411080,33563161,56864092,137794015,150688795,24913074,27698046,45659261,243721842,88507556,247334603,16329577,18812301,12671692,6527442,53981494,22642788,5446992,24619328,47459584,19824430,10371822,117445762,128564422,23534104,22677790,242360696,19891675,54299209,211588974,20065936,74880863,15210284,105445396,93722614,53130511,24844525,14824473,18644548,40277679,159362503,23384516,119079595,20182089,203681891,15093646,15808765,47632905,98425584,15772733,20177097,198106297,14992591,21623531,158412741,23220034,21757965,51224438,15638637,150203689,17612289,104937383,16955870,17299063,68773980,176280674,19278537,95281047,12340562,16440738,54905208,10228272,24024778,19541375,7855622,52254824,237562389,267847170,202594374,8180562,258968955,110757459,151688371,54290523,4262791,6793362,198571335,215396550,144121755,248390035,216216791,119169760,124635414,238144920,236108642,199730264,46812100,220882223,143822047,15693634,172717529,147249049,133727476,203728509,731573,801202,38299186,239200202,125492611,110905955,137751549,71918994,205370456,81189096,37083812,37267497,52167970,4778211,175083160,8105482,118256463,132704571,16160650,114510872,994431,10052972,16194793,13860082,31237389,48920331,54578941,38100497,140935414,47850508,122975302,170316873,45497378,3551701,150115256,72972715,148939636,1120611,136414500,140538938,167624175,174027585,27641388,62015732,70346167,780159,175881965,37093819,19646485,140856303,77338274,187246188,122783603,50471406,11202862,151198642,76326090,104861015,133722469,121613012,47616949,14161128,819256,221019516,9766032,13836402,162104820,618593,47402960,85851159,142684449,63125213,166510517,240615854,269394406,252644913,38371041,136609376,76282043,70418699,185493136,17960806,38727269,234587458,31088763,147886032,265651031,212940155,40565664,201000503,197539670,170474186,263060411,254725084,89866540,130782156,223976506,260391020,19390708,124856935,99698716,52256215,260728760,156769030,40866430,35282604,261151943,58777905,150957265,86351265,179719319,189842436,204918670,203483715,258220853,261297092,204444285,54171939,214020774,89187513,170581751,42612282,44337867,212342630,105542341,190776107,36919153,134781128,47125833,244901765,242245196,194186291,242048469,245106788,215967395,140766042,158795894,116688855,19975710,109892942,95666954,32848956,240815975,77977289,216335730,256471145,247004791,175867315,247424331,107021348,29730710,38146165,237670656,191072401,125075147,15417049,219936176,41190759,179884440,41132274,253520310,73191618,40850899,42981176,112184431,87767749,162022136,193022239,186967108,84145752,227058289,641253,95994080,158848407,742143,41842768,242201454,22240612,19386622,23144737,20379995,254377876,16489591,30335156,13363282,249529225,17625292,151662442,245935903,82274636,15426758,17918808,136358789,169793441,117724817,15769161,157613794,20696985,216577574,116559622,130681675,74594552,7081242,189276359,96265654,20460757,14749606,231362108,222343099,235389922,36050060,44518888,115689700,235295798,189621457,226370793,15960627,20596269,6324592,82778942,43192807,34716594,36803580,52481056,11891542,199187259,111059621,36355699,56758031,15313857,17637614,20646277,100570170,24193017,117984119,22692199,6033532,116828327,20731304,106565458,83883736,17637553,23679582,29256864,214499442,140964771,15888570,44062282,18713,106145765,15255133,43333230,20998647,31311757,15855630,21008297,20334391,39348813,41264383,20660560,19343562,23116280,111444854,8039442,143454922,267206035,142977623,101879887,17565514,28932160,238480106,253632034,87471023,184910040,19226054,160817572,87650675,70246837,36600922,240860245,160926944,41329637,40981798,46345843,30563759,114225051,219920508,18918698,189283341,72694909,15790423,22440038,235120811,222658804,72547608,68838777,233749221,215952307,26668282,151895837,25456478,216995705,87416722,17612928,176417690,41218028,46715620,104571475,91390383,124283714,19026618,49393335,18950761,21526685,134206307,90098789,34192959,186865762,50600062,126418609,135303236,14522546,64220324,30517544,24083587,21258330,38475205,18953259,101192354,8973062,18676177,55544943,34108173,32360145,140894997,19818494,12085042,40389429,19982092,19524590,31239408,21869297,24450456,250463314,238295737,236917131,32397000,151631699,29257521,17235825,82960113,42491255,18044981,181988782,40058557,63350773,19320490,27246723,17119099,95921754,23069301,51865695,143549770,84460847,223724339,93904007,199555388,81929393,131727620,82444043,162580589,171998025,198334225,91171720,201426746,166316761,231206630,125382460,195885307,113678606,103000558,19019263,87307633,18860342,14705240,29160552,2885851,16185801,210693321,15276510,27762127,17147539,21019628,74712538,131315786,15012486,51193546,74735391,18335809,24796185,123331001,73134924,17193794,19432557,24872865,32128783,16622151,83939091,15321447,19733892,14517899,18140091,106506464,24663730,18145841,31282081,16324533,40894238,36893069,74502636,36975611,21375959,29337732,89642493,41407443,12138162,19198326,20692503,41135906,23532159,18148242,14326661,14423603,39806829,133966853,176467382,14463913,52185212,60680365,46743554,87989022,36143299,18815810,34002530,41888469,125415597,35667714,191250892,177343973,165802132,191798541,253342708,232878783,72168780,239613729,136353114,112937614,132618751,190475202,188579340,241837295,241805454,147609274,87871306,235433784,128295129,86566274,162571150,99002460,169719009,240858138,131246928,83900124,168410930,33623402,129254485,113593692,84358766,229643491,50841062,32730931,112074504,138871289,130734452,239623882,162254325,83237549,92817267,197874533,145476646,235792533,178941141,215819570,78145300,148504230,51646374,44758790,206922924,63896875,91437380,105320972,208777407,103080971,231279879,101844353,102607698,26395300,30152403,28050738,94225402,220252762,29091792,24967668,37986775,42082057,74853246,122158761,16234406,31115275,27195114,15942017,51158957,22498452,32482524,40286007,16870682,23618512,23832022,59286800,36860144,30022428,180999717,19028953,53153263,31668714,24621955,100611064,220675169,79551906,41676395,66714703,111673770,97807215,23819304,28605533,25695492,29537319,224325820,30810014,151291323,22100074,78687792,128472974,30113082,18803364,66003255,50572630,213840716,17915334,255591392,198613270,249359076,28110915,30630900,158887925,47945698,36425615,30689667,21541542,118612929,60401740,42203744,101578775,233697048,24293519,19042478,183879398,38773605,101424299,113508007,42371693,28702869,201445961,27060993,59606882,94370073,197220548,121269271,233395335,145742095,237955563,36897911,106375253,154423460,39002053,52963095,30441816,48649566,93928566,27198826,247495144,212479278,90721189,28303502,158393753,24100737,164818150,87623304,201989827,128078136,25465958,110473412,42099565,213707240,19312934,29533792,45419359,63311667,36126354,30228023,70343136,22733444,160027905,49666182,47562163,243010689,78507740,243516650,51219197,20955713,121512266,179767114,94784820,110585067,23586754,132223511,62644624,8713702,2148071,616113,7315022,822112,13348,16354128,45547548,85616044,10085882,21126003,16613870,65640210,11557642,14208914,14270999,20399609,99961552,32472561,39274758,19624506,3144281,90421077,7857652,46072516,4560311,97023892,76058815,5876342,29949063,1424831,10094,63133380,21786260,75616437,9745092,19226849,40880426,18484100,94587503,34406259,1614411,10016972,14140571,16044191,113116513,20169131,20217201,79502328,792646,270668475,53533566,157070412,59444261,39902637,165326620,123102622,265000864,52597678,3346971,49205840,112432370,167033855,129017028,132005200,58334111,93516296,168841469,229710631,175338338,241205936,116510779,84457604,264366428,52322656,234006638,258428607,168537997,95800032,143442028,82430417,84034563,13041552,185030770,32014425,85450815,29736553,53424957,172178623,14120187,67132984,74610929,73215961,140722325,65364950,71112180,112307482,122231942,189320995,178118343,251231223,266200838,70305266,131440156,262900579,249631963,33258173,34784263,105597663,101387077,37245723,48630048,143505274,153245594,61723876,39140545,215858327,168243936,30380192,39187698,235394328,62052874,143257853,77395953,26162191,191685671,40405121,267433029,50856543,168696795,84677497,30681499,261294795,265696487,183144850,127040414,29910549,50390358,30587056,78740274,19804553,53323885,32618047,49900650,144021058,241008487,237897985,75037065,203771716,42104946,111489389,227209417,44120318,234212815,38073608,97557168,162545959,108457267,261884913,204209879,129816753,121186812,161576069,38285013,30995591,60353632,30799610,181080474,239217629,254707723,212340162,46930437,257939339,32820524,89568746,16978961,235381394,93460034,21786322,175291914,185343485,76187493,36216833,46093969,164180736,95261792,25947430,168695261,156972971,196630780,102900059,62863521,25742007,171240557,35332069,50052920,127788481,111398486,254240011,32218646,176536528,139822631,98766044,19487670,236787450,137340994,128642272,20979106,225911802,248794623,75130364,264408558,15947602,225126598,237003654,19595532,265987070,66437032,14082828,11599312,253284344,78436108,122694066,17805567,207612806,34925804,171514924,188028765,122459707,95534157,217073451,20769938,223842388,29717916,259108421,240182448,248314631,73073062,164401022,57924802,20703607,251219812,198240811,40010200,228682873,47914608,777554,243833570,196167925,44877467,18212565,110950574,253664708,20767246,201747039,143415291,79743060,127635571,246738309,121845291,20508222,123979525,47650483,242655310,252488810,253069382,15020865,166926744,89702218,20081488,90276942,174717541,219716235,15906176,125378095,22707793,53454398,141598993,232821256,20906409,27066806,227631390,58842819,54937915,63968007,18764912,246263280,243920442,37376499,207707329,135178205,139853350,236581562,244593709,146787633,20373762,122714443,52487416,192218013,125820691,238956217,148354084,7064802,61133,14298281,20490861,809760,1493121,14336349,38073,14876536,6790312,205700196,200199318,15649570,15184418,51800278,19644571,29432246,18058704,175899673,89998965,51988166,87795815,14570292,15272882,19105442,7035532,112490145,13768732,47700913,241475497,157821181,14562275,20142938,180086129,237325382,16818325,21873261,241043955,16527490,240810497,115410741,150323560,36609309,29093456,17673732,16096448,216338061,235317040,63585408,236819345,93010530,190357418,240594439,239807683,165693574,211897738,112489818,47932614,55923778,225657320,186155101,16352458,240402102,71239119,34474398,239348795,234645414,240223810,141222424,240180276,15036744,94973632,54699775,196811548,141857015,238673608,97616524,224969703,238109234,231493350,239309716,238638637,218570379,39514187,158951763,16403358,239477369,154541100,239609301,239399824,121708861,195121864,72931614,142039374,239127196,233688482,107780778,20606286,65719885,162778946,95200214,215707078,156950036,15003756,7150932,71343661,11935332,200513252,79144576"
else:
sql = "SELECT `accounts`.`external_id` FROM `peepbuzz`.`curators` LEFT JOIN `peepbuzz`.`accounts` ON `accounts`.`account_id` = `curators`.`account_id` WHERE `accounts`.`external_id` IS NOT NULL AND `accounts`.`stream_id` = 1"
curatorsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
users = None
while True:
curator = curatorsQ.fetch_row(1,1)
if curator == ():
break
if users == None:
users = curator[0]['external_id']
else:
users = users + "," + curator[0]['external_id']
return [u for u in users.split(',')]
def getUserList(demo=False):
if demo:
users = "133880286,149198901,101695592,77779709,145125358,135421739,24417275,50393960,131805814,8161232,31348594,128155589,132385468,38403110,123287206,22184352,109275593,59808841,24929621,18839785,97169575,87118217,87170183,101311381,113419517,66606024,56631494,23047158,813286,70179948,24705126,23088711,17842366,81634880,73571482,55593594,80648576,65076023,67227697,83683860,82550972,71281224,31711374,67003520,80932205,58270815,78589660,84770446,43047397,66974773,65560119,51807659,71785951,36736716,81037355,85037143,38323168,17109111,25764613,15989467,23406744,21801881,21424347,22765820,21043020,25966862,17386471,21575516,27606345,30632568,39157919,25477690,23728664,29498919,26911579,20752370,24110202,24326028,25574567,20061192,16354940,19429373,21990842,29625563,30204765,23328321,18320874,30313925,21370475,16434558,23233419,27488712,22543979,23210425,18346856,244655353,27474384,14171957,27583520,131144091,31732799,23551895,27554803,17494118,8316102,42132842,28682172,32613916,25768462,29971169,24721128,19885070,29297677,18287787,15764644,44621097,18278092,6250582,17672407,21807738,20314162,28637856,16627822,18501269,17104244,21225694,28353149,14959344,17825734,17429421,18824182,25914362,4358521,23532875,25154750,14142269,15090987,19415715,10552522,24914623,18444963,30341243,14715999,18663330,23686579,19099133,24333126,29103898,27982454,15690193,23237369,16672734,33087265,31580676,31108370,21113062,158769098,16031927,20714308,6017542,1880781,78945513,202022019,21313855,24618431,20073874,41634245,11134252,3135241,147725246,56413858,20545835,14345062,40116885,10774652,16180004,114655637,60680268,39308549,69773058,73402979,196168350,37764422,103060329,19416225,16555200,18451200,20094138,28195764,4898091,16311797,65146567,19658614,1917731,90725320,19659031,86364947,15450996,7228682,18956073,2704951,46002396,18513241,19611483,66369181,21186125,10126672,16244647,8775672,31310158,28576135,36681590,5988062,25395727,16869718,23725283,63150856,15988710,108351678,20024413,2467791,15918353,63395170,30411537,87103895,15985455,14669951,14844867,57428429,45190383,9300262,10168082,30864583,19417492,16467567,17546958,19553409,15918708,18229117,18216752,26533227,7713202,5357812,9508922,18916432,18217624,18906561,1367531,15675138,3108351,16032925,18646108,23970102,25320005,17454769,22701818,19899088,17844611,156783914,21721528,43008595,26429932,41073372,14553905,71028123,16305330,39803397,159875784,17855061,34346559,16643805,256865137,21987613,13601372,23388896,28706024,51893943,19475309,88663606,138135579,19317307,20532520,55656355,39880788,54579254,15416739,6604072,195223934,65629866,74231747,15883275,47956412,13783772,17998573,189181305,36775887,14619211,37360911,11640472,14089195,9695312,21052372,126707999,6836602,18185605,21466569,113436270,234814458,14085798,23023236,70858251,93819415,153830702,9636632,15430198,134537602,61621735,29899836,15394688,46241448,52786408,73060532,26765003,39588173,78377692,4517721,240892859,22696335,14253479,214677862,10122292,262246353,18510860,180505807,146993421,38883394,33696409,16476000,213796133,243361655,17667607,7995512,44366931,19374728,246913104,141335663,15066958,224300799,150723226,45564482,19195914,140882350,134758540,15117711,18990065,14411731,176936320,225847395,82149912,121836937,26520417,10876852,8383592,1319181,40519997,32518531,35677276,140496824,14079576,755756,2363571,811303,1586501,12798822,23838203,14981380,211891109,59477295,164725958,48499629,14514804,12101862,8237482,67358777,199097888,196352451,19658826,41187603,145753059,34147744,28520903,18051038,15846407,124245139,15591578,67140324,14109167,1284921,29631095,15891023,115230067,16204491,114870386,14179105,15719490,15645093,11336782,150990112,1652541,18164420,34228779,115699970,158865339,18956903,26642006,2195241,22569599,30815404,765564,40283581,19289684,14019652,16361348,19548547,14138514,18875991,138594660,60025762,16460799,15155313,13027572,71051839,6129732,11880802,18130749,18201016,20689408,11348282,16376711,17904776,15205798,46249360,26921818,132811099,52848021,5360012,14281466,22802398,43194755,13058232,12925072,104528362,16302090,135962109,9855132,45577446,24223641,121812725,65987618,21414576,17439893,22737278,11178672,14150661,56792882,24807616,16029780,30544925,33342154,972651,7581972,115710614,3939681,14173084,21875834,14266598,15639642,16012783,27911608,10235102,16661296,7215512,19621110,38218953,62591681,807095,14634720,27740227,19058681,24457744,16356190,15224867,10340482,89309995,17220934,12726792,15458181,54176768,56495963,23596644,15223265,18713552,13201312,37703408,15575625,33248363,107318424,133093395,19929890,56687786,142722336,53556894,52339151,45796568,57928790,106034413,171117515,21425125,137616033,104740065,44599982,70903729,67712707,53355887,108936259,95917731,18681139,97865628,15037033,16425333,56336341,57998991,124258971,171145913,86254626,43017766,99642673,77888423,148248527,42908132,88856792,87484569,111615736,122995784,40884854,36412963,56312411,56685329,56304605,27602673,155151279,37168231,177643755,73635568,24167314,6708952,116763916,19915362,75069067,169021842,14499829,140114710,36683668,15173291,17232815,17539739,68307227,34889448,19918047,39511166,95938895,71873770,18038892,94143715,5120691,12369372,19038395,16120265,18091904,24741685,19683378,39514461,18225966,16348549,20602394,43849753,37926315,4432431,27830610,97219875,33933259,808627,15234407,42226885,35593203,16190898,20214495,20032100,816653,749963,14511951,5763262,14293310,17074440,9460682,21457289,17899109,14874480,20280065,14761795,17004618,114423418,75050496,20224928,6799042,38087951,2184141,190929822,16303106,163866503,62770378,22484810,105675866,17940246,216004224,15485441,125463397,177168334,41814169,181936784,87429873,12044602,66883090,107544554,24671422,140055917,116290235,109740182,111186088,111499749,105122834,26530138,115474823,122474200,122062166,122553145,120641563,96400556,112065034,14435000,15677252,119557197,56115426,16684951,122405991,49989191,28573657,69570423,107139187,45553679,41355103,37632896,43275808,56378022,19337439,78262617,21235661,38143441,52437460,51096668,16350228,63681020,17536755,28114190,23775585,19734494,91083585,20626359,17872077,15378041,81690879,16454856,5172121,16489986,14866691,16232553,15745139,7744882,16591424,18302410,22927130,28333890,65276277,24447643,5768872,19655584,32561466,26208862,19076596,45567297,36021653,29525603,33235771,35846777,15364197,11396522,21695241,21466703,19529816,14322435,20598533,16694619,51376979,195703895,181809164,135456572,228240073,34131007,34197952,37034483,428333,5402612,16542390,112683235,94816809,28694130,70931004,90684808,85312583,90680573,18134734,80543346,17231318,14582285,16143542,14868835,40885516,97611573,115341081,132842874,132094623,96283286,121440641,132859239,94163409,123774113,48511486,83611972,18872373,76294950,198478747,24177024,98932084,42604232,61755650,16542626,22262999,50243043,53718666,113278275,138002210,146385708,130949035,33891322,29951362,50544680,20609518,159039098,154493261,24008967,73638336,145338127,68399705,119326030,114785416,119327640,25478634,102594253,18863815,63438962,117732057,43568964,15044593,56935361,92211391,96101258,78941611,113681700,4836211,92332298,93622144,65036099,79415553,51690631,42464353,32862653,41097723,71992917,15106168,38479920,30829649,16507391,41799998,23061619,54464822,31109504,47753886,65759352,55520719,28011481,18009267,16417648,14620392,7104672,14774255,33893248,45162347,127034693,194534748,110360770,54902524,129262111,97614907,18628249,18962189,18905875,18955181,21387965,12873942,20139348,18349474,19782758,18908190,17643126,12765022,19977028,14914525,17751959,10401102,15523506,16700571,7799412,16042788,16704472,18536826,8039622,15504971,97937152,137418466,155659213,19895282,117063289,92708272,181383752,48410382,106345557,34130969,15274585,92724677,121046433,69914008,196235064,70613646,215572137,121224923,155842625,222339038,64678911,106996530,149593459,104540706,166158304,211425214,74783784,206871837,154576522,99448420,139022142,108998148,26264871,77222698,129136581,16998020,103770785,119601029,30631766,87174678,13838562,97396580,21453086,20136728,66778288,35104773,56761423,48277760,19183953,233287824,18222378,245891639,291,13,20,12,261467278,266860992,28904686,259601739,82952848,269649865,57865199,177468546,29784580,22223740,254211714,54794774,258539150,156828337,34775386,21200792,96821243,24966698,25253084,21070607,112517482,227152801,258691740,37620532,266259739,32837358,263319842,36139712,265820934,247424639,237279585,175157275,23779324,51670525,23621049,159544785,150120631,20456522,31141686,239189534,26589987,79940688,232378553,216299334,21532770,95386169,246319243,253784164,40262755,262191109,191690176,221123236,183023359,258928180,123044254,42663256,97199688,34811069,122160076,47339690,231216037,28041769,97804036,25932078,35280333,259506017,39933869,259379883,258969640,15676578,253322314,253223778,257970633,45302570,55353207,160127025,22330908,237043022,257911756,5490392,191640178,25169275,112057008,136943034,173182035,170079806,70157977,23277580,122826599,99167992,44785212,236021571,32453930,45025729,250551703,95386370,14813320,243070104,17082958,156132825,153684144,23588838,211894963,239160456,81761893,131228928,110365072,44592558,243865694,188039706,242567007,22256645,126177871,20750019,245950301,23125912,23741531,127493777,16857864,21450669,63909970,20014300,180326389,152696429,215138502,112596025,219243547,90700803,187412435,202469077,194748700,205545055,91041719,175298380,124556528,176700507,122292598,175334528,116438547,155443008,15439395,174866331,130684475,181547288,182401865,138108708,154843423,85436197,29576445,76643486,153264772,97893963,135323671,150619089,142598575,21926199,98969077,29098641,33443003,142957691,105971448,59281659,33551876,16297426,30440025,25807119,42610327,21469205,33458339,84542880,28534252,21089219,112578012,85170052,63172203,47571031,26682924,106961801,92424604,80808661,16128694,9237112,47940349,161679647,19675870,112099774,116412043,188273028,127169087,66334209,56637669,52690690,80018213,14087270,39736606,56039856,21957826,100462369,69303631,51416299,70652594,141797364,178060134,177449553,105468025,94263219,58173133,120998613,58809331,144755081,20346956,95377752,111871312,44588485,57107167,100123541,17717614,53790896,103603559,101314413,89202391,24775528,21078156,79915337,64160613,20033989,92726549,22910295,90413046,90222460,82949914,44345790,75714954,10810102,27238177,17461978,48990944,53555106,41364508,66075817,44635677,61197829,44409004,116683144,146169170,67551899,40441273,17685964,25082055,23592970,75854417,114501238,17753033,67602060,58153957,57947582,66421178,14075928,107012281,252008932,220228167,47127694,643443,110843872,43598400,184095031,35927341,118201804,15093629,32613163,128260991,115313754,116586108,15588657,88065828,96827376,83529586,156888235,116677732,121131508,123255923,122928781,146066986,15941979,92753384,6509832,78526519,113586227,120786203,108725613,54387680,70565268,99693145,54829997,50374439,39730630,26565946,94743748,91855786,92732248,58166482,33801019,72857723,42581523,19397785,184966814,84554399,102261535,188161678,77324878,52369385,11695892,104864058,83200505,50745015,84276772,151554374,21380243,134805090,134805458,149186514,99735551,52171246,33465070,15639696,17774849,68663578,262625092,260839582,17482727,8933572,120965579,101779040,14407089,46073276,84352985,250495841,138822469,242253020,17293897,17160680,18745932,122994639,18549724,17342842,34351681,11589192,219617448,77739297,118757160,41094453,15355775,53712478,47685065,17152685,53256946,12716032,80330381,50973651,68164273,16237067,29173795,21014078,50944856,14582087,30486243,110384744,46704429,12757912,36916922,17381587,63129578,18185582,6463042,4047721,43533997,37629039,75896448,19329136,8805142,84620593,50026402,18639738,16832632,18267528,17289530,44582818,44317593,10807392,12954612,21213956,22179994,23326165,21324258,1468401,11347122,15234546,21131816,20067354,14094741,19538986,19923144,19637934,12687952,14862077,158132144,48100887,265084763,130894721,46611379,183764416,143244854,25521487,249308339,113425681,63796828,249492639,231653600,140223443,16913521,94618543,66237835,35080643,29461062,25798558,113101267,17874544,30464067,104249727,23018333,16667262,18220175,71876190,23511303,38002432,74021398,20495104,27041120,19359780,16410830,14302111,15235581,19343576,59542606,15185743,15846527,93122069,76396464,159959486,18432691,14668524,21724225,24868643,18458309,15774871,14872190,23713968,16037825,30915787,205558062,20691661,164912274,166679800,24148980,14841371,16364793,14456675,15573613,120194563,32588032,14702835,162501940,24072419,85339466,16909049,93306677,154511355,98929990,20789748,24647812,41822025,26095370,30006160,153407964,23314049,15907720,20155760,32661253,24019955,25705373,61903300,648,193402132,48115928,119547230,159190716,7885972,16007259,15924107,133238164,73057335,67024873,81731553,78727701,76439500,76089009,72358084,47938250,14402059,73585885,47859817,49548303,27216225,14909713,52612298,65380162,68141256,68909328,50648473,31693895,15354216,38088246,64666523,48588200,68619238,57591415,70175558,91935208,19394188,87021276,72506502,20536157,17728580,156347053,70300310,59019406,80629654,213988401,103061572,23006794,135109677,136594811,123822668,176295495,18735898,181041916,41552881,65659343,106310615,165109462,161548943,108252113,118893858,49657979,25365536,92284903,58547872,40255499,79708561,66306024,95385188,60011582,56913467,68098838,41790315,55778775,58387890,22868769,52603975,37750850,27205294,63982262,73303195,67836471,16423109,19662280,22563769,14506253,10350,24019308,23680913,9930742,989,20109973,22461427,18002296,62295319,14824849,16829827,22940219,15916582,244766451,97720704,157439536,86250341,29856819,71735224,83482914,76975035,20453602,122584206,92820744,81162841,118649107,49320694,88127502,118462618,115946251,119298031,18100055,100784944,77436150,80215266,58813207,60162223,62909470,80857366,80346657,66153834,75647231,75044398,189484942,69826593,183147986,166985108,158639282,149003180,150970545,18306168,131258077,140393642,102979395,101028230,89251621,85329365,59776155,23375688,16409683,14230524,85603854,3004231,65992743,57024049,159992786,21289311,16933337,67995848,21447363,50751556,70659694,114259282,16023803,69382909,101675562,60349064,57405030,71298473,71150249,67224001,65026150,33931356,45866936,36022940,43871900,45001048,46619479,21986512,18626887,116423810,22179561,41025510,15499660,16736293,58460325,100808813,45766975,70631526,39430669,39081659,53041067,76496142,66099575,76271580,198608682,43151567,18924866,17344361,18929773,15749983,14462907,16548435,18393089,15290123,14244769,188265738,14587429,212099989,97867221,27385146,45533787,145773163,87735340,48928735,40056109,61232019,62011530,39459292,26705065,62678144,69813431,20357019,79406971,40536441,13262442,20108560,46079391,55840387,15521075,54952792,52671787,43869144,18850433,18105473,52309424,51964156,783214,15825547,7108192,43542679,46103496,48059907,32825358,37224882,34284467,37382893,42825044,131442917,87942403,68217476,78659919,87994171,94523999,118649170,106309287,92945681,184861637,103913173,168606635,133690223,45022448,28344257,75226979,75196163,71837973,70023070,69044253,67902548,67091953,66981966,22913560,18174776,22125285,23476572,32479720,18175567,47174655,46085006,37352470,28290073,62552583,58136738,99925059,96751605,80285804,77445473,86896314,97186138,100778293,51475215,46392686,17800797,17006157,48410093,187507648,21551205,27781484,15076390,16578309,55422711,23524507,68359712,68071114,17732661,76986804,15766655,16324504,48907063,80864151,117333266,24409746,98816681,129724841,165174290,88287823,147293037,96500633,128124036,156003699,162913443,79453919,56143594,70307067,20584873,72756542,61703078,52655094,77447919,16382194,65054736,67799531,42826666,28824737,185122050,147001819,131607491,88718155,38740909,63509154,95943913,112508240,22534086,18393773,22548403,18131039,15573707,10798802,158645880,18286505,19399719,153051851,117426684,47715374,20326436,33402909,30055564,17600124,25053299,25166595,22536055,5920532,68509753,31194444,44462811,18450106,11018442,20040003,69615397,22171728,50687788,15162193,41338503,65104337,204491797,60054156,24047612,19628693,29271362,42712551,75014376,16147150,14620190,205789121,14275290,79320096,14365068,18735040,58582341,18639734,18665800,759251,19790261,91478624,98841303,113676911,42519612,21008866,18673446,18222770,18222599,15781432,14379501,18845207,22912596,208019751,20727198,18127783,5503832,268163148,29483675,109713729,42858171,23362483,37440098,53756003,249941736,18704659,184647530,122444371,21464713,90420314,106292326,111463209,25013763,100381915,62718650,22110491,93935921,94360081,95515846,96851347,111690277,212782910,22992332,23520739,21267612,32087193,58513684,46570247,115796187,28638191,110886577,110867752,48363171,32192094,26507551,27592116,43768449,56783491,32749378,91242884,55403897,76052943,17372853,22307857,46297163,19383317,27778893,24285686,19329393,21111883,17919972,36726043,28094035,192747213,44924709,257262765,263359642,262734281,90537810,17512682,128744229,22270337,22964152,21950364,47462001,102441113,45215683,247008365,14746298,83723250,1503121,22135992,102476260,247752114,225919331,16971999,131979670,14409497,15054622,170186151,14864693,21597063,242153407,248815930,20756490,157778204,15261298,260064723,21633857,20212370,254186356,17694756,37396006,17911421,124763921,13821862,21062471,24363945,153168449,14260037,21303400,44494276,5568292,77026540,18025954,16898332,12372812,126722715,116387194,22876454,178071411,110905570,17235724,22505100,239571244,136568901,15447566,249854076,17783446,23334601,142901066,57358233,65533134,41911874,112740986,232880095,17599795,17890304,157380172,87313007,258014885,17416700,14198654,259345913,1240681,33935084,21217402,32623377,27275087,15771056,92394644,16522352,1947301,91518617,188154135,241201438,197594920,13623532,36719281,188591875,234826866,138787319,211921304,9855382,153149305,115911638,166252256,19674502,86697288,135230178,153810519,106448460,48706233,13491312,104324860,82161632,117821354,72631996,47279525,18990471,66369206,67378554,18215973,110823581,113420831,105174965,63246339,54546430,40300027,61853389,21857760,76936279,78408666,76348185,62595437,49706919,54655632,61762853,43920155,44615672,48398586,78138151,33998183,76091017,21783950,59204932,44777959,24752484,39657463,14224719,14091091,17961886,16245822,16116713,14350393,17409240,6450322,14079839,9109712,14287409,10894562,14616022,14479720,14505838,15170346,16028241,15647676,16669075,16027546,15649433,18020120,17451742,8487622,15460572,15743810,19109172,18923032,18058609,20456198,19006707,20436059,20179628,20437286,17967675,17974455,17968875,16878156,266830495,14858000,176944763,128225158,16571326,123974480,20776147,16625082,51462013,48029363,65502117,27311044,43881089,243000634,136389258,238259855,160276009,248945322,29235589,20221159,23199697,63877097,36804447,132986633,130945778,17856665,15826822,222140591,27948604,166608401,164383046,244212645,13215382,21844854,24404039,11856892,143798379,180090338,9717322,20733972,33919343,71022729,17494010,18365936,16664681,12701412,26837501,18080708,3952661,52954222,76707735,57770539,234979566,15017893,174263663,106514937,61220477,115979444,43409523,66873399,46469610,189550099,14226495,123738314,6449282,16125224,57768217,14552725,21768766,49620730,15399512,40260612,146167871,196824385,48388417,7212562,16326882,48043084,16081831,23176276,134843187,97474887,55355654,15670644,52774637,173999134,16261445,190298721,4207961,21840737,30907284,136550204,16224666,21146211,258928816,241990563,28260687,259905595,18727438,15371215,196631065,41219840,95493980,15498429,102786512,180859549,14677751,20818801,16250929,93069110,18891923,19670455,8370082,16640598,93876666,47688440,29795115,14327933,34022952,17391507,224464606,32155953,22957302,5392522,46612327,114497182,109287731,17169320,33986037,14129299,222205782,224424419,225234560,15265641,21178616,22771961,18873711,14275426,24295482,21457205,15901113,20713061,16789970,73238146,69181624,22772264,79743108,47694175,21674448,138479037,18686907,24892146,49337217,21699092,17786474,69218324,49573859,48038030,171129408,14804435,28479007,142419647,198504506,46764631,68801460,16340501,14413466,4119741,82643763,89269664,85591466,8979992,42860402,39524002,46161313,21334893,40906518,255762643,96496611,76653027,14836128,22602881,22295473,77215251,28428838,121265704,47136500,95243532,59738746,18773422,50156221,37016784,234293544,32631520,252726343,24954899,118805793,15244981,38236362,31072439,43907446,17373943,20317201,31079712,70758813,33838795,27233809,51258903,37951729,50415064,37395827,30022594,17214648,88295976,20651632,16123781,119957997,24024655,33601431,23090525,37683259,27678902,43766620,17721514,15353981,18619979,31058793,39578080,22571950,20854518,19330213,36106105,47401289,22780988,25324156,22646228,29711814,31468242,41438347,29785315,31606227,46229563,31967467,33651080,39726700,24949726,15353815,25425425,23629827,22485793,20537454,14325369,19728030,22044289,24927543,28393637,29867516,20829172,15386108,22303692,21128035,21699429,17198500,15131310,5741722,5248441,237488632,20407539,17797448,30704824,153741488,15809249,29028408,19212009,56051834,14062236,65495877,112624301,28408675,59058446,33755676,84370522,264459407,86387057,17837853,24741917,118752616,223100019,93783780,66593978,53162803,192019265,17782766,249253283,22330646,199941365,102271048,43018238,20012204,108797271,31119337,153474021,235083242,109660695,49860231,2425151,190170608,49861057,169686021,17771860,49669034,122737411,66891455,188468721,50557402,29840295,229965178,222708016,244243477,15934926,184596818,65773523,15169907,110840720,68524064,129575180,101928415,25626894,246163327,207010069,40735690,230547432,141326970,28669612,39015718,128090469,49197437,163126581,107818778,16929701,50531062,53193168,39936193,44821096,46829683,46313918,49034233,30124262,33009321,28067175,35624959,34097876,26892157,30043495,21119636,79039503,17869798,21619519,40047532,32765534,14412533,14246001,50325797,14529929,12811952,7313362,14688489,245510831,14498484,53197137,16212685,47964412,41144996,816214,15008449,24784891,27589818,39008816,113880573,930061,87904422,1365841,1501471,14413715,1242511,14326840,20333461,657863,1835411,16219126,3829151,109275829,81890700,30345764,8442372,41269920,18840896,101844850,8953122,209792690,40168339,18915795,15664289,133578965,13213122,263767927,71294756,120943272,137466464,40553282,135897307,21348880,18227519,21111896,52772661,91888403,25355724,247033019,25527966,83590349,117928875,21251668,234629912,62915920,17095452,14245378,23778898,196272316,248427077,36554387,224119738,17291393,80638910,40892016,19107878,7380872,56700964,50073507,63004503,21268897,24431892,21307076,21982720,18791763,27901418,18172905,9532402,16037856,85131054,15210410,51263592,43556096,26624619,130617778,52070270,158414847,14268812,92079350,24816848,50323173,187005921,183665224,234521957,47260400,18678924,7651862,58601997,23544596,33792634,91181033,14603515,19071682,20064236,7768572,149217510,188975706,22169931,145492157,14677919,114589771,14693823,15519544,201870414,17781165,14702533,13784592,39050020,167181650,153051008,14861285,30261067,44177383,65647594,28143425,15194013,18959163,28144343,16228479,18901122,36958781,187319644,118022756,16129106,54885400,1068831,68539526,15339201,1344951,2890961,119525215,85634177,7152572,185644175,5561412,14434063,11270992,15024563,16256269,21157904,15572155,31182553,15309804,16160352,7334402,16129920,17912982,16598226,35373847,56705653,250831586,27529145,26618468,238073909,40626514,234132208,235120302,20402945,216431881,35989792,229686366,23539358,169608502,268123287,225278024,261835843,28376631,79293791,6446742,56518880,18915240,180188862,17200116,32948225,25398075,25677409,41737887,18987651,29952857,15537451,24322371,19877186,23967578,46252298,28471026,27406401,33549858,14817982,40141037,24472704,14860448,26933384,82688781,35133435,264938614,172562639,4170491,17292143,16347506,143664435,23065354,19942991,19410817,229129090,237620586,228296619,30959811,21656599,261555701,29365013,105916405,188639874,173581919,22831662,47218790,37367041,16302242,39288259,15126367,25482459,19867571,229390349,35586563,105029433,183089853,23184896,38127255,16966162,151159882,217231723,48488561,23065057,231389485,58294430,254179370,127750483,5162861,187279871,190064796,78575667,21077000,25174913,94216840,108788135,111635980,23593446,267854863,78445977,232992031,163570705,150669746,14342564,19658936,19418459,40959360,84388888,30343716,260845829,263815367,193732179,221792092,19716781,38254095,73181712,65707359,9624742,234489403,248735463,168673083,247486443,249410485,30912937,249348006,246088673,235373000,246357149,210926192,158398519,137823987,33530012,28602948,236511574,234002627,153944899,85396297,240812994,233783568,199325935,119250381,237770636,161791703,242892689,239949176,242426145,229873610,227341953,216503958,24721108,132201224,217547360,208692908,48086014,30216513,80612021,126017061,148453195,192957076,24745957,188019606,24232525,140519774,103922067,28582749,39249305,16583468,15022633,75124560,125445568,34972633,84119348,107545479,138770045,36948268,115111016,153865861,28599820,161411080,33563161,56864092,137794015,150688795,24913074,27698046,45659261,243721842,88507556,247334603,16329577,18812301,12671692,6527442,53981494,22642788,5446992,24619328,47459584,19824430,10371822,117445762,128564422,23534104,22677790,242360696,19891675,54299209,211588974,20065936,74880863,15210284,105445396,93722614,53130511,24844525,14824473,18644548,40277679,159362503,23384516,119079595,20182089,203681891,15093646,15808765,47632905,98425584,15772733,20177097,198106297,14992591,21623531,158412741,23220034,21757965,51224438,15638637,150203689,17612289,104937383,16955870,17299063,68773980,176280674,19278537,95281047,12340562,16440738,54905208,10228272,24024778,19541375,7855622,52254824,237562389,267847170,202594374,8180562,258968955,110757459,151688371,54290523,4262791,6793362,198571335,215396550,144121755,248390035,216216791,119169760,124635414,238144920,236108642,199730264,46812100,220882223,143822047,15693634,172717529,147249049,133727476,203728509,731573,801202,38299186,239200202,125492611,110905955,137751549,71918994,205370456,81189096,37083812,37267497,52167970,4778211,175083160,8105482,118256463,132704571,16160650,114510872,994431,10052972,16194793,13860082,31237389,48920331,54578941,38100497,140935414,47850508,122975302,170316873,45497378,3551701,150115256,72972715,148939636,1120611,136414500,140538938,167624175,174027585,27641388,62015732,70346167,780159,175881965,37093819,19646485,140856303,77338274,187246188,122783603,50471406,11202862,151198642,76326090,104861015,133722469,121613012,47616949,14161128,819256,221019516,9766032,13836402,162104820,618593,47402960,85851159,142684449,63125213,166510517,240615854,269394406,252644913,38371041,136609376,76282043,70418699,185493136,17960806,38727269,234587458,31088763,147886032,265651031,212940155,40565664,201000503,197539670,170474186,263060411,254725084,89866540,130782156,223976506,260391020,19390708,124856935,99698716,52256215,260728760,156769030,40866430,35282604,261151943,58777905,150957265,86351265,179719319,189842436,204918670,203483715,258220853,261297092,204444285,54171939,214020774,89187513,170581751,42612282,44337867,212342630,105542341,190776107,36919153,134781128,47125833,244901765,242245196,194186291,242048469,245106788,215967395,140766042,158795894,116688855,19975710,109892942,95666954,32848956,240815975,77977289,216335730,256471145,247004791,175867315,247424331,107021348,29730710,38146165,237670656,191072401,125075147,15417049,219936176,41190759,179884440,41132274,253520310,73191618,40850899,42981176,112184431,87767749,162022136,193022239,186967108,84145752,227058289,641253,95994080,158848407,742143,41842768,242201454,22240612,19386622,23144737,20379995,254377876,16489591,30335156,13363282,249529225,17625292,151662442,245935903,82274636,15426758,17918808,136358789,169793441,117724817,15769161,157613794,20696985,216577574,116559622,130681675,74594552,7081242,189276359,96265654,20460757,14749606,231362108,222343099,235389922,36050060,44518888,115689700,235295798,189621457,226370793,15960627,20596269,6324592,82778942,43192807,34716594,36803580,52481056,11891542,199187259,111059621,36355699,56758031,15313857,17637614,20646277,100570170,24193017,117984119,22692199,6033532,116828327,20731304,106565458,83883736,17637553,23679582,29256864,214499442,140964771,15888570,44062282,18713,106145765,15255133,43333230,20998647,31311757,15855630,21008297,20334391,39348813,41264383,20660560,19343562,23116280,111444854,8039442,143454922,267206035,142977623,101879887,17565514,28932160,238480106,253632034,87471023,184910040,19226054,160817572,87650675,70246837,36600922,240860245,160926944,41329637,40981798,46345843,30563759,114225051,219920508,18918698,189283341,72694909,15790423,22440038,235120811,222658804,72547608,68838777,233749221,215952307,26668282,151895837,25456478,216995705,87416722,17612928,176417690,41218028,46715620,104571475,91390383,124283714,19026618,49393335,18950761,21526685,134206307,90098789,34192959,186865762,50600062,126418609,135303236,14522546,64220324,30517544,24083587,21258330,38475205,18953259,101192354,8973062,18676177,55544943,34108173,32360145,140894997,19818494,12085042,40389429,19982092,19524590,31239408,21869297,24450456,250463314,238295737,236917131,32397000,151631699,29257521,17235825,82960113,42491255,18044981,181988782,40058557,63350773,19320490,27246723,17119099,95921754,23069301,51865695,143549770,84460847,223724339,93904007,199555388,81929393,131727620,82444043,162580589,171998025,198334225,91171720,201426746,166316761,231206630,125382460,195885307,113678606,103000558,19019263,87307633,18860342,14705240,29160552,2885851,16185801,210693321,15276510,27762127,17147539,21019628,74712538,131315786,15012486,51193546,74735391,18335809,24796185,123331001,73134924,17193794,19432557,24872865,32128783,16622151,83939091,15321447,19733892,14517899,18140091,106506464,24663730,18145841,31282081,16324533,40894238,36893069,74502636,36975611,21375959,29337732,89642493,41407443,12138162,19198326,20692503,41135906,23532159,18148242,14326661,14423603,39806829,133966853,176467382,14463913,52185212,60680365,46743554,87989022,36143299,18815810,34002530,41888469,125415597,35667714,191250892,177343973,165802132,191798541,253342708,232878783,72168780,239613729,136353114,112937614,132618751,190475202,188579340,241837295,241805454,147609274,87871306,235433784,128295129,86566274,162571150,99002460,169719009,240858138,131246928,83900124,168410930,33623402,129254485,113593692,84358766,229643491,50841062,32730931,112074504,138871289,130734452,239623882,162254325,83237549,92817267,197874533,145476646,235792533,178941141,215819570,78145300,148504230,51646374,44758790,206922924,63896875,91437380,105320972,208777407,103080971,231279879,101844353,102607698,26395300,30152403,28050738,94225402,220252762,29091792,24967668,37986775,42082057,74853246,122158761,16234406,31115275,27195114,15942017,51158957,22498452,32482524,40286007,16870682,23618512,23832022,59286800,36860144,30022428,180999717,19028953,53153263,31668714,24621955,100611064,220675169,79551906,41676395,66714703,111673770,97807215,23819304,28605533,25695492,29537319,224325820,30810014,151291323,22100074,78687792,128472974,30113082,18803364,66003255,50572630,213840716,17915334,255591392,198613270,249359076,28110915,30630900,158887925,47945698,36425615,30689667,21541542,118612929,60401740,42203744,101578775,233697048,24293519,19042478,183879398,38773605,101424299,113508007,42371693,28702869,201445961,27060993,59606882,94370073,197220548,121269271,233395335,145742095,237955563,36897911,106375253,154423460,39002053,52963095,30441816,48649566,93928566,27198826,247495144,212479278,90721189,28303502,158393753,24100737,164818150,87623304,201989827,128078136,25465958,110473412,42099565,213707240,19312934,29533792,45419359,63311667,36126354,30228023,70343136,22733444,160027905,49666182,47562163,243010689,78507740,243516650,51219197,20955713,121512266,179767114,94784820,110585067,23586754,132223511,62644624,8713702,2148071,616113,7315022,822112,13348,16354128,45547548,85616044,10085882,21126003,16613870,65640210,11557642,14208914,14270999,20399609,99961552,32472561,39274758,19624506,3144281,90421077,7857652,46072516,4560311,97023892,76058815,5876342,29949063,1424831,10094,63133380,21786260,75616437,9745092,19226849,40880426,18484100,94587503,34406259,1614411,10016972,14140571,16044191,113116513,20169131,20217201,79502328,792646,270668475,53533566,157070412,59444261,39902637,165326620,123102622,265000864,52597678,3346971,49205840,112432370,167033855,129017028,132005200,58334111,93516296,168841469,229710631,175338338,241205936,116510779,84457604,264366428,52322656,234006638,258428607,168537997,95800032,143442028,82430417,84034563,13041552,185030770,32014425,85450815,29736553,53424957,172178623,14120187,67132984,74610929,73215961,140722325,65364950,71112180,112307482,122231942,189320995,178118343,251231223,266200838,70305266,131440156,262900579,249631963,33258173,34784263,105597663,101387077,37245723,48630048,143505274,153245594,61723876,39140545,215858327,168243936,30380192,39187698,235394328,62052874,143257853,77395953,26162191,191685671,40405121,267433029,50856543,168696795,84677497,30681499,261294795,265696487,183144850,127040414,29910549,50390358,30587056,78740274,19804553,53323885,32618047,49900650,144021058,241008487,237897985,75037065,203771716,42104946,111489389,227209417,44120318,234212815,38073608,97557168,162545959,108457267,261884913,204209879,129816753,121186812,161576069,38285013,30995591,60353632,30799610,181080474,239217629,254707723,212340162,46930437,257939339,32820524,89568746,16978961,235381394,93460034,21786322,175291914,185343485,76187493,36216833,46093969,164180736,95261792,25947430,168695261,156972971,196630780,102900059,62863521,25742007,171240557,35332069,50052920,127788481,111398486,254240011,32218646,176536528,139822631,98766044,19487670,236787450,137340994,128642272,20979106,225911802,248794623,75130364,264408558,15947602,225126598,237003654,19595532,265987070,66437032,14082828,11599312,253284344,78436108,122694066,17805567,207612806,34925804,171514924,188028765,122459707,95534157,217073451,20769938,223842388,29717916,259108421,240182448,248314631,73073062,164401022,57924802,20703607,251219812,198240811,40010200,228682873,47914608,777554,243833570,196167925,44877467,18212565,110950574,253664708,20767246,201747039,143415291,79743060,127635571,246738309,121845291,20508222,123979525,47650483,242655310,252488810,253069382,15020865,166926744,89702218,20081488,90276942,174717541,219716235,15906176,125378095,22707793,53454398,141598993,232821256,20906409,27066806,227631390,58842819,54937915,63968007,18764912,246263280,243920442,37376499,207707329,135178205,139853350,236581562,244593709,146787633,20373762,122714443,52487416,192218013,125820691,238956217,148354084,7064802,61133,14298281,20490861,809760,1493121,14336349,38073,14876536,6790312,205700196,200199318,15649570,15184418,51800278,19644571,29432246,18058704,175899673,89998965,51988166,87795815,14570292,15272882,19105442,7035532,112490145,13768732,47700913,241475497,157821181,14562275,20142938,180086129,237325382,16818325,21873261,241043955,16527490,240810497,115410741,150323560,36609309,29093456,17673732,16096448,216338061,235317040,63585408,236819345,93010530,190357418,240594439,239807683,165693574,211897738,112489818,47932614,55923778,225657320,186155101,16352458,240402102,71239119,34474398,239348795,234645414,240223810,141222424,240180276,15036744,94973632,54699775,196811548,141857015,238673608,97616524,224969703,238109234,231493350,239309716,238638637,218570379,39514187,158951763,16403358,239477369,154541100,239609301,239399824,121708861,195121864,72931614,142039374,239127196,233688482,107780778,20606286,65719885,162778946,95200214,215707078,156950036,15003756,7150932,71343661,11935332,200513252,79144576"
else:
sql = "SELECT `accounts`.`external_id` FROM `peepbuzz`.`curators` LEFT JOIN `peepbuzz`.`accounts` ON `accounts`.`account_id` = `curators`.`account_id` WHERE `accounts`.`external_id` IS NOT NULL AND `accounts`.`stream_id` = 1"
curatorsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
users = None
while True:
curator = curatorsQ.fetch_row(1, 1)
if curator == ():
break
if users == None:
users = curator[0]['external_id']
else:
users = users + "," + curator[0]['external_id']
return [u for u in users.split(',')]
def checkBlacklist(user_id=None, account_id=None):
# Make sure we have a user to check against
if not user_id or not account_id:
return False
if infoModule.info.site['dblink'] == None:
return False
# Find which field we are checking
account_where = "`account_id` = '" + str(account_id) + "'"
# Assemble and run the query
checkAccountQ = mysql_tools.mysqlQuery(
"SELECT * FROM `peepbuzz`.`blocked_accounts` WHERE " + account_where +
" AND `user_id` = '" + str(user_id) + "'",
infoModule.info.site['dblink'])
# Now check to see if a record exists
if checkAccountQ.num_rows() > 0:
# Looks like they blocked this account
return True
else:
# No blacklist record was found
return False
def checkForHash(hashtag, type, type_short, block):
# type: {daily, hourly}
# block : time/secs
query = (
"select score from peepbuzz.hashtag_"
+ type
+ "_stats where "
+ type_short
+ '_block="'
+ block
+ '" and hashtag="'
+ hashtag
+ '"'
)
hashQ = mysql_tools.mysqlQuery(query, infoModule.info.site["dblink"])
if hashQ.num_rows() > 0:
while 1:
row = hashQ.fetch_row(1, 1)
if row == ():
break
score = row[0]["score"]
return score
else:
return False
def outboundLinks(html):
siteDB = infoModule.info.site['database']
blbQ = mysql_tools.mysqlQuery(
"select * from " + siteDB + ".blindLinkBlacklist",
infoModule.info.site['dblink'])
blindLinkBlacklist = []
while True:
blb = blbQ.fetch_row(1, 1)
if blb == ():
break
blindLinkBlacklist.append(blb[0]['url'])
URLParts = urlparse.urlparse(infoModule.info.page['url'])
myHost = URLParts[1]
thisHostArray = myHost.split('.')
matchHost = thisHostArray[-2:-1][0] + '.' + thisHostArray[-1]
log.plog('host name for self link matching: ' + matchHost, 2)
#blocked URLS
urlBlockerQ = mysql_tools.mysqlQuery(
"select * from " + siteDB + ".urlBlocker",
infoModule.info.site['dblink'])
blockedURLs = []
while True:
urlBlocker = urlBlockerQ.fetch_row(1, 1)
if urlBlocker == ():
break
blockedURLs.append(urlBlocker[0]['regex'])
#known urls for outbound links
oblQuery = mysql_tools.mysqlQuery(
"select source_id, url_regex from " + siteDB +
".sources where url_regex != '' and url_regex not like '%*%' order by length(url_regex) desc",
infoModule.info.site['dblink'])
oblRegexQuery = mysql_tools.mysqlQuery(
"select source_id, url_regex from " + siteDB +
".sources where url_regex like '%*%' order by length(url_regex) desc",
infoModule.info.site['dblink'])
#differentiate between non-regex and regex urls to speed things up.
knownURLs = []
knownRegexURLs = []
regexToSource = []
regexToSourceRegex = []
pageLinks = []
while True:
obl = oblQuery.fetch_row(1, 1)
if obl == ():
break
knownURLs.append(obl[0]['url_regex'])
regexToSource.append(obl[0]['source_id'])
while True:
obl = oblRegexQuery.fetch_row(1, 1)
if obl == ():
break
knownRegexURLs.append(obl[0]['url_regex'])
regexToSourceRegex.append(obl[0]['source_id'])
while True:
if isinstance(html, str) == False:
log.plog('html in links is not a string', 4)
return False
foundLinks = re.search("href=\"(http.*?)\"", html, re.I)
if foundLinks == None:
break
html = html.replace(foundLinks.group(0), 'x')
#match root domain to avoid bloomberg.com matching search.bloomberg.com
selfUrl = re.search(matchHost, foundLinks.group(1), re.I)
if selfUrl == None:
#url does not contain my own host
pageLinks.append(foundLinks.group(1))
else:
log.plog(foundLinks.group(1) + " is current page's own url", 2)
for link in pageLinks:
# a number of tests learned from experience. Don't question them
log.plog('link is ' + link, 2)
test1 = re.search('mt-search', link, re.I)
if test1 != None:
pageLinks.remove(link)
continue
test1 = re.search('eedproxy.google', link, re.I)
if test1 != None:
pageLinks.remove(link)
continue
test1 = re.search('eeds.usmagazine', link, re.I)
if test1 != None:
pageLinks.remove(link)
continue
#forbidden extensions
forbiddenExtensions = [
'css', 'pdf', 'flv', 'zip', 'mov', 'wav', 'ico', 'mp3', 'xml',
'gif', 'jpg', 'png'
]
isExtensionForbidden = False
for extension in forbiddenExtensions:
if str.lower(link[-4:]) == '.' + extension:
isExtensionForbidden = True
if isExtensionForbidden:
#no reason to include css or pdf files
pageLinks.remove(link)
continue
#block urls that match url blocker
blocked = False
for blockedRegex in blockedURLs:
blockTest = re.search(blockedRegex, link)
if blockTest != None:
log.plog('url ' + link + " matches urlBlocker " + blockedRegex,
2)
blocked = True
break
if blocked == True:
continue
linkParts = urlparse.urlparse(link)
if linkParts[1] in blindLinkBlacklist:
pageLinks.remove(link)
log.plog(
linkParts[1] +
" not added as outbound link because of blacklist", 2)
continue
#do not read links that have only one string in them
shortPath = re.search('^/\w+/*$', linkParts[2])
lp = linkParts[2]
if shortPath != None:
log.plog(
"link excluded because it only has a short path of characters: %s"
% linkParts[2], 2)
continue
#how old is story based on url?
possibleAgeInDays = dateGuesser.urlDateGuesser(link)
if possibleAgeInDays != None:
if int(possibleAgeInDays) > 5:
log.plog(
"story is " + str(possibleAgeInDays) +
" days old. Not adding", 2)
continue
if 'read_blind_stories' in infoModule.info.site and infoModule.info.site[
'read_blind_stories']:
urlRecognized = True
else:
urlRecognized = False
urlSource = 0
#go through all regex known urls
for i in range(len(knownRegexURLs)):
match = re.search(knownRegexURLs[i], link)
if match != None and regexToSourceRegex[
i] != infoModule.info.source['source_id']:
urlRecognized = True
urlSource = regexToSourceRegex[i]
log.plog("Text Link: " + link, 2)
log.plog(
"matched known blog via regex: " + knownURLs[i] +
" source_id is " + urlSource, 2)
break
#if not found go through all non-regex url lookups (no '*' character in the url)
if urlRecognized == False:
for i in range(len(knownURLs)):
if linkParts[1] == knownURLs[i]:
if regexToSource[i] != infoModule.info.source['source_id']:
urlRecognized = True
urlSource = regexToSource[i]
log.plog("Text Link: " + link, 2)
log.plog(
"matched known blog via regex: " + knownURLs[i] +
" source_id is " + urlSource, 2)
break
if urlRecognized and len(linkParts[2]) > 1:
#link must have path component so that base urls (yahoo.com) aren't pulled in
alreadyThere = False
URIParts = link.partition('#')
URLOnly = URIParts[0]
URLOnly = URLOnly.replace("'", "\\'")
link = re.sub('\.html\?.*', '.html', link)
link = link.replace("'", "\\'")
sql = "select url, sub_id from " + siteDB + ".subs where url like '" + URLOnly + "%'"
log.plog(sql, 2)
alreadyInSubsQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInSubsQ.num_rows() > 0:
alreadyThere = True
log.plog('url is already in subs', 2)
sql = "select url, sub_id from " + siteDB + ".newsroom where url like '" + URLOnly + "%'"
log.plog(sql, 2)
alreadyInNewsroomQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInNewsroomQ.num_rows() > 0:
alreadyThere = True
log.plog('url is already in newsroom', 2)
#for now, checking against exact url too.
sql = "select url, sub_id from " + siteDB + ".subs where url = '" + link + "'"
log.plog(sql, 2)
alreadyInSubsQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInSubsQ.num_rows() > 0:
alreadyThere = True
log.plog('url is already in subs', 2)
sql = "select url, sub_id from " + siteDB + ".newsroom where url = '" + link + "'"
log.plog(sql, 2)
alreadyInNewsroomQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInNewsroomQ.num_rows() > 0:
alreadyThere = True
log.plog('url is already in newsroom', 2)
sql = "select guid from " + siteDB + ".feedGuids where guid = '" + link + "'"
alreadyInGuidQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInGuidQ.num_rows() > 0:
alreadyThere = True
log.plog('url is already in feedGuids', 2)
sql = "select guid from " + siteDB + ".feedGuids where guid = '" + link + "'"
alreadyInGuidQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInGuidQ.num_rows() > 0:
alreadyThere = True
log.plog('url is already in feedGuids', 2)
# require two hits to make story go live.
sql = "select url, idx, submittingURL, promoter, hits from " + siteDB + ".sourceReaderQueue where url like '" + URLOnly + "%'"
alreadyInQueueQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInQueueQ.num_rows() > 0:
alreadyThere = True
SRID = alreadyInQueueQ.fetch_row(1, 1)
# need to search through voting_urls instead and add 1 if it isn't there for this idx
log.plog(
'url is already in source reader queue, checking host', 2)
submittingURLParts = urlparse.urlparse(
SRID[0]['submittingURL'])
if myHost == submittingURLParts[1]:
log.plog('no point added to link, host is the same', 2)
else:
log.plog('adding point to link in sourceReaderQueue', 2)
log.plog(
'current hit count for this link is ' +
str(SRID[0]['hits']), 2)
if int(SRID[0]['hits']
) + 1 > infoModule.info.site['votesToGoLive']:
sql = "update " + siteDB + ".sourceReaderQueue hits=hits+1, go_live=1 where idx=" + SRID[
0]['idx']
log.plog(sql, 2)
log.plog(
"setting go_live to 1 for sourceReaderQueue id:" +
SRID[0]['idx'])
log.plog('url to go live is ' + SRID[0]['url'])
else:
sql = "update " + siteDB + ".sourceReaderQueue hits=hits+1 where idx=" + SRID[
0]['idx']
sql2 = "insert into " + siteDB + ".voting_urls set promoter=" + SRID[
0]['promoter'] + ", submittingURL = " + SRID[0][
'submittingURL'] + ", idx=" + SRID[0][
'idx'] + ", added=NOW()"
if 'debug_mode' not in infoModule.info.site or infoModule.info.site[
'debug_mode'] == False:
mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
mysql_tools.mysqlQuery(sql2,
infoModule.info.site['dblink'])
if alreadyThere == False:
sql = "insert into " + siteDB + ".sourceReaderQueue set added = now(), url='" + link + "', publishDate=now(), source_id=" + str(
urlSource
) + ", potentialTitle='', potentialOutline='', promoter=" + infoModule.info.source[
'source_id'] + ", submittingURL='" + infoModule.info.page[
'url'] + "', hits=1"
if 'debug_mode' not in infoModule.info.site or infoModule.info.site[
'debug_mode'] == False:
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
idx = infoModule.info.site['dblink'].insert_id()
sql2 = "insert into " + siteDB + ".voting_urls set added = NOW(), promoter=" + infoModule.info.source[
'source_id'] + ", submittingURL='" + infoModule.info.page[
'url'] + "', idx = " + str(idx)
log.plog(
"**** adding url " + link + " into sourceReaderQueue",
2)
log.plog("****\n" + sql2 + "\n****")
mysql_tools.mysqlQuery(sql2,
infoModule.info.site['dblink'])
def linkScoring(html, workingTable):
# func: findLinks
# param: html, workingTable
# desc: Scans passed html for links to stories that we already have in
# order to add score to stories. workingTable is either subs
# or newsroom. If newsroom, and there's a hit, make it go live
# ret: none
# auth: esr
if 'url' not in infoModule.info.page or infoModule.info.page['url'] == '':
log.plog('no url for this story in findLinks', 4)
return False
URLParts = urlparse.urlparse(infoModule.info.page['url'])
myHost = URLParts[1]
pageLinks = []
siteDB = infoModule.info.site['database']
if 'debug_mode' in infoModule.info.site and infoModule.info.site[
'debug_mode'] == True:
debugMode = True
else:
debugMode = False
while True:
if isinstance(html, str) == False:
log.plog('html in links is not a string', 4)
return False
foundLinks = re.search("href=\"(http.*?)\"", html, re.I)
if foundLinks == None:
break
html = html.replace(foundLinks.group(0), 'x')
#match root domain to avoid bloomberg.com matching search.bloomberg.com
thisHostArray = myHost.split('.')
matchHost = thisHostArray[-2:-1][0] + '.' + thisHostArray[-1]
selfUrl = re.search(matchHost, foundLinks.group(1), re.I)
if selfUrl == None:
#url does not contain my own host
pageLinks.append(foundLinks.group(1))
for link in pageLinks:
link = link.replace("'", "\\'")
## if link has nothing but hostname then it's garbage
linkParts = urlparse.urlparse(link)
if linkParts[2] == '' or linkParts[2] == '/':
log.plog('link ' + link + ' has no path component. Skipping', 2)
continue
sql = "select sub_id, source_id from " + siteDB + "." + workingTable + " where url like '" + link + "%'"
isLinkQuery = mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
if isLinkQuery.num_rows() > 0:
isLink = isLinkQuery.fetch_row(1, 1)
log.plog("found link to one of our stories: " + link, 2)
addPoint = True
### check against network. If in network, do not add point
if 'parent' in infoModule.info.source and infoModule.info.source[
'parent'] != '0':
sql = "select * from " + siteDB + ".networkSourceLinks where source_id=" + infoModule.info.source[
'parent']
else:
sql = "select * from " + siteDB + ".networkSourceLinks where source_id=" + infoModule.info.source[
'source_id']
net1Q = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if net1Q.num_rows() > 0:
log.plog('source is in a network', 2)
networks = ''
sep = ''
while True:
net1 = net1Q.fetch_row(1, 1)
if net1 == ():
break
networks = networks + sep + net1[0]['networkIdx']
sep = ','
sql = "select * from " + siteDB + ".networkSourceLinks where source_id=" + isLink[
0]['source_id'] + " and networkIdx in (" + networks + ")"
net2Q = mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
if net2Q.num_rows() > 0:
log.plog("in network. Do not add point", 2)
addPoint = False
## don't add point if there's already a linkHistory entry
sql = "select * from " + siteDB + ".linkHistory where sub_id=" + isLink[
0]['sub_id'] + " and feedIdx=" + infoModule.info.source[
'source_id'] + " and linkerURL='" + infoModule.info.page[
'url'] + "'"
alreadyInHistoryQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInHistoryQ != False and alreadyInHistoryQ.num_rows() > 0:
log.plog("already in link history. Do not add point", 2)
addPoint = False
#is linker base url already in link history? do not add point
urlHost = URLParts[1].replace("'", "\\'")
sql = "select * from " + siteDB + ".linkHistory where sub_id=" + isLink[
0]['sub_id'] + " and linkerUrl like '" + URLParts[
0] + "://" + urlHost + "%'"
alreadyInHistoryQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if alreadyInHistoryQ != False and alreadyInHistoryQ.num_rows() > 0:
log.plog("base url already in link history. Do not add point",
2)
addPoint = False
if addPoint:
#everything checks out. add point to existing story in subs
log.plog("adding point to sub_id " + isLink[0]['sub_id'], 2)
sql = "update " + siteDB + "." + workingTable + " set score = score + 1 where sub_id=" + isLink[
0]['sub_id']
if debugMode == False:
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
#link history
sql = "insert into " + siteDB + ".linkHistory set sub_id=" + isLink[
0]['sub_id'] + ", feedIdx=" + infoModule.info.source[
'source_id'] + ", linker_id=-1, linkerURL='" + infoModule.info.page[
'url'] + "', linkedOn=now(), userVote=false"
if debugMode == False:
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if workingTable == 'newsroom':
## send to subs
if debugMode == False:
mysql_tools.mysqlQuery(
"update " + siteDB +
".newsroom set status='A' where sub_id=" +
isLink[0]['sub_id'],
infoModule.info.site['dblink'])
newsroomToSubs.promoteFromNewsroom(isLink[0]['sub_id'])
import urllib
import pprint
link = mysql_tools.mysqlConnect("192.168.0.29", "rw_failover", "f@ilfa1l")
if link == False:
print "no connection"
sys.exit(0)
# set database
infoModule.info.site["database"] = "db_sportifi"
infoModule.info.site["dblink"] = link
infoModule.info.site["read_blind_stories"] = True
infoModule.info.site["debug_mode"] = False
infoModule.info.site["log_priority_threshold"] = 1
mysql_tools.mysqlQuery("set wait_timeout = 600", link)
mysql_tools.mysqlQuery("set interactive_timeout = 600", link)
sql = "select 'http://www.rantsports.com/redzonetalk/20100910-a-proposition-for-denver-broncos-tim-tebow/' as url, 6367 as source_id"
randomStoriesQ = mysql_tools.mysqlQuery(sql, infoModule.info.site["dblink"])
ctr = 0
while True:
randomStory = randomStoriesQ.fetch_row(1, 1)
if randomStory == ():
break
print "=================================== " + str(ctr) + " ====================================="
ctr += 1
infoModule.info.page["url"] = randomStory[0]["url"]
sourceQ = mysql_tools.mysqlQuery(
"select * from " + infoModule.info.site["database"] + ".sources where source_id=" + randomStory[0]["source_id"],
import strip_html
import infoModule
import entities
import pprint
import sys
import find_title
import body_extractor
import urllib
from alogClient import *
import find_story
link = mysql_tools.mysqlConnect('192.168.0.115', 'root', 'datafl0w')
if link == False :
print "no connection"
sys.exit(0)
mysql_tools.mysqlQuery("set wait_timeout = 600", link)
mysql_tools.mysqlQuery("set interactive_timeout = 600", link)
# set globals for site
infoModule.info.site['database'] = 'db_politifi'
infoModule.info.site['dblink'] = link
infoModule.info.site['read_blind_stories'] = True
infoModule.info.site['debug_mode'] = False
infoModule.info.site['log_priority_threshold'] = 1
infoModule.info.site['imageReceiver'] = "http://dev.celebrifi.com/"
infoModule.info.site['maxEntities'] = 15
infoModule.info.site['overrideImageMinSize'] = False
infoModule.info.site['imageMinSize'] = 300 * 255
infoModule.info.site['dblink'] = link
def get_links(user_id):
sql = "SELECT block_regex, link_regex, url, account_id FROM peepbuzz.curated_sites WHERE user_id = %s" % user_id
siteQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
row = siteQ.fetch_row(1,1)
if row == ():
infoModule.info.errorList.append("ERROR: something weird happened in get_curated_site_links.py, didn't find user_id %s" % user_id)
return False
# pprint.pprint(row)
url = row[0]['url']
# get site domain to prevent grabbing links in site domain
domain_search = re.search('(https?:\/\/)?(.+)', url)
domain = domain_search.group(2)
account_id = row[0]['account_id']
# block regex should always be in db
block_regex = re.compile(row[0]['block_regex'], re.I | re.S)
# link regex may not be specified
link_regex_str = row[0]['link_regex']
# use link_regex in db, otherwise use default
link_regex_str = link_regex_str if link_regex_str else '<a href="(https?:\/\/.+?)">'
link_regex = re.compile(link_regex_str, re.I | re.S)
## for debug: hacker news anchor tag right after "title" should grab link body and not header or upvote
## block_regex_str = 'class="title">\s*(.+)<\/table>'
## url = 'http://news.ycombinator.com'
if not url:
infoModule.info.errorList.append("No URL associated with curated_site_id %s, dying now") % curated_site_id
sys.exit()
else:
http_req = urllib2.urlopen(url)
http_res = http_req.read()
# first define html block to search for links in
block = re.search(block_regex, http_res)
# then get links in that block
matchlist = link_regex.finditer(block.group(1))
page_info = []
for match in matchlist:
match_index = match.lastindex
page_link = match.group(match_index)
# if link in site's domain, we don't want it
if domain in page_link:
continue
# build external_id for homepage links
m = md5.new()
m.update(page_link)
md5_url = m.hexdigest()
external_id = "homepage_%(user_id)s_%(md5_url)s" % { 'user_id': user_id, 'md5_url': md5_url }
# build list to pass to func
url_data = [page_link, external_id, account_id]
page_info.append(url_data)
zezted_filament_links = get_zezted_filament_links(page_info)
return zezted_filament_links
def main():
sleep = 1 * 60 # Seconds to sleep
pidPath = "/tmp/twitterUserStream.pid"
streamPath = "twitterUserStream.py"
pid = None
userCount = None
lastUserCount = None
noCheck = False
running = False
# Create a loop
while True:
infoModule.info.site['dblink'] = mysql_tools.db_connect()
# New Loop so move the user counts
lastUserCount = userCount
userCount = None
# Check the file for a PID
try:
file = open(pidPath)
while True:
line = file.readline()
if not line:
break
pid = line
if pid:
# Get the status of the PID
try:
os.kill(int(pid), 0)
except OSError:
running = False
else:
running = True
print str(pid) + " - is running?: " + str(running)
else:
running = False
except IOError:
# We dont care if the file does not exist, since it will get created the first time around
# So we can treat it as the deamon is not running
running = False
pid = 0
# Get the count for how many users we are following
sql = "SELECT count(*) as `userCount` FROM `peepbuzz`.`curators` LEFT JOIN `peepbuzz`.`accounts` ON `accounts`.`account_id` = `curators`.`account_id` WHERE `accounts`.`external_id` IS NOT NULL AND `accounts`.`stream_id` = 1"
countQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
try:
userCountV = countQ.fetch_row(1, 1)
userCount = userCountV[0]['userCount']
except:
print "Problem fetching the users from the database"
sys.exit()
# if the count has changed from last count OR the PID is not running start a new deamon
if userCount != lastUserCount or not running:
try:
os.kill(int(pid), 9)
except OSError:
# Proc is aready dead
pass
print "Starting the Daemon"
os.system("python2.7 " + streamPath + " &")
infoModule.info.site['dblink'].close()
time.sleep(sleep)
def scanPage():
siteDB = infoModule.info.site['database']
if 'url' not in infoModule.info.page:
log.plog('scan page called without url', 4)
os._exit(0)
urlBlockerQ = mysql_tools.mysqlQuery("select * from " + siteDB + ".urlBlocker", infoModule.info.site['dblink'])
while True:
urlBlocker = urlBlockerQ.fetch_row(1,1)
if urlBlocker == ():
break
blockTest = re.search(urlBlocker[0]['regex'], infoModule.info.page['url'])
if blockTest != None:
log.plog('url ' + infoModule.info.page['url'] + " matches urlBlocker " + urlBlocker[0]['regex'], 2)
os._exit(0)
log.plog("fetching " + infoModule.info.page['url'], 2)
try:
socket = urllib.urlopen(infoModule.info.page['url'])
except IOError:
log.plog('could not open ' + infoModule.info.page['url'], 4)
return False
responseCode = socket.getcode()
log.plog('urllib response code: ' + str(responseCode), 2)
if responseCode != 200 and responseCode != 302 and responseCode != 301 and responseCode != 303:
log.plog('got failure response code from server', 4)
return False
headerInfo = socket.info()
contentType = headerInfo.gettype()
if contentType != 'text/html' and contentType != 'text/html, text/html':
log.plog('content type: ' + contentType + '. not fetching', 4)
return False
# put in to account for WSJ -dpg
if re.search("wsj\.com", infoModule.info.page['url'], re.S | re.M | re.I):
infoModule.info.page['rawHTML'] = wsjAuthHack(infoModule.info.page['url'])
elif re.search("nytimes\.com", infoModule.info.page['url'], re.S | re.M | re.I):
infoModule.info.page['rawHTML'] = nytAuthHack(infoModule.info.page['url'])
else:
infoModule.info.page['rawHTML'] = socket.read()
redirURL = socket.geturl()
if redirURL != infoModule.info.page['url']:
log.plog('redirected to ' + redirURL, 2)
infoModule.info.page['url'] = redirURL
#redirected urls need to be blocked too
urlBlockerQ = mysql_tools.mysqlQuery("select * from " + siteDB + ".urlBlocker", infoModule.info.site['dblink'])
while True:
urlBlocker = urlBlockerQ.fetch_row(1,1)
if urlBlocker == ():
break
blockTest = re.search(urlBlocker[0]['regex'], infoModule.info.page['url'])
if blockTest != None:
log.plog('url ' + infoModule.info.page['url'] + " matches urlBlocker " + urlBlocker[0]['regex'], 2)
os._exit(0)
### and short url needs to be blocked
#do not read links that have only one string in them
linkParts = urlparse.urlparse(infoModule.info.page['url'])
shortPath = re.search('^/\w+/*$', linkParts[2])
lp = linkParts[2]
if shortPath != None:
log.plog("link excluded because it only has a short path of characters: %s" % linkParts[2], 2)
os._exit(0)
## anything in htmlBlacklist?
htmlBlacklistQ = mysql_tools.mysqlQuery("select regex from " + siteDB + ".htmlBlacklist", infoModule.info.site['dblink'])
while True:
htmlBlacklist = htmlBlacklistQ.fetch_row(1,1)
if htmlBlacklist == ():
break
badSeedHTML = re.search(htmlBlacklist[0]['regex'], infoModule.info.page['rawHTML'])
if badSeedHTML != None:
log.plog('html matches htmlBlocker regex: ' + htmlBlacklist[0]['regex'], 3)
os._exit(0)
###################################
#special case for feedburner sources
#ernst does not like special cases
###################################
infoModule.info.page['url'] = re.sub('\?.*utm_source.*$', '', infoModule.info.page['url'])
#check AGAIN to see if url is already in system
escURL = infoModule.info.page['url'].replace("'", "\\'")
urlCheckQ = mysql_tools.mysqlQuery("select sub_id from " + siteDB + ".newsroom where url='" + escURL + "'", infoModule.info.site['dblink'])
#don't exit, return false so that a new story can be tried
if urlCheckQ.num_rows() > 0:
log.plog("scanpage-url already in newsroom: %s" % infoModule.info.page['url'] , 2)
log.plog("newsroom_id: " + str(urlCheckQ.fetch_row(1,1)))
return False
urlCheckQ = mysql_tools.mysqlQuery("select sub_id from " + siteDB + ".subs where url='" + escURL + "'", infoModule.info.site['dblink'])
if urlCheckQ.num_rows() > 0:
log.plog("scanpage-url already in subs: %s" % infoModule.info.page['url'], 2)
log.plog("sub_id: " + str(urlCheckQ.fetch_row(1,1)))
return False
## if source is '0', try to find source
if infoModule.info.source['source_id'] == '0':
sourceRegexQ = mysql_tools.mysqlQuery("select * from " + siteDB + ".sources where url_regex != ''", infoModule.info.site['dblink'])
while True:
sourceRegex = sourceRegexQ.fetch_row(1,1)
if sourceRegex == ():
break
urlTest = re.search(sourceRegex[0]['url_regex'], infoModule.info.page['url'])
if urlTest != None:
log.plog('found source via regex: ' + sourceRegex[0]['title'], 2)
infoModule.info.source = sourceRegex[0]
for i in infoModule.info.source.keys():
## this is sort of hack-y, but stupid python returns None for null
if infoModule.info.source[i] == None:
infoModule.info.source[i] = ''
break
## maybe check last modified header and don't get stories older than 7 days?
'''possibleAgeInDays = dateGuesser.urlDateGuesser(infoModule.info.page['url'])
if possibleAgeInDays != None:
log.plog("age of story might be: " + str(possibleAgeInDays) + " based on " + infoModule.info.page['url'], 2)
if int(possibleAgeInDays) > 5:
log.plog("story is " + str(possibleAgeInDays) + " days old. Not reading", 2)
return False
'''
if len(infoModule.info.page['rawHTML']) > 500000:
log.plog("article length exceeds 500k, probably not html", 2)
os._exit(0)
#add meta description into the mix
infoModule.info.page['meta_description'] = ''
meta_search = re.search('meta name="description" content="(.*?\s+.*?\s+.*?\s+.*?\s+).*?"', infoModule.info.page['rawHTML'], re.I | re.S)
if meta_search != None:
infoModule.info.page['meta_description'] = meta_search.group(1).decode('utf-8')
log.plog("meta_description: " + infoModule.info.page['meta_description'], 2)
log.plog('======================================= TITLE ================================', 2)
# get title
#set HTMLTitle first
HTMLTitle = re.search('<title>(.*?)<\/title>', infoModule.info.page['rawHTML'], re.S | re.I)
if HTMLTitle != None:
infoModule.info.page['HTMLTitle'] = HTMLTitle.group(1)
log.plog('html title found: ' + infoModule.info.page['HTMLTitle'], 2)
else:
infoModule.info.page['HTMLTitle'] = ""
title = find_title.findTitle()
if title != False:
infoModule.info.page['title'] = title
log.plog('title from regex', 2)
if 'potential_title' in infoModule.info.page and len(infoModule.info.page['potential_title']) > 0:
infoModule.info.page['title'] = strip_html.clearHTML(infoModule.info.page['potential_title'])
log.plog('title from potential_title', 2)
else:
infoModule.info.page['title'] = real_title2.realTitle()
if infoModule.info.page['title'] == False:
infoModule.info.page['title'] = infoModule.info.page['HTMLTitle']
log.plog('using html title', 2)
else:
log.plog('title from realTitle', 2)
if infoModule.info.page['title'] == '':
log.plog('could not find title for page. Setting to HTML Title', 4)
infoModule.info.page['title'] = infoModule.info.page['HTMLTitle']
#clear html from title
infoModule.info.page['title'] = strip_html.clearHTML(infoModule.info.page['title'])
#also titleCase the title
#infoModule.info.page['title'] = infoModule.info.page['title'].title()
log.plog('final title: ' + infoModule.info.page['title'], 2)
log.plog('======================================= OUTLINE ================================', 2)
## fetch outline
if 'featured_source' in infoModule.info.source and infoModule.info.source['featured_source'] == '1':
infoModule.info.page['plainText'] = strip_html.clearHTMLFeatures(infoModule.info.page['rawHTML'])
else:
infoModule.info.page['plainText'] = strip_html.clearHTML(infoModule.info.page['rawHTML'])
outline = False
#this toggle allows for ignoring regex in favor of body_extractor
if infoModule.info.site['skipBodyRegex'] == False:
storySearch = timeout.TimeoutFunction(find_story.findStoryViaRegex, 2)
try:
outline = storySearch()
#set html block used for imaage, author and links to be what outline returns
if outline != False:
infoModule.info.page['imageHTML'] = infoModule.info.page['rawHTML'];
infoModule.info.page['rawHTML'] = outline
except TimeoutFunctionException:
outline = False
log.plog("ERROR regex timed out for %s" % infoModule.info.source['story_start_marker'], 5)
#outline = find_story.findStoryViaRegex()
if outline != False:
if infoModule.info.page['promoter'] == '0' and infoModule.info.source['source_id'] != '0' and 'source_format' in infoModule.info.source and len(infoModule.info.source['source_format']) > 0:
#link scoring only happens on rss feeds
## parse links in page only in regex block if we have regex
log.plog('======================================= LINK SCORING ================================', 2)
links.linkScoring(outline, 'subs')
links.linkScoring(outline, 'newsroom')
log.plog('======================================= OUTBOUND LINKS ================================', 2)
#don't go more than one level deep on blind stories
links.outboundLinks(outline)
if 'featured_source' in infoModule.info.source and infoModule.info.source['featured_source'] == '1':
infoModule.info.page['outline'] = strip_html.clearHTMLFeatures(outline)
else:
infoModule.info.page['outline'] = strip_html.clearHTML(outline)
else:
log.plog('searching for body using body extractor', 2)
outline = body_extractor.extract(infoModule.info.page['plainText'])
if outline != False:
infoModule.info.page['imageHTML'] = infoModule.info.page['rawHTML'];
abbreviatedHTML = html_body_extractor.html_body_extractor(infoModule.info.page['rawHTML'], outline)
if abbreviatedHTML != None:
infoModule.info.page['rawHTML'] = abbreviatedHTML
infoModule.info.page['outline'] = outline
else:
log.plog('could not create an outline for this story!', 5)
os._exit(0)
## parse links in page - no regex, so look in rawHTML for links
## if there are widgetBlockers, first clear them from the html
linkHTML = infoModule.info.page['rawHTML']
widgetBlacklistQ = mysql_tools.mysqlQuery("select * from " + siteDB + ".widgetBlacklist", infoModule.info.site['dblink'])
while True:
widgetBlacklist = widgetBlacklistQ.fetch_row(1,1)
if widgetBlacklist == ():
break
if isinstance(linkHTML, str) == False:
log.plog('linkHTML is not string', 5)
os._exit(0)
wblMatch = re.search(widgetBlacklist[0]['start_text'] + '.*?' + widgetBlacklist[0]['end_text'], linkHTML, re.S | re.I)
if wblMatch != None:
log.plog("found widget blacklist for " + widgetBlacklist[0]['start_text'] + '.*?' + widgetBlacklist[0]['end_text'], 2)
linkHTML = linkHTML.replace(wblMatch.group(0), '')
mysql_tools.mysqlQuery("update " + siteDB + ".widgetBlacklist set hits=hits+1 where widget_id=" + widgetBlacklist[0]['widget_id'], infoModule.info.site['dblink'])
if infoModule.info.page['promoter'] == '0' and infoModule.info.source['source_id'] != '0' and 'source_format' in infoModule.info.source and len(infoModule.info.source['source_format']) > 0:
#link scoring only happens on rss feeds
log.plog('======================================= LINK SCORING ================================', 2)
links.linkScoring(linkHTML, 'subs')
links.linkScoring(linkHTML, 'newsroom')
log.plog('======================================= OUTBOUND LINKS ================================', 2)
#don't go more than one level deep on blind stories
links.outboundLinks(linkHTML)
log.plog('======================================= IMAGES ================================', 2)
#find images
if 'image_start_marker' in infoModule.info.source:
image_start_marker = infoModule.info.source['image_start_marker']
else:
image_start_marker = ''
if 'image_end_marker' in infoModule.info.source:
image_end_marker = infoModule.info.source['image_end_marker']
else:
image_end_marker = ''
imageArray = find_images.findImages(infoModule.info.page['imageHTML'], image_start_marker, image_end_marker)
if imageArray == None:
log.plog('could not find image', 3)
else:
x = imageArray[0]
y = imageArray[1]
imageURL = imageArray[2]
if imageURL == '':
log.plog('could not find image', 3)
else:
log.plog('image found: ' + imageURL, 2)
infoModule.info.page['largestImage'] = imageURL
infoModule.info.page['maxSize'] = x * y
log.plog('======================================= IMAGE CREDIT ================================', 2)
## image credit if any
infoModule.info.page['imageSource'] = ''
if 'image_source_start_marker' in infoModule.info.source and 'image_source_end_marker' in infoModule.info.source:
imageSource = find_credit.findCredit(infoModule.info.page['rawHTML'], infoModule.info.source['image_source_start_marker'], infoModule.info.source['image_source_end_marker'])
if imageSource != False:
infoModule.info.page['imageSource'] = imageSource
log.plog('======================================= VIDEOS ================================', 2)
###look for videos
videoLink = find_video.findVideoEmbed(infoModule.info.page['rawHTML'])
if videoLink == False:
infoModule.info.page['vlink'] = ''
else:
log.plog('found video embed', 2)
infoModule.info.page['vlink'] = videoLink
vthumb = find_video.findVideoThumb(videoLink)
if vthumb == False:
infoModule.info.page['vthumb'] = ''
else:
log.plog('found video thumb', 2)
infoModule.info.page['vthumb'] = vthumb
log.plog('======================================= AUTHOR ================================', 2)
##author in story?
if 'author_start_marker' in infoModule.info.source and 'author_end_marker' in infoModule.info.source:
author = find_author.findAuthor()
if author != False:
author = strip_html.clearHTML(author)
infoModule.info.page['author'] = author
else:
infoModule.info.page['author'] = ''
else:
infoModule.info.page['author'] = ''
log.plog('======================================= ENTITIES ================================', 2)
#### find entities
entities.entityFinder(infoModule.info.page['title'] + ' ' + infoModule.info.page['outline'], True)
nickname = False
while nickname is False:
try:
entities.nicknameFinder(infoModule.info.page['title'] + ' ' + infoModule.info.page['outline'], True)
nickname = True
except:
pass
## test cityToTeam
#cityToTeam.getRelevantEntity()
entities.setPrimo()
#### chop outline to 500 chars unless featured
if 'featured_source' not in infoModule.info.source or infoModule.info.source['featured_source'] == '0':
infoModule.info.page['outline'] = infoModule.info.page['outline'][0:500] + '...'
if len(infoModule.info.entityList) < 1:
log.plog("no entities found in story!", 5)
os._exit(0)
log.plog('======================================= UNKNOWN ENTITIES ================================', 2)
## any unknown entities?
entityFixedString = infoModule.info.page['title'] + ' ' + infoModule.info.page['outline']
entityFixedString = entityFixedString.replace("'s", "")
entityFixedString = re.sub('\W+', ' ', entityFixedString)
find_new_entities.find_new_entities(entityFixedString)
## page must have at least one non-hidden entity
invisibleTypesQuery = mysql_tools.mysqlQuery("select mptype_id from db_topics.mptypes where visibility='invisible'", infoModule.info.site['dblink'])
invisibleTypes = ''
sep = ''
while True:
oneType = invisibleTypesQuery.fetch_row(1,1)
if oneType == ():
break
invisibleTypes = invisibleTypes + sep + oneType[0]['mptype_id']
sep = ','
sep = ''
cclist = ''
for eKey in infoModule.info.entityList.keys():
cclist = cclist + sep + str(eKey)
sep = ','
sql = "select celeb_id from db_topics.celebs where celeb_id in (" + cclist + ") and mptype_id not in (" + invisibleTypes + ")"
nonHiddenEntitiesQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if nonHiddenEntitiesQ.num_rows() == 0:
log.plog('no non-hidden entities found in story!', 4)
os._exit(0)
newSubId = addStory.addStory()
if newSubId == False:
log.plog('addStory failed', 5)
else:
log.plog("Story added. New sub_id: " + str(newSubId), 2)
os._exit(0)
'''
findCelebVerticals
Takes entities
for each entity, comes up with a percentage for each vertical. celebrifi / total, politifi / total, sportifi / total
then we add up the percentages for each entity, highest percentage and that's the vertical we get
'''
link = mysql_tools.mysqlConnect('192.168.0.99', 'gaga', 'badromance')
#link = mysql_tools.mysqlConnect(user="******", password="******")
if link == False:
print "no connection"
sys.exit(0)
infoModule.info.site['dblink'] = link
infoModule.info.site['log_priority_threshold'] = 3
mysql_tools.mysqlQuery("set wait_timeout = 600", link)
mysql_tools.mysqlQuery("set interactive_timeout = 600", link)
'''
getEntityTotals
takes a single entity id
returns a dict with vertical as key, and value as another dict of stories_total, and storiesWeighted_total
'''
def getEntityTotals(entity_id):
entDict = {}
sql = 'select celeb_id, stories_total, storiesWeighted_total, vertical from db_topics.celebStatsTotals where celeb_id =' + str(entity_id)
statsQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while (1):
row = statsQ.fetch_row(1,1)
if row == ():
def promoteFromNewsroom(sub_id):
###################################################################
##func: promoteFromNewsroom
##param: sub_id
##desc: takes newsroom story and makes it life
##ret: bool
##auth: esr
##################################################################
# if debugmode, this func doesn't run at all
if 'debug_mode' in infoModule.info.site and infoModule.info.site[
'debug_mode'] == True:
log.plog("debug mode. Not running promoteFromNewsroom", 3)
return True
# if newsroom sdatetime > 0000 then copy that to sdatetime, not now()
siteDB = infoModule.info.site['database']
if 'source_id' not in infoModule.info.source:
log.plog("cannot convert newsroom to subs without source_id", 5)
return False
sub_id = str(sub_id)
#check to make sure this newsroom id exists
sql = "select sub_id from " + siteDB + ".newsroom where sub_id=" + sub_id
sanityCheckQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if sanityCheckQ == False or sanityCheckQ.num_rows() == 0:
log.plog(
"no such sub_id " + sub_id + " when converting newsroom to sub", 5)
return False
log.plog('promoting ' + str(sub_id) + ' from newsroom', 3)
sql = "insert into " + siteDB + ".subs (status, user_id, sdatetime, firstPosted, votes, score, imgsrc, image_id, title, outline, url, vlink, topPick, vthumb, source_id, sourceType, celebrifier, siteMatrix, title_source, site_1, site_2, site_3, site_4, site_5, site_6, site_7, site_8, site_9, site_10, site_11, site_12, site_13, site_14, site_15, author) select status, user_id, sdatetime, firstPosted, votes, 1, imgsrc, image_id, title, outline, url, vlink, topPick, vthumb, source_id, 'feed', " + infoModule.info.source[
'source_id'] + ", siteMatrix, title_source, site_1, site_2, site_3, site_4, site_5, site_6, site_7, site_8, site_9, site_10, site_11, site_12, site_13, site_14, site_15, author from " + siteDB + ".newsroom where sub_id=" + str(
sub_id)
log.plog(sql, 2)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
newSubId = infoModule.info.site['dblink'].insert_id()
if newSubId == 0:
log.plog("error creating sub from newsroom id " + sub_id, 5)
return False
newSubId = str(newSubId)
log.plog("@~!!~@ : new sub id: " + newSubId, 2)
#copy over new subs_sites_x sub_site joiner table
#for each table that exists, copy data
sql = "show tables from " + siteDB + " like 'newsroom_sites_%'"
joinerTableQuery = mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
while True:
joiner = joinerTableQuery.fetch_row(1, 1)
if joiner == ():
break
subSiteId = joiner[0]['Tables_in_' + siteDB +
' (newsroom_sites_%)'][15:]
# select from newsroom_sites_x. If newsroom id exists there, insert NEW sub ID into subs_sites_x
sql = "select sub_id from " + siteDB + ".newsroom_sites_" + subSiteId + " where sub_id=" + str(
sub_id)
newsroomSubSiteQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
if newsroomSubSiteQ.num_rows() > 0:
sql = "insert into " + siteDB + ".subs_sites_" + subSiteId + " set sub_id=" + str(
newSubId)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
#and delete old record from newsroom_sites_x
sql = "delete from " + siteDB + ".newsroom_sites_" + subSiteId + " where sub_id=" + str(
sub_id)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "insert into " + siteDB + ".subs_celebs (sub_id, celeb_id, primo) select " + newSubId + ", celeb_id, primo from " + siteDB + ".newsroom_celebs where sub_id=" + sub_id
log.plog(sql, 2)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "select imgsrc, vlink from " + siteDB + ".newsroom where sub_id=" + sub_id
picsVideosFlagsQ = mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
if picsVideosFlagsQ == False:
log.plog("picsVideosFlags query failed", 5)
return False
picsVideosFlags = picsVideosFlagsQ.fetch_row(1, 1)
if picsVideosFlags == ():
log.plog("picsVideosFlags query failed", 5)
return False
sql = "delete from " + siteDB + ".newsroom_celebs where sub_id=" + sub_id
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "select celeb_id, primo from " + siteDB + ".subs_celebs where sub_id=" + newSubId
newStoryEntitiesQ = mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
while True:
newStoryEntity = newStoryEntitiesQ.fetch_row(1, 1)
if newStoryEntity == ():
break
entity = newStoryEntity[0]['celeb_id']
primo = newStoryEntity[0]['primo']
sql = "select celeb_id from " + siteDB + ".celebStats where celeb_id=" + entity
statsExistsQ = mysql_tools.mysqlQuery(sql,
infoModule.info.site['dblink'])
videoPoints = 0
picPoints = 0
if picsVideosFlags[0]['vlink'] != '':
videoPoints = 1
if picsVideosFlags[0]['imgsrc'] == 'U':
picPoints = 1
if statsExistsQ.num_rows > 0:
sql = "update " + siteDB + ".celebStats set photos = photos + " + str(
picPoints) + ", videos = videos + " + str(
videoPoints
) + ", stories = stories + 1 where celeb_id = " + entity
else:
sql = "insert into " + siteDB + ".celebStats set photos = " + str(
picPoints) + ", videos = " + str(
videoPoints) + ", stories = 1, celeb_id = " + entity
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
#update hourlystats here because newsroom hourly stats aren't counted
storiesWeighted = 1
if primo == 'Y':
storiesWeighted = 10
elif primo == '2':
storiesWeighted = 7
elif primo == '3':
storiesWeighted = 5
elif primo == '4':
storiesWeighted = 3
#break up stats time by hourly blocks
hourBlock = int(math.floor(time.time() / 3600))
statsExistsQ = mysql_tools.mysqlQuery(
"select celeb_id from " + siteDB +
".celebHourlyStats where celeb_id=" + str(entity) +
" and hourBlock = " + str(hourBlock),
infoModule.info.site['dblink'])
if statsExistsQ.num_rows() > 0:
sql = "update " + siteDB + ".celebHourlyStats set photos = photos + " + str(
picPoints
) + ", videos = videos + " + str(
videoPoints
) + ", stories = stories + 1, storiesWeighted = storiesWeighted + " + str(
storiesWeighted) + " where celeb_id = " + str(
entity) + " and hourBlock = " + str(hourBlock)
else:
sql = "insert into " + siteDB + ".celebHourlyStats set photos = " + str(
picPoints) + ", videos = " + str(
videoPoints) + ", stories = 1, storiesWeighted = " + str(
storiesWeighted) + ", celeb_id = " + str(
entity) + ", hourBlock = " + str(hourBlock)
log.plog(sql, 2)
#don't count stories towards stats if they are in newsroom
if 'debug_mode' not in infoModule.info.site or infoModule.info.site[
'debug_mode'] == False:
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
#update celeb popularity
sql = "insert into " + siteDB + ".relatedSubs set sub_id1=" + newSubId + ", sub_id2=-1, table1='subs'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "delete from " + siteDB + ".newsroom where sub_id=" + sub_id
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "update " + siteDB + ".relatedSubs set sub_id1 = " + newSubId + ", table1='subs' where sub_id1=" + sub_id + " and table1='newsroom'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "update " + siteDB + ".relatedSubs set sub_id2 = " + newSubId + ", table2='subs' where sub_id2=" + sub_id + " and table2='newsroom'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
#every time a new story is created, build the relationship xml
sql = "insert into " + siteDB + ".atom_queue set placed=now(), sub_id=" + str(
newSubId)
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return True
import sys
import _mysql
import mysql_tools
link = mysql_tools.mysqlConnect('127.0.0.1', 'root', '')
if link == False:
print 'no connection'
sys.exit(0)
sql = 'UPDATE db_topics.celebs_related SET cid_1=cid_2, cid_2=cid_1 where cid_1 > cid_2'
mysql_tools.mysqlQuery(sql, link)
print 'DONE - UPDATE db_topics.celebs_related SET cid_1=cid_2, cid_2=cid_1 where cid_1 > cid_2'
print 'START - deleting dupes'
sql = "SELECT COUNT(*) AS count, CONCAT(cid_1, CONCAT('_', cid_2)) AS concat FROM db_topics.celebs_related GROUP BY CONCAT(cid_1, CONCAT('_', cid_2)) HAVING COUNT(*) > 1"
dupeQ = mysql_tools.mysqlQuery(sql, link)
while (1):
row = dupeQ.fetch_row(1, 1)
if row == ():
break
cid_1, cid_2 = row[0]['concat'].split('_')
count = int(row[0]['count'])
if count > 1:
sql = 'DELETE FROM db_topics.celebs_related WHERE cid_1 = ' + cid_1 + ' AND cid_2 = ' + cid_2 + ' ORDER BY relevance ASC LIMIT ' + str(
count - 1)
mysql_tools.mysqlQuery(sql, link)
print "DONE - deleting dupes"
sql = 'DELETE FROM db_topics.celebs_related WHERE relevance IS NULL'
mysql_tools.mysqlQuery(sql, link)
def addMicrodata(text, entities, fullNames=False):
# mptypes by category
organizations = [37, 43, 44, 48, 49, 66, 76, 79, 104, 110, 120]
if len(entities) == 0:
log.plog("no entities passed to addMicrodata", 5)
return text
if (text == None or text == ''):
log.plog("no text passed to addMicrodata", 5)
return text
#celebList should be sorted from long to short
entityNameArray = []
for entity in entities:
entityNameArray.append(
[entity,
len(entityLib.entityLibrary(entity, 'entityName'))])
entityNameArray = sorted(entityNameArray,
key=lambda nameLen: nameLen[1],
reverse=True)
htmlBlocks = []
#set aside all html so that celeb matches are not made inside
ctr = 0
while True:
reres = re.search('<.*?>', text)
if reres == None:
break
htmlBlocks.append(reres.group(0))
text = text.replace(reres.group(0), "~*~%d~*~" % ctr)
# print"TEXT: " + text
ctr += 1
for entityTuple in entityNameArray:
pprint.pprint(entityTuple)
if entityLib.entityLibrary(entityTuple[0],
'visibility') != 'invisible':
name = entityLib.entityLibrary(entityTuple[0], 'entityName')
name = name.strip()
itemtype = itemprop = None
# Person microdata
if (name != '' and
(entityLib.entityLibrary(entityTuple[0], 'human') == str(1))):
itemtype = 'Person'
itemprop = 'name'
if int(entityLib.entityLibrary(entityTuple[0],
'mptype_id')) in organizations:
itemtype = 'Person'
itemprop = 'affiliation'
if itemtype and itemprop != None:
itemscope = '''<span itemscope itemtype="http://www.data-vocabulary.org/''' + itemtype + '''">'''
step1 = re.search(
"([^\]=-])\\b" + name + "('*s*)\\b([^\[]){1}", text, re.I)
if step1 != None:
text = text.replace(
step1.group(0),
step1.group(1) + itemscope + '<span itemprop="' +
itemprop + '">' + name + '</span></span>' +
step1.group(2) + step1.group(3))
step2 = re.search("^" + name + "(\'*s*)\\b([^\[]){1}", text)
if step2 != None:
text = text.replace(
step2.group(0), itemscope + '<span itemprop="' +
itemprop + '">' + name + '</span></span>' +
step2.group(1) + step2.group(2))
step3 = re.search('\\b' + name + '$', text, re.I)
if step3 != None:
text = text.replace(
step3.group(0), itemscope + '<span itemprop="' +
itemprop + '">' + name + '</span></span>')
# now check by first and last name
#in between passes, alter text to have ~(num)~ where the celeb blocks are
tmpArray = []
tmpCtr = 0
# the num points to the array containing the bit so that it can be rebuilt after nicknames are run.
if fullNames == False:
#only do first and last name if the fullNames (meaning require full name) isn't set to true
while True:
squareBlocks = re.search('<span itemscope.*?<\/span><\/span>',
text)
pprint.pprint(squareBlocks)
if squareBlocks == None:
break
tmpArray.append(squareBlocks.group(0))
text = text.replace(squareBlocks.group(0),
"~#~" + str(tmpCtr) + "~#~")
tmpCtr += 1
for entityTuple in entityNameArray:
pprint.pprint(entityTuple)
if entityLib.entityLibrary(entityTuple[0],
'visibility') != 'invisible':
lname = entityLib.entityLibrary(entityTuple[0], 'lname')
if lname != None:
lname = lname.strip()
fname = entityLib.entityLibrary(entityTuple[0], 'fname')
if fname != None:
fname = fname.strip()
itemtype = itemprop = None
if (lname != '' and (entityLib.entityLibrary(
entityTuple[0], 'human') == str(1))):
itemtype = 'Person'
itemprop = 'name'
if (fname != '' and (entityLib.entityLibrary(
entityTuple[0], 'human') == str(1))):
itemtype = 'Person'
itemprop = 'name'
# affiliation microdata
# Get mytype_id and see if res is in organization array
if int(entityLib.entityLibrary(entityTuple[0],
'mptype_id')) in organizations:
itemtype = 'Person'
itemprop = 'affiliation'
if itemtype and itemprop != None:
itemscope = '''<span itemscope itemtype="http://www.data-vocabulary.org/''' + itemtype + '''">'''
# don't use first and last name if fname or lname is a number
try:
int(fname)
except ValueError:
if fname != None and fname != '':
step1 = re.search(
"([^\]=-])\\b" + fname + "('*s*)\\b([^\[]){1}",
text, re.I)
if step1 != None:
text = text.replace(
step1.group(0),
step1.group(1) + itemscope +
'<span itemprop="' + itemprop + '">' +
fname + '</span></span>' + step1.group(2) +
step1.group(3))
step2 = re.search(
"^" + fname + "(\'*s*)\\b([^\[]){1}", text)
if step2 != None:
text = text.replace(
step2.group(0),
itemscope + '<span itemprop="' + itemprop +
'">' + fname + '</span></span>' +
step2.group(1) + step2.group(2))
step3 = re.search('\\b' + fname + '$', text, re.I)
if step3 != None:
text = text.replace(
step3.group(0),
itemscope + '<span itemprop="' + itemprop +
'">' + fname + '</span></span>')
try:
int(lname)
except ValueError:
if lname != None and lname != '':
step1 = re.search(
"([^\]=-])\\b" + lname + "('*s*)\\b([^\[]){1}",
text, re.I)
if step1 != None:
text = text.replace(
step1.group(0),
step1.group(1) + itemscope +
'<span itemprop="' + itemprop + '">' +
lname + '</span></span>' + step1.group(2) +
step1.group(3))
step2 = re.search(
"^" + lname + "(\'*s*)\\b([^\[]){1}", text)
if step2 != None:
text = text.replace(
step2.group(0),
itemscope + '<span itemprop="' + itemprop +
'">' + lname + '</span></span>' +
step2.group(1) + step2.group(2))
step3 = re.search('\\b' + lname + '$', text, re.I)
if step3 != None:
text = text.replace(
step3.group(0),
itemscope + '<span itemprop="' + itemprop +
'">' + lname + '</span></span>')
#nicknames, after the rest are done
#to prep for nicknames, alter text to have ~(num)~ where the celeb blocks are
# the num points to the array containing the bit so that it can be rebuilt after nicknames are run.
while True:
squareBlocks = re.search('<span itemscope.*?<\/span><\/span>', text)
if squareBlocks == None:
break
tmpArray.append(squareBlocks.group(0))
text = text.replace(squareBlocks.group(0), "~#~" + str(tmpCtr) + "~#~")
tmpCtr += 1
for entityTuple in entityNameArray:
if entityLib.entityLibrary(entityTuple[0],
'visibility') != 'invisible':
itemtype = itemprop = None
if entityLib.entityLibrary(entityTuple[0], 'human') == str(1):
itemtype = 'Person'
itemprop = 'name'
# affiliation microdata
# Get mytype_id and see if res is in organization array
if int(entityLib.entityLibrary(entityTuple[0],
'mptype_id')) in organizations:
itemtype = 'Person'
itemprop = 'affiliation'
if itemtype and itemprop != None:
itemscope = '''<span itemscope itemtype="http://www.data-vocabulary.org/''' + itemtype + '''">'''
nicknamesQ = mysql_tools.mysqlQuery(
"select name, case_sensitive from db_topics.nicknames where cid_1="
+ str(entityTuple[0]) + " or cid_2=" +
str(entityTuple[0]) + " or cid_3=" + str(entityTuple[0]),
infoModule.info.site['dblink'])
while True:
nicknameRow = nicknamesQ.fetch_row(1, 1)
if nicknameRow == ():
break
if nicknameRow[0]['case_sensitive'] == 1:
nicknameMatch = re.search(
'\\b' + nicknameRow[0]['name'] + '\\b', text, re.I)
else:
nicknameMatch = re.search(
'\\b' + nicknameRow[0]['name'] + '\\b', text)
if nicknameMatch and itemtype != None:
text = text.replace(
nicknameMatch.group(0),
itemscope + '<span itemprop="' + itemprop + '">' +
nicknameMatch.group(0) + '</span></span>')
#take it out and store it off to prevent more dupes
while True:
squareBlocks = re.search(
'<span itemscope.*?<\/span><\/span>', text)
if squareBlocks == None:
break
tmpArray.append(squareBlocks.group(0))
text = text.replace(squareBlocks.group(0),
"~#~" + str(tmpCtr) + "~#~")
tmpCtr += 1
while True:
repBlock = re.search('~#~(\d+)~#~', text)
if repBlock == None:
break
text = text.replace(repBlock.group(0),
tmpArray[int(repBlock.group(1))])
# text = text.replace("[/celeb]'s", "'s[/celeb]")
# text = text.replace("[/celeb]s", "s[/celeb]")
for i in range(len(htmlBlocks)):
text = text.replace('~*~' + str(i) + '~*~', htmlBlocks[i])
return text
def getTeamFromCity():
###################################################################
##func: getRelevantEntity
##param: none (takes entities from infoModule.info.entityList)
##desc: determines missing team given city(s) and athlete(s), by
## summing the relevance scores of story city(s)/all teams with
## story athlete(s)/all teams.
##ret: cid of team with highest relevance
##auth: mdk
##################################################################
cityIds = []
athleteIds = []
for cid in infoModule.info.entityList.keys():
cidType = entityLib.entityLibrary(cid, 'celeb_type')
if cidType == 'Team':
return False
if cidType == 'City':
cityIds.append(cid)
elif cidType == 'Athlete':
athleteIds.append(cid)
# else:
# print cidType + " " + str(cid)
# exit if no cities identified
if len(cityIds) == 0:
return False
# exit if no athletes identified
if len(athleteIds) == 0:
return False
#log.plog('cityIds: ' + cityIds + 'athleteIds: ' + athleteIds, 2)
athleteDict = {}
cityDict = {}
if len(cityIds) > 1:
cityidIndexCnt = len(cityIds)
else:
cityidIndexCnt = 1
cityIterator = 1
while cityIterator <= cityidIndexCnt:
for id in cityIds:
cityDict[cityIterator] = {}
# create dict of team => relevance
sql = 'SELECT cid_2, relevance FROM db_topics.celebs_related, db_topics.celebs WHERE celebs_related.cid_2=celebs.celeb_id and cid_1 = ' + id + ' AND mptype_id = 75'
cityTeamRelQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
while (1):
cityTeamRel = cityTeamRelQ.fetch_row(1, 1)
if cityTeamRel == ():
break
key = int(cityTeamRel[0]['cid_2'])
value = int(cityTeamRel[0]['relevance'])
cityDict[cityIterator][key] = value
cityIterator += 1
for id in athleteIds:
sql = 'SELECT cid_2, relevance FROM db_topics.celebs_related, db_topics.celebs WHERE celebs_related.cid_2=celebs.celeb_id and cid_1 = ' + id + ' AND mptype_id = 75'
athleteTeamRelQ = mysql_tools.mysqlQuery(
sql, infoModule.info.site['dblink'])
athleteTeamRel = athleteTeamRelQ.fetch_row(1, 1)
while (1):
athleteTeamRel = athleteTeamRelQ.fetch_row(1, 1)
if athleteTeamRel == ():
break
key = int(athleteTeamRel[0]['cid_2'])
value = int(athleteTeamRel[0]['relevance'])
if key in athleteDict:
athleteDict[key] += value
else:
athleteDict[key] = value
if not athleteDict and not cityDict:
return False
# add cityDict values to athleteDict values to get one master dict of team => relevance
if cityDict and athleteDict:
cityIterator = 1
while cityIterator <= cityidIndexCnt:
for key, value in cityDict[cityIterator].items():
if key in athleteDict:
athleteDict[key] += cityDict[cityIterator][key]
else:
athleteDict[key] = value
sortedFinalDict = sorted(athleteDict.iteritems(),
key=operator.itemgetter(1),
reverse=True)
log.plog('picked team => ' + str(sortedFinalDict[0]), 2)
return sortedFinalDict[0]
import imageCredit
import urllib
## warning. This sucker messes with the stories. Don't test it on the live site.
link = mysql_tools.mysqlConnect('192.168.0.115', 'dev_rw', 'dev_rw_pw')
if link == False:
print "no connection"
sys.exit(0)
# set database
infoModule.info.site['database'] = 'db_celebrifi'
infoModule.info.site['dblink'] = link
randSourceQ = mysql_tools.mysqlQuery(
"select * from db_celebrifi.sources where image_source_start_marker != '' order by rand() limit 1",
link)
randSource = randSourceQ.fetch_row(1, 1)
for key in randSource[0].keys():
infoModule.info.source[key] = randSource[0][key]
#find recent story from that source
randStoryQ = mysql_tools.mysqlQuery(
"select url from db_celebrifi.subs where source_id=" +
randSource[0]['source_id'] + " order by sub_id desc limit 1", link)
randStory = randStoryQ.fetch_row(1, 1)
url = randStory[0]['url']
print url
urlSocket = urllib.urlopen(url)
def addContentsToDB(json_contents):
#pprint.pprint(json_contents)
original_url = json_contents['original_url']
#if return is status error
if 'status' not in json_contents['body_extractor']:
print "ERROR in json_contents format: "
pprint.pprint(json_contents)
if json_contents['body_extractor']['status'] == 'ERROR' and json_contents['body_extractor']['error'] == 'Server Busy':
#server was broken. add to explode queue for later
sql = "INSERT IGNORE INTO peepbuzz.explode_queue SET added=now(), original_url='" + original_url + "'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return False
try:
be = json_contents['body_extractor']['story']
except KeyError:
# failure, so delete story
infoModule.info.errorList.append("could not read " + original_url + ", deleting")
sql = "SELECT story_id FROM peepbuzz.stories where original_url='" + original_url + "'"
filament_query = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
while True:
filament_id = filament_query.fetch_row(1,1)
if filament_id == ():
break
sql = "DELETE FROM peepbuzz.filaments where story_id=" + str(filament_id[0]['story_id'])
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
sql = "DELETE from peepbuzz.stories where original_url='" + original_url + "'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
# log failure to failed_urls
try:
fail_reason = json_contents['body_extractor']['error']
except:
fail_reason = 'json_contents error not available'
if 'first_error' in json_contents['body_extractor']:
fail_reason = json_contents['body_extractor']['first_error'] + ' / ' + fail_reason
fail_reason = fail_reason.replace("'", "\\\'")
sql = "INSERT IGNORE into peepbuzz.failed_urls set original_url='" + original_url + "', failed_on=now(), reason='" + fail_reason + "'"
mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
return False
url = be['url']
title = be['title']
outline = be['outline']
try:
title = title.replace("'", "\\'")
except AttributeError:
return False
try:
outline = outline.replace("'", "\\'")
except AttributeError:
return False
query = u'UPDATE peepbuzz.stories set title=\''+title+'\', body=\'' + outline + '\', url=\'' + url + '\', sdatetime=NOW() where original_url=\''+original_url+'\''
try:
mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
except:
return False
sql = "SELECT story_id from peepbuzz.stories WHERE original_url='" + original_url + "'"
print sql
storyIDQ = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
storyIDRow = storyIDQ.fetch_row(1,1)
if storyIDRow == ():
#major error! No story with that original URL?
return False
story_id = storyIDRow[0]['story_id']
# chck images
if(len(be['images'])>0):
for img in be['images']:
query = u'insert into peepbuzz.story_images (story_id, url, width, height) values ("'+str(story_id)+'","'+str(img['url'])+'","'+str(img['width'])+'","'+str(img['height'])+'")'
print query
try:
mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
except:
return False
# check videos
if(len(be['videos'])>0):
for vid in be['videos']:
#check for dupes
sql = "SELECT video_id FROM peepbuzz.story_videos WHERE story_id=" + str(story_id) + " and url='" + str(vid['url']) + "'"
video_dupe_check_q = mysql_tools.mysqlQuery(sql, infoModule.info.site['dblink'])
if video_dupe_check_q.num_rows() == 0:
embed_code = vid['embed_code'].replace("'", "\\'")
query = u"insert into peepbuzz.story_videos (story_id, url, embed_code, width, height) values ('"+str(story_id)+"','"+str(vid['url'])+"','"+embed_code+"','"+str(vid['width'])+"','"+str(vid['height'])+"')"
try:
mysql_tools.mysqlQuery(query, infoModule.info.site['dblink'])
except:
infoModule.info.errorList.append("failed to add video")
infoModule.info.errorList.append(query)
return True
