def user_network():
"""
Start iteration process for load data to user_network
"""
log = get_logger()
with get_connect() as con, get_connect_iptable() as con_ip:
cur = con.cursor()
cur_ip = con_ip.cursor()
# get settings
settings = get_settings(cur)
load_date_start = datetime.strptime(settings['LOAD_DATE_START'],
'%Y.%m.%d %H:%M:%S.%f')
load_pause = int(settings['LOAD_PAUSE'])
load_pause_empty = int(settings['LOAD_PAUSE_EMPTY'])
cnt_rows = int(settings['LOAD_ROWS'])
while True:
date = user_network_load(cur_ip, cur, load_date_start, cnt_rows)
log.info("Load data. load_date_start: %s", date)
# save date between iterations
if date > load_date_start:
sql = "update load_settings set value=%s where param=%s"
date_str = datetime.strftime(date, '%Y.%m.%d %H:%M:%S.%f')
cur.execute(sql, [date_str, 'LOAD_DATE_START'])
else:
# sleep if not new data
log.debug("No new data sleep( %s )", load_pause_empty)
time.sleep(load_pause_empty)
if load_pause:
log.debug("sleep between loads sleep( %s )", load_pause)
time.sleep(load_pause)
load_date_start = date
def download(request):
#下载分析结果
taskcontent = request.POST.get('taskcontent', '')
filename = taskcontent+'.xls' if taskcontent else 'result.xls'
response = HttpResponse(mimetype="applicationnd.ms-excel")
response['Content-Disposition'] = 'attachment; filename=%s' % filename
if taskcontent:
cursor = get_connect()
sql = '''select b.screen_name, a.* from report a join tasklist b on a.uid=b.uid where a.day=%s'''
cursor.execute(sql, (taskcontent, ))
res = cursor.fetchall()
if res:
wb = write_excel(res)
wb.save(response)
return response
uid, weitext = res
weitext = ast.literal_eval(weitext)
dic = {}
fields = ['id', 'reposts_count', 'comments_count', 'pid']
[dic.update({i:weitext.get(i, 0)}) for i in fields]
user = weitext.get('user', {})
user_fields = ['statuses_count', 'location', 'friends_count', 'profile_image_url', 'screen_name', 'verified_type',
'followers_count', 'favourites_count', 'bi_followers_count', 'gender']
[dic.update({i:user.get(i, '')}) for i in user_fields]
dic['description'] = user.get('description', '')
dic['uid'] = user.get('id', '')
dic['user_created_at'] = fmt_create_at(user.get('created_at', ''))
days2now = int(time.mktime(dic['user_created_at'].timetuple()))
date.update({dic['uid']: [dic['bi_followers_count'], dic['statuses_count'], dic['friends_count'], days2now, dic['followers_count'],
dic['favourites_count'], 0 if dic['verified_type'] == -1 else 1, dic['verified_type'],
0 if dic['gender'] == 'f' else 1, len(dic['description'])]})
n += 1
if n%200 == 0:
update_sql(table_name, date)
n = 0
date = {}
tids = {}
update_sql(table_name, date)
if __name__ == "__main__":
print 'start'
cursor = get_connect()
update(cursor, 'interact_tweet_2013_12')
cursor.close()
print 'end'
[dic.update({i: user.get(i, '')}) for i in user_fields]
dic['description'] = user.get('description', '')
dic['uid'] = user.get('id', '')
dic['user_created_at'] = fmt_create_at(user.get('created_at', ''))
days2now = int(time.mktime(dic['user_created_at'].timetuple()))
date.update({
dic['uid']: [
dic['bi_followers_count'], dic['statuses_count'],
dic['friends_count'], days2now, dic['followers_count'],
dic['favourites_count'],
0 if dic['verified_type'] == -1 else 1, dic['verified_type'],
0 if dic['gender'] == 'f' else 1,
len(dic['description'])
]
})
n += 1
if n % 200 == 0:
update_sql(table_name, date)
n = 0
date = {}
tids = {}
update_sql(table_name, date)
if __name__ == "__main__":
print 'start'
cursor = get_connect()
update(cursor, 'interact_tweet_2013_12')
cursor.close()
print 'end'
def home(request):
cursor = get_connect()
sqlcmd = 'select DISTINCT(`day`) from report ORDER BY day desc'
cursor.execute(sqlcmd)
res = cursor.fetchall()
return render_to_response('tasklist.html', {'rows':res})
