def update_params(users, g, lookup):
try:
for u in users:
#print(data_path)
print(g.keys())
if u in lookup and lookup[u] in g:
#print(u)
datap = 'data/{}/temp_policy.Rdata'.format('user' + u +
'_pooled_params')
mu = g[lookup[u]][0]
sigma_string = g[lookup[u]][1]
#with open('data/errors_pool.txt','w+') as f:
#f.write('{}'.format(sigma_string))
to_save = {'mu': mu}
for i in range(len(sigma_string)):
to_save['sigma{}'.format(i)] = sigma_string[i]
pyreadr.write_rdata(datap, pd.DataFrame(to_save))
#print(sigma_string)
#z = np.array([float(i) for i in j.split(' ') for j in sigma_string])
except Exception as e:
print(e)
with open('data/runerrors_time_pool.txt', 'w+') as f:
f.write('{}'.format(e))
f.write('\n')
to_return = 'Errors'
print(to_return)
with open('data/errors_time_pool.txt', 'w+') as f:
f.write('{}'.format(e))
f.write('\n')
to_return = 'Errors'
print(to_return)
def end (self):
self.dfs = {}
for level in self.headers.keys():
columns=[v if v.startswith('base__') == False else v[6:] for v in self.colnames_to_display[level]]
level_data = pd.DataFrame(self.data[level], columns=columns)
self.filename = f'{self.filename_prefix}.{level}.{self.filename_postfix}'
self.filenames.append(self.filename)
pyreadr.write_rdata(self.filename, level_data, df_name=f'{self.filename_prefix}_{level}')
return self.filenames
import os
## set working directory
os.chdir("/Users/XXXXXXXXXX/")
import functions as fun
import pyreadr
data_folder = os.getcwd() + "/data/" ## designate data store folder
followers_folder = data_folder + "followers_lists/" ## designate folder of followers lists
## load opinion leader data
freq_accounts = pyreadr.read_r(data_folder + "freq_accounts.rdata")["freq_accounts"]
### get a list of followers who follow at least 5 popular accounts
userlsJP, filesListJP, censusJP = fun.getUserlist(followers_folder, freq_accounts, 5)
### get follow matrix
y_imm_JP = fun.convertToMatrix(userlsJP, filesListJP, censusJP)
### conduct Correspondence Analysis
ca_immJP, colcoord_immJP, rowcoord_immJP, res_immJP = fun.conductCA(y_imm_JP)
res_immJP.columns = ["immDim1","immDim2","immDim3","immDim4"]
### transfer to accounts file
freq_accounts = freq_accounts.merge(res_immJP, left_on="screen_name", right_index=True)
## save files
pyreadr.write_rdata(data_folder + "freq_accounts.rdata", freq_accounts, df_name="freq_accounts")
file_name = data_folder + "y_imm_JP.pkl"
y_imm_JP.to_pickle(file_name)
file_name = data_folder + "y_imm_JP.pkl"
y_imm_JP = pd.read_pickle(file_name)
## create rows of the number of accounts each follower follows for each category
lib_follow_all = np.sum(y_imm_JP.loc[:, lib_leaders.values.tolist()],
axis=1) / len(lib_leaders)
con_follow_all = np.sum(y_imm_JP.loc[:, con_leaders.values.tolist()],
axis=1) / len(con_leaders)
rw_follow_all = np.sum(y_imm_JP.loc[:, rw_leaders.values.tolist()],
axis=1) / len(rw_leaders)
media_follow_all = np.sum(y_imm_JP.loc[:, media_leaders.values.tolist()],
axis=1) / len(media_leaders)
matome_follow_all = np.sum(y_imm_JP.loc[:, matome_leaders.values.tolist()],
axis=1) / len(matome_leaders)
## merge them
all_follow = pd.concat([
lib_follow_all, con_follow_all, rw_follow_all, media_follow_all,
matome_follow_all
],
axis=1)
all_follow.columns = [
"liberal", "conservative", "right_wing", "media", "matome"
]
all_follow["id_str"] = all_follow.index
## save file
pyreadr.write_rdata(data_folder + "all_follow.rdata",
all_follow,
df_name="all_follow")
import pymysql
import pandas as pd
import pyreadr
import sqlalchemy as sql
connect_str = 'mysql+pymysql://ubuntu:db5656576@localhost/research'
sql_engine = sql.create_engine(connect_str)
query = "SELECT * FROM comments"
query_color = "SELECT * FROM colours"
df = pd.read_sql_query(query,sql_engine)
print(df)
pyreadr.write_rdata("comments.RData",df)
print("============break line==========")
df_color = pd.read_sql_query(query_color,sql_engine)
pyreadr.write_rdata("colours.RData",df_color)