def __init__(self):
try:
self.params = get_db_config()
self.conn = psycopg2.connect(**self.params)
self.cur = self.conn.cursor()
except (Exception, psycopg2.DatabaseError) as error:
raise (error)
def time_similarity(database, table, field='username'):
db2 = MySQLConnection(**get_db_config(database=database))
cursor2 = db2.cursor()
query = "SELECT GROUP_CONCAT( DISTINCT %s ) FROM %s GROUP BY time" % (
field, table)
cursor2.execute(query)
results = cursor2.fetchall()
time_sim = defaultdict(dict)
for row in results:
reader = csv.reader([row[0]])
values = reader.__next__()
for user1 in values:
for user2 in values:
if user1 != user2:
if (user1, user2) in time_sim:
time_sim[user1, user2] += 1
else:
time_sim[user1, user2] = 1
with open('timesim_%s_%s' % (database, table), mode='w') as f:
for user, count in time_sim.items():
f.write('%s,%s,%d\n' % (user[0], user[1], count))
logging.info('Time similarity done. Database: %s, Table: %s' %
(database, table))
def __init__(self, findex, begin, end):
threading.Thread.__init__(self)
self.begin = begin
self.end = end
self.feature_index = findex
self.samples = samples[begin:end]
self.db = MySQLConnection(**get_db_config(database=db_metapath))
def init_session(self):
cf = config.get_db_config()
connect_string = 'mysql+pymysql://%s:%s@%s:%s/%s' % (
cf['user'], cf['password'], cf['host'], cf['port'], cf['db'])
print("connect_string:", connect_string)
self.engine = create_engine(connect_string)
self.Session = sessionmaker()
self.Session.configure(bind=self.engine)
def set_config(self): try: self.config = get_db_config(self.config, db.read_config()) if self.config['active']: self.active = True else: self.active = False except Exception as e: self.manage_exception(e)
def get_db_connection_string():
if DB_CONNECTION_STRING:
return DB_CONNECTION_STRING
# Only import "config" on demand, as it depends on Datastore packages (and
# GAE). When running via CLI or tests, we'll have this from the environment
# instead (above).
import config
return config.get_db_config()['db_connection_string']
def get_db_pool():
global _db_pool
if not _db_pool:
database_config = config.get_db_config()
_db_pool = pool.SimpleConnectionPool(minconn=1,
maxconn=20,
**database_config)
return _db_pool
def get_db_connection_string(backup=False, instance_name=None):
if DB_CONNECTION_STRING:
return DB_CONNECTION_STRING
# Only import "config" on demand, as it depends on Datastore packages (and
# GAE). When running via CLI or tests, we'll have this from the environment
# instead (above).
import config
connection_string_key = 'backup_db_connection_string' if backup else 'db_connection_string'
result = config.get_db_config()[connection_string_key]
if instance_name:
if backup:
raise Exception(
"backup and instance_name should not be used together")
# Connect to the specified instance.
return result.replace('rdrmaindb', instance_name)
return result
def setUp(self) -> None:
command = ['./control.py', 'print_tokens']
out = subprocess.check_output(command,
cwd=PROJECT_DIR).decode().split('\n')
for line in out:
if not line:
continue
token = line.strip().split(':')[1]
if 'working' in line:
self.working_token = token
else:
self.unreachable_token = token
database_config = config.get_db_config()
database_config['host'] = '127.0.0.1'
self.db_pool = pool.SimpleConnectionPool(minconn=1,
maxconn=20,
**database_config)
def __init__(self):
try:
self.bigquery = bigquery
self.params = get_db_config(section='gcp')
credentials = service_account.Credentials.from_service_account_file(
self.params['credentials_file_path'],
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
self.client = bigquery.Client(
credentials=credentials,
project=credentials.project_id,
)
self.storage = bigquery_storage.BigQueryReadClient(
credentials=credentials)
except Exception as error:
raise (error)
def __init__(self):
self.engine = None
self.config = get_db_config()
from flask import Flask, render_template, Response from flask.ext.mysqldb import MySQL from werkzeug.routing import BaseConverter, ValidationError import ssl from config import get_db_config import json from datetime import datetime, timedelta conf = get_db_config() # SSL if 'certificate' in conf['app'] and 'privkey' in conf['app']: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_context.load_cert_chain(conf['app']['certificate'], conf['app']['privkey']) else: ssl_context = None # Reads config.ini from the directory you execute the application from. app = Flask(__name__) app.config['DEBUG'] = conf['app']['debug'] app.config['MYSQL_HOST'] = conf['mysqld']['host'] app.config['MYSQL_USER'] = conf['mysqld']['user'] app.config['MYSQL_PASSWORD'] = conf['mysqld']['password'] app.config['MYSQL_DB'] = conf['mysqld']['db'] app.config['MYSQL_PORT'] = int(conf['mysqld']['port']) #app.config['MYSQL_UNIX_SOCKET', None) app.config['MYSQL_CONNECT_TIMEOUT'] = 10 #app.config['MYSQL_READ_DEFAULT_FILE', None) app.config['MYSQL_USE_UNICODE'] = True app.config['MYSQL_CHARSET'] = 'utf8'
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
md = ConfigManager()
md.load_db_config(cf.get_db_config())
connect_string = md.get_connect_string()
print("======", connect_string)
config.set_main_option('sqlalchemy.url',
connect_string)
target_metadata = base_model.BaseModel.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def get_db_connection_string():
return DB_CONNECTION_STRING or config.get_db_config(
)['db_connection_string']
def _warmup():
# Load configurations into the cache.
get_config()
get_db_config()
return '{ "success": "true" }'
from mysql.connector import MySQLConnection
from progressbar import Bar, ProgressBar
from progressbar import Percentage
from scipy.sparse import csr_matrix
from sklearn.feature_extraction.text import TfidfVectorizer
from config import get_db_config
logging.basicConfig(level=logging.INFO,
format='[%(levelname)s] %(asctime)s:\t %(message)s',
datefmt='%H:%M:%S')
gamma_a_set = ['80']
gamma_t_set = ['10', '20', '30', '40', '50', '60', '70', '80', '90', '100']
db = MySQLConnection(**get_db_config())
cursor = db.cursor()
t = 'ss'
common = '%s_common' % (t, )
twitter = '%s_twitter' % (t, )
fsquare = '%s_fsquare' % (t, )
similarity = '%s_sim' % (t, )
metapaths = '%s_metapaths' % (t, )
def execute_sim(query):
db2 = MySQLConnection(**get_db_config(database=similarity))
cursor2 = db2.cursor()
cursor2.execute(query)
def __init__(self): self.engine = None self.config = get_db_config()
def execute_sim(query):
db2 = MySQLConnection(**get_db_config(database=similarity))
cursor2 = db2.cursor()
cursor2.execute(query)
import pymysql
import config
dbConfig = config.get_db_config()
def get_connection():
connection = pymysql.Connect(dbConfig.get_url(), dbConfig.get_username(),
dbConfig.get_password(),
dbConfig.get_database())
return connection
from flask import Flask, render_template, Response
from flask.ext.mysqldb import MySQL
from werkzeug.routing import BaseConverter, ValidationError
import ssl
from config import get_db_config
import json
from datetime import datetime, timedelta
conf = get_db_config()
# SSL
if 'certificate' in conf['app'] and 'privkey' in conf['app']:
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.load_cert_chain(conf['app']['certificate'],
conf['app']['privkey'])
else:
ssl_context = None
# Reads config.ini from the directory you execute the application from.
app = Flask(__name__)
app.config['DEBUG'] = conf['app']['debug']
app.config['MYSQL_HOST'] = conf['mysqld']['host']
app.config['MYSQL_USER'] = conf['mysqld']['user']
app.config['MYSQL_PASSWORD'] = conf['mysqld']['password']
app.config['MYSQL_DB'] = conf['mysqld']['db']
app.config['MYSQL_PORT'] = int(conf['mysqld']['port'])
#app.config['MYSQL_UNIX_SOCKET', None)
app.config['MYSQL_CONNECT_TIMEOUT'] = 10
#app.config['MYSQL_READ_DEFAULT_FILE', None)
app.config['MYSQL_USE_UNICODE'] = True
def main():
global samples
global features
global gamma_a
global gamma_t
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s: %(message)s',
datefmt='%H:%M:%S')
start = time.time()
db = MySQLConnection(**get_db_config(database=db_common))
cursor = db.cursor()
cursor.execute('SELECT username,label FROM samples')
result = cursor.fetchall()
labels = []
for row in result:
username = row[0]
label = int(row[1])
samples.append(username)
labels.append(label)
n = len(samples)
m = 8 # number of threads
# try:
# saved = loadmat('dataset_partial')
# findex = saved['fc']
# features = saved['X']
# logging.info('Last saved results loaded. continue from %d' % (findex+1,))
# except FileNotFoundError:
# features = numpy.zeros([n, TOTAL])
# findex = 0
# logging.info('Starting from scratch...')
# gamma_a_set = range(100, 9, -10)
gamma_a_set = ['80']
gamma_t_set = range(90, 9, -10)
for gamma_a in gamma_a_set:
# gamma_t_set = rho_set if gamma_a == '100' else ['100']
# gamma_t_set = ['100']
for gamma_t in gamma_t_set:
features = numpy.zeros([n, TOTAL])
findex = 0
logging.info('Extracting Features for gamma_a=%s , gamma_t=%s' %
(gamma_a, gamma_t))
threads = []
for i in range(0, n, n // m):
threads.append(CounterThread(findex, i, i + n // m))
for t in threads:
t.start()
check_progress(findex, threads)
for t in threads:
t.join()
logging.info('Features Extraction Done. Saving Final Results...')
savemat('dataset_%s_%s_%s' % (tsplit, gamma_a, gamma_t), {
'X': features,
'Y': labels
})
end = time.time()
logging.info('All Done in %d seconds.' % (end - start, ))
def post_similarity(database, table, field='username'):
logging.info('Post similarity for database: %s, table: %s' %
(database, table))
db2 = MySQLConnection(**get_db_config(database=database))
cursor2 = db2.cursor()
logging.info('Connected to Database. Executing query...')
query = "SELECT %s, text FROM %s" % (field, table)
cursor2.execute(query)
logging.info('Query Executed. Processing Posts...')
users = []
documents = []
while True:
try:
row = cursor2.fetchone()
if row is None:
break
username = row[0]
text = row[1]
post = text.replace('_', ' ')
users.append(username)
documents.append(post)
except:
pass
logging.info('Documents are ready. Running TF/IDF analysis...')
vect = TfidfVectorizer()
tfidf = vect.fit_transform(documents)
logging.info('TF/IDF ready. Initializing...')
# savemat('tf-idf_' + database, {'tf_idf': tfidf, 'users': users})
#
# logging.info('Initializing...')
widget = [Bar('=', '[', ']'), ' ', Percentage()]
bar = ProgressBar(maxval=len(users), widgets=widget)
rows, cols = tfidf.shape
simdic = {}
x = csr_matrix((1, cols), dtype=numpy.double)
bar.start()
index = 1
for u in users:
simdic[u] = x
bar.update(index)
index += 1
bar.finish()
logging.info('Summing results for each user...')
bar = ProgressBar(maxval=rows, widgets=widget)
bar.start()
for i in range(rows):
r = tfidf.getrow(i)
u = users[i]
simdic[u] = simdic[u] + r
bar.update(i + 1)
bar.finish()
# logging.info('Saving partial results...')
# try:
# numpy.savez_compressed('user-post-tfidf_%s_%s' % (database, table), simdic=simdic)
# except:
# pass
logging.info('Calculating similarities...')
bar = ProgressBar(maxval=len(simdic)**2, widgets=widget)
bar.start()
index = 1
with open('postsim_%s_%s' % (database, table), mode='w') as f:
items = simdic.items()
for u, x in items:
for v, y in items:
if u != v:
sim = x * y.T
if sim > 0:
f.write('%s,%s,%f\n' % (u, v, sim[0, 0]))
bar.update(index)
index += 1
bar.finish()
logging.info('Post similarity done. Database: %s, Table: %s' %
(database, table))
def execute_path(query):
db2 = MySQLConnection(**get_db_config(database=metapaths))
cursor2 = db2.cursor()
cursor2.execute(query)
