def __init__(self): try: self.params = get_db_config() self.conn = psycopg2.connect(**self.params) self.cur = self.conn.cursor() except (Exception, psycopg2.DatabaseError) as error: raise (error)
def time_similarity(database, table, field='username'): db2 = MySQLConnection(**get_db_config(database=database)) cursor2 = db2.cursor() query = "SELECT GROUP_CONCAT( DISTINCT %s ) FROM %s GROUP BY time" % ( field, table) cursor2.execute(query) results = cursor2.fetchall() time_sim = defaultdict(dict) for row in results: reader = csv.reader([row[0]]) values = reader.__next__() for user1 in values: for user2 in values: if user1 != user2: if (user1, user2) in time_sim: time_sim[user1, user2] += 1 else: time_sim[user1, user2] = 1 with open('timesim_%s_%s' % (database, table), mode='w') as f: for user, count in time_sim.items(): f.write('%s,%s,%d\n' % (user[0], user[1], count)) logging.info('Time similarity done. Database: %s, Table: %s' % (database, table))
def __init__(self, findex, begin, end): threading.Thread.__init__(self) self.begin = begin self.end = end self.feature_index = findex self.samples = samples[begin:end] self.db = MySQLConnection(**get_db_config(database=db_metapath))
def init_session(self): cf = config.get_db_config() connect_string = 'mysql+pymysql://%s:%s@%s:%s/%s' % ( cf['user'], cf['password'], cf['host'], cf['port'], cf['db']) print("connect_string:", connect_string) self.engine = create_engine(connect_string) self.Session = sessionmaker() self.Session.configure(bind=self.engine)
def set_config(self): try: self.config = get_db_config(self.config, db.read_config()) if self.config['active']: self.active = True else: self.active = False except Exception as e: self.manage_exception(e)
def get_db_connection_string(): if DB_CONNECTION_STRING: return DB_CONNECTION_STRING # Only import "config" on demand, as it depends on Datastore packages (and # GAE). When running via CLI or tests, we'll have this from the environment # instead (above). import config return config.get_db_config()['db_connection_string']
def get_db_pool(): global _db_pool if not _db_pool: database_config = config.get_db_config() _db_pool = pool.SimpleConnectionPool(minconn=1, maxconn=20, **database_config) return _db_pool
def get_db_connection_string(backup=False, instance_name=None): if DB_CONNECTION_STRING: return DB_CONNECTION_STRING # Only import "config" on demand, as it depends on Datastore packages (and # GAE). When running via CLI or tests, we'll have this from the environment # instead (above). import config connection_string_key = 'backup_db_connection_string' if backup else 'db_connection_string' result = config.get_db_config()[connection_string_key] if instance_name: if backup: raise Exception( "backup and instance_name should not be used together") # Connect to the specified instance. return result.replace('rdrmaindb', instance_name) return result
def setUp(self) -> None: command = ['./control.py', 'print_tokens'] out = subprocess.check_output(command, cwd=PROJECT_DIR).decode().split('\n') for line in out: if not line: continue token = line.strip().split(':')[1] if 'working' in line: self.working_token = token else: self.unreachable_token = token database_config = config.get_db_config() database_config['host'] = '127.0.0.1' self.db_pool = pool.SimpleConnectionPool(minconn=1, maxconn=20, **database_config)
def __init__(self): try: self.bigquery = bigquery self.params = get_db_config(section='gcp') credentials = service_account.Credentials.from_service_account_file( self.params['credentials_file_path'], scopes=["https://www.googleapis.com/auth/cloud-platform"], ) self.client = bigquery.Client( credentials=credentials, project=credentials.project_id, ) self.storage = bigquery_storage.BigQueryReadClient( credentials=credentials) except Exception as error: raise (error)
def __init__(self): self.engine = None self.config = get_db_config()
from flask import Flask, render_template, Response from flask.ext.mysqldb import MySQL from werkzeug.routing import BaseConverter, ValidationError import ssl from config import get_db_config import json from datetime import datetime, timedelta conf = get_db_config() # SSL if 'certificate' in conf['app'] and 'privkey' in conf['app']: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_context.load_cert_chain(conf['app']['certificate'], conf['app']['privkey']) else: ssl_context = None # Reads config.ini from the directory you execute the application from. app = Flask(__name__) app.config['DEBUG'] = conf['app']['debug'] app.config['MYSQL_HOST'] = conf['mysqld']['host'] app.config['MYSQL_USER'] = conf['mysqld']['user'] app.config['MYSQL_PASSWORD'] = conf['mysqld']['password'] app.config['MYSQL_DB'] = conf['mysqld']['db'] app.config['MYSQL_PORT'] = int(conf['mysqld']['port']) #app.config['MYSQL_UNIX_SOCKET', None) app.config['MYSQL_CONNECT_TIMEOUT'] = 10 #app.config['MYSQL_READ_DEFAULT_FILE', None) app.config['MYSQL_USE_UNICODE'] = True app.config['MYSQL_CHARSET'] = 'utf8'
# this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata md = ConfigManager() md.load_db_config(cf.get_db_config()) connect_string = md.get_connect_string() print("======", connect_string) config.set_main_option('sqlalchemy.url', connect_string) target_metadata = base_model.BaseModel.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc.
def get_db_connection_string(): return DB_CONNECTION_STRING or config.get_db_config( )['db_connection_string']
def _warmup(): # Load configurations into the cache. get_config() get_db_config() return '{ "success": "true" }'
from mysql.connector import MySQLConnection from progressbar import Bar, ProgressBar from progressbar import Percentage from scipy.sparse import csr_matrix from sklearn.feature_extraction.text import TfidfVectorizer from config import get_db_config logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(asctime)s:\t %(message)s', datefmt='%H:%M:%S') gamma_a_set = ['80'] gamma_t_set = ['10', '20', '30', '40', '50', '60', '70', '80', '90', '100'] db = MySQLConnection(**get_db_config()) cursor = db.cursor() t = 'ss' common = '%s_common' % (t, ) twitter = '%s_twitter' % (t, ) fsquare = '%s_fsquare' % (t, ) similarity = '%s_sim' % (t, ) metapaths = '%s_metapaths' % (t, ) def execute_sim(query): db2 = MySQLConnection(**get_db_config(database=similarity)) cursor2 = db2.cursor() cursor2.execute(query)
def execute_sim(query): db2 = MySQLConnection(**get_db_config(database=similarity)) cursor2 = db2.cursor() cursor2.execute(query)
import pymysql import config dbConfig = config.get_db_config() def get_connection(): connection = pymysql.Connect(dbConfig.get_url(), dbConfig.get_username(), dbConfig.get_password(), dbConfig.get_database()) return connection
from flask import Flask, render_template, Response from flask.ext.mysqldb import MySQL from werkzeug.routing import BaseConverter, ValidationError import ssl from config import get_db_config import json from datetime import datetime, timedelta conf = get_db_config() # SSL if 'certificate' in conf['app'] and 'privkey' in conf['app']: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_context.load_cert_chain(conf['app']['certificate'], conf['app']['privkey']) else: ssl_context = None # Reads config.ini from the directory you execute the application from. app = Flask(__name__) app.config['DEBUG'] = conf['app']['debug'] app.config['MYSQL_HOST'] = conf['mysqld']['host'] app.config['MYSQL_USER'] = conf['mysqld']['user'] app.config['MYSQL_PASSWORD'] = conf['mysqld']['password'] app.config['MYSQL_DB'] = conf['mysqld']['db'] app.config['MYSQL_PORT'] = int(conf['mysqld']['port']) #app.config['MYSQL_UNIX_SOCKET', None) app.config['MYSQL_CONNECT_TIMEOUT'] = 10 #app.config['MYSQL_READ_DEFAULT_FILE', None) app.config['MYSQL_USE_UNICODE'] = True
def main(): global samples global features global gamma_a global gamma_t logging.basicConfig(level=logging.INFO, format='[%(asctime)s: %(message)s', datefmt='%H:%M:%S') start = time.time() db = MySQLConnection(**get_db_config(database=db_common)) cursor = db.cursor() cursor.execute('SELECT username,label FROM samples') result = cursor.fetchall() labels = [] for row in result: username = row[0] label = int(row[1]) samples.append(username) labels.append(label) n = len(samples) m = 8 # number of threads # try: # saved = loadmat('dataset_partial') # findex = saved['fc'] # features = saved['X'] # logging.info('Last saved results loaded. continue from %d' % (findex+1,)) # except FileNotFoundError: # features = numpy.zeros([n, TOTAL]) # findex = 0 # logging.info('Starting from scratch...') # gamma_a_set = range(100, 9, -10) gamma_a_set = ['80'] gamma_t_set = range(90, 9, -10) for gamma_a in gamma_a_set: # gamma_t_set = rho_set if gamma_a == '100' else ['100'] # gamma_t_set = ['100'] for gamma_t in gamma_t_set: features = numpy.zeros([n, TOTAL]) findex = 0 logging.info('Extracting Features for gamma_a=%s , gamma_t=%s' % (gamma_a, gamma_t)) threads = [] for i in range(0, n, n // m): threads.append(CounterThread(findex, i, i + n // m)) for t in threads: t.start() check_progress(findex, threads) for t in threads: t.join() logging.info('Features Extraction Done. Saving Final Results...') savemat('dataset_%s_%s_%s' % (tsplit, gamma_a, gamma_t), { 'X': features, 'Y': labels }) end = time.time() logging.info('All Done in %d seconds.' % (end - start, ))
def post_similarity(database, table, field='username'): logging.info('Post similarity for database: %s, table: %s' % (database, table)) db2 = MySQLConnection(**get_db_config(database=database)) cursor2 = db2.cursor() logging.info('Connected to Database. Executing query...') query = "SELECT %s, text FROM %s" % (field, table) cursor2.execute(query) logging.info('Query Executed. Processing Posts...') users = [] documents = [] while True: try: row = cursor2.fetchone() if row is None: break username = row[0] text = row[1] post = text.replace('_', ' ') users.append(username) documents.append(post) except: pass logging.info('Documents are ready. Running TF/IDF analysis...') vect = TfidfVectorizer() tfidf = vect.fit_transform(documents) logging.info('TF/IDF ready. Initializing...') # savemat('tf-idf_' + database, {'tf_idf': tfidf, 'users': users}) # # logging.info('Initializing...') widget = [Bar('=', '[', ']'), ' ', Percentage()] bar = ProgressBar(maxval=len(users), widgets=widget) rows, cols = tfidf.shape simdic = {} x = csr_matrix((1, cols), dtype=numpy.double) bar.start() index = 1 for u in users: simdic[u] = x bar.update(index) index += 1 bar.finish() logging.info('Summing results for each user...') bar = ProgressBar(maxval=rows, widgets=widget) bar.start() for i in range(rows): r = tfidf.getrow(i) u = users[i] simdic[u] = simdic[u] + r bar.update(i + 1) bar.finish() # logging.info('Saving partial results...') # try: # numpy.savez_compressed('user-post-tfidf_%s_%s' % (database, table), simdic=simdic) # except: # pass logging.info('Calculating similarities...') bar = ProgressBar(maxval=len(simdic)**2, widgets=widget) bar.start() index = 1 with open('postsim_%s_%s' % (database, table), mode='w') as f: items = simdic.items() for u, x in items: for v, y in items: if u != v: sim = x * y.T if sim > 0: f.write('%s,%s,%f\n' % (u, v, sim[0, 0])) bar.update(index) index += 1 bar.finish() logging.info('Post similarity done. Database: %s, Table: %s' % (database, table))
def execute_path(query): db2 = MySQLConnection(**get_db_config(database=metapaths)) cursor2 = db2.cursor() cursor2.execute(query)