def test_aud_target(self, target_id):
db_conn = self.get_conn_conf_by_id(target_id)
# make a connection to database
if db_conn:
if 'envtype' in db_conn.keys():
del db_conn['envtype']
target_db = DBManager(True, True, **db_conn)
target_db.execute_query(sqls4mysql['dblist'])
target_db.close()
def audit_target(self, target_id, snapshot_id='',save_audit=False, envtype='', additional_info=''):
def calculate_max_memory(**mem_param_rec):
try:
max_memory = mem_param_rec['key_buffer_size'] + \
mem_param_rec['query_cache_size'] + \
mem_param_rec['tmp_table_size'] + \
mem_param_rec['innodb_buffer_pool_size'] + \
mem_param_rec['innodb_additional_mem_pool_size'] + \
mem_param_rec['innodb_log_buffer_size'] + \
(mem_param_rec['max_connections'] * (mem_param_rec['sort_buffer_size'] + \
mem_param_rec['read_buffer_size'] + \
mem_param_rec['read_rnd_buffer_size'] + \
mem_param_rec['join_buffer_size'] + \
mem_param_rec['thread_stack'] + \
mem_param_rec['binlog_cache_size']))
return str(max_memory)
except:
return '0'
db_conn = self.get_conn_conf_by_id(target_id)
audit_rec = {}
if db_conn:
# first time when we check 'envtype' is provided or not and trying to get it from target's record
if 'envtype' in db_conn.keys():
if not envtype:
envtype = str(list(db_conn['envtype'])[0])
del db_conn['envtype']
db_conn_4disply = db_conn.copy()
# hide the password and then print
if 'password' in db_conn_4disply.keys():
db_conn_4disply['password'] = '******'
print("\nAuditing ID={1}, connecting to [{0}]...".format(db_conn_4disply, target_id))
try:
target_db = DBManager(True, False, **db_conn)
except Exception as err:
print("For ID={0}, audit failed ...".format(target_id))
print(err)
return
additional_info += 'connected to: {0}'.format(db_conn_4disply['host']) if 'host' in db_conn_4disply.keys() else 'connected to: {0}'.format('localhost')
audit_rec['snapshot_id'] = snapshot_id if snapshot_id else datetime.now().strftime('%S%M%H%d%m%Y%f')
audit_rec['host'] = dict(target_db.execute_query(sqls4mysql['host'], True)[0])['host']
audit_rec['osversion'] = dict(target_db.execute_query(sqls4mysql['osversion'], True)[0])['osversion']
envtype = envtype.upper()
audit_rec['envtype'] = self.get_env_type(audit_rec['host']) if not envtype or envtype not in ('PROD','STG','QA','DEV') else envtype
audit_rec['dbversion'] = dict(target_db.execute_query(sqls4mysql['dbversion'], True)[0])['dbversion']
# checking if INNODB is enabled
innodb = str(target_db.get_global_variable('have_innodb')).upper()
additional_info += ', Have_InnoDB is set to ({0})'.format(innodb)
# calculating MAX memory
memory_params = dict([(row[0][1],long(row[1][1])) for row in target_db.execute_query(sqls4mysql['memory_params'],True)])
audit_rec['max_req_memory'] = calculate_max_memory(**memory_params)
audit_rec['additional_info'] = additional_info
for dblist in target_db.execute_query(sqls4mysql['dblist'], True):
dbname = dict(dblist)['Database']
audit_rec['dbname'] = dbname
audit_rec['dbsize'] = dict(target_db.execute_query(sqls4mysql['dbsize'].format(dbname), True)[0])['dbsize']
audit_rec['free_space'] = dict(target_db.execute_query(sqls4mysql['free_space'].format(dict(dblist)['Database']), True)[0])['free_space']
if save_audit:
cmd = sqls4mysql['insert_aud_rec'].format(audit_rec['snapshot_id'],
audit_rec['host'],
audit_rec['osversion'],
audit_rec['envtype'],
audit_rec['dbversion'],
audit_rec['dbname'],
audit_rec['dbsize'],
audit_rec['free_space'],
audit_rec['max_req_memory'],
audit_rec['additional_info'])
self.execute_dml(cmd, False, message='', additional_fail_msg='', additional_ok_msg='')
else:
print(audit_rec)
target_db.close()
print('OK')
else:
print("Couldn't find record with id={0} ...".format(target_id))
# draw_venn("data/corrects.p", "charts/venn.pdf")
# ids, repins = db.get_repins()
# aes_feats, aes_data = db.get_data_aesthetics(db, aes_filter, ids)
# sem_feats, sem_data = db.get_data_semantics(db, sem_filter, ids)
# soc_feats, soc_data = db.get_data_social(db, ids)
# image_grid(db, "/scratch/images")
# spearman('data/rankcor.txt', 'charts/rankcor.pdf')
# boxplots(soc_data, "soc", soc_feats)
# control_followers(db, ids, aes_feats, aes_data, repins, "aes")
# plot_control_followers("aes")
# plot_followers_groups('data/followers_groups_boards.p', 'charts/followers_groups_boards.pdf')
# plot_followers_groups('data/followers_groups_users.p', 'charts/followers_groups_users.pdf')
# plot_followers_x_repins('data/followers_x_repins.txt', 'charts/followers_x_repins.pdf')
# plot_gaps('data/gap_accs.txt', 'charts/gap_accs.pdf')
# plot_gaps('data/gap_acc_residues.txt', 'charts/gap_acc_residues.pdf')
# plot_gaps('data/gap_acc_residues.txt', 'charts/gap_acc_residues.pdf')
# plot_cdfs(db)
db.close()
class ExtractorManager() :
def __init__(self, thread_id, extractors):
'''
Since this is run on the main process, it shouldn't
open connection or file descriptors.
'''
self.thread_id = thread_id
# Features to be extracted
self.extrators = extractors
from warnings import filterwarnings
filterwarnings('ignore')
def init(self):
'''
This is actually the real constructor and is run at the start of the forked process.
'''
# Database connection
self.db = DBManager(host=config.DB_HOST,
user=config.DB_USER,
passwd=config.DB_PASSWD,
db=config.DB_SCHEMA,
socket=config.DB_SOCKET,
table="jobs")
# Create folder for scaled images if necessary
if (not os.path.exists(config.SCALED_FOLDER)) :
os.makedirs(config.SCALED_FOLDER)
# Open log file
log_folder = "log"
if (not os.path.exists(log_folder)) :
os.makedirs(log_folder)
self.log_file = open("%s/%s.%d" % (log_folder, socket.gethostname(), self.thread_id), "a")
def log(self, message, show=False) :
'''
File for multi-threading logging.
'''
print >> self.log_file, message
self.log_file.flush()
if show:
print message
def run(self):
print "Starting %s." % self.thread_id
# Should initiate already in the forked process
self.init()
# Process all users allocated to this thread
while (not utils.inout.stop_file_exists()) :
image_id = None
try :
image_id = self.db.get_next(DBManager.AVAILABLE, self.thread_id)
# Check if there isn't already an scaled version of the image
if (not utils.image.exists_and_valid(config.SCALED_FOLDER, image_id)) :
# Load original sized image
rgb_img = utils.image.load_image(config.IMAGE_FOLDER, image_id)
# Scale it down keeping the aspect ratio
rgb_img = utils.image.scale_down(rgb_img, config.MAX_PIXELS)
# Save a copy on disk
utils.image.save_image(config.SCALED_FOLDER , image_id, rgb_img)
else :
# Load scaled down version of the image
rgb_img = utils.image.load_image(config.SCALED_FOLDER, image_id)
# Process all registered extractors
for extractor in self.extrators :
print extractor.__class__
concepts = extractor.process(rgb_img)
self.db.save_features(extractor.get_table_name(), image_id, concepts)
# Update the pin status to DOWNLOADED
self.db.update_status(image_id, DBManager.COMPLETED)
# Everything went ok if got here
print "%s: OK" % image_id
# Nothing to collect
except NothingToProcessException:
self.log("Nothing to process.", show=True)
break
# Any other exception we log the traceback, update the DB and life goes on.
except Exception:
# Could not even get image id. Must halt.
if image_id is None:
print traceback.format_exc()
break
self.log("%s: ERROR\n%s" % (image_id, traceback.format_exc()), show=True)
self.db.update_status(image_id, DBManager.ERROR)
# Last thing before exiting thread
self.close()
def close(self):
'''Clean up routine'''
self.db.close()
from item import Item
from dbmanager import DBManager
dbManager = DBManager()
dbManager.open()
dbManager.createItemsTable()
it = dbManager.getItem(11272912, 1)
for i in it:
print(i)
dbManager.close()
