source[table][column]['min'], source[table][column][ 'max'] = min, max if __name__ == '__main__': config_items = { 'meta_file': {'value':None}, \ 'size_file': {'value':None,'default':None}, \ 'index_file': {'value':None, 'default':None}, \ } parser = argparse.ArgumentParser() for k, v in sorted(config_items.iteritems()): parser.add_argument("--" + k) args = parser.parse_args() for k, v in config_items.iteritems(): if eval(str("args." + k)) == None: if 'default' in config_items[k].keys(): config_items[k]['value'] = config_items[k]['default'] else: config_items[k]['value'] = raw_input("Enter " + k + ":") else: config_items[k]['value'] = eval(str("args." + k)) json_file = config_items['meta_file']['value'] m = MetaData() m.import_from_file(json_file) update(m, config_items['size_file']['value'], config_items['index_file']['value']) m.export_to_file(json_file + ".updated.json") m.export_to_stdout()
def main(): print("Welcome to Deep Data Bench. version: %s" % __version__) config_items = OrderedDict() config_items['source_mysql_user'] = {'section':'MySQL', 'value':None, 'default' : None} config_items['source_mysql_password'] = {'section':'MySQL', 'value':None, 'default' : None} config_items['source_mysql_socket'] = {'section':'MySQL', 'value':None, 'default' : None} config_items['source_mysql_host'] = {'section':'MySQL', 'value':None, 'default' : None} config_items['source_mysql_port'] = {'section':'MySQL', 'value':None, 'default' : 3306} config_items['source_database'] = {'section':'MySQL', 'value':None, 'default' : None} config_items['destination_mysql_user'] = {'section':'MySQL', 'value':None} config_items['destination_mysql_password'] = {'section':'MySQL', 'value':None} config_items['destination_mysql_socket'] = {'section':'MySQL', 'value':None, 'default' : None} config_items['destination_mysql_host'] = {'section':'MySQL', 'value':None} config_items['destination_mysql_port'] = {'section':'MySQL', 'value':None, 'default' : 3306} config_items['destination_database'] = {'section':'MySQL', 'value':None} config_items['pillar_durations'] = {'section':'Pillar', 'value':None, 'default' : "60s,60s,60s"} config_items['pillars'] = {'section':'Pillar', 'value':None, 'default' : "PureLoad,EvenCRUD,Analytics"} config_items['num_clients'] = {'section':'Pillar', 'value':None, 'default' : "8,8,8"} config_items['show_stats_frequency'] = {'section':'Pillar', 'value':0, 'default':0} config_items['tables'] = {'section':'MySQL', 'value':None, 'default':'*'} config_items['collect_stats'] = {'section':'MySQL', 'value':None, 'default':False} #TODO: Make this a flag and cleanup metadata.py config_items['retain_destination_database'] = {'section':'MySQL', 'value':None, 'default':False} #TODO: Make this a flag config_items['write_sql_to_disk'] = {'section':'Pillar', 'value':None, 'default':False} #TODO: Make this a flag config_items['repeat_x_times'] = {'section':'Pillar', 'value':1, 'default':1} config_items['report_name'] = {'section':'Pillar', 'value':None, 'default':None} config_items['destination_mysql_engine'] = {'section':'MySQL', 'value':None, 'default':None} parser = argparse.ArgumentParser() config = ConfigParser.ConfigParser() parser.add_argument("--config",help="config containing all arguments") for k, v in sorted(config_items.iteritems()): parser.add_argument("--" + k) args = parser.parse_args() if (args.config != None): if os.path.isfile(args.config) and os.access(args.config, os.R_OK): #print "config " + args.config + " exists and is readable" config.read(args.config) for k, v in config_items.iteritems(): if not(config.has_section(v['section']) and config.has_option(v['section'], k)): if eval(str("args."+k)) != None: config_items[k]['value'] = eval(str("args."+k)) elif 'default' in config_items[k].keys(): #print "The config does not contain " + k + " in section " + v['section'] + ". Using default: " + str(config_items[k]['default']) config_items[k]['value'] = config_items[k]['default'] else: print "The config does not contain " + k + " in section " + v['section'] sys.exit(1) else: #make command line override config file if eval(str("args."+k)) != None: config_items[k]['value'] = eval(str("args."+k)) else: config_items[k]['value'] = config.get(v['section'], k) else: print args.config + " is either missing or is not readable." sys.exit(1) else: for k, v in config_items.iteritems(): if eval(str("args."+k)) == None: if 'default' in config_items[k].keys(): config_items[k]['value'] = config_items[k]['default'] else: config_items[k]['value'] = raw_input("Enter " + k + ":") else: config_items[k]['value'] = eval(str("args."+k)) if not config.has_section(config_items[k]['section']): config.add_section(config_items[k]['section']) for k, v in config_items.iteritems(): if config_items[k]['value'] is not None: config.set(v['section'], k, config_items[k]['value']) # Writing our configuration file to with open('my.ini', 'wb') as configfile: config.write(configfile) for k, v in config_items.iteritems(): print k + ":" + str(v['value']) profiles_to_run = config_items['pillars']['value'].split(",") pillar_durations = config_items['pillar_durations']['value'].split(",") if config_items['source_database']['value'] is not None: source_databases = config_items['source_database']['value'].split(",") else: source_databases = ['placeholder'] destination_databases = config_items['destination_database']['value'].split(",") num_clients = config_items['num_clients']['value'].split(",") if config_items['source_database']['value'] is not None: if not (len(source_databases) == len(destination_databases)): print "the source and destination databases must have the same number of comma separated items." sys.exit(1) if not (len(profiles_to_run) == len(pillar_durations) == len(num_clients)): print "pillars, pillar_durations and num_clients must have the same number of comma separated items." sys.exit(1) #if len(profiles_to_run) < len(destination_databases): # destination_databases = [] # for d in destination_databases: # destination_databases.append(config_items['destination_database']['value']) todo = zip(profiles_to_run, pillar_durations,num_clients) for k in range(int(config_items['repeat_x_times']['value'])): print "starting iteration " + str(k) for index, source_database in enumerate(source_databases): print "Source DB: " + source_database + ", Destination DB: " + destination_databases[index] meta = MetaData(profiles_to_run[0]) if os.path.isfile(profiles_to_run[0].split(" ")[0]): meta.import_from_file(profiles_to_run[0].split(" ")[0]) print "imported " + str(profiles_to_run[0].split(" ")[0]) else: meta.load(config_items['source_mysql_user']['value'], config_items['source_mysql_password']['value'], config_items['source_mysql_host']['value'], int(config_items['source_mysql_port']['value']), config_items['source_mysql_socket']['value'], source_database, False) if not config_items['retain_destination_database']['value']: print "dumping into " + destination_databases[index] meta.dump(config_items['destination_mysql_user']['value'], config_items['destination_mysql_password']['value'], config_items['destination_mysql_host']['value'], int(config_items['destination_mysql_port']['value']), config_items['destination_mysql_socket']['value'], destination_databases[index],config_items['destination_mysql_engine']['value']) dict_of_profiles_to_run = {} final_report = PillarReport() final_report.num_tables = meta.getNumberOfTables() final_report.num_columns = meta.getTotalNumberOfColumns() final_report.num_indexes = meta.getTotalNumberOfIndexes() final_report.mysql_variables = meta.mysql_variables final_report.create_tables = meta.create_tables for pillar in todo: profile_file_list = [] for profile in pillar[0].split(" "): m = MetaData(profile) if os.path.isfile(profile.strip()): m.import_from_file(profile) profile_file_list.append(profile) else: if profile == "PureLoad": m.load(config_items['source_mysql_user']['value'], config_items['source_mysql_password']['value'], config_items['source_mysql_host']['value'], int(config_items['source_mysql_port']['value']), config_items['source_mysql_socket']['value'], source_database, config_items['collect_stats']['value']) else: m.load(config_items['source_mysql_user']['value'], config_items['source_mysql_password']['value'], config_items['source_mysql_host']['value'], int(config_items['source_mysql_port']['value']), config_items['source_mysql_socket']['value'], source_database,False) f = os.path.dirname(os.path.realpath(__file__)) + "/meta_data/" + profile + ".json" #TODO add tables somehow to the name? maybe just random name m.export_to_file(f) profile_file_list.append(f) #check for no tables if len(m.meta_data.keys()) == 0: print "profile: " + profile + " has no tables? exiting..." sys.exit(1) pillar_name = pillar[0].replace(" ","_") dict_of_profiles_to_run[pillar_name] = DBAppConductor(pillar[2].split(" "), pillar[1], config_items['destination_mysql_user']['value'], config_items['destination_mysql_password']['value'], config_items['destination_mysql_host']['value'], int(config_items['destination_mysql_port']['value']), config_items['destination_mysql_socket']['value'], destination_databases[index], profile_file_list, config_items['tables']['value'].split(" ") ) print "Running Profile: " + pillar_name + " on database " + destination_databases[index] + " ..." dict_of_profiles_to_run[pillar_name].setShowStatsFrequency(config_items['show_stats_frequency']['value']) dict_of_profiles_to_run[pillar_name].write_sql_to_disk = config_items['write_sql_to_disk']['value'] dict_of_profiles_to_run[pillar_name].go() final_report.feedClientStats(pillar_name,dict_of_profiles_to_run[pillar_name].getClientStats()) final_report.printFullReport() if config_items['report_name']['value'] == None: #f = tempfile.NamedTemporaryFile(mode='wb', delete=False, dir=os.path.dirname(os.path.realpath(__file__)) + "/reports/", suffix='.dump', prefix=source_database+'_report') f = tempfile.NamedTemporaryFile(mode='wb', delete=False, dir=os.getcwd(), suffix='.dump', prefix=destination_databases[index] + '_report') else: #f = open(os.path.dirname(os.path.realpath(__file__)) + "/reports/" + config_items['report_name']['value'] + "_" + str(k) + ".dump",'wb') f = open(os.getcwd() + "/" + config_items['report_name']['value'] + "_" + str(k) + ".dump",'wb') pickle.dump(final_report, f) f.close()
max = min + item('uniqueness') else: min = 0 max = 128 source[table][column]['min'], source[table][column]['max'] = min, max if __name__ == '__main__': config_items = { 'meta_file': {'value':None}, \ 'size_file': {'value':None,'default':None}, \ 'index_file': {'value':None, 'default':None}, \ } parser = argparse.ArgumentParser() for k, v in sorted(config_items.iteritems()): parser.add_argument("--" + k) args = parser.parse_args() for k, v in config_items.iteritems(): if eval(str("args."+k)) == None: if 'default' in config_items[k].keys(): config_items[k]['value'] = config_items[k]['default'] else: config_items[k]['value'] = raw_input("Enter " + k + ":") else: config_items[k]['value'] = eval(str("args."+k)) json_file = config_items['meta_file']['value'] m = MetaData() m.import_from_file(json_file) update(m,config_items['size_file']['value'],config_items['index_file']['value']) m.export_to_file(json_file + ".updated.json") m.export_to_stdout()
class QueryGenerator(multiprocessing.Process): #join_types = ['INNER','LEFT','RIGHT'] join_types = ['INNER','LEFT'] aggregate_functions = ['MIN', 'MAX', 'AVG', 'COUNT', 'SUM'] def __init__(self, pipe_home, mysqluser, mysqlpass, host, port, socket, database, table, duration, profile = "Random", write_sql_to_disk = False): multiprocessing.Process.__init__(self) self.pipe_home = pipe_home self.savepoint_list = [] self.savepoint_counter = 0 self.potential_tables = [] self.__begin_sql_insert = {} self.__insert_sql = {} self.transaction_size = 1 self.iterations_instead_of_duration = True self.write_sql_to_disk = write_sql_to_disk self.update_min_max_counter = 0 self.tables_in_transaction = [] self.num_inserts = 0 self.delta = 0.0 self.start_time = 0.0 self.poison_pill = False self.where_clause_gen_having_trouble = 0 self.where_clause_gen_trying_desc = 0 if os.path.isfile(profile): self.__metadata = MetaData(profile) self.__metadata.import_from_file(profile) self.profilefilename = os.path.splitext(os.path.basename(profile))[0] #print self.name + " loaded " + profile + " from disk" # loaded Profile from disk. else: self.__metadata = MetaData(profile) self.__metadata.load(mysqluser, mysqlpass, host, port, socket, database, True) self.__metadata.export_to_file(os.path.dirname(os.path.realpath(__file__)) + "/meta_data/" + database + "." + profile + ".json") self.profilefilename = profile #print "creating profile on the fly..." self.profile = profile self.write_sql_to_file = os.path.dirname(os.path.realpath(__file__)) + "/meta_data/" + self.name + self.profilefilename + ".sql" duration = str(duration) if duration[-1] == 's': self.iterations_instead_of_duration = False duration = duration[:-1] self._duration = float(duration) else: self.iterations_instead_of_duration = True self._duration = int(duration) self.__mysqluser = mysqluser self.__mysqlpass = mysqlpass self.__host = host self.__port = port self.__socket = socket self.__database = database self.__table = table self.auto_increment_increment = 1 # number of clients self.auto_increment_offset = 1 # client number self.statistics = ClientQueryStats(self.name) #self.random_worker = RandomWorker() self.generated_queries = {} self.connect_to_mysql() self.__initialize_things() def connect_to_mysql(self): try: if self.__socket == None: self.__db = MySQLdb.connect(host=self.__host, port=self.__port, user=self.__mysqluser, passwd=self.__mysqlpass, db=self.__database) else: self.__db = MySQLdb.connect(unix_socket=self.__socket, user=self.__mysqluser, passwd=self.__mysqlpass, db=self.__database) self.__db.autocommit(1) self.__cur = self.__db.cursor() self.__cur.execute("SET @@session.time_zone='+00:00'") #self.__cur.execute('set unique_checks=0') except MySQLdb.Error, e: print "An Error occured in " + self.__class__.__name__ + " %s" %e exit(1)