Example #1
0
    def setUp(self):
        settings.read_properties("pydbcopy.conf")

        self.source_host = MySQLHost(settings.source_host, settings.source_user, \
                                     settings.source_password, settings.source_database)
        self.dest_host = MySQLHost(settings.target_host, settings.target_user, \
                                   settings.target_password, settings.target_database)
    
        #
        # Just in case lets tear down an old or canceled run...
        #
        self.tearDown()
        
        #
        # Bring up the fixture
        #
        c = self.source_host.conn.cursor()
        c.execute("SET AUTOCOMMIT=1")
        c.execute("create table if not exists tmp_pydbcopy_test ( id integer primary key, test_string varchar(50) )")
        c.execute("insert into tmp_pydbcopy_test (id,test_string) values (1,'test')")
        c.execute("create table if not exists tmp_hashed_pydbcopy_test ( id integer primary key, test_string varchar(50), fieldHash varchar(50) )")
        c.execute("insert into tmp_hashed_pydbcopy_test (id,test_string,fieldHash) values (1,'test','123')")
        c.execute("insert into tmp_hashed_pydbcopy_test (id,test_string,fieldHash) values (2,'test1','234')")
        c.execute("insert into tmp_hashed_pydbcopy_test (id,test_string,fieldHash) values (3,'test2','345')")
        c.execute("create table if not exists tmp_pydbcopy_modified_test ( id integer primary key, test_string varchar(50), lastModifiedDate timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP )")
        c.execute("insert into tmp_pydbcopy_modified_test (id,test_string,lastModifiedDate) values (1,'test','2010-11-23 05:00:00')")
        c.close()
Example #2
0
    def setUp(self):
        settings.read_properties("pydbcopy.conf")
        
        self.source_host = MySQLHost(settings.source_host, settings.source_user, \
                            settings.source_password, settings.source_database)
        self.dest_host = MySQLHost(settings.target_host, settings.target_user, \
                          settings.target_password, settings.target_database)

        #
        # Bring up the fixture
        #
        c = self.source_host.conn.cursor()
        c.execute("SET AUTOCOMMIT=1")
        # Create tmp_pydbcopy_test table and a single row
        c.execute("create table if not exists tmp_pydbcopy_test ( id integer primary key, test_string varchar(50) )")
        c.execute("insert into tmp_pydbcopy_test (id,test_string) values (1,'test')")
        # Create tmp_hashed_pydbcopy_test table and three rows
        c.execute("create table if not exists tmp_hashed_pydbcopy_test ( id integer primary key, test_string varchar(50), fieldHash varchar(50) )")
        c.execute("insert into tmp_hashed_pydbcopy_test (id,test_string,fieldHash) values (1,'test','123')")
        c.execute("insert into tmp_hashed_pydbcopy_test (id,test_string,fieldHash) values (2,'test1','234')")
        c.execute("insert into tmp_hashed_pydbcopy_test (id,test_string,fieldHash) values (3,'test2','345')")
        # Create tmp_pydbcopy_modified_table and a single row
        c.execute("create table if not exists tmp_pydbcopy_modified_test ( id integer primary key, test_string varchar(50), lastModifiedDate timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP )")
        c.execute("insert into tmp_pydbcopy_modified_test (id,test_string, lastModifiedDate) values (1,'test', '2010-11-23 05:00:00')")
        c.close()
        
        c = self.dest_host.conn.cursor()
        c.execute("SET AUTOCOMMIT=1")
        c.execute("create table if not exists tmp_pydbcopy_modified_test ( id integer primary key, test_string varchar(50), lastModifiedDate timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP )")
        c.execute("insert into tmp_pydbcopy_modified_test (id,test_string, lastModifiedDate) values (1,'test', '2010-11-22 05:00:00')")
        c.close()
Example #3
0
def main(argv=None):
    """
        This is the main routine for pydbcopy. Pydbcopy copies a set of tables from one 
        database (possibly remote) to a local database. If possible an incremental copy is 
        performed (see the perform_incremental_copy routine below). If an 
        incremental copy cannot be performed a full copy is performed (see the perform_full_copy
        routine below).
        
        This routine reads settings from a prop file and then overrides any properties therein
        with command line args. See the usage output for a description of the arguments. 
        
        This program is multi-process by default but can be switched into single process mode
        for debugging (see the --debug option in the usage). Pydbcopy will use a number of processes 
        equal to the number of cpus detected on the system minus one.
    """
    if argv is None:
        argv = sys.argv

    parser = get_option_parser()
    options = parser.parse_args(argv[1:])[0]

    if options.properties:
        settings.read_properties(options.properties)
    """ Command Line options override properties from the prop file """
    if options.source_host is not None:
        settings.source_host = options.source_host
    if options.source_user is not None:
        settings.source_user = options.source_user
    if options.source_password is not None:
        settings.source_password = options.source_password
    if options.source_database is not None:
        settings.source_database = options.source_database
    if options.target_host is not None:
        settings.target_host = options.target_host
    if options.target_user is not None:
        settings.target_user = options.target_user
    if options.target_password is not None:
        settings.target_password = options.target_password
    if options.target_database is not None:
        settings.target_database = options.target_database
    if options.scp_user is not None: settings.scp_user = options.scp_user
    if options.dump_dir is not None: settings.dump_dir = options.dump_dir
    if options.verify_threshold is not None:
        settings.verify_threshold = options.verify_threshold
    if options.force_full is not None: settings.force_full = options.force_full
    if options.no_last_mod_check is not None:
        settings.no_last_mod_check = options.no_last_mod_check
    if options.debug is not None: settings.debug = options.debug

    if options.tables is not None: settings.tables = options.tables.split()
    if options.tables_to_skip_verification is not None:
        settings.tables_to_skip_verification = options.tables_to_skip_verification.split(
        )

    if options.num_processes is not None and options.num_processes != '':
        settings.num_processes = int(options.num_processes)

    if settings.num_processes == 0:
        settings.num_processes = multiprocessing.cpu_count() - 1

    settings.verbosity = 0
    if options.verbose is not None and options.verbose is True:
        settings.verbosity = 1

    # check that we have a list of tables to copy
    if len(settings.tables) == 0:
        sys.stderr.write("Error: No tables specified.\n")
        return 1

    # check that user specified dump dir is readable/writable by all
    if not os.path.isdir(settings.dump_dir):
        os.makedirs(settings.dump_dir)

    if os.stat(settings.dump_dir)[stat.ST_MODE] != 0777:
        try:
            os.chmod(settings.dump_dir, 0777)
        except OSError:
            sys.stderr.write("Warning: unable to chmod 777 on '%s'. Pydbcopy may not be able to clean up after itself!\n" \
                             % settings.dump_dir)
            pass

    if not os.access(settings.dump_dir, os.F_OK | os.R_OK | os.W_OK):
        sys.stderr.write("Error: unable to find or create a writable dump dir at '%s'\n" \
                         % settings.dump_dir)
        return 1

    # Configure the logging
    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.DEBUG if settings.verbosity else logging.INFO)
    formatter = logging.Formatter(
        "%(asctime)s - %(processName)s - %(levelname)s - %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    logger.setLevel(logging.DEBUG if settings.verbosity else logging.INFO)

    if not settings.debug and settings.num_processes > 1:
        pool = multiprocessing.Pool(settings.num_processes)
        result_list = pool.map(verify_and_copy_table, settings.tables, 1)
    else:
        result_list = map(verify_and_copy_table, settings.tables)

    failed_tables = set()
    invalid_tables = set()
    skipped_tables = set()
    copied_tables = set()
    for result, table in map(None, result_list, settings.tables):
        if result == 1:
            skipped_tables.add(table)
        elif result == -1:
            invalid_tables.add(table)
        elif result < -1:
            failed_tables.add(table)
        else:
            copied_tables.add(table)

    logger.info('Summary for copy from source database %s on %s to target database %s on %s:' % \
                (settings.source_database, settings.source_host, settings.target_database, settings.target_host))
    logger.info('--------------------------------------')
    if len(skipped_tables) > 0:
        logger.info(' Skipped: %s' % ', '.join(skipped_tables))
    if len(copied_tables) > 0:
        logger.info('  Copied: %s' % ', '.join(copied_tables))
    if len(invalid_tables) > 0:
        logger.error('Invalid: %s' % ', '.join(invalid_tables))
    if len(failed_tables) > 0:
        logger.error(' Failed: %s' % ', '.join(failed_tables))
    logger.info('--------------------------------------')

    if len(invalid_tables) > 0 or len(failed_tables) > 0:
        return -1

    return 0
Example #4
0
def main(argv=None):
    """
        This is the main routine for pydbcopy. Pydbcopy copies a set of tables from one 
        database (possibly remote) to a local database. If possible an incremental copy is 
        performed (see the perform_incremental_copy routine below). If an 
        incremental copy cannot be performed a full copy is performed (see the perform_full_copy
        routine below).
        
        This routine reads settings from a prop file and then overrides any properties therein
        with command line args. See the usage output for a description of the arguments. 
        
        This program is multi-process by default but can be switched into single process mode
        for debugging (see the --debug option in the usage). Pydbcopy will use a number of processes 
        equal to the number of cpus detected on the system minus one.
    """
    if argv is None:
        argv = sys.argv
        
    parser = get_option_parser()
    options = parser.parse_args(argv[1:])[0]
    
    if options.properties:
        settings.read_properties(options.properties)

    """ Command Line options override properties from the prop file """
    if options.source_host is not None: settings.source_host = options.source_host 
    if options.source_user is not None: settings.source_user = options.source_user 
    if options.source_password is not None: settings.source_password = options.source_password 
    if options.source_database is not None: settings.source_database = options.source_database 
    if options.target_host is not None: settings.target_host = options.target_host 
    if options.target_user is not None: settings.target_user = options.target_user 
    if options.target_password is not None: settings.target_password = options.target_password 
    if options.target_database is not None: settings.target_database = options.target_database 
    if options.scp_user is not None: settings.scp_user = options.scp_user 
    if options.dump_dir is not None: settings.dump_dir = options.dump_dir 
    if options.verify_threshold is not None: settings.verify_threshold = options.verify_threshold 
    if options.force_full is not None: settings.force_full = options.force_full 
    if options.no_last_mod_check is not None: settings.no_last_mod_check = options.no_last_mod_check 
    if options.debug is not None: settings.debug = options.debug 

    if options.tables is not None: settings.tables = options.tables.split()
    if options.tables_to_skip_verification is not None: settings.tables_to_skip_verification = options.tables_to_skip_verification.split()

    if options.num_processes is not None and options.num_processes != '':
        settings.num_processes = int(options.num_processes)
        
    if settings.num_processes == 0:
        settings.num_processes = multiprocessing.cpu_count() - 1
            
    settings.verbosity = 0
    if options.verbose is not None and options.verbose is True:
        settings.verbosity = 1
        
    # check that we have a list of tables to copy
    if len(settings.tables) == 0:
        sys.stderr.write("Error: No tables specified.\n")
        return 1
        
    # check that user specified dump dir is readable/writable by all
    if not os.path.isdir(settings.dump_dir):
        os.makedirs(settings.dump_dir)
    
    if os.stat(settings.dump_dir)[stat.ST_MODE] != 0777:
        try:
            os.chmod(settings.dump_dir, 0777)
        except OSError:
            sys.stderr.write("Warning: unable to chmod 777 on '%s'. Pydbcopy may not be able to clean up after itself!\n" \
                             % settings.dump_dir)
            pass
        
    if not os.access(settings.dump_dir, os.F_OK | os.R_OK | os.W_OK):
        sys.stderr.write("Error: unable to find or create a writable dump dir at '%s'\n" \
                         % settings.dump_dir)
        return 1
    
    # Configure the logging
    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.DEBUG if settings.verbosity else logging.INFO)
    formatter = logging.Formatter("%(asctime)s - %(processName)s - %(levelname)s - %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    logger.setLevel(logging.DEBUG if settings.verbosity else logging.INFO)
        
    if not settings.debug and settings.num_processes > 1:
        pool = multiprocessing.Pool(settings.num_processes)
        result_list = pool.map(verify_and_copy_table, settings.tables, 1)
    else:
        result_list = map(verify_and_copy_table, settings.tables)
    
    failed_tables = set()
    invalid_tables = set()
    skipped_tables = set()
    copied_tables = set()
    for result, table in map(None, result_list, settings.tables):
        if result == 1:
            skipped_tables.add(table)
        elif result == -1:
            invalid_tables.add(table)
        elif result < -1:
            failed_tables.add(table)
        else:
            copied_tables.add(table)
    
    logger.info('Summary for copy from source database %s on %s to target database %s on %s:' % \
                (settings.source_database, settings.source_host, settings.target_database, settings.target_host))
    logger.info('--------------------------------------')
    if len(skipped_tables) > 0:
        logger.info(' Skipped: %s' % ', '.join(skipped_tables))
    if len(copied_tables) > 0:
        logger.info('  Copied: %s' % ', '.join(copied_tables))
    if len(invalid_tables) > 0:
        logger.error('Invalid: %s' % ', '.join(invalid_tables))
    if len(failed_tables) > 0:
        logger.error(' Failed: %s' % ', '.join(failed_tables))
    logger.info('--------------------------------------')
    
    if len(invalid_tables) > 0 or len(failed_tables) > 0:
        return -1
    
    return 0