def svn_action_rep_branch_detection(self, action, src_docker_cli, act_func, edits_in_other_folder_before_copy): '''test svn->p4 branch point detection script''' test_case = 'svn_action_rep_%s' % action dst_docker_cli = self.docker_p4d_clients['p4d_%s' % action] src_depot_dir = '/reptest_%s' % action trunk, branch = act_func(src_docker_cli, src_depot_dir, edits_in_other_folder_before_copy) trunk_src = os.path.join(trunk, 'src') branch_src = os.path.join(branch, 'src') # replicate trunk src_depot_dir = trunk dst_depot_dir = src_depot_dir dst_depot = '//depot/svn_rep/buildtest_trunk/...' dst_view = ((dst_depot, './...'), ) src_mapping = ((src_depot_dir, ' '), ) dst_mapping = dst_view args = replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) args.target_rep_branch_root = '//depot/svn_rep' args.source_project_dir = branch_src # detect svn branch point branch_from = detect_branch_point(args) return branch_from
def test_svn_action_rep_sample_mapping_exclude_subdir(self): '''Exclude a subdirectory from sample svn repo ''' test_case = 'svn_action_rep_sample_mapping_exclude_subdir' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/bigtop/trunk' exclude_path = '/bigtop/trunk/bigtop-packages' dst_depot = '//depot/buildtest%s/...' % test_case dst_mapping = ((dst_depot, './...'), ) exclude_mapping = '-%s' % exclude_path src_mapping = ( (src_depot_dir, ' '), (exclude_mapping, ' '), ) exclude_dirs = [ 'bigtop-packages', ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_subdirs=exclude_dirs)
def test_svn_action_rep_sample_mapping_exclude_two_subdir(self): '''Exclude two subdirectories from sample svn repo ''' test_case = 'svn_action_rep_sample_mapping_exclude_two_subdir' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/bigtop/trunk' exclude_path_0 = '/bigtop/trunk/bigtop-packages' exclude_path_1 = '/bigtop/trunk/bigtop-test-framework/src/main' dst_depot = '//depot/buildtest%s/...' % test_case dst_mapping = ((dst_depot, './...'), ) exclude_mapping_0 = '-%s' % exclude_path_0 exclude_mapping_1 = '-%s' % exclude_path_1 src_mapping = ( (src_depot_dir, ' '), (exclude_mapping_0, ' '), (exclude_mapping_1, ' '), ) exclude_dirs = [ exclude_path_0[len(src_depot_dir) + 1:], exclude_path_1[len(src_depot_dir) + 1:], ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_subdirs=exclude_dirs)
def test_svn_action_rep_view_mapping_exclude_certain_file_0(self): '''Exclude test_dir_parent/a.* ''' test_case = 'svn_action_rep_view_mapping_exclude_certain_file_0' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/%s' % test_case exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir) for idx, exclude_path in enumerate([exc_path, exc_path + '_1']): dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx) dst_mapping = ((dst_depot, './...'), ) exclude_mapping = '-%s/a.*' % exclude_path src_mapping = ((src_depot_dir, ' '), (exclude_mapping, ' ')) excluded_files = [ os.path.join(exclude_path[len(src_depot_dir) + 1:], fn) for fn in [ 'a.exe', 'a.json', 'a.txt', ] ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_files=excluded_files) obliterate_all_depots(dst_docker_cli)
def test_svn_action_rep_view_mapping_exclude_external_subdirs(self): '''Exclude a whole external: test_dir_parent/deploy ''' test_case = 'svn_action_rep_view_mapping_exclude_externals_subdirs' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/%s' % test_case exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir) for idx, exclude_path in enumerate([exc_path, exc_path + '_1']): dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx) dst_mapping = ((dst_depot, './...'), ) exclude_path = '-%s/deploy' % exclude_path src_mapping = ((src_depot_dir, ' '), (exclude_path, ' ')) verification_exclude = exclude_path[len('-') + len(src_depot_dir):] exclude_subdir = [ verification_exclude, ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_subdirs=exclude_subdir) obliterate_all_depots(dst_docker_cli)
def test_replicate_repos_bigtop_trunk_source_counter(self): '''test incorrect source counter ''' test_case = 'replicate_repos_bigtop_trunk_source_counter' src_depot_dir = '/bigtop/site/trunk/content/xdoc' dst_depot_dir = os.path.basename(src_depot_dir) src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir) svn_revs.insert(0, 0) num_replicate = 2 src_ip = src_docker_cli.get_container_ip_addr() dst_ip = dst_docker_cli.get_container_ip_addr() dst_depot = '//depot/buildtest%s/...' % dst_depot_dir dst_view = ((dst_depot, './...'), ) src_mapping = ((src_depot_dir, ' '), ) dst_mapping = dst_view for idx, src_counter in enumerate(svn_revs[:10:num_replicate]): changelist_offset = 0 if idx == 1: changelist_offset = -1 elif idx == 2: changelist_offset = 1 try: replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=src_counter + changelist_offset, replicate_change_num=num_replicate) except ReplicationException, e: if ('src counter' in str(e) and 'last replicated rev' in str(e) and changelist_offset > 0): break else: raise verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=src_counter, replicate_change_num=num_replicate)
def test_svn_action_rep_view_mapping_rel_externals_include_subsubdir(self): '''Exclude external: test_dir_parent/hadoop/hadoop_src but keep test_dir_parent/hadoop/hadoop_src/resources/images ''' test_case = 'svn_action_rep_view_mapping_rel_externals_include_subsubdir' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/%s' % test_case exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir) for idx, exclude_path in enumerate([exc_path, exc_path + '_1']): dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx) dst_mapping = ((dst_depot, './...'), ) exclude_dir = '-%s/hadoop/hadoop_src' % exclude_path include_dir = '+%s/hadoop/hadoop_src/resources/images' % exclude_path src_mapping = ( (src_depot_dir, ' '), (exclude_dir, ' '), (include_dir, ' '), ) external_files_to_exclude = [ 'resources/css/site.css', 'site.xml', 'xdoc/index.xml', 'xdoc/irc-channel.xml', 'xdoc/issue-tracking.xml', 'xdoc/mail-lists.xml', 'xdoc/release-notes.xml', 'xdoc/team-list.xml', ] excluded_files = [ os.path.join(exclude_path[len(src_depot_dir) + 1:], 'hadoop/hadoop_src/%s' % fn) for fn in external_files_to_exclude ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_files=excluded_files) obliterate_all_depots(dst_docker_cli)
def test_svn_action_rep_sample_mapping_exclude_recursively_same_suffix( self): '''Exclude files with same suffix from sample svn repo ''' test_case = 'svn_action_rep_sample_mapping_exclude_recursively_same_suffix' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/bigtop/trunk' exclude_path = '/bigtop/trunk/bigtop-tests' dst_depot = '//depot/buildtest%s/...' % test_case dst_mapping = ((dst_depot, './...'), ) exclude_mapping = '-%s/.../*.out' % exclude_path src_mapping = ( (src_depot_dir, ' '), (exclude_mapping, ' '), ) rel_exc_files = '''test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-string.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date-export.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-all-tables.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-columns.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-where-clause.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-testtable.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_fp.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_string.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool-export.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-query.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-non-string.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-append.out test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_int.out''' excluded_files = [ os.path.join(exclude_path[len(src_depot_dir) + 1:], fn.strip()) for fn in rel_exc_files.split('\n') ] logger.error('excluded_files: %s' % excluded_files) replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_files=excluded_files)
def test_replicate_repos_bigtop_trunk_ingroups_changing_repoinfo_location( self): '''replicate the whole /repos/bigtop/trunk ''' test_case = 'replicate_repos_bigtop_trunk_ingroups_changing_repoinfo' src_depot_dir = '/bigtop/trunk' dst_depot_dir = os.path.basename(src_depot_dir) src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir) svn_revs.insert(0, 0) num_replicate = 2 src_ip = src_docker_cli.get_container_ip_addr() dst_ip = dst_docker_cli.get_container_ip_addr() dst_depot = '//depot/buildtest%s/...' % dst_depot_dir dst_view = ((dst_depot, './...'), ) src_mapping = ((src_depot_dir, ' '), ) dst_mapping = dst_view prefix_repinfo = False for src_counter in svn_revs[:10:num_replicate]: replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=0, replicate_change_num=num_replicate, prefix_repinfo=prefix_repinfo) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=src_counter, replicate_change_num=num_replicate, prefix_repinfo=prefix_repinfo) prefix_repinfo = not prefix_repinfo obliterate_depots(dst_docker_cli, dst_mapping) logger.passed(test_case)
def test_svn_action_rep_view_mapping_exclude_subdir_inc_one_file(self): test_case = 'svn_action_rep_view_mapping_exclude_subdir_inc_one_file' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/%s' % test_case exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir) #exc_path = exc_path[len(src_depot_dir):] for idx, exclude_path in enumerate([exc_path, exc_path + '_1']): dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx) dst_mapping = ((dst_depot, './...'), ) logger.error(exc_path) exclude_mapping = '-%s/*.exe' % exclude_path include_mapping = '+%s/b.exe' % exclude_path src_mapping = ( (src_depot_dir, ' '), (exclude_mapping, ' '), (include_mapping, ' '), ) excluded_files = [ os.path.join(exclude_path[len(src_depot_dir) + 1:], fn) for fn in [ 'a.exe', 'c.exe/a.json', 'c.exe/a.txt', 'c.exe/b.txt', ] ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_files=excluded_files) obliterate_all_depots(dst_docker_cli)
def replicate_sample_svn_repo(self, src_repos_dir, **kwargs): dst_depot_dir = os.path.basename(src_repos_dir) src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_ip = src_docker_cli.get_container_ip_addr() dst_ip = dst_docker_cli.get_container_ip_addr() dst_depot = '//depot/buildtest%s/...' % dst_depot_dir dst_view = ((dst_depot, './...'), ) src_mapping = ((src_repos_dir, ' '), ) dst_mapping = dst_view replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, **kwargs) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, **kwargs) obliterate_depots(dst_docker_cli, dst_mapping)
def test_replicate_repos_bigtop_trunk_ingroups(self): '''replicate the whole /repos/bigtop/trunk ''' test_case = 'replicate_repos_bigtop_trunk_ingroups' src_depot_dir = '/bigtop/trunk' dst_depot_dir = os.path.basename(src_depot_dir) src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir) svn_revs.insert(0, 0) num_replicate = 2 dst_depot = '//depot/buildtest%s/...' % dst_depot_dir dst_view = ((dst_depot, './...'), ) src_mapping = ((src_depot_dir, ' '), ) dst_mapping = dst_view for src_counter in svn_revs[::num_replicate]: replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=0, replicate_change_num=num_replicate) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=src_counter, replicate_change_num=num_replicate) obliterate_depots(dst_docker_cli, dst_mapping) logger.passed(test_case)
def test_svn_action_rep_view_mapping_externals_include_subdir(self): '''Exclude external: test_dir_parent/tests but keep test_dir_parent/tests/test-artifacts ''' test_case = 'svn_action_rep_view_mapping_externals_include_subdir' src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = self.docker_p4d_clients['p4d_0'] src_depot_dir = '/%s' % test_case exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir) for idx, exclude_path in enumerate([exc_path, exc_path + '_1']): dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx) dst_mapping = ((dst_depot, './...'), ) exclude_dir = '-%s/tests' % exclude_path include_dir = '+%s/tests/test-artifacts' % exclude_path src_mapping = ( (src_depot_dir, ' '), (exclude_dir, ' '), (include_dir, ' '), ) verification_exclude = exclude_dir[len('-') + len(src_depot_dir):] exclude_subdir = [ os.path.join(verification_exclude, 'test-execution'), ] replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, excluded_subdirs=exclude_subdir) obliterate_all_depots(dst_docker_cli)
def svn_action_rep_action(self, action, act_func, **kwargs): '''create test env, run test, and verify replication @param action, string of action to be tested @param act_func, function to be called to create test svn repo @param **kwargs, other arguments used to create test repo ''' test_case = 'svn_action_rep_%s' % action src_depot_dir = '/reptest_%s' % action dst_depot_dir = src_depot_dir src_docker_cli = self.docker_svn_clients['svn_0'] dst_docker_cli = kwargs.get('dst_docker_cli') if not dst_docker_cli: dst_docker_cli = self.docker_p4d_clients['p4d_0'] else: del kwargs['dst_docker_cli'] group_num = kwargs.get('group_num') if group_num: del kwargs['group_num'] test_dir = act_func(src_docker_cli, src_depot_dir, **kwargs) if test_dir: src_depot_dir = test_dir dst_depot = '//depot/buildtest%s/...' % dst_depot_dir dst_view = ((dst_depot, './...'), ) if kwargs.get('replicate_only_branch_dir'): src_depot_dir = os.path.join(src_depot_dir, 'branches') del kwargs['replicate_only_branch_dir'] src_mapping = ((src_depot_dir, ' '), ) dst_mapping = dst_view replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, **kwargs) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, **kwargs) if group_num == None: return dst_depot obliterate_all_depots(dst_docker_cli) svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir) num_replicate = group_num svn_revs.insert(0, 0) for src_counter in svn_revs[::num_replicate]: replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=0, replicate_change_num=num_replicate, **kwargs) verify_replication(src_mapping, dst_mapping, src_docker_cli, dst_docker_cli, src_counter=src_counter, replicate_change_num=num_replicate, **kwargs) logger.passed(test_case) return dst_depot