def test_svn_action_rep_sample_mapping_exclude_subdir(self):
'''Exclude a subdirectory from sample svn repo
'''
test_case = 'svn_action_rep_sample_mapping_exclude_subdir'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/bigtop/trunk'
exclude_path = '/bigtop/trunk/bigtop-packages'
dst_depot = '//depot/buildtest%s/...' % test_case
dst_mapping = ((dst_depot, './...'), )
exclude_mapping = '-%s' % exclude_path
src_mapping = (
(src_depot_dir, ' '),
(exclude_mapping, ' '),
)
exclude_dirs = [
'bigtop-packages',
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_subdirs=exclude_dirs)
def test_svn_action_rep_sample_mapping_exclude_two_subdir(self):
'''Exclude two subdirectories from sample svn repo
'''
test_case = 'svn_action_rep_sample_mapping_exclude_two_subdir'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/bigtop/trunk'
exclude_path_0 = '/bigtop/trunk/bigtop-packages'
exclude_path_1 = '/bigtop/trunk/bigtop-test-framework/src/main'
dst_depot = '//depot/buildtest%s/...' % test_case
dst_mapping = ((dst_depot, './...'), )
exclude_mapping_0 = '-%s' % exclude_path_0
exclude_mapping_1 = '-%s' % exclude_path_1
src_mapping = (
(src_depot_dir, ' '),
(exclude_mapping_0, ' '),
(exclude_mapping_1, ' '),
)
exclude_dirs = [
exclude_path_0[len(src_depot_dir) + 1:],
exclude_path_1[len(src_depot_dir) + 1:],
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_subdirs=exclude_dirs)
def test_svn_action_rep_view_mapping_exclude_certain_file_0(self):
'''Exclude test_dir_parent/a.*
'''
test_case = 'svn_action_rep_view_mapping_exclude_certain_file_0'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/%s' % test_case
exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir)
for idx, exclude_path in enumerate([exc_path, exc_path + '_1']):
dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx)
dst_mapping = ((dst_depot, './...'), )
exclude_mapping = '-%s/a.*' % exclude_path
src_mapping = ((src_depot_dir, ' '), (exclude_mapping, ' '))
excluded_files = [
os.path.join(exclude_path[len(src_depot_dir) + 1:], fn)
for fn in [
'a.exe',
'a.json',
'a.txt',
]
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_files=excluded_files)
obliterate_all_depots(dst_docker_cli)
def test_svn_action_rep_view_mapping_exclude_external_subdirs(self):
'''Exclude a whole external: test_dir_parent/deploy
'''
test_case = 'svn_action_rep_view_mapping_exclude_externals_subdirs'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/%s' % test_case
exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir)
for idx, exclude_path in enumerate([exc_path, exc_path + '_1']):
dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx)
dst_mapping = ((dst_depot, './...'), )
exclude_path = '-%s/deploy' % exclude_path
src_mapping = ((src_depot_dir, ' '), (exclude_path, ' '))
verification_exclude = exclude_path[len('-') + len(src_depot_dir):]
exclude_subdir = [
verification_exclude,
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_subdirs=exclude_subdir)
obliterate_all_depots(dst_docker_cli)
def test_replicate_repos_bigtop_trunk_source_counter(self):
'''test incorrect source counter
'''
test_case = 'replicate_repos_bigtop_trunk_source_counter'
src_depot_dir = '/bigtop/site/trunk/content/xdoc'
dst_depot_dir = os.path.basename(src_depot_dir)
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir)
svn_revs.insert(0, 0)
num_replicate = 2
src_ip = src_docker_cli.get_container_ip_addr()
dst_ip = dst_docker_cli.get_container_ip_addr()
dst_depot = '//depot/buildtest%s/...' % dst_depot_dir
dst_view = ((dst_depot, './...'), )
src_mapping = ((src_depot_dir, ' '), )
dst_mapping = dst_view
for idx, src_counter in enumerate(svn_revs[:10:num_replicate]):
changelist_offset = 0
if idx == 1:
changelist_offset = -1
elif idx == 2:
changelist_offset = 1
try:
replicate_SvnP4Replicate(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=src_counter +
changelist_offset,
replicate_change_num=num_replicate)
except ReplicationException, e:
if ('src counter' in str(e) and 'last replicated rev' in str(e)
and changelist_offset > 0):
break
else:
raise
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=src_counter,
replicate_change_num=num_replicate)
def test_svn_action_rep_view_mapping_rel_externals_include_subsubdir(self):
'''Exclude external: test_dir_parent/hadoop/hadoop_src
but keep test_dir_parent/hadoop/hadoop_src/resources/images
'''
test_case = 'svn_action_rep_view_mapping_rel_externals_include_subsubdir'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/%s' % test_case
exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir)
for idx, exclude_path in enumerate([exc_path, exc_path + '_1']):
dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx)
dst_mapping = ((dst_depot, './...'), )
exclude_dir = '-%s/hadoop/hadoop_src' % exclude_path
include_dir = '+%s/hadoop/hadoop_src/resources/images' % exclude_path
src_mapping = (
(src_depot_dir, ' '),
(exclude_dir, ' '),
(include_dir, ' '),
)
external_files_to_exclude = [
'resources/css/site.css',
'site.xml',
'xdoc/index.xml',
'xdoc/irc-channel.xml',
'xdoc/issue-tracking.xml',
'xdoc/mail-lists.xml',
'xdoc/release-notes.xml',
'xdoc/team-list.xml',
]
excluded_files = [
os.path.join(exclude_path[len(src_depot_dir) + 1:],
'hadoop/hadoop_src/%s' % fn)
for fn in external_files_to_exclude
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_files=excluded_files)
obliterate_all_depots(dst_docker_cli)
def test_svn_action_rep_sample_mapping_exclude_recursively_same_suffix(
self):
'''Exclude files with same suffix from sample svn repo
'''
test_case = 'svn_action_rep_sample_mapping_exclude_recursively_same_suffix'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/bigtop/trunk'
exclude_path = '/bigtop/trunk/bigtop-tests'
dst_depot = '//depot/buildtest%s/...' % test_case
dst_mapping = ((dst_depot, './...'), )
exclude_mapping = '-%s/.../*.out' % exclude_path
src_mapping = (
(src_depot_dir, ' '),
(exclude_mapping, ' '),
)
rel_exc_files = '''test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-string.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date-export.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-all-tables.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-columns.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-where-clause.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-testtable.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_fp.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_string.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool-export.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-query.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-non-string.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-append.out
test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_int.out'''
excluded_files = [
os.path.join(exclude_path[len(src_depot_dir) + 1:], fn.strip())
for fn in rel_exc_files.split('\n')
]
logger.error('excluded_files: %s' % excluded_files)
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_files=excluded_files)
def test_replicate_repos_bigtop_trunk_ingroups_changing_repoinfo_location(
self):
'''replicate the whole /repos/bigtop/trunk
'''
test_case = 'replicate_repos_bigtop_trunk_ingroups_changing_repoinfo'
src_depot_dir = '/bigtop/trunk'
dst_depot_dir = os.path.basename(src_depot_dir)
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir)
svn_revs.insert(0, 0)
num_replicate = 2
src_ip = src_docker_cli.get_container_ip_addr()
dst_ip = dst_docker_cli.get_container_ip_addr()
dst_depot = '//depot/buildtest%s/...' % dst_depot_dir
dst_view = ((dst_depot, './...'), )
src_mapping = ((src_depot_dir, ' '), )
dst_mapping = dst_view
prefix_repinfo = False
for src_counter in svn_revs[:10:num_replicate]:
replicate_SvnP4Replicate(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=0,
replicate_change_num=num_replicate,
prefix_repinfo=prefix_repinfo)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=src_counter,
replicate_change_num=num_replicate,
prefix_repinfo=prefix_repinfo)
prefix_repinfo = not prefix_repinfo
obliterate_depots(dst_docker_cli, dst_mapping)
logger.passed(test_case)
def svn_action_rep_branch_detection(self, action, src_docker_cli, act_func,
edits_in_other_folder_before_copy):
'''test svn->p4 branch point detection script'''
test_case = 'svn_action_rep_%s' % action
dst_docker_cli = self.docker_p4d_clients['p4d_%s' % action]
src_depot_dir = '/reptest_%s' % action
trunk, branch = act_func(src_docker_cli, src_depot_dir,
edits_in_other_folder_before_copy)
trunk_src = os.path.join(trunk, 'src')
branch_src = os.path.join(branch, 'src')
# replicate trunk
src_depot_dir = trunk
dst_depot_dir = src_depot_dir
dst_depot = '//depot/svn_rep/buildtest_trunk/...'
dst_view = ((dst_depot, './...'), )
src_mapping = ((src_depot_dir, ' '), )
dst_mapping = dst_view
args = replicate_SvnP4Replicate(src_mapping, dst_mapping,
src_docker_cli, dst_docker_cli)
verify_replication(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
args.target_rep_branch_root = '//depot/svn_rep'
args.source_project_dir = branch_src
# detect svn branch point
branch_from = detect_branch_point(args)
return branch_from
def test_svn_action_rep_view_mapping_exclude_subdir_inc_one_file(self):
test_case = 'svn_action_rep_view_mapping_exclude_subdir_inc_one_file'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/%s' % test_case
exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir)
#exc_path = exc_path[len(src_depot_dir):]
for idx, exclude_path in enumerate([exc_path, exc_path + '_1']):
dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx)
dst_mapping = ((dst_depot, './...'), )
logger.error(exc_path)
exclude_mapping = '-%s/*.exe' % exclude_path
include_mapping = '+%s/b.exe' % exclude_path
src_mapping = (
(src_depot_dir, ' '),
(exclude_mapping, ' '),
(include_mapping, ' '),
)
excluded_files = [
os.path.join(exclude_path[len(src_depot_dir) + 1:], fn)
for fn in [
'a.exe',
'c.exe/a.json',
'c.exe/a.txt',
'c.exe/b.txt',
]
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_files=excluded_files)
obliterate_all_depots(dst_docker_cli)
def replicate_sample_svn_repo(self, src_repos_dir, **kwargs):
dst_depot_dir = os.path.basename(src_repos_dir)
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_ip = src_docker_cli.get_container_ip_addr()
dst_ip = dst_docker_cli.get_container_ip_addr()
dst_depot = '//depot/buildtest%s/...' % dst_depot_dir
dst_view = ((dst_depot, './...'), )
src_mapping = ((src_repos_dir, ' '), )
dst_mapping = dst_view
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli, **kwargs)
verify_replication(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli, **kwargs)
obliterate_depots(dst_docker_cli, dst_mapping)
def test_replicate_repos_bigtop_trunk_ingroups(self):
'''replicate the whole /repos/bigtop/trunk
'''
test_case = 'replicate_repos_bigtop_trunk_ingroups'
src_depot_dir = '/bigtop/trunk'
dst_depot_dir = os.path.basename(src_depot_dir)
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir)
svn_revs.insert(0, 0)
num_replicate = 2
dst_depot = '//depot/buildtest%s/...' % dst_depot_dir
dst_view = ((dst_depot, './...'), )
src_mapping = ((src_depot_dir, ' '), )
dst_mapping = dst_view
for src_counter in svn_revs[::num_replicate]:
replicate_SvnP4Replicate(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=0,
replicate_change_num=num_replicate)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=src_counter,
replicate_change_num=num_replicate)
obliterate_depots(dst_docker_cli, dst_mapping)
logger.passed(test_case)
def test_svn_action_rep_view_mapping_externals_include_subdir(self):
'''Exclude external: test_dir_parent/tests
but keep test_dir_parent/tests/test-artifacts
'''
test_case = 'svn_action_rep_view_mapping_externals_include_subdir'
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = self.docker_p4d_clients['p4d_0']
src_depot_dir = '/%s' % test_case
exc_path = self.svn_exclusion_construct_source_repos(src_depot_dir)
for idx, exclude_path in enumerate([exc_path, exc_path + '_1']):
dst_depot = '//depot/buildtest%s_%s/...' % (src_depot_dir, idx)
dst_mapping = ((dst_depot, './...'), )
exclude_dir = '-%s/tests' % exclude_path
include_dir = '+%s/tests/test-artifacts' % exclude_path
src_mapping = (
(src_depot_dir, ' '),
(exclude_dir, ' '),
(include_dir, ' '),
)
verification_exclude = exclude_dir[len('-') + len(src_depot_dir):]
exclude_subdir = [
os.path.join(verification_exclude, 'test-execution'),
]
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
excluded_subdirs=exclude_subdir)
obliterate_all_depots(dst_docker_cli)
def svn_action_rep_action(self, action, act_func, **kwargs):
'''create test env, run test, and verify replication
@param action, string of action to be tested
@param act_func, function to be called to create test svn repo
@param **kwargs, other arguments used to create test repo
'''
test_case = 'svn_action_rep_%s' % action
src_depot_dir = '/reptest_%s' % action
dst_depot_dir = src_depot_dir
src_docker_cli = self.docker_svn_clients['svn_0']
dst_docker_cli = kwargs.get('dst_docker_cli')
if not dst_docker_cli:
dst_docker_cli = self.docker_p4d_clients['p4d_0']
else:
del kwargs['dst_docker_cli']
group_num = kwargs.get('group_num')
if group_num:
del kwargs['group_num']
test_dir = act_func(src_docker_cli, src_depot_dir, **kwargs)
if test_dir:
src_depot_dir = test_dir
dst_depot = '//depot/buildtest%s/...' % dst_depot_dir
dst_view = ((dst_depot, './...'), )
if kwargs.get('replicate_only_branch_dir'):
src_depot_dir = os.path.join(src_depot_dir, 'branches')
del kwargs['replicate_only_branch_dir']
src_mapping = ((src_depot_dir, ' '), )
dst_mapping = dst_view
replicate_SvnP4Replicate(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli, **kwargs)
verify_replication(src_mapping, dst_mapping, src_docker_cli,
dst_docker_cli, **kwargs)
if group_num == None:
return dst_depot
obliterate_all_depots(dst_docker_cli)
svn_revs = get_svn_rev_list(src_docker_cli, src_depot_dir)
num_replicate = group_num
svn_revs.insert(0, 0)
for src_counter in svn_revs[::num_replicate]:
replicate_SvnP4Replicate(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=0,
replicate_change_num=num_replicate,
**kwargs)
verify_replication(src_mapping,
dst_mapping,
src_docker_cli,
dst_docker_cli,
src_counter=src_counter,
replicate_change_num=num_replicate,
**kwargs)
logger.passed(test_case)
return dst_depot
