def sync(self, repo_id, url, ca, cert):
start = time.time()
yum_repo_grinder = YumRepoGrinder(repo_id, url, THREADS,
cacert=ca, clicert=cert, packages_location=PKGS_DIR)
self.report = yum_repo_grinder.fetchYumRepo(
"%s/%s" % (REPO_DIR, repo_id),
verify_options=VERIFY_OPTIONS)
end = time.time()
def test_sync_of_repo_no_treeinfo(self):
test_url = "http://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/repo_resync_a/"
temp_label = "test_sync_of_repo_no_treeinfo"
yum_fetch = YumRepoGrinder(temp_label, test_url, 5)
sync_report = yum_fetch.fetchYumRepo(self.temp_dir)
# Ensure that temp files were cleaned up for .treeinfo.part
self.assertFalse(os.path.exists(os.path.join(self.temp_dir, temp_label, "treeinfo.part")))
self.assertFalse(os.path.exists(os.path.join(self.temp_dir, temp_label, ".treeinfo.part")))
def sync(self, repo_id, url, ca, cert):
start = time.time()
yum_repo_grinder = YumRepoGrinder(repo_id, url, THREADS,
cacert=ca, clicert=cert, packages_location=PKGS_DIR)
self.report = yum_repo_grinder.fetchYumRepo(
"%s/%s" % (REPO_DIR, repo_id),
verify_options=VERIFY_OPTIONS)
end = time.time()
def test_sync(self):
test_url = "http://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/pulp_unittest"
temp_label = "test_sync"
yum_fetch = YumRepoGrinder(temp_label, test_url, 5)
sync_report = yum_fetch.fetchYumRepo(self.temp_dir)
distro_tree_files = glob.glob("%s/%s/images/*" % (self.temp_dir, temp_label))
print distro_tree_files
self.assertEquals(len(distro_tree_files), 3)
def test_sync(self):
test_url = "http://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/pulp_unittest"
temp_label = "test_sync"
yum_fetch = YumRepoGrinder(temp_label, test_url, 5)
sync_report = yum_fetch.fetchYumRepo(self.temp_dir)
distro_tree_files = glob.glob("%s/%s/images/*" %
(self.temp_dir, temp_label))
print distro_tree_files
self.assertEquals(len(distro_tree_files), 3)
def testSync(self):
channel_label = "rhel-i386-server-vt-5"
repoURL = "http://mmccune.fedorapeople.org/pulp/fedora/13/x86_64/"
parallel = 8 # number of threads to fetch with
repoGrinder = YumRepoGrinder("test_label_f13_updates", repoURL, parallel,
newest=True)
report = repoGrinder.fetchYumRepo(callback=sampleCallBack)
print "report = %s" % (report)
self.assertTrue(G_STATUS == "FINISHED")
self.assertTrue(G_SIZE_LEFT == 0)
self.assertTrue(G_ITEMS_LEFT == 0)
def test_sync_of_repo_no_treeinfo(self):
test_url = "http://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/repo_resync_a/"
temp_label = "test_sync_of_repo_no_treeinfo"
yum_fetch = YumRepoGrinder(temp_label, test_url, 5)
sync_report = yum_fetch.fetchYumRepo(self.temp_dir)
# Ensure that temp files were cleaned up for .treeinfo.part
self.assertFalse(
os.path.exists(
os.path.join(self.temp_dir, temp_label, "treeinfo.part")))
self.assertFalse(
os.path.exists(
os.path.join(self.temp_dir, temp_label, ".treeinfo.part")))
def testSync(self):
channel_label = "rhel-i386-server-vt-5"
repoURL = "http://mmccune.fedorapeople.org/pulp/fedora/13/x86_64/"
parallel = 8 # number of threads to fetch with
repoGrinder = YumRepoGrinder("test_label_f13_updates",
repoURL,
parallel,
newest=True)
report = repoGrinder.fetchYumRepo(callback=sampleCallBack)
print "report = %s" % (report)
self.assertTrue(G_STATUS == "FINISHED")
self.assertTrue(G_SIZE_LEFT == 0)
self.assertTrue(G_ITEMS_LEFT == 0)
def sync(self, repo_id, url, ca, cert):
print "%s sync(%s, %s)" % (datetime.now(), repo_id, url)
#print "%s sync(%s, %s, %s, %s)" % (datetime.now(), repo_id, url, ca, cert)
#tr = tracker.SummaryTracker()
start = time.time()
yum_repo_grinder = YumRepoGrinder(repo_id, url, THREADS,
cacert=ca, clicert=cert, packages_location=PKGS_DIR)
report = yum_repo_grinder.fetchYumRepo(
"%s/%s" % (REPO_DIR, repo_id),
verify_options=VERIFY_OPTIONS)
end = time.time()
print ("<%s> reported %s successes, %s downloads, %s errors" \
% (repo_id, report.successes, report.downloads, report.errors))
print "<%s> took %s seconds" % (repo_id, end-start)
def _do_command(self):
"""
Executes the command.
"""
self._validate_options()
sslverify=1
if self.options.nosslverify:
sslverify=0
limit = None
if self.options.limit:
limit = int(self.options.limit)
verify_options={}
if self.options.skip_verify_size:
verify_options["size"] = False
if self.options.skip_verify_checksum:
verify_options["checksum"] = False
if self.options.filter:
self.options.filter = Filter(self.options.filter,
regex_list=self.options.filter_regex)
self.yfetch = YumRepoGrinder(
self.options.label, self.options.url,
self.parallel, cacert=self.options.cacert,
clicert=self.options.clicert,
clikey=self.options.clikey,
proxy_url=self.options.proxy_url,
proxy_port=self.options.proxy_port,
proxy_user=self.options.proxy_user,
proxy_pass=self.options.proxy_pass,
sslverify=sslverify, max_speed=limit,
filter=self.options.filter)
if self.options.basepath:
self.yfetch.fetchYumRepo(self.options.basepath, verify_options=verify_options)
else:
self.yfetch.fetchYumRepo(verify_options=verify_options)
def sync(self, repo_id, url, ca, cert):
print "%s sync(%s, %s)" % (datetime.now(), repo_id, url)
#print "%s sync(%s, %s, %s, %s)" % (datetime.now(), repo_id, url, ca, cert)
#tr = tracker.SummaryTracker()
start = time.time()
yum_repo_grinder = YumRepoGrinder(repo_id,
url,
THREADS,
cacert=ca,
clicert=cert,
packages_location=PKGS_DIR)
report = yum_repo_grinder.fetchYumRepo("%s/%s" % (REPO_DIR, repo_id),
verify_options=VERIFY_OPTIONS)
end = time.time()
print ("<%s> reported %s successes, %s downloads, %s errors" \
% (repo_id, report.successes, report.downloads, report.errors))
print "<%s> took %s seconds" % (repo_id, end - start)
def _do_command(self):
"""
Executes the command.
"""
self._validate_options()
sslverify = 1
if self.options.nosslverify:
sslverify = 0
limit = None
if self.options.limit:
limit = int(self.options.limit)
verify_options = {}
if self.options.skip_verify_size:
verify_options["size"] = False
if self.options.skip_verify_checksum:
verify_options["checksum"] = False
if self.options.filter:
self.options.filter = Filter(self.options.filter,
regex_list=self.options.filter_regex)
self.yfetch = YumRepoGrinder(self.options.label,
self.options.url,
self.parallel,
cacert=self.options.cacert,
clicert=self.options.clicert,
clikey=self.options.clikey,
proxy_url=self.options.proxy_url,
proxy_port=self.options.proxy_port,
proxy_user=self.options.proxy_user,
proxy_pass=self.options.proxy_pass,
sslverify=sslverify,
max_speed=limit,
filter=self.options.filter)
if self.options.basepath:
self.yfetch.fetchYumRepo(self.options.basepath,
verify_options=verify_options)
else:
self.yfetch.fetchYumRepo(verify_options=verify_options)
class RepoDriver(CliDriver):
parallel = 5
def __init__(self):
usage = "usage: %prog yum [OPTIONS]"
shortdesc = "Fetches content from a yum source."
desc = "yum"
CliDriver.__init__(self, "yum", usage, shortdesc, desc)
#GrinderLog.setup(self.debug)
self.parser.add_option("--label", dest="label",
help="Label for the content fetched from repository URL")
self.parser.add_option("--limit", dest="limit",
help="Limit bandwidth in KB/sec per thread", default=None)
self.parser.add_option('-U', "--url", dest="url",
help="URL to the repository whose content to fetch")
self.parser.add_option("--cacert", dest="cacert", default=None,
help="Path location to CA Certificate.")
self.parser.add_option("--cert", dest="clicert", default=None,
help="Path location to Client SSl Certificate.")
self.parser.add_option("--key", dest="clikey", default=None,
help="Path location to Client Certificate Key.")
self.parser.add_option("--nosslverify", action="store_true", dest="nosslverify",
help="disable ssl verify of server cert")
self.parser.add_option('-P', "--parallel", dest="parallel",
help="Thread count to fetch the bits in parallel. Defaults to 5")
self.parser.add_option('-b', '--basepath', dest="basepath",
help="Directory path to store the fetched content.Defaults to current working directory")
self.parser.add_option('--proxy_url', dest="proxy_url",
help="proxy url, example 'http://172.31.1.1'", default=None)
self.parser.add_option('--proxy_port', dest="proxy_port",
help="proxy port, default is 3128", default='3128')
self.parser.add_option('--proxy_user', dest="proxy_user",
help="proxy username, if auth is required", default=None)
self.parser.add_option('--proxy_pass', dest="proxy_pass",
help="proxy password, if auth is required", default=None)
self.parser.add_option('--skip_verify_size', action="store_true",
help="skip verify size of existing packages")
self.parser.add_option('--skip_verify_checksum', action="store_true",
help="skip verify checksum of existing packages")
self.parser.add_option('--filter', action="store",
help="add a filter, either whitelist or blacklist")
self.parser.add_option('--filter_regex', action="append",
help="add a filter regex; may be use multiple times")
def _validate_options(self):
if not self.options.label:
print("repo label is required. Try --help.")
sys.exit(-1)
if not self.options.url:
print("No Url specified to fetch content. Try --help")
sys.exit(-1)
if self.options.parallel:
self.parallel = self.options.parallel
if self.options.filter:
if ((self.options.filter != "whitelist") and
(self.options.filter != "blacklist")):
print("--filter=<type> should be either " +
"'whitelist' or 'blacklist'")
sys.exit(-1)
if not self.options.filter_regex:
print("please provide a --filter_regex when using --filter")
sys.exit(-1)
LOG.debug("--filter=%s --filter_regex=%s" %
(self.options.filter,
self.options.filter_regex))
def _do_command(self):
"""
Executes the command.
"""
self._validate_options()
sslverify=1
if self.options.nosslverify:
sslverify=0
limit = None
if self.options.limit:
limit = int(self.options.limit)
verify_options={}
if self.options.skip_verify_size:
verify_options["size"] = False
if self.options.skip_verify_checksum:
verify_options["checksum"] = False
if self.options.filter:
self.options.filter = Filter(self.options.filter,
regex_list=self.options.filter_regex)
self.yfetch = YumRepoGrinder(
self.options.label, self.options.url,
self.parallel, cacert=self.options.cacert,
clicert=self.options.clicert,
clikey=self.options.clikey,
proxy_url=self.options.proxy_url,
proxy_port=self.options.proxy_port,
proxy_user=self.options.proxy_user,
proxy_pass=self.options.proxy_pass,
sslverify=sslverify, max_speed=limit,
filter=self.options.filter)
if self.options.basepath:
self.yfetch.fetchYumRepo(self.options.basepath, verify_options=verify_options)
else:
self.yfetch.fetchYumRepo(verify_options=verify_options)
def stop(self):
self.yfetch.stop()
class RepoDriver(CliDriver):
parallel = 5
def __init__(self):
usage = "usage: %prog yum [OPTIONS]"
shortdesc = "Fetches content from a yum source."
desc = "yum"
CliDriver.__init__(self, "yum", usage, shortdesc, desc)
#GrinderLog.setup(self.debug)
self.parser.add_option(
"--label",
dest="label",
help="Label for the content fetched from repository URL")
self.parser.add_option("--limit",
dest="limit",
help="Limit bandwidth in KB/sec per thread",
default=None)
self.parser.add_option(
'-U',
"--url",
dest="url",
help="URL to the repository whose content to fetch")
self.parser.add_option("--cacert",
dest="cacert",
default=None,
help="Path location to CA Certificate.")
self.parser.add_option("--cert",
dest="clicert",
default=None,
help="Path location to Client SSl Certificate.")
self.parser.add_option("--key",
dest="clikey",
default=None,
help="Path location to Client Certificate Key.")
self.parser.add_option("--nosslverify",
action="store_true",
dest="nosslverify",
help="disable ssl verify of server cert")
self.parser.add_option(
'-P',
"--parallel",
dest="parallel",
help="Thread count to fetch the bits in parallel. Defaults to 5")
self.parser.add_option(
'-b',
'--basepath',
dest="basepath",
help=
"Directory path to store the fetched content.Defaults to current working directory"
)
self.parser.add_option('--proxy_url',
dest="proxy_url",
help="proxy url, example 'http://172.31.1.1'",
default=None)
self.parser.add_option('--proxy_port',
dest="proxy_port",
help="proxy port, default is 3128",
default='3128')
self.parser.add_option('--proxy_user',
dest="proxy_user",
help="proxy username, if auth is required",
default=None)
self.parser.add_option('--proxy_pass',
dest="proxy_pass",
help="proxy password, if auth is required",
default=None)
self.parser.add_option('--skip_verify_size',
action="store_true",
help="skip verify size of existing packages")
self.parser.add_option(
'--skip_verify_checksum',
action="store_true",
help="skip verify checksum of existing packages")
self.parser.add_option(
'--filter',
action="store",
help="add a filter, either whitelist or blacklist")
self.parser.add_option(
'--filter_regex',
action="append",
help="add a filter regex; may be use multiple times")
def _validate_options(self):
if not self.options.label:
print("repo label is required. Try --help.")
sys.exit(-1)
if not self.options.url:
print("No Url specified to fetch content. Try --help")
sys.exit(-1)
if self.options.parallel:
self.parallel = self.options.parallel
if self.options.filter:
if ((self.options.filter != "whitelist")
and (self.options.filter != "blacklist")):
print("--filter=<type> should be either " +
"'whitelist' or 'blacklist'")
sys.exit(-1)
if not self.options.filter_regex:
print("please provide a --filter_regex when using --filter")
sys.exit(-1)
LOG.debug("--filter=%s --filter_regex=%s" %
(self.options.filter, self.options.filter_regex))
def _do_command(self):
"""
Executes the command.
"""
self._validate_options()
sslverify = 1
if self.options.nosslverify:
sslverify = 0
limit = None
if self.options.limit:
limit = int(self.options.limit)
verify_options = {}
if self.options.skip_verify_size:
verify_options["size"] = False
if self.options.skip_verify_checksum:
verify_options["checksum"] = False
if self.options.filter:
self.options.filter = Filter(self.options.filter,
regex_list=self.options.filter_regex)
self.yfetch = YumRepoGrinder(self.options.label,
self.options.url,
self.parallel,
cacert=self.options.cacert,
clicert=self.options.clicert,
clikey=self.options.clikey,
proxy_url=self.options.proxy_url,
proxy_port=self.options.proxy_port,
proxy_user=self.options.proxy_user,
proxy_pass=self.options.proxy_pass,
sslverify=sslverify,
max_speed=limit,
filter=self.options.filter)
if self.options.basepath:
self.yfetch.fetchYumRepo(self.options.basepath,
verify_options=verify_options)
else:
self.yfetch.fetchYumRepo(verify_options=verify_options)
def stop(self):
self.yfetch.stop()