def test_protected_repo(url=None,
label=None,
download_dir=None,
clientcert=None,
cacert=None):
memUsage = MemoryUsage()
print "Initial %s" % (memUsage.get_time_memory_stamp())
if not url:
url = "https://cdn.redhat.com/content/dist/rhel/rhui/server/6/6Server/i386/os"
if not label:
label = "rhel-server-6-6Server-i386"
if not download_dir:
download_dir = "./data/%s" % (label)
if not cacert:
cacert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.ca"
if not clientcert:
clientcert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.cert"
log("Fetching protected metadata for: %s" % (url))
count = 0
while True:
yum_repo_grinder = YumRepoGrinderModified(label,
url,
cacert=cacert,
clicert=clientcert)
report, time_delta = yum_repo_grinder.fetchYumRepo(
basepath=download_dir)
log("Iteration: %s, %s, [%s] in %d seconds" %
(count, memUsage.get_time_memory_stamp(), report, time_delta))
LOG.flush()
count = count + 1
def test_protected_repo(url=None,
label=None,
download_dir=None,
clientcert=None,
cacert=None):
if not url:
#url = "https://cdn.redhat.com/content/dist/rhel/rhui/server/6/6Server/i386/os"
url = "https://cdn.redhat.com/content/dist/rhel/rhui/server/5/5.6/i386/source/SRPMS"
if not label:
#label = "rhel-server-6-6Server-i386"
label = "rhel-server-srpms-5-5.6-i386"
if not download_dir:
download_dir = "./data/%s" % (label)
if not cacert:
#cacert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.ca"
cacert = "/etc/pki/content/rhel-server-srpms-5-5.6-i386/feed-rhel-server-srpms-5-5.6-i386.ca"
if not clientcert:
#clientcert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.cert"
clientcert = "/etc/pki/content/rhel-server-srpms-5-5.6-i386/feed-rhel-server-srpms-5-5.6-i386.cert"
log("Fetching protected metadata for: %s" % (url))
count = 0
memUsage = MemoryUsage()
while True:
length = parse_metadata(label, url, download_dir, cacert, clientcert)
log("%s, %s, %s packages, iteration %s" %
(memUsage.get_time_memory_stamp(), label, length, count))
count = count + 1
def test_from_feed_urls(feed_urls, base_dir="./data"):
feeds = parse_feed_urls(feed_urls)
count = 0
memUsage = MemoryUsage()
while True:
for f in feeds:
download_dir = "./%s/%s" % (base_dir, f["id"])
length = parse_metadata(f["id"], f["url"], download_dir, f["feed_ca"], f["feed_cert"])
log("%s, %s, %s packages, iteration %s" % (memUsage.get_time_memory_stamp(), f["id"], length, count))
count = count + 1
def test_simple_repo(url=None, label=None, download_dir=None):
if not url:
url = "http://repos.fedorapeople.org/repos/pulp/pulp/fedora-15/x86_64/"
if not label:
label = "pulp_f15_x86_64"
if not download_dir:
download_dir = "./data/%s" % (label)
log("Fetching metadata for: %s" % (url))
count = 0
memUsage = MemoryUsage()
while True:
length = parse_metadata(label, url, download_dir)
log("%s, %s, %s packages, iteration %s" % (memUsage.get_time_memory_stamp(), label, length, count))
count = count + 1
def test_simple_repo(url=None, label=None, download_dir=None):
memUsage = MemoryUsage()
print "Initial %s" % (memUsage.get_time_memory_stamp())
if not url:
#url = "http://repos.fedorapeople.org/repos/pulp/pulp/fedora-15/x86_64/"
url = "http://jmatthews.fedorapeople.org/repo_500/"
if not label:
label = "pulp_f15_x86_64"
if not download_dir:
download_dir = "./data/%s" % (label)
log("Fetching metadata for: %s" % (url))
count = 0
while True:
yum_repo_grinder = YumRepoGrinderModified(label, url)
report, time_delta = yum_repo_grinder.fetchYumRepo(basepath=download_dir)
count = count + 1
log("Iteration: %s, %s, [%s] in %d seconds" % (count, memUsage.get_time_memory_stamp(), report, time_delta))
LOG.flush()
def test_protected_repo(url=None, label=None, download_dir=None, clientcert=None, cacert=None):
memUsage = MemoryUsage()
print "Initial %s" % (memUsage.get_time_memory_stamp())
if not url:
url = "https://cdn.redhat.com/content/dist/rhel/rhui/server/6/6Server/i386/os"
if not label:
label = "rhel-server-6-6Server-i386"
if not download_dir:
download_dir = "./data/%s" % (label)
if not cacert:
cacert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.ca"
if not clientcert:
clientcert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.cert"
log("Fetching protected metadata for: %s" % (url))
count = 0
while True:
yum_repo_grinder = YumRepoGrinderModified(label, url, cacert=cacert, clicert=clientcert)
report, time_delta = yum_repo_grinder.fetchYumRepo(basepath=download_dir)
log("Iteration: %s, %s, [%s] in %d seconds" % (count, memUsage.get_time_memory_stamp(), report, time_delta))
LOG.flush()
count = count + 1
def test_protected_repo(url=None, label=None, download_dir=None, clientcert=None, cacert=None):
if not url:
#url = "https://cdn.redhat.com/content/dist/rhel/rhui/server/6/6Server/i386/os"
url = "https://cdn.redhat.com/content/dist/rhel/rhui/server/5/5.6/i386/source/SRPMS"
if not label:
#label = "rhel-server-6-6Server-i386"
label = "rhel-server-srpms-5-5.6-i386"
if not download_dir:
download_dir = "./data/%s" % (label)
if not cacert:
#cacert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.ca"
cacert = "/etc/pki/content/rhel-server-srpms-5-5.6-i386/feed-rhel-server-srpms-5-5.6-i386.ca"
if not clientcert:
#clientcert = "/etc/pki/content/rhel-server-6-6Server-i386/feed-rhel-server-6-6Server-i386.cert"
clientcert = "/etc/pki/content/rhel-server-srpms-5-5.6-i386/feed-rhel-server-srpms-5-5.6-i386.cert"
log("Fetching protected metadata for: %s" % (url))
count = 0
memUsage = MemoryUsage()
while True:
length = parse_metadata(label, url, download_dir, cacert, clientcert)
log("%s, %s, %s packages, iteration %s" % (memUsage.get_time_memory_stamp(), label, length, count))
count = count + 1
while loop_again:
loop_again = False
time.sleep(1)
for t in threads:
if t.finished:
continue
loop_again = True
if __name__ == "__main__":
if len(sys.argv) < 2:
print "Usage: %s FEED_URLS_FILE_LIST" % (sys.argv[0])
sys.exit(1)
feed_urls = parse_feed_urls(sys.argv[1])
log("%s Feed URLs found, may run up to %s YumRepoGrinder jobs at a time" % (len(feed_urls), MAX_SYNC_JOBS))
memUsage = MemoryUsage()
count = 0
time_mem_stamp = memUsage.get_time_memory_stamp()
log("Initial %s" % (time_mem_stamp))
while True:
if MAX_SYNC_JOBS == 1:
for index, repo in enumerate(feed_urls):
t = SyncThread(repo["id"], repo["url"], repo["feed_ca"], repo["feed_cert"])
t.start()
while not t.finished:
time.sleep(2)
report = "%s" % (t.report)
del t
time_mem_stamp = memUsage.get_time_memory_stamp()
log("<%s> %s iteration [%s] %s, %s" % (time_mem_stamp, count, report, repo["id"], repo["url"]))
log("<%s> Completed iteration: %s\n" % (memUsage.get_time_memory_stamp(), count))
while loop_again:
loop_again = False
time.sleep(1)
for t in threads:
if t.finished:
continue
loop_again = True
if __name__ == "__main__":
if len(sys.argv) < 2:
print "Usage: %s FEED_URLS_FILE_LIST" % (sys.argv[0])
sys.exit(1)
feed_urls = parse_feed_urls(sys.argv[1])
log("%s Feed URLs found, may run up to %s YumRepoGrinder jobs at a time" % (len(feed_urls), MAX_SYNC_JOBS))
memUsage = MemoryUsage()
count = 0
time_mem_stamp = memUsage.get_time_memory_stamp()
log("Initial %s" % (time_mem_stamp))
while True:
if MAX_SYNC_JOBS == 1:
for index, repo in enumerate(feed_urls):
t = SyncThread(repo["id"], repo["url"], repo["feed_ca"], repo["feed_cert"])
t.start()
while not t.finished:
time.sleep(2)
report = "%s" % (t.report)
del t
time_mem_stamp = memUsage.get_time_memory_stamp()
log("<%s> %s iteration [%s] %s, %s" % (time_mem_stamp, count, report, repo["id"], repo["url"]))
log("<%s> Completed iteration: %s\n" % (memUsage.get_time_memory_stamp(), count))
import psutil
import threading, time
from memory_usage import MemoryUsage
memory = psutil.virtual_memory()
percentage_threshold = 80.0
WAIT_TIME_SECONDS = 15
ticker = threading.Event()
virtual_memory = MemoryUsage(memory, percentage_threshold)
while not ticker.wait(WAIT_TIME_SECONDS) and True:
virtual_memory.over_threshold()
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from memory_usage import MemoryUsage # Tests for MemoryUsage leng = 10 mem_use = MemoryUsage(1) usage = mem_use.init_state(10, 3) print(usage.shape) print(usage) usage[0, 0] = 20 usage[2, 0] = 2 usage[0, 1:] = 3 usage[2, 2:] = 4 usage[0, leng - 1] = 49494994 print(usage.shape) print(usage) cumprod = mem_use.exclusive_cumprod_temp(usage) print(cumprod.shape) print(cumprod)
time.sleep(1)
for t in threads:
if t.finished:
continue
loop_again = True
if __name__ == "__main__":
if len(sys.argv) < 2:
print "Usage: %s FEED_URLS_FILE_LIST" % (sys.argv[0])
sys.exit(1)
feed_urls = parse_feed_urls(sys.argv[1])
log("%s Feed URLs found, may run up to %s YumRepoGrinder jobs at a time" %
(len(feed_urls), MAX_SYNC_JOBS))
memUsage = MemoryUsage()
count = 0
time_mem_stamp = memUsage.get_time_memory_stamp()
log("Initial %s" % (time_mem_stamp))
while True:
if MAX_SYNC_JOBS == 1:
for index, repo in enumerate(feed_urls):
t = SyncThread(repo["id"], repo["url"], repo["feed_ca"],
repo["feed_cert"])
t.start()
while not t.finished:
time.sleep(2)
report = "%s" % (t.report)
del t
time_mem_stamp = memUsage.get_time_memory_stamp()
log("<%s> %s iteration [%s] %s, %s" %