def __init__(
self,
hg=None, # CONNECT TO hg
repo=None, # CONNECTION INFO FOR ES CACHE
branches=None, # CONNECTION INFO FOR ES CACHE
use_cache=False, # True IF WE WILL USE THE ES FOR DOWNLOADING BRANCHES
timeout=30 * SECOND,
kwargs=None):
if not _hg_branches:
_late_imports()
self.es_locker = Lock()
self.todo = mo_threads.Queue("todo for hg daemon",
max=DAEMON_QUEUE_SIZE)
self.settings = kwargs
self.timeout = Duration(timeout)
# VERIFY CONNECTIVITY
with Explanation("Test connect with hg"):
response = http.head(self.settings.hg.url)
if branches == None:
self.branches = _hg_branches.get_branches(kwargs=kwargs)
self.es = None
return
self.last_cache_miss = Date.now()
set_default(repo, {"schema": revision_schema})
self.es = elasticsearch.Cluster(kwargs=repo).get_or_create_index(
kwargs=repo)
def setup_es(please_stop):
with suppress_exception:
self.es.add_alias()
with suppress_exception:
self.es.set_refresh_interval(seconds=1)
Thread.run("setup_es", setup_es)
self.branches = _hg_branches.get_branches(kwargs=kwargs)
self.timeout = timeout
Thread.run("hg daemon", self._daemon)
def test_memory_cleanup_with_signal(self):
"""
LOOKING FOR A MEMORY LEAK THAT HAPPENS ONLY DURING THREADING
ACTUALLY, THE PARTICULAR LEAK FOUND CAN BE RECREATED WITHOUT THREADS
BUT IT IS TOO LATE TO CHANGE THIS TEST
"""
NUM_CYCLES = 100
gc.collect()
start_mem = psutil.Process(os.getpid()).memory_info().rss
Log.note("Start memory {{mem|comma}}", mem=start_mem)
queue = mo_threads.Queue("", max=1000000)
def _consumer(please_stop):
while not please_stop:
v = queue.pop(till=please_stop)
if Random.int(1000) == 0:
Log.note("got " + v)
def _producer(t, please_stop=None):
for i in range(2):
queue.add(str(t) + ":" + str(i))
Till(seconds=0.01).wait()
consumer = Thread.run("", _consumer)
objgraph.growth(limit=None)
no_change = 0
for g in range(NUM_CYCLES):
mid_mem = psutil.Process(os.getpid()).memory_info().rss
Log.note("{{group}} memory {{mem|comma}}", group=g, mem=mid_mem)
if USE_PYTHON_THREADS:
threads = [
threading.Thread(target=_producer, args=(i, ))
for i in range(500)
]
for t in threads:
t.start()
else:
threads = [Thread.run("", _producer, i) for i in range(500)]
for t in threads:
t.join()
del threads
gc.collect()
results = objgraph.growth(limit=3)
if not results:
no_change += 1
else:
if DEBUG_SHOW_BACKREFS:
for typ, count, delta in results:
Log.note("%-*s%9d %+9d\n" % (18, typ, count, delta))
obj_list = objgraph.by_type(typ)
if obj_list:
obj = obj_list[-1]
objgraph.show_backrefs(obj, max_depth=10)
else:
Log.note("growth = \n{{results}}", results=results)
consumer.please_stop.go()
consumer.join()
self.assertGreater(
no_change, NUM_CYCLES / 2
) # IF MOST CYCLES DO NOT HAVE MORE OBJCETS, WE ASSUME THERE IS NO LEAK
def __init__(
self,
hg=None, # CONNECT TO hg
repo=None, # CONNECTION INFO FOR ES CACHE
use_cache=False, # True IF WE WILL USE THE ES FOR DOWNLOADING BRANCHES
timeout=30 * SECOND,
kwargs=None,
):
if not _hg_branches:
_late_imports()
if not is_text(repo.index):
Log.error("Expecting 'index' parameter")
self.repo_locker = Lock()
self.moves_locker = Lock()
self.todo = mo_threads.Queue("todo for hg daemon",
max=DAEMON_QUEUE_SIZE)
self.settings = kwargs
self.timeout = Duration(timeout)
self.last_cache_miss = Date.now()
# VERIFY CONNECTIVITY
with Explanation("Test connect with hg"):
http.head(self.settings.hg.url)
set_default(repo, {
"type": "revision",
"schema": revision_schema,
})
kwargs.branches = set_default(
{
"index": repo.index + "-branches",
"type": "branch",
},
repo,
)
moves = set_default(
{
"index": repo.index + "-moves",
},
repo,
)
self.branches = _hg_branches.get_branches(kwargs=kwargs)
cluster = elasticsearch.Cluster(kwargs=repo)
self.repo = cluster.get_or_create_index(kwargs=repo)
self.moves = cluster.get_or_create_index(kwargs=moves)
def setup_es(please_stop):
with suppress_exception:
self.repo.add_alias()
with suppress_exception:
self.moves.add_alias()
with suppress_exception:
self.repo.set_refresh_interval(seconds=1)
with suppress_exception:
self.moves.set_refresh_interval(seconds=1)
Thread.run("setup_es", setup_es)
Thread.run("hg daemon", self._daemon)