def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
# TODO: Find a way to exclude all standard library modules from reloading.
exclude = [
# Standard exclusions:
'sys',
'os.path',
builtin_mod_name,
'__main__',
# Test-related exclusions:
'unittest',
'UserDict',
'_collections_abc',
'tokenize',
'collections',
'collections.abc',
'importlib',
'importlib.machinery',
'_imp',
'importlib._bootstrap',
'importlib._bootstrap_external',
'_frozen_importlib',
'_frozen_importlib_external',
]
dreload(numpy, exclude=exclude)
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
exclude = [
# Standard exclusions:
'sys', 'os.path', '__builtin__', '__main__',
# Test-related exclusions:
'unittest', 'UserDict',
]
dreload(numpy, exclude=exclude)
def rebuild():
cwd = os.getcwd()
setup_path = os.path.abspath(pjoin(os.path.split(__file__)[0],".."))
os.chdir(setup_path)
subprocess.call(sys.executable + " setup.py build",shell=True)
subprocess.call(sys.executable + " setup.py install",shell=True)
os.chdir(cwd)
# mute std out for the dreload call
capture = cStringIO.StringIO()
orig_stdout = sys.stdout
sys.stdout = capture
dreload(pylulesh)
sys.stdout = orig_stdout
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
exclude = [
# Standard exclusions:
'sys',
'os.path',
'__builtin__',
'__main__',
# Test-related exclusions:
'unittest',
'UserDict',
]
dreload(numpy, exclude=exclude)
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
# TODO: Find a way to exclude all standard library modules from reloading.
exclude = [
# Standard exclusions:
'sys', 'os.path', builtin_mod_name, '__main__',
# Test-related exclusions:
'unittest', 'UserDict', '_collections_abc', 'tokenize',
'collections', 'collections.abc',
'importlib', 'importlib.machinery', '_imp',
'importlib._bootstrap', 'importlib._bootstrap_external',
'_frozen_importlib', '_frozen_importlib_external',
]
dreload(numpy, exclude=exclude)
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
exclude = [
# Standard exclusions:
'sys', 'os.path', builtin_mod_name, '__main__',
# Test-related exclusions:
'unittest', 'UserDict', '_collections_abc', 'tokenize'
]
# `collections` builtin shall not be reloaded to avoid failure
# of lib.pretty collections related pretty printing.
# of course this make nose crash on Py3 because of Py 2.3 compat...
if not PY3:
exclude.append('collections')
dreload(numpy, exclude=exclude)
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
exclude = [
# Standard exclusions:
'sys', 'os.path', builtin_mod_name, '__main__',
# Test-related exclusions:
'unittest', 'UserDict',
]
# `collections` builtin shall not be reloaded to avoid failure
# of lib.pretty collections related pretty printing.
# of course this make nose crash on Py3 because of Py 2.3 compat...
if not PY3:
exclude.append('collections')
dreload(numpy, exclude=exclude)
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
exclude = [
# Standard exclusions:
'sys',
'os.path',
builtin_mod_name,
'__main__',
# Test-related exclusions:
'unittest',
'UserDict',
'_collections_abc',
'tokenize',
'collections',
'collections.abc',
]
dreload(numpy, exclude=exclude)
def test_deepreload():
"Test that dreload does deep reloads and skips excluded modules."
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
with open(os.path.join(tmpdir, "A.py"), "w") as f:
f.write("class Object(object):\n pass\n")
with open(os.path.join(tmpdir, "B.py"), "w") as f:
f.write("import A\n")
import A
import B
# Test that A is not reloaded.
obj = A.Object()
dreload(B, exclude=["A"])
nt.assert_true(isinstance(obj, A.Object))
# Test that A is reloaded.
obj = A.Object()
dreload(B)
nt.assert_false(isinstance(obj, A.Object))
def test_deepreload_numpy():
"Test that NumPy can be deep reloaded."
import numpy
exclude = [
# Standard exclusions:
"sys",
"os.path",
builtin_mod_name,
"__main__",
# Test-related exclusions:
"unittest",
"UserDict",
"_collections_abc",
"tokenize",
"collections",
"collections.abc",
]
dreload(numpy, exclude=exclude)
def test_deepreload():
"Test that dreload does deep reloads and skips excluded modules."
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
with open(os.path.join(tmpdir, "A.py"), "w") as f:
f.write("class Object(object):\n pass\n")
with open(os.path.join(tmpdir, "B.py"), "w") as f:
f.write("import A\n")
import A
import B
# Test that A is not reloaded.
obj = A.Object()
dreload(B, exclude=["A"])
nt.assert_true(isinstance(obj, A.Object))
# Test that A is reloaded.
obj = A.Object()
dreload(B)
nt.assert_false(isinstance(obj, A.Object))
def test_deepreload():
"Test that dreload does deep reloads and skips excluded modules."
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
tmpdirpath = Path(tmpdir)
with open(tmpdirpath / "A.py", "w") as f:
f.write("class Object(object):\n pass\n")
with open(tmpdirpath / "B.py", "w") as f:
f.write("import A\n")
import A
import B
# Test that A is not reloaded.
obj = A.Object()
dreload(B, exclude=["A"])
assert isinstance(obj, A.Object) is True
# Test that A is reloaded.
obj = A.Object()
dreload(B)
assert isinstance(obj, A.Object) is False
def test_deepreload():
"Test that dreload does deep reloads and skips excluded modules."
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
tmpdirpath = Path(tmpdir)
with open(tmpdirpath / "A.py", "w") as f:
f.write("class Object:\n pass\nok = True\n")
with open(tmpdirpath / "B.py", "w") as f:
f.write("import A\nassert A.ok, 'we are fine'\n")
import A
import B
# Test that A is not reloaded.
obj = A.Object()
dreload(B, exclude=["A"])
assert isinstance(obj, A.Object) is True
# Test that an import failure will not blow-up us.
A.ok = False
with pytest.raises(AssertionError, match="we are fine"):
dreload(B, exclude=["A"])
assert len(modules_reloading) == 0
assert not A.ok
# Test that A is reloaded.
obj = A.Object()
A.ok = False
dreload(B)
assert A.ok
assert isinstance(obj, A.Object) is False
import gmaneLegacy as g, os, pickle, time as T, numpy as n
ENV=os.environ["PATH"]
import importlib
from IPython.lib.deepreload import reload as dreload
importlib.reload(g.pca)
importlib.reload(g.loadMessages)
importlib.reload(g.listDataStructures)
importlib.reload(g.timeStatistics)
importlib.reload(g.agentStatistics)
importlib.reload(g.tableHelpers)
importlib.reload(g.networkEvolution)
importlib.reload(g.evolutionTimelines)
#importlib.reload(g.interactionNetwork)
#importlib.reload(g.networkMeasures)
dreload(g,exclude="pytz")
os.environ["PATH"]=ENV
labels={'gmane.comp.gcc.libstdc++.devel':"CPP", 'gmane.linux.audio.devel':"LAD", 'gmane.linux.audio.users':"LAU", 'gmane.politics.organizations.metareciclagem':"MET"}
##print("initializing")
dl=g.DownloadGmaneData('~/.gmane4/')
TT=T.time()
##print("{0:.2f} for initializing download dir initializing".format(T.time()-TT)); TT=T.time()
def pDump(tobject,tfilename):
with open(tfilename,"wb") as f:
pickle.dump(tobject,f,-1)
def pRead(tfilename):
with open(tfilename,"rb") as f:
tobject=pickle.load(f)
return tobject
##
###### DATA STRUCTURES
import gmaneLegacy as g #, importlib
import multiprocessing as mp
from IPython.lib.deepreload import reload as dreload
#importlib.reload(g)
#importlib.reload(g.download)
dreload(g, exclude=["pytz"])
lm = g.LoadMessages("gmane.ietf.rfc822", 10, basedir="~/.gmane2/")
dl = g.DownloadGmaneData('~/.gmane2/')
dl.getDownloadedLists()
lms = []
for list_id in dl.downloaded_lists[:10]:
print(list_id)
lms.append(g.LoadMessages(list_id, basedir="~/.gmane2/"))
# to download first three lists with the greated number
# of downloaded messages, do:
dl.downloadedStats() # might take a while
lms2 = []
for list_stat in dl.lists[:3]:
list_id = list_stat[0]
lms2.append(g.LoadMessages(list_id, basedir="~/.gmane2/"))
import importlib, os
import multiprocessing as mp
from IPython.lib.deepreload import reload as dreload
import gmane as g, percolation as P
G=g
c=P.utils.check
importlib.reload(g.listDataStructures)
importlib.reload(g.loadMessages)
importlib.reload(g.triplifyList)
importlib.reload(P.rdf)
importlib.reload(P.utils)
importlib.reload(g.utils)
dreload(g,exclude="pytz")
#lm=g.LoadMessages("gmane.ietf.rfc822",10,basedir="~/.gmane2/")
#ds=g.ListDataStructures(lm)
#
#dl=g.DownloadGmaneData(dpath)
#dl.downloadedStats() # might take a while
dpath='/home/r/.gmane4/'
dpath='/home/r/.gmane/'
dpath='/disco/.gmane/'
load_msgs=[]
data_structs=[]
scriptpath=os.path.realpath(__file__)
fpath="./publishing/"
umbrella_dir="gmane2/"
#for list_stat in dl.lists:
# list_id=list_stat[0]
#for list_id in ['gmane.comp.gcc.libstdc++.devel']:
#for list_id in ['gmane.comp.java.hadoop.hive.user']:
import gmaneLegacy as g, importlib
import multiprocessing as mp
from IPython.lib.deepreload import reload as dreload
#importlib.reload(g)
#importlib.reload(g.download)
dreload(g)
dl = g.DownloadGmaneData('~/.gmane2/')
dl.downloadListIDS()
#for list_id in loads.list_ids[9:]:
# loads.downloadListMessages(list_id,end=100000)
# download all files from list with first ID of list_ids
# define uma função que baixa, que já é o loads.download
# manda baixar async umas 30 listas e ve o que rola
pool = mp.Pool(processes=100)
args_ = tuple((i, 1, 1000000, 100) for i in dl.list_ids[:200])
#results___=pool.map_async(loads.downloadListMessages,args_)
results__ = [pool.apply_async(dl.downloadListMessages, args=x) for x in args_]
import social as S, os ENV=os.environ["PATH"] import importlib from IPython.lib.deepreload import reload as dreload importlib.reload(S.triplification.participaTriplification) #dreload(g,exclude="pytz") dreload(S.triplification,exclude="rdflib") trip=S.triplification.ParticipaTriplification()
def test_not_in_sys_modules():
fake_module = types.ModuleType("fake_module")
with pytest.raises(ImportError, match="not in sys.modules"):
dreload(fake_module)
