import gmaneLegacy as g, os
ENV = os.environ["PATH"]
import importlib
from IPython.lib.deepreload import reload as dreload
importlib.reload(g.loadMessages)
importlib.reload(g.listDataStructures)
importlib.reload(g.timeStatistics)
#importlib.reload(g.interactionNetwork)
#importlib.reload(g.networkMeasures)
dreload(g, exclude="pytz")
os.environ["PATH"] = ENV
dl = g.DownloadGmaneData('~/.gmane2/')
dl.downloadedStats() # might take a while
lm = g.LoadMessages(dl.lists[0][0], basedir="~/.gmane2/")
print("loaded messages")
ds = g.ListDataStructures(lm)
print("made datastructures")
ts = g.TimeStatistics(ds)
print("made overall activity statistics along time")
import gmaneLegacy as g #, importlib
import multiprocessing as mp
from IPython.lib.deepreload import reload as dreload
#importlib.reload(g)
#importlib.reload(g.download)
dreload(g, exclude=["pytz"])
lm = g.LoadMessages("gmane.ietf.rfc822", 10, basedir="~/.gmane2/")
dl = g.DownloadGmaneData('~/.gmane2/')
dl.getDownloadedLists()
lms = []
for list_id in dl.downloaded_lists[:10]:
print(list_id)
lms.append(g.LoadMessages(list_id, basedir="~/.gmane2/"))
# to download first three lists with the greated number
# of downloaded messages, do:
dl.downloadedStats() # might take a while
lms2 = []
for list_stat in dl.lists[:3]:
list_id = list_stat[0]
lms2.append(g.LoadMessages(list_id, basedir="~/.gmane2/"))
import gmaneLegacy as g, os
ENV = os.environ["PATH"]
import importlib
from IPython.lib.deepreload import reload as dreload
#importlib.reload(g.listDataStructures)
#importlib.reload(g.loadMessages)
importlib.reload(g.interactionNetwork)
importlib.reload(g.networkMeasures)
dreload(g, exclude="pytz")
os.environ["PATH"] = ENV
lm = g.LoadMessages("gmane.ietf.rfc822", 10, basedir="~/.gmane2/")
ds = g.ListDataStructures(lm)
iN = g.InteractionNetwork(ds)
nm = g.NetworkMeasures(iN)
dl = g.DownloadGmaneData('~/.gmane2/')
dl.downloadedStats() # might take a while
lm = g.LoadMessages(dl.lists[0][0], basedir="~/.gmane2/")
print("loaded messages")
ds = g.ListDataStructures(lm)
print("made datastructures")
iN = g.InteractionNetwork(ds)
print("made interaction network")
nm = g.NetworkMeasures(iN)
print("network mesaures")
#lm=g.LoadMessages("gmane.ietf.rfc822",10,basedir="~/.gmane2/")
#ds=g.ListDataStructures(lm)
#
#dl=g.DownloadGmaneData(dpath)
#dl.downloadedStats() # might take a while
dpath='/disco/.gmane/'
dpath='/home/r/.gmane4/'
dpath='/home/r/.gmane/'
load_msgs=[]
data_structs=[]
scriptpath=os.path.realpath(__file__)
fpath="./publishing/"
umbrella_dir="gmane1/"
#for list_stat in dl.lists:
# list_id=list_stat[0]
#for list_id in ['gmane.comp.gcc.libstdc++.devel']:
#for list_id in ['gmane.comp.java.hadoop.hive.user']:
#for list_id in ['gmane.politics.organizations.metareciclagem', 'gmane.comp.gcc.libstdc++.devel', 'gmane.linux.audio.devel', 'gmane.linux.audio.users']:
for list_id in ['gmane.comp.web.egroupware.user', 'gmane.culture.language.basque.eibartarrak','gmane.org.operators.nznog', 'gmane.science.nmr.relax.scm',"gmane.linux.fbdev.devel",]:
# lm=g.LoadMessages(list_id,basedir=dpath,n_messages=20000)
# lm=g.LoadMessages(list_id,basedir=dpath,n_messages=200)
lm=g.LoadMessages(list_id,basedir=dpath)
ds=g.ListDataStructures(lm)
foo=G.triplifyList.makeRepo(ds,fpath,dpath+list_id,"Linked data of the email list with Gmane id: {}".format(list_id),scriptpath=scriptpath,umbrella_dir=umbrella_dir)
mm= ds.messages
ids=ds.message_ids
print("first: ", mm[ids[0]][2], "last:", mm[ids[-1]][2])
def hardClean(text):
return "".join(c for c in text if c.isalnum() or c in allowed)
import gmaneLegacy as g, mass as m, os
ENV = os.environ["PATH"]
import importlib
#from IPython.lib.deepreload import reload as dreload
importlib.reload(g.evolutionMusic)
importlib.reload(m.pieces.fourHubsDance)
dreload(g, exclude="pytz")
os.environ["PATH"] = ENV
#dreload(m)
#os.environ["PATH"]=ENV
#dl=g.DownloadGmaneData('/disco/.gmane2/')
#dl.downloadedStats() # might take a while
#print("made liststats")
lm = g.LoadMessages("gmane.linux.audio.users", 14000, basedir="/disco/.gmane/")
#lm=g.LoadMessages("gmane.linux.audio.users",800,basedir="/disco/.gmane/")
print("loaded messages")
#
##ne=g.NetworkEvolution(step_size=20)
##ne.evolveRaw(lm.messages[:500])
##ne.makeVideo(framerate=12)
#
ne = g.NetworkEvolution(window_size=400, step_size=40)
print("evolution started")
ne.evolveRaw(lm.messages, imagerate=4, erdos_sectors=True)
print("network evolved")
em = g.EvolutionMusic()
print("music is done")
for key in keys:
if ("gmane" in key):
del sys.modules[key]
if ("musicLegagy" in key):
try:
del sys.modules[key]
except:
pass
import gmaneLegacy as g, os, musicLegacy as m
#dl=g.DownloadGmaneData('/disco/.gmane2/')
#dl.downloadedStats() # might take a while
#print("made liststats")
lm = g.LoadMessages("cpp",
4000,
basedir="~/repos/versinus/data/gmaneMessages/mbox/")
#lm=g.LoadMessages("gmane.linux.audio.users",800,basedir="/disco/.gmane/")
print("loaded messages")
#
##ne=g.NetworkEvolution(step_size=20)
##ne.evolveRaw(lm.messages[:500])
##ne.makeVideo(framerate=12)
#
ne = g.NetworkEvolution(window_size=40, step_size=40)
print("evolution started")
ne.evolveRaw(lm.messages, imagerate=4, erdos_sectors=True)
print("network evolved")
em = g.EvolutionMusic()
print("music is done")
#pDump(dl,"{}dl.pickle".format(PDIR))
dl = pRead("{}dl.pickle".format(PDIR))
###### DATA STRUCTURES
TOTAL_M = 200
dss = []
iNs = []
nms = []
tss = []
nps = []
PDIR = "pickledir/"
for lid in dl.lists[4:6]:
lid = lid[0]
print("\n", lid)
# label=labels[lid]
lm = g.LoadMessages(lid, TOTAL_M, basedir="~/.gmane3/")
print(lid + "{0:.2f} for loading messages".format(T.time() - TT))
TT = T.time()
ds = g.ListDataStructures(lm, text="yes")
print(lid + "{0:.2f} for data structures".format(T.time() - TT))
TT = T.time()
dss.append(ds)
pDump(ds, "{}ds{}.pickle".format(PDIR, lid))
ts = g.TimeStatistics(ds)
print("{0:.2f} for statistics along time".format(T.time() - TT))
TT = T.time()
tss.append(ts)
pDump(ts, "{}ts{}.pickle".format(PDIR, lid))
