import gmaneLegacy as g, os ENV = os.environ["PATH"] import importlib from IPython.lib.deepreload import reload as dreload importlib.reload(g.loadMessages) importlib.reload(g.listDataStructures) importlib.reload(g.timeStatistics) #importlib.reload(g.interactionNetwork) #importlib.reload(g.networkMeasures) dreload(g, exclude="pytz") os.environ["PATH"] = ENV dl = g.DownloadGmaneData('~/.gmane2/') dl.downloadedStats() # might take a while lm = g.LoadMessages(dl.lists[0][0], basedir="~/.gmane2/") print("loaded messages") ds = g.ListDataStructures(lm) print("made datastructures") ts = g.TimeStatistics(ds) print("made overall activity statistics along time")
import gmaneLegacy as g #, importlib import multiprocessing as mp from IPython.lib.deepreload import reload as dreload #importlib.reload(g) #importlib.reload(g.download) dreload(g, exclude=["pytz"]) lm = g.LoadMessages("gmane.ietf.rfc822", 10, basedir="~/.gmane2/") dl = g.DownloadGmaneData('~/.gmane2/') dl.getDownloadedLists() lms = [] for list_id in dl.downloaded_lists[:10]: print(list_id) lms.append(g.LoadMessages(list_id, basedir="~/.gmane2/")) # to download first three lists with the greated number # of downloaded messages, do: dl.downloadedStats() # might take a while lms2 = [] for list_stat in dl.lists[:3]: list_id = list_stat[0] lms2.append(g.LoadMessages(list_id, basedir="~/.gmane2/"))
import gmaneLegacy as g, os ENV = os.environ["PATH"] import importlib from IPython.lib.deepreload import reload as dreload #importlib.reload(g.listDataStructures) #importlib.reload(g.loadMessages) importlib.reload(g.interactionNetwork) importlib.reload(g.networkMeasures) dreload(g, exclude="pytz") os.environ["PATH"] = ENV lm = g.LoadMessages("gmane.ietf.rfc822", 10, basedir="~/.gmane2/") ds = g.ListDataStructures(lm) iN = g.InteractionNetwork(ds) nm = g.NetworkMeasures(iN) dl = g.DownloadGmaneData('~/.gmane2/') dl.downloadedStats() # might take a while lm = g.LoadMessages(dl.lists[0][0], basedir="~/.gmane2/") print("loaded messages") ds = g.ListDataStructures(lm) print("made datastructures") iN = g.InteractionNetwork(ds) print("made interaction network") nm = g.NetworkMeasures(iN) print("network mesaures")
#lm=g.LoadMessages("gmane.ietf.rfc822",10,basedir="~/.gmane2/") #ds=g.ListDataStructures(lm) # #dl=g.DownloadGmaneData(dpath) #dl.downloadedStats() # might take a while dpath='/disco/.gmane/' dpath='/home/r/.gmane4/' dpath='/home/r/.gmane/' load_msgs=[] data_structs=[] scriptpath=os.path.realpath(__file__) fpath="./publishing/" umbrella_dir="gmane1/" #for list_stat in dl.lists: # list_id=list_stat[0] #for list_id in ['gmane.comp.gcc.libstdc++.devel']: #for list_id in ['gmane.comp.java.hadoop.hive.user']: #for list_id in ['gmane.politics.organizations.metareciclagem', 'gmane.comp.gcc.libstdc++.devel', 'gmane.linux.audio.devel', 'gmane.linux.audio.users']: for list_id in ['gmane.comp.web.egroupware.user', 'gmane.culture.language.basque.eibartarrak','gmane.org.operators.nznog', 'gmane.science.nmr.relax.scm',"gmane.linux.fbdev.devel",]: # lm=g.LoadMessages(list_id,basedir=dpath,n_messages=20000) # lm=g.LoadMessages(list_id,basedir=dpath,n_messages=200) lm=g.LoadMessages(list_id,basedir=dpath) ds=g.ListDataStructures(lm) foo=G.triplifyList.makeRepo(ds,fpath,dpath+list_id,"Linked data of the email list with Gmane id: {}".format(list_id),scriptpath=scriptpath,umbrella_dir=umbrella_dir) mm= ds.messages ids=ds.message_ids print("first: ", mm[ids[0]][2], "last:", mm[ids[-1]][2]) def hardClean(text): return "".join(c for c in text if c.isalnum() or c in allowed)
import gmaneLegacy as g, mass as m, os ENV = os.environ["PATH"] import importlib #from IPython.lib.deepreload import reload as dreload importlib.reload(g.evolutionMusic) importlib.reload(m.pieces.fourHubsDance) dreload(g, exclude="pytz") os.environ["PATH"] = ENV #dreload(m) #os.environ["PATH"]=ENV #dl=g.DownloadGmaneData('/disco/.gmane2/') #dl.downloadedStats() # might take a while #print("made liststats") lm = g.LoadMessages("gmane.linux.audio.users", 14000, basedir="/disco/.gmane/") #lm=g.LoadMessages("gmane.linux.audio.users",800,basedir="/disco/.gmane/") print("loaded messages") # ##ne=g.NetworkEvolution(step_size=20) ##ne.evolveRaw(lm.messages[:500]) ##ne.makeVideo(framerate=12) # ne = g.NetworkEvolution(window_size=400, step_size=40) print("evolution started") ne.evolveRaw(lm.messages, imagerate=4, erdos_sectors=True) print("network evolved") em = g.EvolutionMusic() print("music is done")
for key in keys: if ("gmane" in key): del sys.modules[key] if ("musicLegagy" in key): try: del sys.modules[key] except: pass import gmaneLegacy as g, os, musicLegacy as m #dl=g.DownloadGmaneData('/disco/.gmane2/') #dl.downloadedStats() # might take a while #print("made liststats") lm = g.LoadMessages("cpp", 4000, basedir="~/repos/versinus/data/gmaneMessages/mbox/") #lm=g.LoadMessages("gmane.linux.audio.users",800,basedir="/disco/.gmane/") print("loaded messages") # ##ne=g.NetworkEvolution(step_size=20) ##ne.evolveRaw(lm.messages[:500]) ##ne.makeVideo(framerate=12) # ne = g.NetworkEvolution(window_size=40, step_size=40) print("evolution started") ne.evolveRaw(lm.messages, imagerate=4, erdos_sectors=True) print("network evolved") em = g.EvolutionMusic() print("music is done")
#pDump(dl,"{}dl.pickle".format(PDIR)) dl = pRead("{}dl.pickle".format(PDIR)) ###### DATA STRUCTURES TOTAL_M = 200 dss = [] iNs = [] nms = [] tss = [] nps = [] PDIR = "pickledir/" for lid in dl.lists[4:6]: lid = lid[0] print("\n", lid) # label=labels[lid] lm = g.LoadMessages(lid, TOTAL_M, basedir="~/.gmane3/") print(lid + "{0:.2f} for loading messages".format(T.time() - TT)) TT = T.time() ds = g.ListDataStructures(lm, text="yes") print(lid + "{0:.2f} for data structures".format(T.time() - TT)) TT = T.time() dss.append(ds) pDump(ds, "{}ds{}.pickle".format(PDIR, lid)) ts = g.TimeStatistics(ds) print("{0:.2f} for statistics along time".format(T.time() - TT)) TT = T.time() tss.append(ts) pDump(ts, "{}ts{}.pickle".format(PDIR, lid))