def round_generate(self):
"""
Generate objects from the scraped data.
"""
socgr = Graph.Read(self.fp_i("soc.graphml"))
gumap = dict_load(self.fp_i("group-user.map"))
pddb = self.db("prod-doc")
dppb = self.db("doc-prod")
dtdb = self.db("doc-tag")
tcdb = self.db("tag-cluster")
phdb = self.db("p_idx", lrusize=self.cache)
phsb = self.db("p_idx_s")
pgdb = self.db("p_tgr", lrusize=self.cache)
pgsb = self.db("p_tgr_s")
FILE_IDX = "idx.graphml"
FILE_CMM = "communities.map"
FILE_TGR = "tgr.graphml"
FILE_PTB = "ptb.graphml"
FILE_PTB_U = "ptables.map"
sg = SampleGenerator(socgr, gumap, pddb, dppb, dtdb, tcdb, phdb, phsb, pgdb, pgsb)
# indexes
if not self.fp_exists(FILE_IDX):
sg.generateIndexes()
sg.prodgr.write(self.fp_o(FILE_IDX))
else:
sg.prodgr = Graph.Read(self.fp_i(FILE_IDX))
# communities
if not self.fp_exists(FILE_CMM):
sg.generateCommunities()
dict_save(dict(enumerate(sg.comm)), self.fp_o(FILE_CMM))
else:
sg.comm = [v for k, v in sorted(dict_load(self.fp_i(FILE_CMM)).iteritems())]
# tgraphs
if not self.fp_exists(FILE_TGR):
sg.generateTGraphs()
sg.sprdgr.write(self.fp_o(FILE_TGR))
else:
sg.sprdgr = Graph.Read(self.fp_i(FILE_TGR))
# ptables
if not self.fp_exists(FILE_PTB):
sg.generatePTables()
sg.ptabgr.write(self.fp_o(FILE_PTB))
dict_save(sg.ptbmap, self.fp_o(FILE_PTB_U))
else:
sg.ptabgr = Graph.Read(self.fp_i(FILE_PTB))
sg.ptbmap = dict_load(self.fp_i(FILE_PTB_U))
LOG.info("generation complete; don't forget to run `postgen -d %s`" % self.base)
if self.interact: code.interact(banner=self.banner(locals()), local=locals())
def round_group(self):
"""
Scrape the group network from the social network.
"""
users = Graph.Read(self.fp_i("soc.graphml")).vs["id"]
gumap = self.ff.scrapeGroups(users)
dict_save(gumap, self.fp_o("group-user.map"))
if self.interact: code.interact(banner=self.banner(locals()), local=locals())
def round_photo(self):
"""
Scrape photos of the collected producers.
"""
socgr = Graph.Read(self.fp_i("soc.graphml"))
gumap = dict_load(self.fp_i("group-user.map"))
pddb = self.db("prod-doc")
self.ff.commitUserPhotos(socgr.vs["id"], pddb)
self.ff.commitGroupPhotos(gumap, pddb)
self.ff.pruneProducers(socgr, gumap, pddb)
socgr.write_graphml(self.fp_o("soc.graphml"))
dict_save(gumap, self.fp_o("group-user.map"))
if self.interact: code.interact(banner=self.banner(locals()), local=locals())
