def round_generate(self): """ Generate objects from the scraped data. """ socgr = Graph.Read(self.fp_i("soc.graphml")) gumap = dict_load(self.fp_i("group-user.map")) pddb = self.db("prod-doc") dppb = self.db("doc-prod") dtdb = self.db("doc-tag") tcdb = self.db("tag-cluster") phdb = self.db("p_idx", lrusize=self.cache) phsb = self.db("p_idx_s") pgdb = self.db("p_tgr", lrusize=self.cache) pgsb = self.db("p_tgr_s") FILE_IDX = "idx.graphml" FILE_CMM = "communities.map" FILE_TGR = "tgr.graphml" FILE_PTB = "ptb.graphml" FILE_PTB_U = "ptables.map" sg = SampleGenerator(socgr, gumap, pddb, dppb, dtdb, tcdb, phdb, phsb, pgdb, pgsb) # indexes if not self.fp_exists(FILE_IDX): sg.generateIndexes() sg.prodgr.write(self.fp_o(FILE_IDX)) else: sg.prodgr = Graph.Read(self.fp_i(FILE_IDX)) # communities if not self.fp_exists(FILE_CMM): sg.generateCommunities() dict_save(dict(enumerate(sg.comm)), self.fp_o(FILE_CMM)) else: sg.comm = [v for k, v in sorted(dict_load(self.fp_i(FILE_CMM)).iteritems())] # tgraphs if not self.fp_exists(FILE_TGR): sg.generateTGraphs() sg.sprdgr.write(self.fp_o(FILE_TGR)) else: sg.sprdgr = Graph.Read(self.fp_i(FILE_TGR)) # ptables if not self.fp_exists(FILE_PTB): sg.generatePTables() sg.ptabgr.write(self.fp_o(FILE_PTB)) dict_save(sg.ptbmap, self.fp_o(FILE_PTB_U)) else: sg.ptabgr = Graph.Read(self.fp_i(FILE_PTB)) sg.ptbmap = dict_load(self.fp_i(FILE_PTB_U)) LOG.info("generation complete; don't forget to run `postgen -d %s`" % self.base) if self.interact: code.interact(banner=self.banner(locals()), local=locals())
def round_group(self): """ Scrape the group network from the social network. """ users = Graph.Read(self.fp_i("soc.graphml")).vs["id"] gumap = self.ff.scrapeGroups(users) dict_save(gumap, self.fp_o("group-user.map")) if self.interact: code.interact(banner=self.banner(locals()), local=locals())
def round_photo(self): """ Scrape photos of the collected producers. """ socgr = Graph.Read(self.fp_i("soc.graphml")) gumap = dict_load(self.fp_i("group-user.map")) pddb = self.db("prod-doc") self.ff.commitUserPhotos(socgr.vs["id"], pddb) self.ff.commitGroupPhotos(gumap, pddb) self.ff.pruneProducers(socgr, gumap, pddb) socgr.write_graphml(self.fp_o("soc.graphml")) dict_save(gumap, self.fp_o("group-user.map")) if self.interact: code.interact(banner=self.banner(locals()), local=locals())