def runSampler(sample): sample.init() searches = sample.loadSearches() logging.info("Total search size [%s]" % len(searches)) filtered = sample.filterSearches(searches) logging.info("Filtered search size [%s]" % len(filtered)) logging.info("Logging in as [%s]" % sample.getUserName()) SessionManager.doLogin(sample.getUserName()) #for k,v in filtered.iteritems(): # logging.info("[%32s] => [%2d][%s]" % (k,v.Match,v.UserName)) sample.crawlProfiles([x.UserName for x in filtered.itervalues()])
def init(self): logging.info("Initializing Experiment [%s]" % self.__expName) bigConfigPath = os.path.join(self.__rootPath,"Config","experiments.ini") if not os.path.exists(bigConfigPath): raise RuntimError,"Global experiment list not found" bigConfig = ConfigParser.ConfigParser() bigConfig.optionxform=str bigConfig.read(bigConfigPath) if not bigConfig.has_section(self.__expName): raise RuntimeError,"No such experiment [%s] in global list" % self.__expName if not os.path.exists(self.__dataPath): logging.warn("Data path does not exist, creating") os.mkdir(self.__dataPath) if os.path.exists(self.__expPath): logging.warn("Old experiment directory exists, clearing") shutil.rmtree(self.__expPath) os.mkdir(self.__expPath) os.mkdir(self.__searchPath) os.mkdir(self.__profilePath) self.__config.add_section("Settings") self.__config.add_section("Locations") self.__config.add_section("User") self.__config.add_section("Timestamps") for k,v in MinerExperiment.PROPERTIES.iteritems(): if bigConfig.has_option(self.__expName,k): self.__config.set("Settings",k,bigConfig.get(self.__expName,k)) else: self.__config.set("Settings",k,v) SessionManager.doLogin(self.getUserName()) self.__fillUserProfile() if bigConfig.has_option(self.__expName,"Locations"): parser = shlex.shlex(bigConfig.get(self.__expName,"Locations")) parser.whitespace += ',' for location in parser: if location == "Anywhere": self.__config.set("Locations",location,"%s,0" % self.__getLocation(location)) else: self.__config.set("Locations",location,"%s,25" % self.__getLocation(location)) else: self.__config.set("Locations","Near me","%s,25" % self.__getLocation("Near me")) self.saveConfig()