def runSampler(sample):
sample.init()
searches = sample.loadSearches()
logging.info("Total search size [%s]" % len(searches))
filtered = sample.filterSearches(searches)
logging.info("Filtered search size [%s]" % len(filtered))
logging.info("Logging in as [%s]" % sample.getUserName())
SessionManager.doLogin(sample.getUserName())
#for k,v in filtered.iteritems():
# logging.info("[%32s] => [%2d][%s]" % (k,v.Match,v.UserName))
sample.crawlProfiles([x.UserName for x in filtered.itervalues()])
def init(self):
logging.info("Initializing Experiment [%s]" % self.__expName)
bigConfigPath = os.path.join(self.__rootPath,"Config","experiments.ini")
if not os.path.exists(bigConfigPath):
raise RuntimError,"Global experiment list not found"
bigConfig = ConfigParser.ConfigParser()
bigConfig.optionxform=str
bigConfig.read(bigConfigPath)
if not bigConfig.has_section(self.__expName):
raise RuntimeError,"No such experiment [%s] in global list" % self.__expName
if not os.path.exists(self.__dataPath):
logging.warn("Data path does not exist, creating")
os.mkdir(self.__dataPath)
if os.path.exists(self.__expPath):
logging.warn("Old experiment directory exists, clearing")
shutil.rmtree(self.__expPath)
os.mkdir(self.__expPath)
os.mkdir(self.__searchPath)
os.mkdir(self.__profilePath)
self.__config.add_section("Settings")
self.__config.add_section("Locations")
self.__config.add_section("User")
self.__config.add_section("Timestamps")
for k,v in MinerExperiment.PROPERTIES.iteritems():
if bigConfig.has_option(self.__expName,k):
self.__config.set("Settings",k,bigConfig.get(self.__expName,k))
else:
self.__config.set("Settings",k,v)
SessionManager.doLogin(self.getUserName())
self.__fillUserProfile()
if bigConfig.has_option(self.__expName,"Locations"):
parser = shlex.shlex(bigConfig.get(self.__expName,"Locations"))
parser.whitespace += ','
for location in parser:
if location == "Anywhere":
self.__config.set("Locations",location,"%s,0" % self.__getLocation(location))
else:
self.__config.set("Locations",location,"%s,25" % self.__getLocation(location))
else:
self.__config.set("Locations","Near me","%s,25" % self.__getLocation("Near me"))
self.saveConfig()
