def _create_dam(self, symbol): dam_name = self.config.get(CONF_ANALYZER_SECTION, CONF_INPUT_DAM) input_db = self.config.get(CONF_ANALYZER_SECTION, CONF_INPUT_DB) dam = DAMFactory.createDAM(dam_name, {'db': input_db}) dam.symbol = symbol return dam
def getOptions(self): ''' crawling data and save to hbase ''' parser = optparse.OptionParser("Usage: %prog [options]") parser.add_option("-f", "--symbolFile", dest="symbolFile", type="string", help="file that contains symbols for each line") parser.add_option("-o", "--outputDAM", dest="outputDAM", default='sql', type="string", help="output dam, e.g. sql|hbase") (options, _) = parser.parse_args() # get symbols if options.symbolFile is None or not path.exists(options.symbolFile): print("Please provide valid file: %s" % options.symbolFile) exit(4) # get all symbols with open(options.symbolFile, 'r') as f: for line in f.readlines(): self.symbols.append(line.strip()) if not self.symbols: print("No symbols provided in file %s" % options.symbolFile) exit(4) # set output dam if options.outputDAM not in ["hbase", "sql"]: print("Please provide valid outputDAM %s" % options.outputDAM) exit(4) if 'sql' == options.outputDAM: sqlLocation = 'sqlite:///%s' % self.getOutputSql() print("Sqlite location: %s" % sqlLocation) setting = {'db': sqlLocation} # set google and output dam self.googleDAM = DAMFactory.createDAM("google") self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
def getOptions(self): ''' crawling data and save to hbase ''' parser=optparse.OptionParser("Usage: %prog [options]") parser.add_option("-f", "--symbolFile", dest="symbolFile", type="string", help="file that contains symbols for each line") parser.add_option("-o", "--outputDAM", dest="outputDAM", default='sql', type="string", help="output dam, e.g. sql|hbase") (options, _)=parser.parse_args() # get symbols if options.symbolFile is None or not path.exists(options.symbolFile): print("Please provide valid file: %s" % options.symbolFile) exit(4) # get all symbols with open(options.symbolFile, 'r') as f: for line in f.readlines(): self.symbols.append(line.strip()) if not self.symbols: print("No symbols provided in file %s" % options.symbolFile) exit(4) # set output dam if options.outputDAM not in ["hbase", "sql"]: print("Please provide valid outputDAM %s" % options.outputDAM) exit(4) if 'sql' == options.outputDAM: sqlLocation='sqlite:///%s' % self.getOutputSql() print("Sqlite location: %s" % sqlLocation) setting={'db': sqlLocation} # set google and output dam self.googleDAM=DAMFactory.createDAM("google") self.outputDAM=DAMFactory.createDAM(options.outputDAM, setting)