def _create_dam(self, symbol):
dam_name = self.config.get(CONF_ANALYZER_SECTION, CONF_INPUT_DAM)
input_db = self.config.get(CONF_ANALYZER_SECTION, CONF_INPUT_DB)
dam = DAMFactory.createDAM(dam_name, {'db': input_db})
dam.symbol = symbol
return dam
def _create_dam(self, symbol):
dam_name = self.config.get(CONF_ANALYZER_SECTION, CONF_INPUT_DAM)
input_db = self.config.get(CONF_ANALYZER_SECTION, CONF_INPUT_DB)
dam = DAMFactory.createDAM(dam_name, {'db': input_db})
dam.symbol = symbol
return dam
def getOptions(self):
''' crawling data and save to hbase '''
parser = optparse.OptionParser("Usage: %prog [options]")
parser.add_option("-f",
"--symbolFile",
dest="symbolFile",
type="string",
help="file that contains symbols for each line")
parser.add_option("-o",
"--outputDAM",
dest="outputDAM",
default='sql',
type="string",
help="output dam, e.g. sql|hbase")
(options, _) = parser.parse_args()
# get symbols
if options.symbolFile is None or not path.exists(options.symbolFile):
print("Please provide valid file: %s" % options.symbolFile)
exit(4)
# get all symbols
with open(options.symbolFile, 'r') as f:
for line in f.readlines():
self.symbols.append(line.strip())
if not self.symbols:
print("No symbols provided in file %s" % options.symbolFile)
exit(4)
# set output dam
if options.outputDAM not in ["hbase", "sql"]:
print("Please provide valid outputDAM %s" % options.outputDAM)
exit(4)
if 'sql' == options.outputDAM:
sqlLocation = 'sqlite:///%s' % self.getOutputSql()
print("Sqlite location: %s" % sqlLocation)
setting = {'db': sqlLocation}
# set google and output dam
self.googleDAM = DAMFactory.createDAM("google")
self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
def getOptions(self):
''' crawling data and save to hbase '''
parser=optparse.OptionParser("Usage: %prog [options]")
parser.add_option("-f", "--symbolFile", dest="symbolFile", type="string",
help="file that contains symbols for each line")
parser.add_option("-o", "--outputDAM", dest="outputDAM",
default='sql', type="string",
help="output dam, e.g. sql|hbase")
(options, _)=parser.parse_args()
# get symbols
if options.symbolFile is None or not path.exists(options.symbolFile):
print("Please provide valid file: %s" % options.symbolFile)
exit(4)
# get all symbols
with open(options.symbolFile, 'r') as f:
for line in f.readlines():
self.symbols.append(line.strip())
if not self.symbols:
print("No symbols provided in file %s" % options.symbolFile)
exit(4)
# set output dam
if options.outputDAM not in ["hbase", "sql"]:
print("Please provide valid outputDAM %s" % options.outputDAM)
exit(4)
if 'sql' == options.outputDAM:
sqlLocation='sqlite:///%s' % self.getOutputSql()
print("Sqlite location: %s" % sqlLocation)
setting={'db': sqlLocation}
# set google and output dam
self.googleDAM=DAMFactory.createDAM("google")
self.outputDAM=DAMFactory.createDAM(options.outputDAM, setting)
