def parseConfig(self):
toml_dict = toml.loads(TS_MS_CONFIG)
toml_dict[INPUTS][TS_NAME]['consumer_key'] = TwitterSourceConfig.consumer_key
toml_dict[INPUTS][TS_NAME]['consumer_secret'] = TwitterSourceConfig.consumer_secret
toml_dict[INPUTS][TS_NAME]['access_token'] = TwitterSourceConfig.access_token
toml_dict[INPUTS][TS_NAME]['access_token_secret'] = TwitterSourceConfig.access_token_secret
results = ConfigParser.parse_components_dict(toml_dict)
return results
def run_component(component_name, config_file):
results = ConfigParser.parse_components(config_file)
components = extract_components(results)
item = components[component_name]
logging.debug("Starting consumer type: %s" % (component_name))
item.periodic_consume(2.0)
while True:
time.sleep(10)
def parseBasicConfig(self):
toml_dict = toml.loads(BASIC_TS_CONFIG)
# thoughtful protection of secrets
toml_dict[INPUTS][TS_NAME][
'consumer_key'] = TwitterSourceConfig.consumer_key
toml_dict[INPUTS][TS_NAME][
'consumer_secret'] = TwitterSourceConfig.consumer_secret
toml_dict[INPUTS][TS_NAME][
'access_token'] = TwitterSourceConfig.access_token
toml_dict[INPUTS][TS_NAME][
'access_token_secret'] = TwitterSourceConfig.access_token_secret
inputs_dict = toml_dict.get(INPUTS)
inputs = ConfigParser.parse_inputs(inputs_dict, CLASS_MAPS)
return inputs
def run_twitter01(inputs_dict):
global KEEP_RUNNING, TW_G
KEEP_RUNNING = True
inputs = ConfigParser.parse_inputs(inputs_dict, CLASS_MAPS)
twitter01 = inputs.get('twitter01')
TW_G = twitter01
while KEEP_RUNNING:
try:
posts = twitter01.consume_and_publish()
time.sleep(5.0)
if posts is None:
continue
num_msgs = sum([len(i) for i in posts.values()])
print("Found %d posts for %d handle" % (num_msgs, len(posts)))
except KeyboardInterrupt:
KEEP_RUNNING = False
print("Ending execution")
break
KEYS = ('keys', [])
MINE_NOT_YOURS = 'internal_keys.toml'
config = toml.load(open("internal_keys.toml"))
keys = config.get(KEYS[0], KEYS[1])
for k in keys:
classname, attr, value = config.get(k)
# this works because imported all names/class
# for the test modules into the local space
klass = locals()[classname]
# Set the static attributes in the class
setattr(klass, attr, value)
TS_NAME = 'twitter01'
INPUTS = 'inputs'
toml_dict = toml.loads(BASIC_TS_CONFIG)
# thoughtful protection of secrets
toml_dict[INPUTS][TS_NAME]['consumer_key'] = TwitterSourceConfig.consumer_key
toml_dict[INPUTS][TS_NAME]['consumer_secret'] = TwitterSourceConfig.consumer_secret
toml_dict[INPUTS][TS_NAME]['access_token'] = TwitterSourceConfig.access_token
toml_dict[INPUTS][TS_NAME]['access_token_secret'] = TwitterSourceConfig.access_token_secret
inputs_dict = toml_dict.get(INPUTS)
inputs = ConfigParser.parse_inputs(inputs_dict, CLASS_MAPS)
twitter01 = inputs[TS_NAME]
started = twitter01.svc.start()
logging.error("[XXXXXXX] Starting")
if __name__ == '__main__':
args = parser.parse_args()
if args.config is None:
parser.print_help()
sys.exit(1)
try:
os.stat(args.config)
except:
logging.error("[X] Error file does not exist")
sys.exit(1)
component_procs = {}
results = ConfigParser.parse_components(args.config)
components = extract_components(results)
if args.reset:
reset_components(components)
cname = None
if args.name != "ALL":
cname = args.name
logging.info("Creating process for %s"%cname)
component_procs[cname] = multiprocessing.Process(name=cname, target=run_component, args=(cname, args.config))
logging.info("Starting %s"%cname)
component_procs[cname].start()
else:
# start all the components in separate procs
for name, item in components.items():
def parseConfig(self):
toml_dict = toml.loads(TP_CONFIG)
results = ConfigParser.parse_components_dict(toml_dict)
return results