def get_monitor_data(combinations, title, source, keywords, data_dir,
progress):
print progress + " monitors: Adding monitor for " + title + "..."
# add buzz monitor
new_monitor = Monitor(title=title,
sources=source,
languages=languages,
keywords=keywords,
start=start,
end=end)
monitor_id = new_monitor.id
# for each combination
for combi_index, combination in enumerate(combinations):
progress = "[" + str(combi_index + 1) + "/" + str(
len(combinations)) + "]"
# make keyword query
keyword_query = get_keyword_query(combination)
# make short query
short_query = get_short_query(combination)
# make a data directory
query_dir = make_query_dir(data_dir, short_query)
# save keyword query in txt-file
with open(query_dir + "keywords.txt", "w+") as txt_file:
txt_file.write(keyword_query)
# get data for combination (sequential)
get_filter_data(monitor_id, keyword_query, short_query, query_dir,
progress)
# delete buzz monitor
new_monitor.delete()
def test_add_delete_monitor(delete_monitor_keys):
"""Tests an API call to add and delete a monitor"""
response_add = Monitor(title="Test Title",
sources=["twitter"],
keywords="Twitter",
start="2016-01-01",
end="2016-01-03")
assert isinstance(response_add, Monitor)
assert isinstance(response_add.id, int)
response_delete = response_add.delete()
assert isinstance(response_delete, dict)
assert set(delete_monitor_keys).issubset(
response_delete.keys()), "All keys should be in the response"
def test_monitor_get_age_data(age_data_keys):
"""Tests an API call to get age counts for specific monitor"""
response = Monitor(16180624304).get_age_data(start="2016-01-01",
end="2016-01-03")
assert isinstance(response, list)
assert isinstance(response[0], dict)
assert set(age_data_keys).issubset(
response[0].keys()), "All keys should be in the response"
def get_monitor_data(combinations, title, source, keywords, data_dir,
progress):
print progress + " monitors: Adding monitor for " + title + "..."
# add buzz monitor
new_monitor = Monitor(title=title,
sources=source,
languages=languages,
keywords=keywords,
start=start,
end=end)
monitor_id = new_monitor.id
# start worker threads for parallel processing
pool = multiprocessing.dummy.Pool(processes=len(combinations))
res_list = []
# for each combination
for combi_index, combination in enumerate(combinations):
progress = "[" + str(combi_index + 1) + "/" + str(
len(combinations)) + "]"
# make keyword query
keyword_query = get_keyword_query(combination)
# make short query
short_query = get_short_query(combination)
# make a data directory
query_dir = make_query_dir(data_dir, short_query)
# save keyword query in txt-file
with open(query_dir + "keywords.txt", "w+") as txt_file:
txt_file.write(keyword_query)
# get data for combination (parallel)
res = pool.apply_async(
add_filter, (monitor_id, keyword_query, short_query, progress))
res_list.append((res, query_dir))
for (res, query_dir) in res_list:
filter = res.get()
get_filter_data(filter, query_dir)
pool.close()
pool.join()
# delete buzz monitor
new_monitor.delete()