def run():
# create topic
topic_example = TwTopic()
topic_example.title = "migration"
topic, created = TwTopic.objects.get_or_create(topic_example.__dict__)
# create query string with well known migration organisations and their twitter accounts
settings_dir = os.path.dirname(__file__)
project_root = Path(os.path.dirname(settings_dir)).absolute()
twitter_addresses_yaml = Path.joinpath(project_root, "twitter\\twitter_addresses.yaml")
with open(twitter_addresses_yaml) as f:
# use safe_load instead load
data_map = yaml.safe_load(f)
class TwitterAddresses:
def __init__(self, entries):
self.__dict__.update(entries)
addresses = TwitterAddresses(data_map)
pp = pprint.PrettyPrinter(indent=4)
# query_string = ""
# for (institution in addresses.institutions):
# query_string+= institution.twitter_account
# institutions = list(map(lambda x: list(x.values())[0], addresses.institutions))
institutions = addresses.institutions
twitter_accounts = map(lambda x: x.get('institution').get('twitter_account').replace("@", ""), institutions)
twitter_accounts_query_1 = map(lambda x: "(from:{}) OR ".format(x), twitter_accounts)
twitter_accounts_query_2 = reduce(lambda x, y: x + y, twitter_accounts_query_1)
twitter_accounts_query_2 += "#immigration lang:en"
pp.pprint(twitter_accounts_query_2)
params = {'query': '{}'.format(twitter_accounts_query_2), 'max_results': '500'}
search_url = "https://api.twitter.com/2/tweets/search/all"
connector = TwitterConnector(1)
json_result = connector.get_from_twitter(search_url, params, True)
# print(json.dumps(json_result, indent=4, sort_keys=True))
twitter_data: list = json_result.get("data")
for tweet_raw in twitter_data:
try:
tweet = Tweet()
tweet.topic = topic
tweet.query_string = twitter_accounts_query_2
tweet.text = tweet_raw.get("text")
tweet.twitter_id = tweet_raw.get("id")
tweet.save()
except IntegrityError:
pass
def run():
# create topic
topic_example = TwTopic()
topic_example.title = "random"
topic, created = TwTopic.objects.get_or_create(topic_example.__dict__)
count = 500
connector = TwitterConnector(1)
random_stream_url = "https://api.twitter.com/2/tweets/sample/stream"
headers = connector.create_headers()
params = {'tweet.fields': 'lang'}
response = requests.request("GET",
random_stream_url,
headers=headers,
stream=True,
params=params)
print("is connected {}".format(response.status_code))
if response.status_code != 200:
raise Exception("Request returned an error: {} {}".format(
response.status_code, response.text))
for response_line in response.iter_lines():
if response_line:
json_response = json.loads(response_line)
while True:
if count == 0:
break
twitter_data: list = json_response.get("data")
if twitter_data.get("lang") == "en":
print(json.dumps(json_response, indent=4, sort_keys=True))
count += -1
try:
tweet = Tweet()
tweet.topic = topic
tweet.query_string = topic_example.title
tweet.text = twitter_data.get("text")
tweet.twitter_id = twitter_data.get("id")
tweet.save()
except IntegrityError:
pass
