def run(): # create topic topic_example = TwTopic() topic_example.title = "migration" topic, created = TwTopic.objects.get_or_create(topic_example.__dict__) # create query string with well known migration organisations and their twitter accounts settings_dir = os.path.dirname(__file__) project_root = Path(os.path.dirname(settings_dir)).absolute() twitter_addresses_yaml = Path.joinpath(project_root, "twitter\\twitter_addresses.yaml") with open(twitter_addresses_yaml) as f: # use safe_load instead load data_map = yaml.safe_load(f) class TwitterAddresses: def __init__(self, entries): self.__dict__.update(entries) addresses = TwitterAddresses(data_map) pp = pprint.PrettyPrinter(indent=4) # query_string = "" # for (institution in addresses.institutions): # query_string+= institution.twitter_account # institutions = list(map(lambda x: list(x.values())[0], addresses.institutions)) institutions = addresses.institutions twitter_accounts = map(lambda x: x.get('institution').get('twitter_account').replace("@", ""), institutions) twitter_accounts_query_1 = map(lambda x: "(from:{}) OR ".format(x), twitter_accounts) twitter_accounts_query_2 = reduce(lambda x, y: x + y, twitter_accounts_query_1) twitter_accounts_query_2 += "#immigration lang:en" pp.pprint(twitter_accounts_query_2) params = {'query': '{}'.format(twitter_accounts_query_2), 'max_results': '500'} search_url = "https://api.twitter.com/2/tweets/search/all" connector = TwitterConnector(1) json_result = connector.get_from_twitter(search_url, params, True) # print(json.dumps(json_result, indent=4, sort_keys=True)) twitter_data: list = json_result.get("data") for tweet_raw in twitter_data: try: tweet = Tweet() tweet.topic = topic tweet.query_string = twitter_accounts_query_2 tweet.text = tweet_raw.get("text") tweet.twitter_id = tweet_raw.get("id") tweet.save() except IntegrityError: pass
def run(): # create topic topic_example = TwTopic() topic_example.title = "random" topic, created = TwTopic.objects.get_or_create(topic_example.__dict__) count = 500 connector = TwitterConnector(1) random_stream_url = "https://api.twitter.com/2/tweets/sample/stream" headers = connector.create_headers() params = {'tweet.fields': 'lang'} response = requests.request("GET", random_stream_url, headers=headers, stream=True, params=params) print("is connected {}".format(response.status_code)) if response.status_code != 200: raise Exception("Request returned an error: {} {}".format( response.status_code, response.text)) for response_line in response.iter_lines(): if response_line: json_response = json.loads(response_line) while True: if count == 0: break twitter_data: list = json_response.get("data") if twitter_data.get("lang") == "en": print(json.dumps(json_response, indent=4, sort_keys=True)) count += -1 try: tweet = Tweet() tweet.topic = topic tweet.query_string = topic_example.title tweet.text = twitter_data.get("text") tweet.twitter_id = twitter_data.get("id") tweet.save() except IntegrityError: pass