def test_nested_objects(self):
json_data = json.loads('{"something":{"property":["a","b","c"]}}')
retriever = Query().get("something").get("property").index(
2).finalize()
value = retriever.get_from(json_data)
self.assertEqual('c', value)
def tweet_url(obj):
screen_name = Query().get('user').get('screen_name').finalize().get_from(
obj)
tweet_id = Query().get('id_str').finalize().get_from(obj)
if screen_name is None or tweet_id is None:
return None
return "https://twitter.com/" + screen_name + "/status/" + tweet_id
def test_custom_callback(self):
def custom_callback(obj):
return obj + "bc"
json_data = json.loads('{"value":"a"}')
retriever = Query().get("value").custom(custom_callback).finalize()
value = retriever.get_from(json_data)
self.assertEqual('abc', value)
def test_attribute(self):
class TestObj:
def __init__(self):
self.SomeValue = 20
test_obj = TestObj()
retriever = Query().get('SomeValue').finalize()
value = retriever.get_from(test_obj)
self.assertEqual(20, value)
def get_tweet_type(obj):
retweeted_status = Query().get('retweeted_status').finalize().get_from(obj)
is_quote_status = Query().get('is_quote_status').finalize().get_from(obj)
in_reply_to_status_id = Query().get(
'in_reply_to_status_id_str').finalize().get_from(obj)
if retweeted_status is not None:
return 'Retweet'
if is_quote_status is not None and is_quote_status is True:
return 'Retweet with Comment'
if in_reply_to_status_id is not None:
return 'Reply'
return 'Tweet'
def test_data_map(self):
class DataSetter:
def __init__(self):
self.Key1 = None
self.Key2 = None
def set_key1(self, value):
self.Key1 = value
def set_key2(self, value):
self.Key2 = value
consumer = DataSetter()
data_map = DataMap({
consumer.set_key1: Query().get("Key1").finalize(),
consumer.set_key2: Query().get("Key2").finalize()
})
json_data = json.loads('{"Key1":"abc", "Key2": 20}')
data_map.transfer(json_data)
self.assertEqual('abc', consumer.Key1)
self.assertEqual(20, consumer.Key2)
def get_original_tweet(obj):
retweeted_status = Query().get('retweeted_status').finalize().get_from(obj)
if retweeted_status is not None:
return retweeted_status
quoted_status = Query().get('quoted_status').finalize().get_from(obj)
if quoted_status is not None:
return quoted_status
reply_id = Query().get('in_reply_to_status_id_str').finalize().get_from(
obj)
reply_user_id = Query().get(
'in_reply_to_status_user_id_str').finalize().get_from(obj)
reply_screen_name = Query().get(
'in_reply_to_status_screen_name').finalize().get_from(obj)
return {
"id_str": reply_id,
"user": {
"id_str": reply_user_id,
"screen_name": reply_screen_name,
}
}
def get_full_text(obj):
retweeted_status = Query().get('retweeted_status').finalize().get_from(obj)
if retweeted_status is not None:
return None
return Query().get('full_text').finalize().get_from(obj)
def orig_tweet():
return Query().custom(get_original_tweet)
def pi_push_all_records(self, n_record_limit: int) -> bool:
def orig_tweet():
return Query().custom(get_original_tweet)
data_mapper = AyxDataMap(
self.alteryx_engine, self.label, {
('Id', FieldType.String):
Query().get('id_str').finalize(),
('TweetType', FieldType.String):
Query().custom(get_tweet_type).finalize(),
('CreatedAt', FieldType.Datetime):
Query().get('created_at').finalize(),
('Text', FieldType.String):
Query().custom(get_full_text).finalize(),
('Source', FieldType.String):
Query().get('source').finalize(),
('UserId', FieldType.String):
Query().get('user').get('id_str').finalize(),
('ScreenName', FieldType.String):
Query().get('user').get('screen_name').finalize(),
('UserLocation', FieldType.String):
Query().get('user').get('location').finalize(),
('UserDescription', FieldType.String):
Query().get('user').get('description').finalize(),
('UserVerified', FieldType.Bool):
Query().get('user').get('verified').finalize(),
('UserFollowers', FieldType.Integer):
Query().get('user').get('followers_count').finalize(),
('UserFriends', FieldType.Integer):
Query().get('user').get('friends_count').finalize(),
('UserFavorites', FieldType.Integer):
Query().get('user').get('favourites_count').finalize(),
('UserTweets', FieldType.Integer):
Query().get('user').get('statuses_count').finalize(),
('UserCreatedAt', FieldType.Datetime):
Query().get('user').get('created_at').finalize(),
('RetweetCount', FieldType.Integer):
Query().get('retweet_count').finalize(),
('FavoriteCount', FieldType.Integer):
Query().get('favorite_count').finalize(),
('PossiblySensitive', FieldType.Bool):
Query().get('possibly_sensitive').finalize(),
('Language', FieldType.String):
Query().get('lang').finalize(),
('Url', FieldType.String):
Query().custom(tweet_url).finalize(),
('OriginalTweetId', FieldType.String):
orig_tweet().get('id_str').finalize(),
('OriginalTweetCreatedAt', FieldType.Datetime):
orig_tweet().get('created_at').finalize(),
('OriginalTweetText', FieldType.String):
orig_tweet().custom(get_full_text).finalize(),
('OriginalTweetSource', FieldType.String):
orig_tweet().get('source').finalize(),
('OriginalTweetUserId', FieldType.String):
orig_tweet().get('user').get('id_str').finalize(),
('OriginalTweetScreenName', FieldType.String):
orig_tweet().get('user').get('screen_name').finalize(),
('OriginalTweetUserLocation', FieldType.String):
orig_tweet().get('user').get('location').finalize(),
('OriginalTweetUserDescription', FieldType.String):
orig_tweet().get('user').get('description').finalize(),
('OriginalTweetUserVerified', FieldType.Bool):
orig_tweet().get('verified').finalize(),
('OriginalTweetUserFollowers', FieldType.Integer):
orig_tweet().get('user').get('followers_count').finalize(),
('OriginalTweetUserFriends', FieldType.Integer):
orig_tweet().get('user').get('friends_count').finalize(),
('OriginalTweetUserFavorites', FieldType.Integer):
orig_tweet().get('user').get('favourites_count').finalize(),
('OriginalTweetUserTweets', FieldType.Integer):
orig_tweet().get('user').get('statuses_count').finalize(),
('OriginalTweetUserCreatedAt', FieldType.Datetime):
orig_tweet().get('user').get('created_at').finalize(),
('OriginalTweetRetweetCount', FieldType.Integer):
orig_tweet().get('retweet_count').finalize(),
('OriginalTweetFavoriteCount', FieldType.Integer):
orig_tweet().get('favorite_count').finalize(),
('OriginalTweetPossiblySensitive', FieldType.Bool):
orig_tweet().get('possibly_sensitive').finalize(),
('OriginalTweetLanguage', FieldType.String):
orig_tweet().get('lang').finalize(),
('OriginalTweetUrl', FieldType.String):
orig_tweet().custom(tweet_url).finalize(),
})
self.Output.init(data_mapper.Info)
if self.alteryx_engine.get_init_var(self.n_tool_id,
"UpdateOnly") == "True":
self.Output.close()
return True
consumer_key = self.alteryx_engine.decrypt_password(self.ConsumerKey)
consumer_secret = self.alteryx_engine.decrypt_password(
self.ConsumerSecret)
access_token = self.alteryx_engine.decrypt_password(self.AccessToken)
access_secret = self.alteryx_engine.decrypt_password(self.AccessSecret)
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth, wait_on_rate_limit=True)
for raw_tweet in tweepy.Cursor(api.search,
q=self.Query,
tweet_mode="extended").items():
blob = data_mapper.transfer(raw_tweet)
self.Output.push_record(blob)
self.Output.close()
return True
def test_convert_datetime(self):
json_data = json.loads('{"value":"2020-01-02 03:04:05"}')
retriever = Query().get("value").to_datetime(
"%Y-%m-%d %H:%M:%S").finalize()
value = retriever.get_from(json_data)
self.assertEqual(datetime.datetime(2020, 1, 2, 3, 4, 5), value)
def test_json_property(self):
json_data = json.loads('{"property":"abc"}')
retriever = Query().get('property').finalize()
value = retriever.get_from(json_data)
self.assertEqual('abc', value)
def test_retrieve_none(self):
data = None
retriever = Query().get("something").finalize()
value = retriever.get_from(data)
self.assertIsNone(value)
def test_get_last_item(self):
json_data = json.loads('["a","b","c"]')
retriever = Query().last().finalize()
value = retriever.get_from(json_data)
self.assertEqual('c', value)
def test_missing_attribute(self):
json_data = json.loads('{"something":1}')
retriever = Query().get("la la la").finalize()
value = retriever.get_from(json_data)
self.assertIsNone(value)
def test_invalid_index(self):
json_data = json.loads('["a","b","c"]')
retriever = Query().index(10).finalize()
value = retriever.get_from(json_data)
self.assertIsNone(value)
def test_get_index(self):
json_data = json.loads('["a","b","c"]')
retriever = Query().index(1).finalize()
value = retriever.get_from(json_data)
self.assertEqual('b', value)