def parse(cls, api, json):
status = cls(api)
setattr(status, '_json', json)
for k, v in json.items():
if k == 'user':
user_model = getattr(api.parser.model_factory, 'user') if api else User
user = user_model.parse(api, v)
setattr(status, 'author', user)
setattr(status, 'user', user) # DEPRECIATED
elif k == 'created_at':
setattr(status, k, parse_datetime(v))
elif k == 'source':
if '<' in v:
setattr(status, k, parse_html_value(v))
setattr(status, 'source_url', parse_a_href(v))
else:
setattr(status, k, v)
setattr(status, 'source_url', None)
elif k == 'retweeted_status':
setattr(status, k, Status.parse(api, v))
elif k == 'place':
if v is not None:
setattr(status, k, Place.parse(api, v))
else:
setattr(status, k, None)
else:
setattr(status, k, v)
return status
def parse(cls, api, json):
status = cls(api)
setattr(status, "_json", json)
for k, v in json.items():
if k == "user":
user_model = getattr(api.parser.model_factory, "user") if api else User
user = user_model.parse(api, v)
setattr(status, "author", user)
setattr(status, "user", user) # DEPRECIATED
elif k == "created_at":
setattr(status, k, parse_datetime(v))
elif k == "source":
if "<" in v:
setattr(status, k, parse_html_value(v))
setattr(status, "source_url", parse_a_href(v))
else:
setattr(status, k, v)
setattr(status, "source_url", None)
elif k == "retweeted_status":
setattr(status, k, Status.parse(api, v))
elif k == "place":
if v is not None:
setattr(status, k, Place.parse(api, v))
else:
setattr(status, k, None)
else:
setattr(status, k, v)
return status
def parse(cls, api, json):
ss = cls(api)
for k, v in json.items():
if k == 'created_at':
setattr(ss, k, parse_datetime(v))
else:
setattr(ss, k, v)
return ss
def parse(cls, api, json):
ss = cls(api)
for k, v in json.items():
if k == 'created_at':
setattr(ss, k, parse_datetime(v))
else:
setattr(ss, k, v)
return ss
def parse(cls, api, json):
lst = List(api)
for k, v in json.items():
if k == 'user':
setattr(lst, k, User.parse(api, v))
elif k == 'created_at':
setattr(lst, k, parse_datetime(v))
else:
setattr(lst, k, v)
return lst
def parse(cls, api, json):
dm = cls(api)
for k, v in json.items():
if k == 'sender' or k == 'recipient':
setattr(dm, k, User.parse(api, v))
elif k == 'created_at':
setattr(dm, k, parse_datetime(v))
else:
setattr(dm, k, v)
return dm
def parse(cls, api, json):
lst = List(api)
for k, v in json.items():
if k == 'user':
setattr(lst, k, User.parse(api, v))
elif k == 'created_at':
setattr(lst, k, parse_datetime(v))
else:
setattr(lst, k, v)
return lst
def parse(cls, api, json):
dm = cls(api)
for k, v in json.items():
if k == 'sender' or k == 'recipient':
setattr(dm, k, User.parse(api, v))
elif k == 'created_at':
setattr(dm, k, parse_datetime(v))
else:
setattr(dm, k, v)
return dm
def parse(cls, api, json):
dm = cls(api)
for k, v in pyjson.loads(dm._api.last_response.text).items():
if k == 'sender' or k == 'recipient':
setattr(dm, k, User.parse(api, v))
elif k == 'created_at':
setattr(dm, k, parse_datetime(v))
else:
setattr(dm, k, v)
return dm
def parse(cls, api, json):
dm = cls(api)
dm._json = json
for k, v in json.items():
if k == "sender" or k == "recipient":
setattr(dm, k, User.parse(api, v))
elif k == "created_at":
setattr(dm, k, parse_datetime(v))
else:
setattr(dm, k, v)
return dm
def __init__(self, data):
self._data = data
self._sentiment = None
self._user_keys = ('followers_count', 'friends_count', 'location')
self._sentiment_keys = ('sentiment', 'pos', 'neu', 'neg')
try:
ts = parse_datetime(data['created_at'])
data['created_at'] = ts
except KeyError:
print(data)
raise
def parse(cls, api, json):
lst = List(api)
lst._json = json
for k, v in json.items():
if k == "user":
setattr(lst, k, User.parse(api, v))
elif k == "created_at":
setattr(lst, k, parse_datetime(v))
else:
setattr(lst, k, v)
return lst
def __init__(self, data):
self.data = data
self.id = data["id"]
self.options = data["options"]
self.duration_minutes = data.get("duration_minutes")
self.end_datetime = data.get("end_datetime")
if self.end_datetime is not None:
self.end_datetime = parse_datetime(self.end_datetime)
self.voting_status = data.get("voting_status")
def __init__(self, data):
self.data = data
self.id = data["id"]
self.state = data["state"]
self.created_at = data.get("created_at")
if self.created_at is not None:
self.created_at = parse_datetime(self.created_at)
self.ended_at = data.get("ended_at")
if self.ended_at is not None:
self.ended_at = parse_datetime(self.ended_at)
self.host_ids = data.get("host_ids", [])
self.lang = data.get("lang")
self.is_ticketed = data.get("is_ticketed")
self.invited_user_ids = data.get("invited_user_ids", [])
self.participant_count = data.get("participant_count")
self.scheduled_start = data.get("scheduled_start")
if self.scheduled_start is not None:
self.scheduled_start = parse_datetime(self.scheduled_start)
self.speaker_ids = data.get("speaker_ids", [])
self.started_at = data.get("started_at")
if self.started_at is not None:
self.started_at = parse_datetime(self.started_at)
# https://twittercommunity.com/t/missing-documentation-for-new-space-object-subscriber-count-field-on-space-object-page/166943
self.subscriber_count = data.get("subscriber_count")
self.title = data.get("title")
self.topic_ids = data.get("topic_ids", [])
self.updated_at = data.get("updated_at")
if self.updated_at is not None:
self.updated_at = parse_datetime(self.updated_at)
def __init__(self, data):
self.data = data
self.id = data["id"]
self.name = data["name"]
self.created_at = data.get("created_at")
if self.created_at is not None:
self.created_at = parse_datetime(self.created_at)
self.description = data.get("description")
self.follower_count = data.get("follower_count")
self.member_count = data.get("member_count")
self.private = data.get("private")
self.owner_id = data.get("owner_id")
def __init__(self, data):
self.data = data
self.id = data["id"]
self.state = data["state"]
self.created_at = data.get("created_at")
if self.created_at is not None:
self.created_at = parse_datetime(self.created_at)
self.ended_at = data.get("ended_at")
if self.ended_at is not None:
self.ended_at = parse_datetime(self.ended_at)
self.host_ids = data.get("host_ids", [])
self.lang = data.get("lang")
self.is_ticketed = data.get("is_ticketed")
self.invited_user_ids = data.get("invited_user_ids", [])
self.participant_count = data.get("participant_count")
self.subscriber_count = data.get("subscriber_count")
self.scheduled_start = data.get("scheduled_start")
if self.scheduled_start is not None:
self.scheduled_start = parse_datetime(self.scheduled_start)
self.speaker_ids = data.get("speaker_ids", [])
self.started_at = data.get("started_at")
if self.started_at is not None:
self.started_at = parse_datetime(self.started_at)
self.title = data.get("title")
self.topic_ids = data.get("topic_ids", [])
self.updated_at = data.get("updated_at")
if self.updated_at is not None:
self.updated_at = parse_datetime(self.updated_at)
def parse(cls, api, json):
user = cls(api)
for k, v in json.items():
if k == 'created_at':
setattr(user, k, parse_datetime(v))
elif k == 'status':
setattr(user, k, Status.parse(api, v))
elif k == 'following':
# twitter sets this to null if it is false
if v is True:
setattr(user, k, True)
else:
setattr(user, k, False)
else:
setattr(user, k, v)
return user
def parse(cls, api, json):
user = cls(api)
for k, v in json.items():
if k == "created_at":
setattr(user, k, parse_datetime(v))
elif k == "status":
setattr(user, k, Status.parse(api, v))
elif k == "following":
# twitter sets this to null if it is false
if v is True:
setattr(user, k, True)
else:
setattr(user, k, False)
else:
setattr(user, k, v)
return user
def save_tweet(tweetobj):
twitter_id = tweetobj["id"]
username = tweetobj["user"]["name"]
screenname = tweetobj["user"]["screen_name"]
text = tweetobj["text"]
created_at = parse_datetime(tweetobj["created_at"])
created_at = timezone.make_aware(created_at, timezone=pytz.UTC) # tweets are stored int UTC
image_urls = []
try:
for media in tweetobj["entities"]["media"]:
if media["type"] == "photo":
image_urls.append(media["media_url"] + ":large")
# cut image url from tweet text
text = text.replace(media["url"], "")
except KeyError:
pass
# print("no picture")
# return # no picture
# if len(image_list) < 2:
# print ("less than 2 pictures" + str(len(image_list)))
# return
# create tweet
newtweet = Tweet(
twitter_id=twitter_id,
username=username,
screenname=screenname,
text=text,
created_at=created_at,
from_twitter=True,
)
newtweet.save()
if image_urls:
for image_url in image_urls:
# print(image_url)
image = retrieve_image(image_url)
image = process_image(image) # returns jpg
image_name = "tmp.jpg" # will be renamed by model save function
newpic = TweetPic()
newpic.tweet = newtweet
newpic.picture.save(image_name, image)
newpic.save()
generate_all_aliases(newpic.picture, include_global=True) # create thumbnails
print("saved tweet with id %s" % (str(twitter_id)))
def parse(cls, api, json):
user = cls(api)
user._json = json
for k, v in json.items():
if k == 'created_at':
setattr(user, k, parse_datetime(v))
elif k == 'status':
setattr(user, k, Status.parse(api, v))
elif k == 'following':
# twitter sets this to null if it is false
if v is True:
setattr(user, k, True)
else:
setattr(user, k, False)
else:
setattr(user, k, v)
return user
def __init__(self, data):
self.data = data
self.id = int(data["id"])
self.text = data["text"]
self.attachments = data.get("attachments")
self.author_id = data.get("author_id")
if self.author_id is not None:
self.author_id = int(self.author_id)
self.context_annotations = data.get("context_annotations", [])
self.conversation_id = data.get("conversation_id")
if self.conversation_id is not None:
self.conversation_id = int(self.conversation_id)
self.created_at = data.get("created_at")
if self.created_at is not None:
self.created_at = parse_datetime(self.created_at)
self.entities = data.get("entities")
self.geo = data.get("geo")
self.in_reply_to_user_id = data.get("in_reply_to_user_id")
if self.in_reply_to_user_id is not None:
self.in_reply_to_user_id = int(self.in_reply_to_user_id)
self.lang = data.get("lang")
self.non_public_metrics = data.get("non_public_metrics")
self.organic_metrics = data.get("organic_metrics")
self.possibly_sensitive = data.get("possibly_sensitive")
self.promoted_metrics = data.get("promoted_metrics")
self.public_metrics = data.get("public_metrics")
self.referenced_tweets = None
if "referenced_tweets" in data:
self.referenced_tweets = [
ReferencedTweet(referenced_tweet)
for referenced_tweet in data["referenced_tweets"]
]
self.reply_settings = data.get("reply_settings")
self.source = data.get("source")
self.withheld = data.get("withheld")
def parse_follow_unfollow_event(self, data):
"""
if data is follow or unfollow event-object then return parsed event-object,
else return False.
"""
if '"event":"follow"' not in data and '"event":"unfollow"' not in data:
return False
jobj = json.loads(data)
if 'event' not in jobj or 'created_at' not in jobj or 'target' not in jobj:
return False
event_name = jobj['event']
if event_name != 'follow' and event_name != 'unfollow':
return False
target = User.parse(self.api, jobj['target'])
if event_name == 'follow' and target.id == self.my_id: # User is followed
return False
created_at = parse_datetime(jobj['created_at'])
return dict(event_name=event_name, created_at=created_at, target=target)
def save_tweet(tweetobj):
twitter_id = tweetobj['id']
username = tweetobj['user']['name']
screenname = tweetobj['user']['screen_name']
text = tweetobj['text']
created_at = parse_datetime(tweetobj['created_at'])
created_at = timezone.make_aware(
created_at, timezone=pytz.UTC) #tweets are stored int UTC
image_urls = []
try:
for media in tweetobj['entities']['media']:
if media['type'] == 'photo':
image_urls.append(media['media_url'] + ":large")
# cut image url from tweet text
text = text.replace(media['url'], "")
except KeyError:
pass
# print("no picture")
# return # no picture
# if len(image_urls) < 2:
# print ("less than 2 pictures" + str(len(image_urls)))
# return
# create tweet
newtweet, created = Tweet.objects.get_or_create(twitter_id=twitter_id,
username=username,
screenname=screenname,
text=text,
created_at=created_at,
from_twitter=True)
if created:
if image_urls:
for image_url in image_urls:
image = retrieve_image(image_url)
image = process_image(image) # returns jpg
image_name = "tmp.jpg" # will be renamed by model save function
newpic = TweetPic()
newpic.tweet = newtweet
newpic.picture.save(image_name, image, save=False)
newpic.save()
print("saved tweet with id %s" % (str(twitter_id)))
else:
print(" tweet with id %s already exists" % (str(twitter_id)))
def parse_status_event(self, data):
"""
if status contains user-object then return parsed user-object,
else return False.
** NOTE **
this method treats tweet and RT,
but other data also contains '"in_reply_to_status_id":', e.g. fav.
"""
if '"in_reply_to_status_id":' not in data:
return False
jobj = json.loads(data)
if 'user' not in jobj or 'created_at' not in jobj:
return False
user = User.parse(self.api, jobj['user'])
if user.id != self.my_id and user.id not in self.friends:
return False
created_at = parse_datetime(jobj['created_at'])
return dict(user=user, created_at=created_at)
def parse(cls, api, json):
status = cls(api)
for k, v in json.items():
if k == 'user':
user = User.parse(api, v)
setattr(status, 'author', user)
setattr(status, 'user', user) # DEPRECIATED
elif k == 'created_at':
setattr(status, k, parse_datetime(v))
elif k == 'source':
if '<' in v:
setattr(status, k, parse_html_value(v))
setattr(status, 'source_url', parse_a_href(v))
else:
setattr(status, k, v)
elif k == 'retweeted_status':
setattr(status, k, Status.parse(api, v))
else:
setattr(status, k, v)
return status
def parse(cls, api, json):
event = cls(api)
for k, v in json.items():
if k == 'target':
user_model = getattr(api.parser.model_factory, 'user')
user = user_model.parse(api, v)
setattr(event, 'target', user)
elif k == 'source':
user_model = getattr(api.parser.model_factory, 'user')
user = user_model.parse(api, v)
setattr(event, 'source', user)
elif k == 'created_at':
setattr(event, k, parse_datetime(v))
elif k == 'target_object':
setattr(event, 'target_object', v)
elif k == 'event':
setattr(event, 'event', v)
else:
setattr(event, k, v)
return event
def parse(cls, api, json):
status = cls(api)
for k, v in json.items():
if k == 'user':
user = User.parse(api, v)
setattr(status, 'author', user)
setattr(status, 'user', user) # DEPRECIATED
elif k == 'created_at':
setattr(status, k, parse_datetime(v))
elif k == 'source':
if '<' in v:
setattr(status, k, parse_html_value(v))
setattr(status, 'source_url', parse_a_href(v))
else:
setattr(status, k, v)
elif k == 'retweeted_status':
setattr(status, k, User.parse(api, v))
else:
setattr(status, k, v)
return status
def first_parse(cls, api, json):
status = cls(api)
for k, v in json.items():
if k == 'user':
user_model = getattr(api.parser.model_factory, 'user')
user = user_model.parse(api, v)
setattr(status, 'author', user)
setattr(status, 'user', user) # DEPRECIATED
elif k == 'created_at':
setattr(status, k, parse_datetime(v))
elif k == 'source':
if '<' in v:
setattr(status, k, parse_html_value(v))
setattr(status, 'source_url', parse_a_href(v))
else:
setattr(status, k, v)
setattr(status, 'source_url', None)
elif k == 'retweeted_status':
setattr(status, k, Status.parse(api, v))
else:
setattr(status, k, v)
return status
def save_tweet(tweetobj):
twitter_id = tweetobj['id']
username = tweetobj['user']['name']
screenname = tweetobj['user']['screen_name']
text = tweetobj['text']
created_at = parse_datetime(tweetobj['created_at'])
created_at = timezone.make_aware(created_at, timezone=pytz.UTC) #tweets are stored int UTC
image_urls = []
try:
for media in tweetobj['entities']['media']:
if media['type'] == 'photo':
image_urls.append(media['media_url'] + ":large")
# cut image url from tweet text
text = text.replace(media['url'],"")
except KeyError:
pass
# print("no picture")
# return # no picture
# if len(image_urls) < 2:
# print ("less than 2 pictures" + str(len(image_urls)))
# return
# create tweet
newtweet, created = Tweet.objects.get_or_create(twitter_id=twitter_id, username=username, screenname=screenname, text=text, created_at=created_at, from_twitter=True)
if created:
if image_urls:
for image_url in image_urls:
image = retrieve_image(image_url)
image = process_image(image) # returns jpg
image_name = "tmp.jpg" # will be renamed by model save function
newpic = TweetPic()
newpic.tweet = newtweet
newpic.picture.save(image_name,image,save=False)
newpic.save()
print("saved tweet with id %s" %(str(twitter_id)))
else:
print(" tweet with id %s already exists" %(str(twitter_id)))
def __init__(self, data):
self.data = data
self.id = int(data["id"])
self.name = data["name"]
self.username = data["username"]
self.created_at = data.get("created_at")
if self.created_at is not None:
self.created_at = parse_datetime(self.created_at)
self.description = data.get("description")
self.entities = data.get("entities")
self.location = data.get("location")
self.pinned_tweet_id = data.get("pinned_tweet_id")
if self.pinned_tweet_id is not None:
self.pinned_tweet_id = int(self.pinned_tweet_id)
self.profile_image_url = data.get("profile_image_url")
self.protected = data.get("protected")
self.public_metrics = data.get("public_metrics")
self.url = data.get("url")
self.verified = data.get("verified")
self.withheld = data.get("withheld")
def on_data(self, data):
statuse = json.loads(data)
if 'delete' in statuse:
return True # keep stream alive
if 'id' in statuse:
statuse_quoted_text = None
geoloc = None
url_expanded = None
url_media = None
type_media = None
text = None
location = None
description = None
name = None
date = None
app = None
try:
id_tweet = statuse['id']
recent_tweet = id_tweet
profile_user = statuse['user']
if 'quoted_status_id' in statuse:
print statuse['quoted_status_id']
if 'quoted_status' in statuse:
statuse_quoted = statuse['quoted_status']
if 'text' in statuse_quoted:
statuse_quoted_text = statuse_quoted['text']
statuse_quoted_text = re.sub(
'[\r\n\t]+', ' ', statuse_quoted_text)
print 'tweet nested', statuse_quoted_text
if 'coordinates' in statuse:
coordinates = statuse['coordinates']
if coordinates != None:
list_geoloc = coordinates['coordinates']
geoloc = '%s, %s' % (list_geoloc[0], list_geoloc[1])
if 'entities' in statuse:
entities = statuse['entities']
urls = entities['urls']
if len(urls) > 0:
url = urls[0]
url_expanded = url['expanded_url']
text = re.sub('[\r\n\t]+', ' ', statuse['text'])
if profile_user['location'] != None:
location = re.sub('[\r\n\t]+', ' ',
profile_user['location'], re.UNICODE)
if profile_user['description'] != None:
description = re.sub('[\r\n\t]+', ' ',
profile_user['description'],
re.UNICODE)
if profile_user['name'] != None:
name = re.sub('[\r\n\t]+', ' ', profile_user['name'],
re.UNICODE)
date = parse_datetime(statuse['created_at'])
app = parse_html_value(statuse['source'])
tweet = '%s\t%s\t@%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (
id_tweet, date, profile_user['screen_name'], text, app,
profile_user['id'], profile_user['followers_count'],
profile_user['friends_count'],
profile_user['statuses_count'], location, url_expanded,
geoloc, name, description, url_media, type_media,
statuse_quoted_text)
self.f_out.write(tweet)
print '---->collected tweet', id_tweet
except:
text_error = '---------------> parser error at %s, id-tweet %s\n' % (
datetime.datetime.now(), statuse)
self.f_log.write(text_error)
pass
else:
text_error = '---------------> message no expected %s, %s\n' % (
datetime.datetime.now(), data)
self.f_log.write(text_error)
return True # keep stream alive
def on_data(self, data):
statuse = json.loads(data)
if 'delete' in statuse:
return True # keep stream alive
if 'id' in statuse:
statuse_quoted_text = None
geoloc = None
url_expanded = None
url_media = None
type_media = None
text = None
location = None
description = None
name = None
date = parse_datetime(statuse['created_at'])
app = parse_html_value(statuse['source'])
entities = None
relation = None
quoted_id = None
replied_id = None
retweeted_id = None
user_replied = None
user_quoted = None
user_retweeted = None
first_HT = None
#get interactions Ids
try:
id_tweet = statuse['id_str']
if statuse['in_reply_to_status_id_str'] != None:
relation = 'reply'
replied_id = statuse['in_reply_to_status_id_str']
user_replied = statuse['in_reply_to_screen_name']
if 'quoted_status' in statuse:
relation = 'quote'
quoted_id = statuse['quoted_status_id_str']
user_quoted = statuse['quoted_status']['user'][
'screen_name']
elif 'retweeted_status' in statuse:
relation = 'RT'
retweeted_id = statuse['retweeted_status']['id_str']
user_retweeted = statuse['retweeted_status']['user'][
'screen_name']
if 'quoted_status' in statuse['retweeted_status']:
quoted_id = statuse['retweeted_status'][
'quoted_status']['id_str']
user_quoted = statuse['retweeted_status'][
'quoted_status']['user']['screen_name']
except:
text_error = '---------------->bad interactions ids, id tweet %s at %s\n' % (
id_tweet, time.asctime())
self.f_log.write(text_error)
#get geolocation
if 'coordinates' in statuse:
coordinates = statuse['coordinates']
if coordinates != None:
try:
if 'coordinates' in coordinates:
list_geoloc = coordinates['coordinates']
print list_geoloc
geoloc = '%s, %s' % (list_geoloc[0],
list_geoloc[1])
except:
text_error = '---------------->bad coordinates, id tweet %s at %s\n' % (
id_tweet, datetime.datetime.now())
self.f_log.write(text_error)
#get entities
if 'entities' in statuse:
entities = statuse['entities']
if 'extended_tweet' in statuse:
entities = statuse['extended_tweet']['entities']
if 'retweeted_status' in statuse:
if 'entities' in statuse['retweeted_status']:
entities = statuse['retweeted_status']['entities']
if 'extended_tweet' in statuse['retweeted_status']:
entities = statuse['retweeted_status']['extended_tweet'][
'entities']
if entities != None:
try:
urls = entities['urls']
if len(urls) > 0:
url_expanded = urls[0]['expanded_url']
except:
text_error = '---------------->bad enttity urls, id tweet %s at %s\n' % (
id_tweet, datetime.datetime.now())
self.f_log.write(text_error)
try:
if 'media' in entities:
list_media = entities['media']
if len(list_media) > 0:
url_media = list_media[0]['media_url']
type_media = list_media[0]['type']
except:
text_error = '---------------->bad entity media, at %s id tweet %s \n' % (
datetime.datetime.now(), id_tweet)
self.f_log.write(text_error)
try:
if 'hashtags' in entities:
HTs = entities['hashtags']
if len(HTs) > 0:
first_HT = HTs[0]['text']
except:
text_error = '---------------->bad entity HT, id tweet %s at %s\n' % (
id_tweet, time.asctime())
self.f_log.write(text_error)
#get text
try:
if 'text' in statuse:
text = re.sub('[\r\n\t]+', ' ', statuse['text'])
if 'extended_tweet' in statuse:
text = re.sub('[\r\n\t]+', ' ',
statuse['extended_tweet']['full_text'])
if 'retweeted_status' in statuse:
statuse_RT = statuse['retweeted_status']
if 'text' in statuse_RT:
RT_expand = re.sub('[\r\n\t]+', ' ',
statuse_RT['text'])
if 'extended_tweet' in statuse_RT:
extended_RT = statuse_RT['extended_tweet']
RT_expand = re.sub('[\r\n\t]+', ' ',
extended_RT['full_text'])
RT = re.match(r'(^RT @\w+: )', text)
if RT:
text = RT.group(1) + RT_expand
except:
text_error = '---------------->bad tweet text, at %s id tweet %s \n' % (
datetime.datetime.now(), id_tweet)
self.f_log.write(text_error)
#get quoted if exist
try:
if 'quoted_status' in statuse:
if 'text' in statuse['quoted_status']:
statuse_quoted_text = statuse['quoted_status']['text']
if 'extended_tweet' in statuse['quoted_status']:
statuse_quoted_text = statuse['quoted_status'][
'extended_tweet']['full_text']
statuse_quoted_text = re.sub('[\r\n\t]+', ' ',
statuse_quoted_text)
elif 'retweeted_status' in statuse:
if 'quoted_status' in statuse['retweeted_status']:
if 'text' in statuse['retweeted_status'][
'quoted_status']:
statuse_quoted_text = statuse['retweeted_status'][
'quoted_status']['text']
if 'extended_tweet' in statuse['retweeted_status'][
'quoted_status']:
statuse_quoted_text = statuse['retweeted_status'][
'quoted_status']['extended_tweet']['full_text']
statuse_quoted_text = re.sub('[\r\n\t]+', ' ',
statuse_quoted_text)
except:
text_error = '---------------->bad quoted, at %s id tweet %s \n' % (
datetime.datetime.now(), id_tweet)
self.f_log.write(text_error)
#get user profile
if 'user' in statuse:
try:
if 'location' in statuse['user']:
if statuse['user']['location'] != None:
location = re.sub('[\r\n\t]+', ' ',
statuse['user']['location'],
re.UNICODE)
except:
text_error = '---------------->bad user location:%s , at %s id tweet %s \n' % (
datetime.datetime.now(), statuse['user']['location'],
id_tweet)
self.f_log.write(text_error)
try:
if 'description' in statuse['user']:
if statuse['user']['description'] != None:
description = re.sub(
'[\r\n\t]+', ' ',
statuse['user']['description'], re.UNICODE)
except:
text_error = '---------------->bad user description, at %s id tweet %s \n' % (
datetime.datetime.now(), id_tweet)
self.f_log.write(text_error)
try:
if 'name' in statuse['user']:
if statuse['user']['name'] != None:
name = re.sub('[\r\n\t]+', ' ',
statuse['user']['name'], re.UNICODE)
except:
text_error = '---------------->bad user name, at %s id tweet %s \n' % (
datetime.datetime.now(), id_tweet)
self.f_log.write(text_error)
try:
link_tweet = 'https://twitter.com/%s/status/%s' % (
statuse['user']['screen_name'], id_tweet)
tweet = '%s\t%s\t@%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (
id_tweet, date, statuse['user']['screen_name'], text, app,
statuse['user']['id'], statuse['user']['followers_count'],
statuse['user']['friends_count'],
statuse['user']['statuses_count'], location, url_expanded,
geoloc, name, description, url_media, type_media,
statuse_quoted_text, relation, replied_id, user_replied,
retweeted_id, user_retweeted, quoted_id, user_quoted,
first_HT, statuse['lang'],
parse_datetime(statuse['user']['created_at']),
statuse['user']['verified'],
statuse['user']['profile_image_url_https'], link_tweet)
self.f_out.write(tweet)
print '---->collected tweet', id_tweet
except:
text_error = '---------------> format error at %s, id-tweet %s\n' % (
datetime.datetime.now(), id_tweet)
self.f_log.write(text_error)
pass
else:
text_error = '---------------> message no expected %s, %s\n' % (
datetime.datetime.now(), data)
self.f_log.write(text_error)
return True # keep stream alive
