def _heartbeat(self):
with self.lock:
mylog.debug('pump: ' + self.name)
self.connection.process_data_events()
# RabbitMQ expects a heartbeat every 60 seconds, so send one every 30 seconds.
timer = threading.Timer(30, self._heartbeat_wrap)
timer.daemon = True
timer.start()
def _heartbeat(self):
with self.lock:
mylog.debug('pump: ' + self.name)
self.connection.process_data_events()
# RabbitMQ expects a heartbeat every 60 seconds, so send one every 30 seconds.
timer = threading.Timer(30, self._heartbeat_wrap)
timer.daemon = True
timer.start()
def parse_tweet_status(status):
# mylog.debug('Now parsing status {}'.format(status))
try:
report = dict()
report['uid'] = status.user.id_str
hh = status.entities["hashtags"]
# For each element, only view 'text' subelement
report['hashtags'] = []
for h in hh:
report['hashtags'].append(h['text'])
report['content'] = status.text
report['username'] = status.user.screen_name
report['userscreen'] = status.user.name
# Slap Twitter-developer for not delivering a 'timestamp_ms' property
report['time'] = datetime_to_unix(status.created_at) * 1000
report['tweet_id'] = status.id_str
report['profile_img'] = status.user.profile_image_url_https
if 'retweeted_status' in status.__dict__:
# Actual "Retweet" (retweet with no additional text or media)
sub = status.retweeted_status
report['retweet'] = dict(
# Skip uid (available on request!)
# Skip hashtags (available on request!)
content=sub.text,
username=sub.user.screen_name,
userscreen=sub.user.name,
# Skip time (available on request -- but difficult for quotes, see below)
tweet_id=sub.id_str,
profile_img=status.user.profile_image_url_https,
# No recursive retweet detection -- would require POST requests.
)
elif 'quoted_status' in status.__dict__:
# "Quote" (retweet with some additional text or media)
# Dear tweepy, why don't you parse this JSON?!
# I would create a new issue, but it's "no longer maintained" anyway.
sub = status.quoted_status
report['retweet'] = dict(
# Skip uid (available on request!)
# Skip hashtags (available on request!)
content=sub['text'],
username=sub['user']['screen_name'],
userscreen=sub['user']['name'],
# Skip time (available on request -- but would need non-trivial parsing)
tweet_id=sub['id_str'],
profile_img=sub['user']['profile_image_url_https'],
)
# else: not a retweet in either meaning.
# For initial compatibility with the current tests: throw away most of the info.
report['retweet'] = report.get('retweet') is not None
return report
except (KeyError, AttributeError) as e:
mylog.debug('Failed to parse status {}'.format(status))
mylog.error('Something when wrong while parsing status: {}'.format(e))
return None
def parse_tweet_status(status):
# mylog.debug('Now parsing status {}'.format(status))
try:
report = dict()
report['uid'] = status.user.id_str
hh = status.entities["hashtags"]
# For each element, only view 'text' subelement
report['hashtags'] = []
for h in hh:
report['hashtags'].append(h['text'])
report['content'] = status.text
report['username'] = status.user.screen_name
report['userscreen'] = status.user.name
# Slap Twitter-developer for not delivering a 'timestamp_ms' property
report['time'] = datetime_to_unix(status.created_at) * 1000
report['tweet_id'] = status.id_str
report['profile_img'] = status.user.profile_image_url_https
if 'retweeted_status' in status.__dict__:
# Actual "Retweet" (retweet with no additional text or media)
sub = status.retweeted_status
report['retweet'] = dict(
# Skip uid (available on request!)
# Skip hashtags (available on request!)
content=sub.text,
username=sub.user.screen_name,
userscreen=sub.user.name,
# Skip time (available on request -- but difficult for quotes, see below)
tweet_id=sub.id_str,
profile_img=status.user.profile_image_url_https,
# No recursive retweet detection -- would require POST requests.
)
elif 'quoted_status' in status.__dict__:
# "Quote" (retweet with some additional text or media)
# Dear tweepy, why don't you parse this JSON?!
# I would create a new issue, but it's "no longer maintained" anyway.
sub = status.quoted_status
report['retweet'] = dict(
# Skip uid (available on request!)
# Skip hashtags (available on request!)
content=sub['text'],
username=sub['user']['screen_name'],
userscreen=sub['user']['name'],
# Skip time (available on request -- but would need non-trivial parsing)
tweet_id=sub['id_str'],
profile_img=sub['user']['profile_image_url_https'],
)
# else: not a retweet in either meaning.
# For initial compatibility with the current tests: throw away most of the info.
report['retweet'] = report.get('retweet') is not None
return report
except (KeyError, AttributeError) as e:
mylog.debug('Failed to parse status {}'.format(status))
mylog.error('Something when wrong while parsing status: {}'.format(e))
return None
def maybe_log_message(self, msg):
import os.path
if self.log_file is None:
return
if not WRITE_TWEETS:
return
mylog.debug('Writing to {}: {}'.format(self.log_file, msg))
old = []
# Let's hope there are never two backend running simultaneously
if os.path.exists(self.log_file):
with open(self.log_file, 'r') as fp:
old = json.load(fp)
old.append(msg)
with open(self.log_file, 'w') as fp:
json.dump(old, fp, sort_keys=True, indent=1)
def maybe_log_message(self, msg):
import os.path
if self.log_file is None:
return
if not WRITE_TWEETS:
return
mylog.debug('Writing to {}: {}'.format(self.log_file, msg))
old = []
# Let's hope there are never two backend running simultaneously
if os.path.exists(self.log_file):
with open(self.log_file, 'r') as fp:
old = json.load(fp)
old.append(msg)
with open(self.log_file, 'w') as fp:
json.dump(old, fp, sort_keys=True, indent=1)
def consumeTweet(self, tweet):
self.prev_msg_id += 1
mylog.info("(" * 80)
mylog.info("Received tweet #{msg_id}:".format(msg_id=self.prev_msg_id))
mylog.debug(tweet)
# Boring stuff
msg = dict()
msg['content'] = tweet['content']
msg['hashtags'] = tweet['hashtags']
msg['id'] = self.prev_msg_id
msg['image'] = tweet['profile_img']
msg['name'] = tweet['userscreen']
msg['retweet'] = tweet['retweet'] or tweet['content'].startswith('RT ')
msg['time'] = tweet['time']
msg['twitterName'] = tweet['username']
poli = self.pb.getPolitician(tweet['uid'])
citi = self.tw.getCitizen(tweet['uid'])
# Resolve politician/citizen specifics
if poli is not None:
mylog.info("This is definitely a politician.")
msg['poli'] = poli['pid']
birds = self.handle_poli(tweet, msg, poli)
elif citi is not None:
mylog.info("This is definitely a citizen.")
msg['poli'] = None
birds = self.handle_citizen(citi, msg)
else:
mylog.info("Outdated tweet by no-longer citizen {}".format(
tweet['uid']))
birds = None
# Make a sound
if birds is None:
mylog.info("=> drop tweet, DONE")
mylog.info(")" * 80)
return
cBird, pBird = birds
msg['sound'] = generate_sound(tweet['content'], tweet['retweet'],
cBird, pBird)
# Send it
self.sendingQueue.post(msg)
mylog.info("Done with this tweet, DONE")
mylog.info(")" * 80)
def consumeTweet(self, tweet):
self.prev_msg_id += 1
mylog.info("(" * 80)
mylog.info("Received tweet #{msg_id}:".format(msg_id=self.prev_msg_id))
mylog.debug(tweet)
# Boring stuff
msg = dict()
msg['content'] = tweet['content']
msg['hashtags'] = tweet['hashtags']
msg['id'] = self.prev_msg_id
msg['image'] = tweet['profile_img']
msg['name'] = tweet['userscreen']
msg['retweet'] = tweet['retweet'] or tweet['content'].startswith('RT ')
msg['time'] = tweet['time']
msg['twitterName'] = tweet['username']
poli = self.pb.getPolitician(tweet['uid'])
citi = self.tw.getCitizen(tweet['uid'])
# Resolve politician/citizen specifics
if poli is not None:
mylog.info("This is definitely a politician.")
msg['poli'] = poli['pid']
birds = self.handle_poli(tweet, msg, poli)
elif citi is not None:
mylog.info("This is definitely a citizen.")
msg['poli'] = None
birds = self.handle_citizen(citi, msg)
else:
mylog.info("Outdated tweet by no-longer citizen {}".format(tweet['uid']))
birds = None
# Make a sound
if birds is None:
mylog.info("=> drop tweet, DONE")
mylog.info(")" * 80)
return
cBird, pBird = birds
msg['sound'] = generate_sound(tweet['content'], tweet['retweet'], cBird, pBird)
# Send it
self.sendingQueue.post(msg)
mylog.info("Done with this tweet, DONE")
mylog.info(")" * 80)
def addCitizen(self, twittername, birdid, tid=None):
if tid is None:
tid = self.twitter.resolve_name(twittername)
if tid is None:
mylog.warning("citizen user ignored, invalid name: " + twittername)
self.consumer_updates.updateShortpoll(twittername, "unknown-user")
return
if self.polBack.getPolitician(tid) is not None:
self.consumer_updates.updateShortpoll(twittername, "is-politician")
return
if birdid not in self.birdBack.bJson:
mylog.warning("citizen user ignored, invalid bird: " + birdid)
self.consumer_updates.updateShortpoll(twittername, "unknown-bird")
return
with self.lock:
if tid in self.citizens:
entry = self.citizens[tid]
mylog.info(
"Updating existing citizen's bird from {}".format(entry))
else:
mylog.info("Creating new citizen's bird")
entry = dict()
entry["userId"] = tid
entry["party"] = 'neutral'
self.citizens[tid] = entry
# Even if a tweet comes in instantly, getCitizen syncs on
# self.lock, so it's fine. That's also why getCitizen() will
# never see an incomplete citizen.
self.twitter.register_shortlived(tid, twittername)
entry["birdId"] = birdid
token = poll_counter()
entry["token"] = token
mylog.debug("Resulting citizen entry: {}".format(entry))
timer = threading.Timer(REMOVE_CITIZEN_TIME,
self._remove_citizen_wrap, [tid, token])
# Don't prevent shutting down
timer.daemon = True
timer.start()
self.consumer_updates.updateShortpoll(twittername, "succ-resolved")
return
def addCitizen(self, twittername, birdid, tid=None):
if tid is None:
tid = self.twitter.resolve_name(twittername)
if tid is None:
mylog.warning("citizen user ignored, invalid name: " + twittername)
self.consumer_updates.updateShortpoll(twittername, "unknown-user")
return
if self.polBack.getPolitician(tid) is not None:
self.consumer_updates.updateShortpoll(twittername, "is-politician")
return
if birdid not in self.birdBack.bJson:
mylog.warning("citizen user ignored, invalid bird: " + birdid)
self.consumer_updates.updateShortpoll(twittername, "unknown-bird")
return
with self.lock:
if tid in self.citizens:
entry = self.citizens[tid]
mylog.info("Updating existing citizen's bird from {}".format(entry))
else:
mylog.info("Creating new citizen's bird")
entry = dict()
entry["userId"] = tid
entry["party"] = 'neutral'
self.citizens[tid] = entry
# Even if a tweet comes in instantly, getCitizen syncs on
# self.lock, so it's fine. That's also why getCitizen() will
# never see an incomplete citizen.
self.twitter.register_shortlived(tid, twittername)
entry["birdId"] = birdid
token = poll_counter()
entry["token"] = token
mylog.debug("Resulting citizen entry: {}".format(entry))
timer = threading.Timer(REMOVE_CITIZEN_TIME,
self._remove_citizen_wrap, [tid, token])
# Don't prevent shutting down
timer.daemon = True
timer.start()
self.consumer_updates.updateShortpoll(twittername, "succ-resolved")
return
def poll_user(self, e):
query_params = dict(user_id=e['user'], count=2)
seen_tweets = [] # Collect string representation of Tweet IDs
if 'last_id' in e:
query_params['since_id'] = e['last_id']
is_new = False
seen_tweets.append(e['last_id'])
else:
is_new = True
mylog.debug('[POLL] on REST with {} (new={})'.format(query_params, is_new))
statuses = self.api.user_timeline(**query_params)
for status in statuses:
tweet = parse_tweet_status(status)
seen_tweets.append(tweet['tweet_id'])
if not is_new:
self.consumer_tweets.consumeTweet(tweet)
assert len(seen_tweets) > 0, e
# Oh god. I'm so sorry.
e['last_id'] = str(max([int(tid) for tid in seen_tweets]))
if is_new:
self.consumer_updates.updateShortpoll(e['name'], 'succ-firstpoll')
def poll_user(self, e):
query_params = dict(user_id=e['user'], count=2)
seen_tweets = [] # Collect string representation of Tweet IDs
if 'last_id' in e:
query_params['since_id'] = e['last_id']
is_new = False
seen_tweets.append(e['last_id'])
else:
is_new = True
mylog.debug('[POLL] on REST with {} (new={})'.format(
query_params, is_new))
statuses = self.api.user_timeline(**query_params)
for status in statuses:
tweet = parse_tweet_status(status)
seen_tweets.append(tweet['tweet_id'])
if not is_new:
self.consumer_tweets.consumeTweet(tweet)
assert len(seen_tweets) > 0, e
# Oh god. I'm so sorry.
e['last_id'] = str(max([int(tid) for tid in seen_tweets]))
if is_new:
self.consumer_updates.updateShortpoll(e['name'], 'succ-firstpoll')
if mType == "SNP":
resultStr = SNP_MIP_Gap(chrome, start, end, min_hom_length,
max_hom_length, tm_min, tm_max,
gc_threshold_min, gc_threshold_max, ref, alt)
else:
print("Right now, it only supports SNP")
continue
#elif mType == "MNP":
# resultStr = MNP_MIP(chrome,start,end,min_hom_length,max_hom_length,tm_min,tm_max,gc_threshold_min,gc_threshold_max,ref,alt)
if resultStr != "":
output_fh.write(resultStr)
output_fh.write("\n")
else:
debug("Failed " + "|".join(parts[:3]))
"""
result_snp_on_gf = SNP_on_GF(chrome,start,end,min_hom_length,max_hom_length,tm_min,tm_max,gc_threshold_min,gc_threshold_max,ref,alt)
if result_snp_on_gf !="" :
output_fh.write(result_snp_on_gf)
result_snp_on_H2 = SNP_on_H2(chrome,start,end,min_hom_length,max_hom_length,tm_min,tm_max,gc_threshold_min,gc_threshold_max,ref,alt)
if result_snp_on_H2 !="" :
output_fh.write(result_snp_on_H2)
result_snp_on_H1 = SNP_on_H1(chrome,start,end,min_hom_length,max_hom_length,tm_min,tm_max,gc_threshold_min,gc_threshold_max,ref,alt)
if result_snp_on_H1 !="" :
output_fh.write(result_snp_on_H1)
"""
output_fh.close()