def run(self):
"""Runs a continuous loop, waiting for records to appear in the queue,
then processing them.
"""
while True :
while True:
record = self.queue.get()
if record is None:
return
if self.queue.qsize() <= self.max_backlog:
break
if self.skip_this_post(record['dateTime']):
continue
try:
self.process_record(record)
except BadLogin, e:
logging.error("restx: %s: bad login; waiting 60 minutes then retrying" % self.protocol_name)
time.sleep(3600)
except FailedPost, e:
time_str = timestamp_to_string(record['dateTime'])
logging.error("restx: %s: Failed to publish record %s: %s" % (self.protocol_name, time_str, e))
def new_loop_packet(self, event):
"""Apply quality check to the data in a LOOP packet"""
data_dict = to_ENGLISH(event.packet)
for obs_type in self.min_max_dict:
if data_dict.has_key(obs_type) and data_dict[obs_type] is not None:
if not self.min_max_dict[obs_type][0] <= data_dict[obs_type] <= self.min_max_dict[obs_type][1]:
logging.info("QC: %s LOOP value '%s' %s outside limits (%s, %s)" %
(timestamp_to_string(data_dict['dateTime']),
obs_type, data_dict[obs_type],
self.min_max_dict[obs_type][0], self.min_max_dict[obs_type][1]))
event.packet[obs_type] = None
def skip_this_post(self, time_ts):
if self.post_interval is not None:
how_long = time_ts - self.lastpost
if how_long < self.post_interval:
logging.debug("restx: %s: wait interval (%d < %d) has not passed for record %s" %
(self.protocol_name,
how_long, self.post_interval,
timestamp_to_string(time_ts)))
return True
self.lastpost = time_ts
return False
def print_results(self):
print ''
print '*************************'
print '|| RESULTS ||'
print '*************************'
print 'Note - Everything listed is detected without any hard-coding or web scraping'
print ''
print 'Event:', self.event_name
print 'Date:', util.timestamp_to_string(self.start_time, '%x')
print 'Start time:', util.timestamp_to_string(self.start_time,
'%I:%M %Z')
print ''
print self.get_name_list(self.hosts, 'Hosts')
print ''
self.display_winners()
print ''
print self.get_name_list(self.best_dressed, 'Best Dressed')
print ''
print self.get_name_list(self.worst_dressed, 'Worst Dressed')
print ''
return
def soundTheAlarm(self, timestamp):
t_str = timestamp_to_string(timestamp)
logging.info("lowBattery: Low battery alarm sounded at %s." % t_str)
for number in self.numbers:
msg_text = """Hello %s! The battery for weather station %s is low!""" % (number, self.station_name)
self.client.messages.create(
to = self.numbers[number],
from_="+15125803368",
body=msg_text
)
def display_winners(self):
print '[AWARDS]'
for i in range(len(self.winners)):
print ' Title:', self.winners[i][1]
print ' Time:', util.timestamp_to_string(self.winners[i][2],
'%H:%M:%S')
if self.presenters[i]:
print ' Presenter',
if len(self.presenters[i]) > 1:
print 's',
print ':', self.get_name_list(self.presenters[i])
if self.nominees[i]:
print ' Nominees:', self.get_name_list(self.nominees[i])
print ' Winner:', self.winners[i][0]
print ''
def generate_torrent_info_message(torrent_data):
name = torrent_data['name']
percentage_done = '%.2f' % ((1.0 - float(torrent_data['leftUntilDone']) / float(torrent_data['totalSize'])) * 100.0)
size = '%.2f' % (float(torrent_data['totalSize'] / (1000.0 * 1000.0 * 1000.0)))
added = util.timestamp_to_string(util.epoch_to_timestamp(torrent_data['addedDate']))
status = transmission_driver.get_torrent_status(torrent_data['status'])
message = 'Name: {name}\nSize: {size} GB\nStatus: {status}\nDone: {percentage_done}%\nAdded: {added}'.format(
name = name,
percentage_done = percentage_done,
size = size,
added = added,
status = status
)
return message
def run(self):
while True:
record = self.queue.get()
if record is None:
return
if self.skip_this_post(record['dateTime']):
continue
try:
self.process_record(record)
except BadLogin:
logging.error("restx: %s: bad login; waiting 60 minutes then retrying" % self.protocol_name)
time.sleep(3600)
except FailedPost, e:
time_str = timestamp_to_string(record['dateTime'])
logging.error("restx: %s: Failed to publish record %s: %s" % (self.protocol_name, time_str, e))
except Exception, e:
logging.critical("restx: %s: Unexpected exception of type %s: %s" % (self.protocol_name, type(e), e))
def soundTheAlarm(self, timestamp, battery_status, alarm_count):
"""This function is called when the low battery alarm has been sounded."""
t_str = timestamp_to_string(timestamp)
logging.info("lowBattery: Low battery alarm sounded at %s." % t_str)
msg_text = """The low battery alarm has been triggered.\n\n"""\
"""Alarm sounded at %s\n\n""" % t_str
msg = MIMEText(msg_text)
msg['Subject'] = self.SUBJECT
msg['From'] = self.FROM
msg['To'] = self.TO
s = smtplib.SMTP(self.smtp_host)
try:
# Some servers (eg, gmail) require encrypted transport.
# Be prepared to catch an exception if the server
# doesn't support it.
s.ehlo()
s.starttls()
s.ehlo()
logging.debug(" **** using encrypted transport")
except smtplib.SMTPException:
logging.debug(" **** using unencrypted transport")
try:
if self.smtp_user:
s.login(self.smtp_user, self.smtp_password)
logging.info(" **** logged in with user name %s" % (self.smtp_user,))
s.sendmail(msg['From'], self.TO, msg.as_string())
s.quit()
except Exception, e:
logging.error("lowBattery: SMTP mailer refused message with error %s" % (e,))
raise
def compile_autograder_result(self):
self.autograder_result = {
'metadata': {
'year': util.timestamp_to_string(self.start_time, '%Y'),
'names': {
'hosts': {
'method':
'detected',
'method_description':
'The tweets are filtered first by the regex \'host\' and second by \n'
'a regex we wrote to extract names. These names are placed into a \n'
'dictionary, which maintains the popularity of each name. The \n'
'names are sorted by popularity, and the ones that are most often \n'
'mentioned are returned.'
},
'nominees': {
'method':
'detected',
'method_description':
'For every award, an estimated time of conferral is used as an \n'
'anchor time to collect a cursor of tweets which are matched first \n'
'to the regex \'nominees\' and \n then to the regex for names to \n'
'extract probable nominees. The most popular names that are not \n'
'hosts or winners are selected.'
},
'awards': {
'method':
'detected',
'method_description':
'The awards are detected in conjunction with the winners. First, \n'
'the tweets are filtered by a regex that checks if the tweet \n'
'contains a form of the word \'win\' and \'best\'. These tweets \n'
'are further filtered by removing those using subjunctive tense \n'
'or occurring before the ceremony\'s start time, which is also \n'
'detected. The remaining tweets are matched against several \n'
'language models that attempt to pull out winner and award names. \n'
'The awards are grouped by winner (as the dictionary key), which \n'
'goes through a couple consolidation steps. The top winners are \n'
'extracted, and the most popular award name per winner bin is \n'
'added to the award list.'
},
'presenters': {
'method':
'detected',
'method_description':
'Detected at the same time as nominees, using an identical method'
}
},
'mappings': {
'nominees': {
'method':
'detected',
'method_description':
'Both nominees and presenters are mapped to their awards by \n'
'maintaining the timestamps on tweets throughout the program. \n'
'We are thus able to detect when an award was given. These times \n'
'are passed into a function that procedurally generates regular \n'
'expressions that match tweets that were tweeted around a certain \n'
'time. We look at a window of 6 minutes after a winner is \n'
'announced for the nominees and map the results to the winner\'s \n'
'award.'
},
'presenters': {
'method':
'detected',
'method_description':
'Same as nominees, but with a window of 3 minutes before an award \n'
'is announced.'
}
}
},
'data': {
'unstructured': {
'hosts':
self.hosts,
'winners':
[winner for winner, award, time in self.winners],
'awards': [award for winner, award, time in self.winners],
'presenters':
list(itertools.chain.from_iterable(self.presenters)),
'nominees':
list(itertools.chain.from_iterable(self.nominees))
},
'structured': {}
}
}
# Structured
for i in range(len(self.winners)):
self.autograder_result['data']['structured'][
self.winners[i][1]] = {
'nominees': list(self.nominees[i]),
'winner': self.winners[i][0],
'presenters': list(self.presenters[i])
}