def refresh(self, json): self.thread = json try: debug = DebugLog("debug.log") except: raise for posts in self.thread['posts']: try: # skip if record found in dictionary no = posts['no'] # Post is OP try: if posts['resto'] is 0: self.tdict['OP'] = {'no': posts['no'], 'sub': posts['sub'].encode('utf-8'), 'semantic_url': posts['semantic_url'].encode('utf-8')} except Exception as e: debug.msg("Exception:" + e.msg() + "\n") raise if no in self.tdict: continue name = posts['name'] time = datetime.datetime.fromtimestamp(posts['time']).strftime('%H:%M') except: continue curses.use_default_colors() for i in range(0, curses.COLORS): # @UndefinedVariable curses.init_pair(i + 1, i, -1) # assign color to post number color = randint(2, 255) try: country = posts['country'] except: country = "" try: com = posts['com'] com = re.sub('<br>', ' ', com) refposts = "" refposts = re.findall('>>(\d+)', com) com = re.sub('>>(\d+)', '\g<1> ', com) com = re.sub(''', '\'', com) com = re.sub('>', '>', com) com = re.sub('<', '<', com) com = re.sub('"', '"', com) com = re.sub('<[^<]+?>', '', com) except: com = "[File only]" try: trip = posts['trip'] except: trip = "" self.tdict[no] = {'country':country, 'name':name, 'time':time, 'com':com, 'trip':trip, 'color':color} # try: # line = u' '.join((time, ">>" + str(no), country, "<" + name + ">", com)).encode('utf-8') # except: # raise try: self.bp.addstr("", curses.color_pair(color)) self.bp.addstr(time) self.bp.addstr(" >>" + str(no), curses.color_pair(color)) self.bp.addstr(" " + country) if re.match(self.nickname, name) is not None: self.bp.addstr(" <" + self.nickname + "> ", curses.A_BOLD) else: self.bp.addstr(" <" + name.encode('utf8') + "> ") #self.bp.addstr(com.encode('utf8')) comlist = com.split() try: for word in comlist: if word not in refposts: self.bp.addstr(u''.join((word + " ")).encode('utf8')) else: # Comment and reference color encoding try: refcolor = self.tdict[int(word)]['color'] self.bp.addstr(">>" + word + " ", curses.color_pair(refcolor)) # if reference points to nickname, higligt the name if re.match(self.tdict[int(word)]['name'], self.nickname): self.bp.addstr("(You) ", curses.A_BOLD | curses.color_pair(221)) Notifier.send(com) # if re.match(word, threadno): # self.bp.addstr("(OP) ", curses.A_BOLD | curses.color_pair(197)) except: self.bp.addstr(word) except: self.bp.addstr("[File only]") except: raise self.bp.addstr("\n") try: self.title = self.tdict['OP']['sub'] except Exception as e: debug.msg("Couldn't set title" + str(e) + "\n") pass
# parameters to the scanner facilityID = jRocksCamp.getFacilityID() startDate = "01/05/2020" endDate = "01/07/2020" scanner = CampScanner(facilityID, startDate, endDate) # keep scanning until an available campsite is found. while (True): scanner.scanCampground() if scanner.getAvailableCampSites(): break time.sleep(900) # wait 900 secs/15 mins before scanning again print("Campsites Available. Sending notification...") # parameters to Notifier password = "******" # put sending email password here sEmailAddr = "*****@*****.**" # sending email address here rEmailAddr = "*****@*****.**" # receiving email address to send to here subject = "Available Sites!" emailMsgBdy = ("Campsites are available at " + facilityName + ". Navigate to " "https://www.recreation.gov/camping/campgrounds/" + facilityID + "/availability" " to view the availability table.") notifier = Notifier(password, sEmailAddr, rEmailAddr, subject, emailMsgBdy) notifier.buildMessage() notifier.send()
def refresh(self, jsonobj, jsonpages=None): self.thread = jsonobj self.pages = jsonpages #self.tdict['OP'] = {'no': 213, 'com': 'unset', 'sub': 'unset'.encode('utf-8'), 'semantic-url': 'unset'.encode('utf-8')} # Get some information about the thread (post count, reply count, etc) from the first post self.bp.stdscr.noutrefresh() try: self.originalpost.update({'no': self.thread['posts'][0]['no']}) try: self.originalpost.update( {'replies': self.thread['posts'][0]['replies']}) self.originalpost.update( {'images': self.thread['posts'][0]['images']}) self.originalpost.update( {'bumplimit': self.thread['posts'][0]['bumplimit']}) self.originalpost.update( {'imagelimit': self.thread['posts'][0]['imagelimit']}) try: self.originalpost.update( {'archived': self.thread['posts'][0]['archived']}) except KeyError: self.originalpost['archived'] = 0 self.bp.tb.bumplimit = self.originalpost['bumplimit'] self.bp.tb.imagelmit = self.originalpost['imagelimit'] self.bp.tb.images = self.originalpost['images'] self.bp.tb.replies = self.originalpost['replies'] try: self.originalpost.update( {'unique_ips': self.thread['posts'][0]['unique_ips']}) if int(self.originalpost['unique_ips']) > int( self.bp.tb.unique_ips ) and self.bp.tb.unique_ips != 0: self.bp.tb.unique_ips_changed = True else: self.bp.tb.unique_ips_changed = False self.bp.tb.unique_ips = self.originalpost['unique_ips'] # Archived threads do not set unique_ips except: self.bp.tb.unique_ips = "?" # Look up catalog page the thread is currently in self.refresh_pages(jsonpages) except Exception as e: self.dlog.warn(e, logLevel=3, msg=">>>in DictOutput.refresh() -> OP") try: self.originalpost.update({ 'semantic_url': self.thread['posts'][0]['semantic_url'].encode('utf-8') }) except: pass try: com = self.clean_html(self.thread['posts'][0]['com']) self.originalpost.update({'com': com.encode('utf-8')}) self.title = self.originalpost['com'] except: pass try: sub = self.clean_html(self.thread['posts'][0]['sub']) self.originalpost.update({'sub': sub.encode('utf-8')}) self.title = self.originalpost['sub'] except: pass except Exception as e: self.dlog.warn(e, msg=">>>in DictOutput.refresh()", cn=self.__class__.__name__) raise db_posts = [] try: # Array containing posts from previous sessions db_posts = self.db.get_postno_from_threadno( self.originalpost['no']) except Exception as e: self.dlog.excpt(e, msg=">>>in refresh() -> db_posts", cn=self.__class__.__name__) for posts in self.thread['posts']: # skip post if base data can't be processed try: no = posts['no'] # skip post if it was already processed if no in self.tdict: continue self.no_abs_to_rel_dict[no] = self.no_rel name = posts['name'][:16] time = datetime.datetime.fromtimestamp( posts['time']).strftime('%H:%M') except: continue #color = randint(11, 240) color = self.no_rel % 244 + 11 # re-assign black color # TODO is this term/color-independent? if color % 244 == 26: color = 11 refposts = "" # posts referenced in a reply try: country = posts['country'] except: country = "" try: com = posts['com'] com = re.sub('<br>', ' ', com) # save all post quotes in a list (without >>) refposts = re.findall('>>(\d+)', com) # remove >> from quotes for now com = re.sub('>>(\d+)', '\g<1>', com) #com = re.sub(''', '\'', com) #com = re.sub('>', '>', com) #com = re.sub('<', '<', com) #com = re.sub('"', '"', com) #com = re.sub('<[^<]+?>', '', com) com = self.clean_html(com) except: com = "[File only]" # @UndefinedVariable try: trip = posts['trip'] except: trip = "" try: filename = self.clean_html(posts['filename']) except: filename = "" try: fsize = posts['fsize'] except: fsize = 0 try: tim = posts['tim'] except: tim = "" try: ext = self.clean_html(posts['ext']) file_ext_short = ext[1:2].upper() except: ext = "" file_ext_short = " " # Compare comment content to recently posted comment try: marked = False if self.comment_tbm_timeout > 0: self.dlog.msg( "comment wrote: " + str(com) + " comment to be matched: " + str(self.comment_tbm), 4) # TODO regex match with n percentage accuracy should work better if com == self.comment_tbm: marked = True self.comment_tbm = None self.comment_tbm_timeout = 0 self.bp.update_db(no) self.bp.threadFetcher.update_n = 3 self.comment_tbm_timeout -= 1 if no in db_posts: marked = True except: pass try: # TODO maybe just use the structure from 4chan's json. Maybe. self.tdict[no] = { 'country': country, 'name': name, 'time': time, 'com': com, 'trip': trip, 'color': color, 'filename': filename, 'tim': tim, 'ext': ext, 'marked': marked, 'refposts': refposts, 'fsize': fsize } def filter_scan(filterdict): filter_matched = False for rule in json.loads(filterdict): for section in json.loads(rule['filter']): try: test = json.loads(rule['filter']) except: pass if self.tdict[no][section] in test[ section] and self.tdict[no][section]: self.dlog.msg( "--Matched: " + str(self.tdict[no][section]) + " in " + str(test[section]), 4) filter_matched = True elif self.tdict[no][section] not in test[section]: self.dlog.msg( "--Not matched: " + str(self.tdict[no][section]) + " in " + str(test[section]), 4) filter_matched = False break else: self.dlog.msg( "--Not matched (section does not exist): " + str(section), 4) filter_matched = False break self.dlog.msg( "--Matched (" + str(rule) + "): " + str(filter_matched), 4) return filter_matched # Scan for matched filters filter_matched = False if self.cfg.get('filter.ignore.list') and filter_scan( self.cfg.get('filter.ignore.list')): filter_matched = True if self.cfg.get('filter.except.list') and filter_scan( self.cfg.get('filter.except.list')): filter_matched = False # ####### FIXME: Remove. This is to test filtering # pattern = re.compile(ur'Japanese', re.UNICODE) # filter_matched = False # if country in ["DE", "TH"] and self.bp.board == "int" and name == "Anonymous" \ # and trip == "" and pattern.search(self.originalpost['com']): # # filter_matched = True # # # Exception for filter # for letter in com: # if unicodedata.east_asian_width(letter) in ['W', 'F']: # filter_matched = False # break # elif country == "US" and name == "Anonymous": # filter_matched = True if filter_matched: self.dlog.msg("--FILTER TEST: Skipping comment: " + str(no)) continue except Exception as err: self.dlog.warn(err, msg=">>>in DictOutput.refresh() (-> filter)") try: #if filter_matched: # # continue ####### End of filter test ########### # Write [TIME] <Name> self.bp.addstr("", curses.color_pair(color)) # @UndefinedVariable self.bp.addstr(time) if self.cfg.get("board.postno.style") is 'relative': self.bp.addstr( " >>" + str(self.no_rel).zfill(3), curses.color_pair(color)) # @UndefinedVariable else: self.bp.addstr( " >>" + str(no), curses.color_pair(color)) # @UndefinedVariable # Add country code self.bp.addstr(" " + country) # Count of characters that are in every new BoardPad line (for indentation on nl in bp) indent = len(time) + len(str(no)) + len(country) + 4 # Make own nickname bold if re.match(str(self.nickname), name) or marked: # if not self.nickname: # nick = "Anonymous" # else: # nick = self.nickname self.bp.addstr(" <" + file_ext_short + name + "> ", curses.A_BOLD) # @UndefinedVariable # Make name decoration stand out if file is attached else: self.bp.addstr( " <", curses.color_pair(250)) # @UndefinedVariable self.bp.addstr(file_ext_short, curses.color_pair(250) | curses.A_BOLD) # @UndefinedVariable self.bp.addstr(name.encode('utf8'), curses.A_DIM) # @UndefinedVariable if self.cfg.get("board.postno.style") is 'hybrid': self.bp.addstr( "-" + str(self.no_rel).zfill(3), curses.color_pair(color)) # @UndefinedVariable self.bp.addstr( "> ", curses.color_pair(250)) # @UndefinedVariable # width of name including unicode east asian characters + len("< > ") == 5 indent += self.bp.calcline(name.encode('utf8')) + 5 comlist = com.split() try: # # Post filename if exists # try: # if filename: # self.bp.addstr("[", curses.A_BOLD) # @UndefinedVariable # self.bp.addstr(filename[:12]) # self.bp.addstr("] ", curses.A_BOLD) # @UndefinedVariable # except: # pass # Iterate over every word in a comment for word in comlist: # refposts contains all >>(\d+) in a comment if word not in refposts: # Output non-reference self.bp.addstr(u''.join( (word + " ")).encode('utf8'), curses.A_NORMAL, indent) # @UndefinedVariable # Handle references (>>(\d+)) else: # Comment and reference color encoding try: refcolor = self.tdict[int(word)]['color'] if self.cfg.get("board.postno.style" ) is not 'absolute': self.bp.addstr( ">>" + str(self.no_abs_to_rel_dict[int(word)]) + " ", curses.color_pair(refcolor), indent) # @UndefinedVariable else: self.bp.addstr( ">>" + word + " ", curses.color_pair(refcolor), indent) # @UndefinedVariable # Add (You) to referenced posts written by self.nickname if re.match(self.tdict[int(word)]['name'], str(self.nickname)) or self.tdict[ int(word)]['marked'] == True: self.bp.addstr( "(You) ", curses.A_BOLD | curses.color_pair(221), indent, mentioned=True) # @UndefinedVariable try: Notifier.send(name, com) except Exception as e: self.dlog.excpt( e, msg=">>>in DictOutput.refresh()", cn=self.__class__.__name__) raise # highlight OP reference if re.match(word, str(self.originalpost['no'])): try: self.bp.addstr( "(OP) ", curses.A_BOLD | curses.color_pair(4), indent) # @UndefinedVariable except: raise except KeyError: self.bp.addstr(word + " ", curses.A_DIM, indent) # @UndefinedVariable except Exception as err: self.dlog.excpt( err, msg=">>>in DictOutput.filter_scan()", cn=self.__class__.__name__) except: self.bp.addstr("[File only]", curses.A_DIM, indent) # @UndefinedVariable except: raise self.bp.addstr("\n", curses.A_NORMAL, indent) # @UndefinedVariable self.no_rel += 1 try: if self.bp.subtitle.append_to_subfile: self.bp.subtitle.subfile_append(com) except AttributeError: pass # refetch entire thread on post count mismatch try: if len(self.tdict) != int(self.thread['posts'][0]['replies']) + 1: self.bp.update_thread(notail=True) except Exception as e: self.dlog.excpt(e, msg=">>>in DictOutput.refresh() ->refetch", cn=self.__class__.__name__) if self.originalpost['archived']: self.bp.threadFetcher.stop() self.bp.sb.setStatus("ARCHIVED") curses.doupdate() # @UndefinedVariable