def main(): from optparse import OptionParser parser = OptionParser(usage="%prog [options] wiki.html") parser.add_option("-d", "--destination", dest="dest", default="txt", help="the directory to write text files to") parser.add_option("-v", "--vim-notes", dest="vim_notes", default=False, action="store_true", help="format output for vim-notes.") opts, args = parser.parse_args() if not args: parser.error("you must specify the path to the TiddlyWiki file") wiki = open(args[0]).read().strip() parser = WikiParser() parser.feed(wiki) import os.path as P import os from formatter import vim_notes_format if not P.isdir(opts.dest): os.makedirs(opts.dest) for t in parser.tiddlers: fname = P.join(opts.dest, t["title"]) + ".txt" fout = codecs.open(fname, "w", "utf-8") title, text, tags = t["title"], t["text"], t["tags"] if opts.vim_notes: fout.write(vim_notes_format(title, text, tags)) else: fout.write("\n".join([title] + text + ["tags: " + tags ])) fout.close() print "wrote [%s] to [%s]" % (t["title"], fname)
def main(): parser = OptionParser() parser.add_option('-a', '--author', dest='author', help='Sets song performer to AUTHOR', metavar='AUTHOR') parser.add_option('-t', '--title', dest='title', help='Sets the title of the song to TITLE', metavar='TITLE') parser.add_option('-l', '--lyrics', dest='lyrics', help='Sets the lyrics of the song to LYRICS', metavar='LYRICS') parser.add_option('-d', '--debug', action='store_true', dest='debug', default=False, help='Runs in debug mode') (options, args) = parser.parse_args() debug = options.debug artist = options.author title = options.title lyrics = options.lyrics furigana_lyrics_html = furiganize_lyrics(preprocess_lyrics(lyrics)) parser = MyHTMLParser() parser.feed(furigana_lyrics_html) if debug: print(lyrics) print(furigana_lyrics_html) print(parser.get_lyrics()) clipboard.set(fill_tex_template(artist, title, parser.get_lyrics())) webbrowser.open('shortcuts://')
def main(): usage_str = "%prog begin_yyyymmddhhmm end_yyyymmddhhmm" parser = OptionParser(usage = usage_str) parser.add_option("-n", "--nothing", dest="nothing", action="store_true", help="nothing at all") parser.add_option("-c", "--config_file", dest="config", help="read specified configuration file") parser.add_option("-l", "--log", dest="log", help="write log messages to specified file") (options, args) = parser.parse_args() if len(args) != 2: parser.print_help() sys.exit(2) begin_yyyymmddhhmm = args[0] end_yyyymmddhhmm = args[1] if options.log: logg = log_msg.LogMessage(options.log) else: logg = log_msg.LogMessage("") logg.write_starting() begin_time = calendar.timegm(time.strptime(begin_yyyymmddhhmm, "%Y%m%d%H%M")) end_time = calendar.timegm(time.strptime(end_yyyymmddhhmm, "%Y%m%d%H%M")) for time_value in range(begin_time, end_time, 60): time_string = time.strftime("%Y%m%d%H%M", time.gmtime(time_value)) #response = urllib2.urlopen('http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s&Length=1' % time_string) #print 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s&Length=1' % time_string #http_cmd = 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s&Length=1' % time_string response = urllib2.urlopen('http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s UTC&Length=1' % time_string) print 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s UTC&Length=1' % time_string http_cmd = 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s UTC&Length=1' % time_string logg.write_info("Executing: %s" % (http_cmd)) html = response.read() parser = MyHTMLParser() parser.feed(html) time.sleep(1) logg.write_ending()
def main(): from optparse import OptionParser parser = OptionParser(usage="%prog [options] wiki.html") parser.add_option("-d", "--destination", dest="dest", default="txt", help="the directory to write text files to") parser.add_option("-v", "--vim-notes", dest="vim_notes", default=False, action="store_true", help="format output for vim-notes.") opts, args = parser.parse_args() if not args: parser.error("you must specify the path to the TiddlyWiki file") wiki = open(args[0]).read().strip() parser = WikiParser() parser.feed(wiki) import os.path as P import os from formatter import vim_notes_format if not P.isdir(opts.dest): os.makedirs(opts.dest) for t in parser.tiddlers: title_fixed = re.sub( "[/><]", "", t["title"] ) # Remove "/ > <" characters from title, for use in the next line fname = P.join(opts.dest, title_fixed) + ".txt" fout = codecs.open(fname, "w", "utf-8") title, text, tags = t["title"], t["text"], t["tags"] if opts.vim_notes: fout.write(vim_notes_format(title, text, tags)) else: fout.write("\n".join([title] + text + ["tags: " + tags])) fout.close() print "wrote [%s] to [%s]" % (t["title"], fname)
def handle(self, url, **optz): # Check if subscriber sites and filters can be found if optz.get("subscribe"): try: subscribe = list(models.Site.get_by_string(name) for name in optz["subscribe"]) except (ObjectDoesNotExist, MultipleObjectsReturned) as err: raise CommandError(err.message) else: subscribe = list() if optz.get("filter"): try: filters = list(models.Filter.objects.get(id=fid) for fid in optz["filter"]) except ObjectDoesNotExist as err: raise CommandError(err.message) else: filters = list() # Fill in missing feed name fields if not optz.get("name"): fpf = feedparser.parse(url, agent=fjupdate.USER_AGENT) optz["name"] = fpf.feed.get("title", "")[:200] if not optz["name"]: raise CommandError("Failed to acquire name from the feed ({0})".format(url)) if not optz.get("shortname"): optz["shortname"] = optz["name"][:50] # Add Feed / Subscriber objects with transaction.commit_on_success(): feed = models.Feed(feed_url=url, name=optz["name"], shortname=optz["shortname"]) if optz.get("field_data"): try: import yaml as s except ImportError: import json as s import io for k, v in s.load(io.BytesIO(optz["field_data"])).iteritems(): setattr(feed, k, v) feed.save() for f in filters: feed.filters.add(f) for site in subscribe: models.Subscriber.objects.create(feed=feed, site=site) # Perform the initial fetch, if requested if optz.get("initial_fetch"): fetch_optz, fetch_argz = OptionParser(option_list=fjupdate.make_cli_option_list()).parse_args(list()) fetch_optz.feed = [feed.id] fetch_optz.hidden = optz.get("fetch_hidden", False) fjupdate.main(fetch_optz)
def run(self): """ Main entry point """ # Read command line usage = "%prog [options] <url> <address> [<address2>] [<address...>]" parser = OptionParser(usage=usage, version=self.NAME + ' ' + self.VERSION) parser.add_option("--http-user", dest="http_user", help="Username for HTTP POST authentication") parser.add_option("--http-pass", dest="http_pass", help="Password for HTTP POST authentication") parser.add_option("-s", "--smtp", dest="smtp", help="SMTP server address. Default: localhost", default='localhost') parser.add_option("--smtp-user", dest="smtp_user", help="Username for SMTP authentication") parser.add_option("--smtp-pass", dest="smtp_pass", help="Password for SMTP authentication") parser.add_option("-c", "--cc", dest="cc", help="Carbon Copy recipient") parser.add_option("-f", "--from", dest="sender", help="eMail sender. Default: emailer@localhost", default="emailer@localhost") parser.add_option( "-j", "--subject", dest="subject", help="eMail Subject. Default: MailWWW Autogenerated Mail", default="MailWWW Autogenerated Mail") parser.add_option("-n", "--no-css", dest="nocss", help="Disable embedding of linked Style Sheets", default=False, action="store_true") parser.add_option( "-m", "--multiple", dest="multiple", help= "Send multiple emails: one for each recipient (Cc field is ignored)", default=False, action="store_true") parser.add_option("-v", "--verbose", dest="verbose", help="Show progress information", default=False, action="store_true") (options, args) = parser.parse_args() # Parse mandatory arguments if len(args) < 2: parser.error("unvalid number of arguments") dest = [] i = 0 for a in args: if i == 0: url = a else: dest.append(a) i += 1 # Parse optional arguments http_user = options.http_user http_pass = options.http_pass cc = [] if options.cc: cc.append(options.cc) host = options.smtp port = 25 user = options.smtp_user pwd = options.smtp_pass sender = options.sender subject = options.subject nocss = options.nocss multiple = options.multiple verbose = options.verbose logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) # Opens URL logging.info('Fetching url %s', url) data = None if http_user or http_pass: # Use POST authentication data = urllib.urlencode({ 'username': http_user, 'password': http_pass, 'login': True }) f = urllib.urlopen(url, data) html = f.read() # Search for meta content-type tag, use this encoding when found encre = re.compile( r'<meta\s+http-equiv=(?:"|\')Content-Type(?:"|\')\s+content=(?:"|\')([^\'"]*)(?:"|\')\s*/>', re.I | re.M) match = encre.search(html) if match: encoding = self.__parseEncoding(match.group(1)) try: html = unicode(html, encoding, errors='replace') except LookupError as e: encoding = self.__parseEncoding(f.headers['content-type']) html = unicode(html, encoding, errors='replace') else: encoding = self.__parseEncoding(f.headers['content-type']) html = unicode(html, encoding, errors='replace') logging.info('Detected charset: %s', encoding) f.close() # Retrieve linked style sheets if not nocss: logging.info('Fetching Style Sheets...') parser = CSSLister(url) parser.feed(html) parser.close() for search, replace in parser.get_replacements(): html = html.replace(search, replace, 1) # Prepare mail msg = MIMEMultipart() msg['Date'] = formatdate(localtime=True) msg['Message-ID'] = make_msgid('emailer') msg['Subject'] = subject msg['From'] = sender if cc and not multiple: msg['Cc'] = ', '.join(cc) msg.preamble = 'This is a milti-part message in MIME format.' txt = MIMEText(html.encode('utf-8'), 'html', 'utf-8') msg.attach(txt) if not multiple: msg['To'] = ', '.join(dest) # Sends message smtp = smtplib.SMTP() smtp.connect(host, port) if user: smtp.login(user, pwd) if multiple: for d in dest: del msg['To'] msg['To'] = d logging.info('Sending mail to: %s', d) smtp.sendmail(sender, d, msg.as_string()) else: logging.info('Sending mail to: %s, Cc: %s', dest, cc) smtp.sendmail(sender, dest + cc, msg.as_string()) smtp.quit()
connection.request(self.method, self.url, self.body, headers) response = connection.getresponse() if response.status == self.expectedCode: cookie_store.add_cookies(response.getheader('Set-Cookie')) if self.next_request: response.read() return self.next_request.process(connection, cookie_store, self.url) else: return response.read() else: raise FailedException(self, response.status, response.reason, response.read()) request = ChainedHttpRequest('GET', '/' + dico + '/' + word, httplib.OK) request = ChainedHttpRequest('GET', '/', httplib.OK, request) store = CookieStore() connection = httplib.HTTPConnection('www.wordreference.com') connection.set_debuglevel(9) res = request.process(connection, store, 'http://www.wordreference.com/') parser = WordReferenceHTMLParser() parser.feed(res)
sys.exit(2) else: url = options.url #Reading Configurations readConfig() #Checking Arguments url,port = parseURL(url) url_final = BASE_URL % (url,port) #Downloading HTML file html_response = getHTML(url_final) if html_response == '' or 'Axis2: Services' not in html_response: rc = RC_CRIT msg = "There is no Configuration in URL or is not a service " else: parser = MyHTMLParser() parser.feed(html_response) msg = "" rc = RC_UNKNOWN for service in SERVICES: if not checkService(service): rc = RC_CRIT msg += " %s not in configuration " % (service) if msg == "": rc = RC_OK msg = "All Services are up" exit(rc,msg)
options.path = '/'+options.path conn = httplib.HTTPConnection("localhost:29392") conn.request(method,options.action+options.path) resp = conn.getresponse() data = resp.read() if options.action == '/download': print sys.getsizeof(data) exit() parser = ET.XMLTreeBuilder() parser.feed(data) tree = ET.ElementTree(parser.close()) root = tree.getroot() #print root.text ET.dump(tree) if options.action == '/list' and method == "GET": print "File listing: " file_listing(options.path, root) if options.action == '/tagdata' and method == "GET": print "Tag Data: " tag_data("/", root, options)
st=False for post in posts: if st:break post['author']=authors[post['author']]['name'] code=SWG_CODE for k,v in post.items(): code=code.replace("%"+k+"_I%",str(v)) if o.SAVEIMG: if o.WWW==None: print "[X] -w REQUIRED" st=True break else: parser=ParseImages() parser.feed(code) parser.close() for img in parser.getImgs(): dl=o.WWW+"/"+img if not o.WWW in img else img name=str(time.time())+"_import"+os.path.splitext(img)[1] dest="images/"+name code=code.replace(img,"/"+dest) print "[!] Downloading "+dl+" in "+dest+"..." urllib.urlretrieve(dl,dest) print "[O] Done." nc="" i=0 for c in code: nc+=c i+=1 if i==o.BREAK:
start_time = timeit.default_timer() # Get the html for the specific page data = getPageNum(pageNum) # The html page returns data in unicode, which the HTML Parser can't handle # replace all unknown characters and convert to ascii. data = data.decode('ascii', 'replace') dbQueryCurlTime = dbQueryCurlTime + timeit.default_timer( ) - start_time start_time = timeit.default_timer() # Parse the contents of the received HTML page parser = MyHTMLParser() parser.feed(data) # Keep track of the stats from the html parsing newAds = parser.getNewAds() lastExistingAds = parser.getLastExistingAds() totalNewAds = totalNewAds + newAds pageNum = pageNum + 1 parser.adReset() print "We got [%d] new ads" % (newAds) dbQueryParseTime = dbQueryParseTime + timeit.default_timer( ) - start_time # Option to read database and return new ads if options.readDb: start_time = timeit.default_timer()
Logger.info('Starting MX validation for %s'%(options.server)) page = mechanize.Browser() page.open(base_url) assert page.viewing_html() # Fills form and submit page.select_form(nr=0) page["IP"] = options.server response = page.submit() if debug : Logger.debug("Getting data") # Get result page html = response.get_data() # Parse results and retrieve images url parser = DnsblParser(base_url) if debug : Logger.debug('Feeding') parser.feed(html) # Check MX reputation status = BlValidator.validate_parallel(parser.bl_list) # Prints out result if status : sys.stdout.write('OK\n') sys.exit(0) else : sys.stdout.write('ERROR, check %s\n' % (base_url)) sys.exit(2)
Logger.info('Starting MX validation for %s' % (options.server)) page = mechanize.Browser() page.open(base_url) assert page.viewing_html() # Fills form and submit page.select_form(nr=0) page["IP"] = options.server response = page.submit() if debug: Logger.debug("Getting data") # Get result page html = response.get_data() # Parse results and retrieve images url parser = DnsblParser(base_url) if debug: Logger.debug('Feeding') parser.feed(html) # Check MX reputation status = BlValidator.validate_parallel(parser.bl_list) # Prints out result if status: sys.stdout.write('OK\n') sys.exit(0) else: sys.stdout.write('ERROR, check %s\n' % (base_url)) sys.exit(2)
def run(self): """ Main entry point """ # Read command line usage = "%prog [options] <url> <address> [<address2>] [<address...>]" parser = OptionParser(usage=usage, version=self.NAME + ' ' + self.VERSION) parser.add_option("--http-user", dest="http_user", help="Username for HTTP POST authentication") parser.add_option("--http-pass", dest="http_pass", help="Password for HTTP POST authentication") parser.add_option("-s", "--smtp", dest="smtp", help="SMTP server address. Default: localhost", default='localhost') parser.add_option("--smtp-user", dest="smtp_user", help="Username for SMTP authentication") parser.add_option("--smtp-pass", dest="smtp_pass", help="Password for SMTP authentication") parser.add_option("-c", "--cc", dest="cc", help="Carbon Copy recipient") parser.add_option("-f", "--from", dest="sender", help="eMail sender. Default: emailer@localhost", default="emailer@localhost") parser.add_option("-j", "--subject", dest="subject", help="eMail Subject. Default: MailWWW Autogenerated Mail", default="MailWWW Autogenerated Mail") parser.add_option("-n", "--no-css", dest="nocss", help="Disable embedding of linked Style Sheets", default=False, action="store_true") parser.add_option("-m", "--multiple", dest="multiple", help="Send multiple emails: one for each recipient (Cc field is ignored)", default=False, action="store_true") parser.add_option("-v", "--verbose", dest="verbose", help="Show progress information", default=False, action="store_true") (options, args) = parser.parse_args() # Parse mandatory arguments if len(args) < 2: parser.error("unvalid number of arguments") dest = [] i = 0 for a in args: if i == 0: url = a else: dest.append(a) i += 1 # Parse optional arguments http_user = options.http_user http_pass = options.http_pass cc = [] if options.cc: cc.append(options.cc) host = options.smtp port = 25 user = options.smtp_user pwd = options.smtp_pass sender = options.sender subject = options.subject nocss = options.nocss multiple = options.multiple verbose = options.verbose logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) # Opens URL logging.info('Fetching url %s', url) data = None if http_user or http_pass: # Use POST authentication data = urllib.urlencode({ 'username': http_user, 'password': http_pass, 'login': True }) f = urllib.urlopen(url, data) html = f.read() # Search for meta content-type tag, use this encoding when found encre = re.compile(r'<meta\s+http-equiv=(?:"|\')Content-Type(?:"|\')\s+content=(?:"|\')([^\'"]*)(?:"|\')\s*/>', re.I | re.M) match = encre.search(html) if match: encoding = self.__parseEncoding(match.group(1)) try: html = unicode(html, encoding, errors='replace') except LookupError as e: encoding = self.__parseEncoding(f.headers['content-type']) html = unicode(html, encoding, errors='replace') else: encoding = self.__parseEncoding(f.headers['content-type']) html = unicode(html, encoding, errors='replace') logging.info('Detected charset: %s', encoding) f.close() # Retrieve linked style sheets if not nocss: logging.info('Fetching Style Sheets...') parser = CSSLister(url) parser.feed(html) parser.close() for search, replace in parser.get_replacements(): html = html.replace(search, replace, 1) # Prepare mail msg = MIMEMultipart() msg['Date'] = formatdate(localtime=True) msg['Message-ID'] = make_msgid('emailer') msg['Subject'] = subject msg['From'] = sender if cc and not multiple: msg['Cc'] = ', '.join(cc) msg.preamble = 'This is a milti-part message in MIME format.' txt = MIMEText(html.encode('utf-8'), 'html', 'utf-8') msg.attach(txt) if not multiple: msg['To'] = ', '.join(dest) # Sends message smtp = smtplib.SMTP() smtp.connect(host, port) if user: smtp.login(user, pwd) if multiple: for d in dest: del msg['To'] msg['To'] = d logging.info('Sending mail to: %s', d) smtp.sendmail(sender, d, msg.as_string()) else: logging.info('Sending mail to: %s, Cc: %s', dest, cc) smtp.sendmail(sender, dest+cc, msg.as_string()) smtp.quit()