def main():
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options] wiki.html")
    parser.add_option("-d", "--destination", dest="dest", default="txt", help="the directory to write text files to")
    parser.add_option("-v", "--vim-notes", dest="vim_notes", default=False, action="store_true", help="format output for vim-notes.")
    opts, args = parser.parse_args()
    if not args:
        parser.error("you must specify the path to the TiddlyWiki file")
    wiki = open(args[0]).read().strip()
    parser = WikiParser()
    parser.feed(wiki)

    import os.path as P
    import os
    from formatter import vim_notes_format

    if not P.isdir(opts.dest):
        os.makedirs(opts.dest)

    for t in parser.tiddlers:
        fname = P.join(opts.dest, t["title"]) + ".txt"
        fout = codecs.open(fname, "w", "utf-8")
        title, text, tags = t["title"], t["text"], t["tags"]
        if opts.vim_notes:
            fout.write(vim_notes_format(title, text, tags))
        else:
            fout.write("\n".join([title] + text + ["tags: " + tags ]))
        fout.close()
        print "wrote [%s] to [%s]" % (t["title"], fname)
Beispiel #2
0
def main():
    parser = OptionParser()
    parser.add_option('-a',
                      '--author',
                      dest='author',
                      help='Sets song performer to AUTHOR',
                      metavar='AUTHOR')
    parser.add_option('-t',
                      '--title',
                      dest='title',
                      help='Sets the title of the song to TITLE',
                      metavar='TITLE')
    parser.add_option('-l',
                      '--lyrics',
                      dest='lyrics',
                      help='Sets the lyrics of the song to LYRICS',
                      metavar='LYRICS')
    parser.add_option('-d',
                      '--debug',
                      action='store_true',
                      dest='debug',
                      default=False,
                      help='Runs in debug mode')
    (options, args) = parser.parse_args()

    debug = options.debug
    artist = options.author
    title = options.title
    lyrics = options.lyrics

    furigana_lyrics_html = furiganize_lyrics(preprocess_lyrics(lyrics))

    parser = MyHTMLParser()
    parser.feed(furigana_lyrics_html)

    if debug:
        print(lyrics)
        print(furigana_lyrics_html)
        print(parser.get_lyrics())

    clipboard.set(fill_tex_template(artist, title, parser.get_lyrics()))
    webbrowser.open('shortcuts://')
Beispiel #3
0
def main():

    usage_str = "%prog begin_yyyymmddhhmm end_yyyymmddhhmm"
    parser = OptionParser(usage = usage_str)
    parser.add_option("-n", "--nothing", dest="nothing", action="store_true", help="nothing at all")
    parser.add_option("-c", "--config_file", dest="config", help="read specified configuration file")
    parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")

    (options, args) = parser.parse_args()

    if len(args) != 2:
        parser.print_help()
        sys.exit(2)

    begin_yyyymmddhhmm = args[0]
    end_yyyymmddhhmm = args[1]

    if options.log:
        logg = log_msg.LogMessage(options.log)
    else:
        logg = log_msg.LogMessage("")
        
    logg.write_starting()
    
    begin_time = calendar.timegm(time.strptime(begin_yyyymmddhhmm, "%Y%m%d%H%M"))
    end_time = calendar.timegm(time.strptime(end_yyyymmddhhmm, "%Y%m%d%H%M"))
    for time_value in range(begin_time, end_time, 60):
        time_string = time.strftime("%Y%m%d%H%M", time.gmtime(time_value))
        #response = urllib2.urlopen('http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s&Length=1' % time_string)
        #print 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s&Length=1' % time_string
        #http_cmd = 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s&Length=1' % time_string
        response = urllib2.urlopen('http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s UTC&Length=1' % time_string)
        print 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s UTC&Length=1' % time_string
        http_cmd = 'http://mndotcam.ameritrak.biz/getATImageFiles.php?Agency=NCAR&BeginTime=%s UTC&Length=1' % time_string
        logg.write_info("Executing: %s" % (http_cmd))
        html = response.read()
        parser = MyHTMLParser()
        parser.feed(html)
        time.sleep(1)

    logg.write_ending()
Beispiel #4
0
def main():
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options] wiki.html")
    parser.add_option("-d",
                      "--destination",
                      dest="dest",
                      default="txt",
                      help="the directory to write text files to")
    parser.add_option("-v",
                      "--vim-notes",
                      dest="vim_notes",
                      default=False,
                      action="store_true",
                      help="format output for vim-notes.")
    opts, args = parser.parse_args()
    if not args:
        parser.error("you must specify the path to the TiddlyWiki file")
    wiki = open(args[0]).read().strip()
    parser = WikiParser()
    parser.feed(wiki)

    import os.path as P
    import os
    from formatter import vim_notes_format

    if not P.isdir(opts.dest):
        os.makedirs(opts.dest)

    for t in parser.tiddlers:
        title_fixed = re.sub(
            "[/><]", "", t["title"]
        )  # Remove "/ > <" characters from title, for use in the next line
        fname = P.join(opts.dest, title_fixed) + ".txt"
        fout = codecs.open(fname, "w", "utf-8")
        title, text, tags = t["title"], t["text"], t["tags"]
        if opts.vim_notes:
            fout.write(vim_notes_format(title, text, tags))
        else:
            fout.write("\n".join([title] + text + ["tags: " + tags]))
        fout.close()
        print "wrote [%s] to [%s]" % (t["title"], fname)
Beispiel #5
0
    def handle(self, url, **optz):
        # Check if subscriber sites and filters can be found
        if optz.get("subscribe"):
            try:
                subscribe = list(models.Site.get_by_string(name) for name in optz["subscribe"])
            except (ObjectDoesNotExist, MultipleObjectsReturned) as err:
                raise CommandError(err.message)
        else:
            subscribe = list()
        if optz.get("filter"):
            try:
                filters = list(models.Filter.objects.get(id=fid) for fid in optz["filter"])
            except ObjectDoesNotExist as err:
                raise CommandError(err.message)
        else:
            filters = list()

        # Fill in missing feed name fields
        if not optz.get("name"):
            fpf = feedparser.parse(url, agent=fjupdate.USER_AGENT)
            optz["name"] = fpf.feed.get("title", "")[:200]
            if not optz["name"]:
                raise CommandError("Failed to acquire name from the feed ({0})".format(url))
        if not optz.get("shortname"):
            optz["shortname"] = optz["name"][:50]

            # Add Feed / Subscriber objects
        with transaction.commit_on_success():
            feed = models.Feed(feed_url=url, name=optz["name"], shortname=optz["shortname"])
            if optz.get("field_data"):
                try:
                    import yaml as s
                except ImportError:
                    import json as s
                import io

                for k, v in s.load(io.BytesIO(optz["field_data"])).iteritems():
                    setattr(feed, k, v)
            feed.save()
            for f in filters:
                feed.filters.add(f)
            for site in subscribe:
                models.Subscriber.objects.create(feed=feed, site=site)

                # Perform the initial fetch, if requested
        if optz.get("initial_fetch"):
            fetch_optz, fetch_argz = OptionParser(option_list=fjupdate.make_cli_option_list()).parse_args(list())
            fetch_optz.feed = [feed.id]
            fetch_optz.hidden = optz.get("fetch_hidden", False)
            fjupdate.main(fetch_optz)
Beispiel #6
0
    def run(self):
        """ Main entry point """

        # Read command line
        usage = "%prog [options] <url> <address> [<address2>] [<address...>]"
        parser = OptionParser(usage=usage,
                              version=self.NAME + ' ' + self.VERSION)
        parser.add_option("--http-user",
                          dest="http_user",
                          help="Username for HTTP POST authentication")
        parser.add_option("--http-pass",
                          dest="http_pass",
                          help="Password for HTTP POST authentication")
        parser.add_option("-s",
                          "--smtp",
                          dest="smtp",
                          help="SMTP server address. Default: localhost",
                          default='localhost')
        parser.add_option("--smtp-user",
                          dest="smtp_user",
                          help="Username for SMTP authentication")
        parser.add_option("--smtp-pass",
                          dest="smtp_pass",
                          help="Password for SMTP authentication")
        parser.add_option("-c",
                          "--cc",
                          dest="cc",
                          help="Carbon Copy recipient")
        parser.add_option("-f",
                          "--from",
                          dest="sender",
                          help="eMail sender. Default: emailer@localhost",
                          default="emailer@localhost")
        parser.add_option(
            "-j",
            "--subject",
            dest="subject",
            help="eMail Subject. Default: MailWWW Autogenerated Mail",
            default="MailWWW Autogenerated Mail")
        parser.add_option("-n",
                          "--no-css",
                          dest="nocss",
                          help="Disable embedding of linked Style Sheets",
                          default=False,
                          action="store_true")
        parser.add_option(
            "-m",
            "--multiple",
            dest="multiple",
            help=
            "Send multiple emails: one for each recipient (Cc field is ignored)",
            default=False,
            action="store_true")
        parser.add_option("-v",
                          "--verbose",
                          dest="verbose",
                          help="Show progress information",
                          default=False,
                          action="store_true")

        (options, args) = parser.parse_args()

        # Parse mandatory arguments
        if len(args) < 2:
            parser.error("unvalid number of arguments")
        dest = []
        i = 0
        for a in args:
            if i == 0:
                url = a
            else:
                dest.append(a)
            i += 1

        # Parse optional arguments
        http_user = options.http_user
        http_pass = options.http_pass
        cc = []
        if options.cc:
            cc.append(options.cc)
        host = options.smtp
        port = 25
        user = options.smtp_user
        pwd = options.smtp_pass
        sender = options.sender
        subject = options.subject
        nocss = options.nocss
        multiple = options.multiple
        verbose = options.verbose

        logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)

        # Opens URL
        logging.info('Fetching url %s', url)
        data = None
        if http_user or http_pass:
            # Use POST authentication
            data = urllib.urlencode({
                'username': http_user,
                'password': http_pass,
                'login': True
            })
        f = urllib.urlopen(url, data)
        html = f.read()
        # Search for meta content-type tag, use this encoding when found
        encre = re.compile(
            r'<meta\s+http-equiv=(?:"|\')Content-Type(?:"|\')\s+content=(?:"|\')([^\'"]*)(?:"|\')\s*/>',
            re.I | re.M)
        match = encre.search(html)
        if match:
            encoding = self.__parseEncoding(match.group(1))
            try:
                html = unicode(html, encoding, errors='replace')
            except LookupError as e:
                encoding = self.__parseEncoding(f.headers['content-type'])
                html = unicode(html, encoding, errors='replace')
        else:
            encoding = self.__parseEncoding(f.headers['content-type'])
            html = unicode(html, encoding, errors='replace')
        logging.info('Detected charset: %s', encoding)
        f.close()

        # Retrieve linked style sheets
        if not nocss:
            logging.info('Fetching Style Sheets...')
            parser = CSSLister(url)
            parser.feed(html)
            parser.close()
            for search, replace in parser.get_replacements():
                html = html.replace(search, replace, 1)

        # Prepare mail
        msg = MIMEMultipart()
        msg['Date'] = formatdate(localtime=True)
        msg['Message-ID'] = make_msgid('emailer')
        msg['Subject'] = subject
        msg['From'] = sender

        if cc and not multiple:
            msg['Cc'] = ', '.join(cc)
        msg.preamble = 'This is a milti-part message in MIME format.'

        txt = MIMEText(html.encode('utf-8'), 'html', 'utf-8')
        msg.attach(txt)

        if not multiple:
            msg['To'] = ', '.join(dest)

        # Sends message
        smtp = smtplib.SMTP()
        smtp.connect(host, port)
        if user:
            smtp.login(user, pwd)
        if multiple:
            for d in dest:
                del msg['To']
                msg['To'] = d
                logging.info('Sending mail to: %s', d)
                smtp.sendmail(sender, d, msg.as_string())
        else:
            logging.info('Sending mail to: %s, Cc: %s', dest, cc)
            smtp.sendmail(sender, dest + cc, msg.as_string())
        smtp.quit()
Beispiel #7
0
        connection.request(self.method, self.url, self.body, headers)
        response = connection.getresponse()

        if response.status == self.expectedCode:
            cookie_store.add_cookies(response.getheader('Set-Cookie'))

            if self.next_request:
                response.read()
                return self.next_request.process(connection, cookie_store,
                                                 self.url)
            else:
                return response.read()
        else:
            raise FailedException(self, response.status, response.reason,
                                  response.read())


request = ChainedHttpRequest('GET', '/' + dico + '/' + word, httplib.OK)
request = ChainedHttpRequest('GET', '/', httplib.OK, request)

store = CookieStore()

connection = httplib.HTTPConnection('www.wordreference.com')
connection.set_debuglevel(9)

res = request.process(connection, store, 'http://www.wordreference.com/')

parser = WordReferenceHTMLParser()
parser.feed(res)
     sys.exit(2)
 else: 
     url = options.url
 #Reading Configurations
 readConfig()
 #Checking Arguments      
 url,port = parseURL(url)
 url_final = BASE_URL % (url,port)    
 #Downloading HTML file
 html_response = getHTML(url_final)
 if html_response == '' or 'Axis2: Services' not in html_response:
     rc = RC_CRIT
     msg = "There is no Configuration in URL or is not a service "        
 else:
     parser = MyHTMLParser()    
     parser.feed(html_response)
     msg = ""
     rc = RC_UNKNOWN
     for service in SERVICES:            
         if not checkService(service):            
             rc = RC_CRIT            
             msg += " %s not in configuration " % (service)
     if msg == "":
         rc = RC_OK
         msg = "All Services are up"
 exit(rc,msg)
 
 
 
 
 
Beispiel #9
0
	options.path = '/'+options.path


conn = httplib.HTTPConnection("localhost:29392")
conn.request(method,options.action+options.path)

resp = conn.getresponse()

data = resp.read()

if options.action == '/download':
	print sys.getsizeof(data)
	exit()

parser = ET.XMLTreeBuilder()
parser.feed(data)

tree = ET.ElementTree(parser.close())

root = tree.getroot()
#print root.text
ET.dump(tree)

if options.action == '/list' and method == "GET":
	print "File listing: "
	file_listing(options.path, root)

if options.action == '/tagdata' and method == "GET":
	print "Tag Data: "
	tag_data("/", root, options)
st=False
for post in posts:
    if st:break
    post['author']=authors[post['author']]['name']
    code=SWG_CODE
    for k,v in post.items():
        code=code.replace("%"+k+"_I%",str(v))
    if o.SAVEIMG:
        if o.WWW==None:
            print "[X] -w REQUIRED"
            st=True
            break
        else:
            parser=ParseImages()
            parser.feed(code)
            parser.close()
            for img in parser.getImgs():
                dl=o.WWW+"/"+img if not o.WWW in img else img
                name=str(time.time())+"_import"+os.path.splitext(img)[1]
                dest="images/"+name
                code=code.replace(img,"/"+dest)
                print "[!] Downloading "+dl+" in "+dest+"..."
                urllib.urlretrieve(dl,dest)
                print "[O] Done."
    nc=""
    i=0
    for c in code:
        nc+=c
        i+=1
        if i==o.BREAK:
Beispiel #11
0
            start_time = timeit.default_timer()

            # Get the html for the specific page
            data = getPageNum(pageNum)

            # The html page returns data in unicode, which the HTML Parser can't handle
            # replace all unknown characters and convert to ascii.
            data = data.decode('ascii', 'replace')
            dbQueryCurlTime = dbQueryCurlTime + timeit.default_timer(
            ) - start_time

            start_time = timeit.default_timer()

            # Parse the contents of the received HTML page
            parser = MyHTMLParser()
            parser.feed(data)

            # Keep track of the stats from the html parsing
            newAds = parser.getNewAds()
            lastExistingAds = parser.getLastExistingAds()
            totalNewAds = totalNewAds + newAds
            pageNum = pageNum + 1
            parser.adReset()
            print "We got [%d] new ads" % (newAds)

            dbQueryParseTime = dbQueryParseTime + timeit.default_timer(
            ) - start_time

    # Option to read database and return new ads
    if options.readDb:
        start_time = timeit.default_timer()
Beispiel #12
0
Logger.info('Starting MX validation for %s'%(options.server))
page = mechanize.Browser()
page.open(base_url)

assert page.viewing_html()

# Fills form and submit
page.select_form(nr=0)
page["IP"] = options.server
response = page.submit()

if debug : Logger.debug("Getting data")
# Get result page
html = response.get_data()
   
# Parse results and retrieve images url
parser = DnsblParser(base_url)
if debug : Logger.debug('Feeding')
parser.feed(html)

# Check MX reputation
status = BlValidator.validate_parallel(parser.bl_list)

# Prints out result
if status :
    sys.stdout.write('OK\n')
    sys.exit(0)
else :
    sys.stdout.write('ERROR, check %s\n' % (base_url))
    sys.exit(2)
Beispiel #13
0
Logger.info('Starting MX validation for %s' % (options.server))
page = mechanize.Browser()
page.open(base_url)

assert page.viewing_html()

# Fills form and submit
page.select_form(nr=0)
page["IP"] = options.server
response = page.submit()

if debug: Logger.debug("Getting data")
# Get result page
html = response.get_data()

# Parse results and retrieve images url
parser = DnsblParser(base_url)
if debug: Logger.debug('Feeding')
parser.feed(html)

# Check MX reputation
status = BlValidator.validate_parallel(parser.bl_list)

# Prints out result
if status:
    sys.stdout.write('OK\n')
    sys.exit(0)
else:
    sys.stdout.write('ERROR, check %s\n' % (base_url))
    sys.exit(2)
Beispiel #14
0
    def run(self):
        """ Main entry point """

        # Read command line
        usage = "%prog [options] <url> <address> [<address2>] [<address...>]"
        parser = OptionParser(usage=usage, version=self.NAME + ' ' + self.VERSION)
        parser.add_option("--http-user", dest="http_user",
            help="Username for HTTP POST authentication")
        parser.add_option("--http-pass", dest="http_pass",
            help="Password for HTTP POST authentication")
        parser.add_option("-s", "--smtp", dest="smtp",
            help="SMTP server address. Default: localhost",
            default='localhost')
        parser.add_option("--smtp-user", dest="smtp_user",
            help="Username for SMTP authentication")
        parser.add_option("--smtp-pass", dest="smtp_pass",
            help="Password for SMTP authentication")
        parser.add_option("-c", "--cc", dest="cc",
            help="Carbon Copy recipient")
        parser.add_option("-f", "--from", dest="sender",
            help="eMail sender. Default: emailer@localhost",
            default="emailer@localhost")
        parser.add_option("-j", "--subject", dest="subject",
            help="eMail Subject. Default: MailWWW Autogenerated Mail",
            default="MailWWW Autogenerated Mail")
        parser.add_option("-n", "--no-css", dest="nocss",
            help="Disable embedding of linked Style Sheets",
            default=False, action="store_true")
        parser.add_option("-m", "--multiple", dest="multiple",
            help="Send multiple emails: one for each recipient (Cc field is ignored)",
            default=False, action="store_true")
        parser.add_option("-v", "--verbose", dest="verbose",
            help="Show progress information",
            default=False, action="store_true")

        (options, args) = parser.parse_args()

        # Parse mandatory arguments
        if len(args) < 2:
            parser.error("unvalid number of arguments")
        dest = []
        i = 0
        for a in args:
            if i == 0:
                url = a
            else:
                dest.append(a)
            i += 1

        # Parse optional arguments
        http_user = options.http_user
        http_pass = options.http_pass
        cc = []
        if options.cc:
            cc.append(options.cc)
        host = options.smtp
        port = 25
        user = options.smtp_user
        pwd = options.smtp_pass
        sender = options.sender
        subject = options.subject
        nocss = options.nocss
        multiple = options.multiple
        verbose = options.verbose

        logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)

        # Opens URL
        logging.info('Fetching url %s', url)
        data = None
        if http_user or http_pass:
            # Use POST authentication
            data = urllib.urlencode({ 'username': http_user, 'password': http_pass, 'login': True })
        f = urllib.urlopen(url, data)
        html = f.read()
        # Search for meta content-type tag, use this encoding when found
        encre = re.compile(r'<meta\s+http-equiv=(?:"|\')Content-Type(?:"|\')\s+content=(?:"|\')([^\'"]*)(?:"|\')\s*/>',
            re.I | re.M)
        match = encre.search(html)
        if match:
            encoding = self.__parseEncoding(match.group(1))
            try:
                html = unicode(html, encoding, errors='replace')
            except LookupError as e:
                encoding = self.__parseEncoding(f.headers['content-type'])
                html = unicode(html, encoding, errors='replace')
        else:
            encoding = self.__parseEncoding(f.headers['content-type'])
            html = unicode(html, encoding, errors='replace')
        logging.info('Detected charset: %s', encoding)
        f.close()

        # Retrieve linked style sheets
        if not nocss:
            logging.info('Fetching Style Sheets...')
            parser = CSSLister(url)
            parser.feed(html)
            parser.close()
            for search, replace in parser.get_replacements():
                html = html.replace(search, replace, 1)

        # Prepare mail
        msg = MIMEMultipart()
        msg['Date'] = formatdate(localtime=True)
        msg['Message-ID'] = make_msgid('emailer')
        msg['Subject'] = subject
        msg['From'] = sender

        if cc and not multiple:
            msg['Cc'] = ', '.join(cc)
        msg.preamble = 'This is a milti-part message in MIME format.'

        txt = MIMEText(html.encode('utf-8'), 'html', 'utf-8')
        msg.attach(txt)

        if not multiple:
            msg['To'] = ', '.join(dest)

        # Sends message
        smtp = smtplib.SMTP()
        smtp.connect(host, port)
        if user:
            smtp.login(user, pwd)
        if multiple:
            for d in dest:
                del msg['To']
                msg['To'] = d
                logging.info('Sending mail to: %s', d)
                smtp.sendmail(sender, d, msg.as_string())
        else:
            logging.info('Sending mail to: %s, Cc: %s', dest, cc)
            smtp.sendmail(sender, dest+cc, msg.as_string())
        smtp.quit()