def main(): import optparse from sonet.lib import SonetOption p = optparse.OptionParser( usage="usage: %prog [options] input_file dictionary output_file", option_class=SonetOption ) p.add_option('-v', action="store_true", dest="verbose", default=False, help="Verbose output (like timings)") p.add_option('-T', "--timeout", action="store", dest="timeout", type=float, default=0.5, help="Diff timeout (default=0.5, 0=no timeout)") p.add_option('-c', '--clean', action="store_true", dest="clean", default=False, help="Cleans HTML, wiki syntax, acronyms and emoticons") p.add_option('-S', '--detailed-start', action="store", dest='detailed_start', type="yyyymmdd", metavar="YYYYMMDD", default=None, help="Detailed output start date") p.add_option('-E', '--detailed-end', action="store", dest='detailed_end', type="yyyymmdd", metavar="YYYYMMDD", default=None, help="Detailed output end date") p.add_option('-n', '--detailed-namespace', action="store", dest="detailed_ns", default="Normal", help="Namespace of desired detailed data (default: Normal)") opts, files = p.parse_args() if len(files) != 3: p.error("Wrong parameters") if opts.verbose: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') xml = files[0] dic = files[1] output = files[2] dumps_checker(xml) lang, _, _ = explode_dump_filename(xml) deflate, _lineno = lib.find_open_for_this_file(xml) if _lineno: src = deflate(xml, 51) else: src = deflate(xml) translation = get_translations(src) tag = get_tags(src, tags=('page,title,revision,timestamp,text,redirect,' 'contributor,username,ip')) namespaces = [x[1] for x in [(0, "Normal")] + mwlib.get_namespaces(src)] src.close() src = deflate(xml) if os.path.exists(output): logging.error("File %s already exists!", output) sys.exit(0) out = open(output, 'w') processor = PyWCProcessor(tag=tag, lang=lang, dic=dic, output=out, userns=translation['User']) processor.namespaces = namespaces processor.diff_timeout = opts.timeout processor.clean = opts.clean processor.pywc.clean_wiki = processor.pywc.clean_html = opts.clean if opts.detailed_start and opts.detailed_end: print """ You are going to run the script with detailed output on %d days. This is going to produce some CSV files on your disk, one for each day. Is this want you really want to do? [press enter to continue] """ % (opts.detailed_end - opts.detailed_start).days raw_input() processor.pywc.detailed = True processor.detailed_start = opts.detailed_start processor.detailed_end = opts.detailed_end processor.detailed_ns = opts.detailed_ns with Timr('Processing'): processor.start(src) # PROCESSING processor.flush() out.close()
def main(): import optparse p = optparse.OptionParser( usage="usage: %prog [options] input_file dictionary output_file") p.add_option('-t', '--type', action="store", dest="type", default="all", help="Type of page to analize (content|talk|all)") p.add_option('-e', '--encoding', action="store", dest="encoding", default="latin-1", help="encoding of the desired_list file") p.add_option('-v', action="store_true", dest="verbose", default=False, help="Verbose output (like timings)") p.add_option('-T', "--timeout", action="store", dest="timeout", type=float, default=0.5, help="Diff timeout (default=0.5, 0=no timeout)") p.add_option('-c', '--clean', action="store_true", dest="clean", default=False, help="Cleans HTML, wiki syntax, acronyms and emoticons") p.add_option('-C', '--charlimit', action="store", dest="charlimit", type="int", default=100000, help="Maximim characters per line (default=100000)") p.add_option('-r', action="store_true", dest="regex", default=False, help="Use a dictionary composed by regex (default=false)") opts, files = p.parse_args() if len(files) != 3: p.error("Wrong parameters") if opts.verbose: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') xml = files[0] dic = files[1] output = files[2] dumps_checker(xml) lang, _, _ = explode_dump_filename(xml) deflate, _lineno = lib.find_open_for_this_file(xml) if _lineno: src = deflate(xml, 51) else: src = deflate(xml) translation = get_translations(src) tag = get_tags(src, tags='page,title,revision,timestamp,text,redirect') namespaces = [x[1] for x in [(0, "Normal")] + mwlib.get_namespaces(src)] src.close() src = deflate(xml) out = open(output, 'w') processor = PyWCProcessor(tag=tag, lang=lang, dic=dic, output=out, userns=translation['User']) processor.namespaces = namespaces if opts.type == 'talk': processor.get_articles = False elif opts.type == 'content': processor.get_talks = False processor.diff_timeout = opts.timeout processor.clean = opts.clean processor.pywc.clean_wiki = processor.pywc.clean_html = opts.clean with Timr('Processing'): processor.start(src) ## PROCESSING processor.flush() out.close()