예제 #1
0
파일: process.py 프로젝트: embr/nonce
def parse_args():

    parser = SquidArgumentParser(description='Process a collection of squid logs and write certain extracted metrics to file')
    parser.add_argument('providers', 
                        metavar='PROVIDER_IDENTIFIER',
                        nargs='*',
                        default=DEFAULT_PROVIDERS,
                        help='list of provider identifiers used in squid log file names')
    parser.add_argument('--name_format',
                        dest='name_format',
                        type=str,
                        default='%s.log-%.gz',
                        help='a printf style format string which is formatted with the tuple: (provider_name, date_representation')
    parser.set_defaults(datadir='/a/squid/archive/zero')


    args = parser.parse_args()
    # custom logic for which files to grab
    prov_files = {}
    for prov in args.providers:
        args.basename = prov
        logging.info('args prior to ge_files: %s', pprint.pformat(args.__dict__))
        prov_files[prov] = SquidArgumentParser.get_files(args)
    setattr(args, 'squid_files', prov_files)

    
    logging.info(pprint.pformat(args.__dict__))
    return args
예제 #2
0
#!/usr/bin/python
from squid import count_files, write_counts
from squid.util import SquidArgumentParser
import logging
import pprint

logger = logging.getLogger(__name__)

if __name__ == '__main__':
    parser = SquidArgumentParser()
    parser.set_defaults(basename='zero-.*', datadir='/a/squid/archive/zero')
    args = parser.parse_args()
    logger.info(pprint.pformat(vars(args)))


    criteria = [
            lambda r : r.site() in ['M', 'Z'],
            lambda r : r.old_init_request(),
            lambda r : r.project == 'wikipedia',
            ]

    fields = ['date', 'language', 'project', 'site', 'na', 'provider_from_file']
    
    counts = count_files(args.squid_files,
            fields,
            criteria,
            nproc=10,
            limit=args.max_lines,
            fname='carrier_counts_cidr_all.incremental.csv')

    write_counts(counts, 'carrier_counts_cidr_all.counts.csv')