#check the data for every patent that shows up in the log

# maybe the patent after them too

# or buffer the patn search

#BUGBUG look into pickling again (highest protocol slowest? cause it's slow as **fuck**)

#fewer lines in poolsize

#nPatnsEachIPC-A needs labels -- cumhits!

#make a bunch of the perQuarter graphs peryear

# esp. nisdPerQuarter

#make dictQ FIFO to make auditting easier

# -> figure out why totals aren't right!

#ExpIPC w/uspc

#remake act figs with coloring reflecting degree of separation

# Notes:

# to double-check counts in xml files do something like:

# grep -c '^<us-patent-grant .* file="US0[0-9]*-' *.xml

# leaving off the us-patent-grant bit finds image tags as well

# nPatns found in files 1976.dat through ipgb20101228.xml

# grep '^WKU \+[0-9]' *.dat | wc -l

# 2892360

# grep '^<us-patent-grant.*file="US0' *.xml | wc -l

# 1020707

# = 3913067

# open `svn st | gawk '{printf("%s ",$2)}'`

# todo:

# fix live re-running, can't be copying patns to each new process

# look into using split and re.split for the file parsing

# look at using SAX for the XML

# look at controlling swapping out

# talk with Noah re: old stuff

import os, time, logging, datetime, inspect

import multiprocessing, Queue

import DATParser, XMLParser, Patent

from numpy import array, unique

fnLog = 'patents.log'

frOutputData = 'html/data/'

logFormat = "%(asctime)s %(levelname)s %(processName)s\t%(message)s"

if not 'patns' in dir(): # assume this is first time running

# don't overwrite patns!

patns = dict()

# no need to re-run logging config

logging.basicConfig(filename=fnLog, level=logging.NOTSET, format = logFormat)

# purposely leave errorQ untouched on rerun, ditto filelists

fileQ = multiprocessing.JoinableQueue()

dictQ = multiprocessing.JoinableQueue()

errorQ = multiprocessing.JoinableQueue()

# xmlfilelist = [XMLParser.fr + x for x in os.listdir(XMLParser.fr) if x[-4:] == '.xml']

# datfilelist = [DATParser.fr + x for x in os.listdir(DATParser.fr) if x[-4:] == '.dat']

xmlfilelist = []

# DATParser.fr = './'

# datfilelist = [DATParser.fr + x for x in ['2000.dat', '2001.dat']]

datfilelist = [DATParser.fr + x for x in ['1976.dat', '1977.dat', '1978.dat', '1979.dat', '1980.dat', '1981.dat', '1982.dat', '1983.dat', '1984.dat', '1985.dat', '1986.dat', '1987.dat', '1988.dat', '1989.dat', '1990.dat', '1991.dat', '1992.dat', '1993.dat', '1994.dat', '1995.dat', '1996.dat', '1997.dat', '1998.dat', '1999.dat', '2000.dat', '2001.dat']]

else:

logging.info('Found existing patns dict, continuing')

# because having trouble with exec('checkCoverage.py')...

def checkCoverage(patns):

return missing

def loadPatnFiles(patns, fl):

logging.info("done reading files after %.2f minutes.", (time.time()-tStart)/60)

def sanityCheck(patns):

# <main-group/> : 89 from 2005 - 20101228

def populateCites(patns):

logging.info("done after %.2f minutes.", (time.time()-tStart)/60)

logging.info("-------------------------------------")

logging.info("$Id: readPatnsFromFiles.py 293 2011-04-04 20:45:34Z andy $")

#xmlfilelist = [frXML + 'ipgb20081104.xml', frXML + 'ipgb20091215.xml']

#datfilelist = [datfilelist[-1]]

#xmlfilelist = ['test.xml']

#datfilelist = ['test.dat']

print 'loading...'

loadPatnFiles(patns, xmlfilelist + datfilelist)

print 'sanity...'

sanityCheck(patns)

print 'cites...'

populateCites(patns)

#########################################################

# Load the tfidf stuff

import sys

sys.path = sys.path+['../gensim/']

from gensim.corpora import Dictionary,MmCorpus

import pickle as pk

logging.info("-------------------------------------")

logging.info("loading dictionary and corpus")

tStart=time.time()

dic = Dictionary.load('trim1000.dic')

corp = MmCorpus('tfidftrim1kall__tfidf.mm')

logging.info("done after %.2f minutes.", (time.time()-tStart)/60.0)

logging.info("loading patnums")

tStart=time.time()

patnums = pk.load(open('patnums.pickle'))

patnums = array(patnums)

logging.info("done after %.2f minutes.", (time.time()-tStart)/60.0)

logging.info("putting the tfidf info into the patns...")

tStart=time.time()

i = -1

cntgot=0

cntnotgot = 0

newkeys = []

for k in patnums:

i += 1

try:

patns[k]

patns[k].tfidf = corp[i]

cntgot +=1

newkeys.append(k)

except:

cntnotgot += 1

logging.info("done after %.2f minutes.", (time.time()-tStart)/60)

logging.info("got %d, skipped %d not in patns or without tfidf",cntgot, cntnotgot)

# <codecell>

logging.info("%d duplicates keys",len(newkeys)-len(unique(newkeys)))

newkeys = unique(newkeys)

logging.info("%d unique keys stored in newkeys",len(newkeys))