Exemplo n.º 1
0
def main(year, doctype='grant'):
    # get patents as iterator to save memory
    # use subqueryload to get better performance by using less queries on the backend:
    # --> http://docs.sqlalchemy.org/en/latest/orm/tutorial.html#eager-loading
    session = alchemy.fetch_session(dbtype=doctype)
    schema = alchemy.schema.Patent
    if doctype == 'application':
        schema = alchemy.schema.App_Application
    if year:
        patents = (p for p in session.query(schema).filter(extract('year', schema.date) == gyear).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('classes')).yield_per(1))
    else:
        patents = (p for p in session.query(schema).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('classes')).yield_per(1))
    i = 0
    for patent in patents:
        i += 1
        if i % 100000 == 0:
          print i, datetime.now()
        try:
          # create common dict for this patent
          loc = patent.rawinventors[0].rawlocation
          mainclass = patent.classes[0].mainclass_id if patent.classes else ''
          subclass = patent.classes[0].subclass_id if patent.classes else ''
          row = {'number': patent.number,
                 'mainclass': mainclass,
                 'subclass': subclass,
                 'state': loc.state if loc else '',
                 'country': loc.country if loc else '',
                 'city': loc.city if loc else '',
                 }
          row['assignee'] = get_assignee_id(patent.assignees[0]) if patent.assignees else ''
          row['rawassignee'] = get_assignee_id(patent.rawassignees[0]) if patent.rawassignees else ''
          # generate a row for each of the inventors on a patent
          for ri in patent.rawinventors:
              namedict = {'name_first': ri.name_first, 'uuid': ri.uuid}
              raw_name = ri.name_last.split(' ')
              # name_last is the last space-delimited word. Middle name is everything before that
              name_middle, name_last = ' '.join(raw_name[:-1]), raw_name[-1]
              namedict['name_middle'] = name_middle
              namedict['name_last'] = name_last
              tmprow = row.copy()
              tmprow.update(namedict)
              newrow = normalize_utf8(ROW(tmprow))
              with codecs.open('disambiguator.csv', 'a', encoding='utf-8') as csv:
                  csv.write(newrow)
        except Exception as e:
          print e
          continue
Exemplo n.º 2
0
def main(year, doctype):
    # get patents as iterator to save memory
    # use subqueryload to get better performance by using less queries on the backend:
    # --> http://docs.sqlalchemy.org/en/latest/orm/tutorial.html#eager-loading
    session = alchemy.fetch_session(dbtype=doctype)
    schema = alchemy.schema.Patent
    if doctype == 'application':
        schema = alchemy.schema.App_Application
    if year:
        patents = (p for p in session.query(schema).filter(extract('year', schema.date) == year).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('classes')).yield_per(1))
    else:
        patents = (p for p in session.query(schema).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('classes')).yield_per(1))
    i = 0
    for patent in patents:
        i += 1
        if i % 100000 == 0:
          print i, datetime.now()
        try:
          # create common dict for this patent
          primrawloc = patent.rawinventors[0].rawlocation
          if primrawloc:
            primloc = patent.rawinventors[0].rawlocation.location
          else:
            primloc = primrawloc
          mainclass = patent.classes[0].mainclass_id if patent.classes else ''
          subclass = patent.classes[0].subclass_id if patent.classes else ''
          row = {'number': patent.id,
                 'mainclass': mainclass,
                 'subclass': subclass,
                 'ignore': 0,
                 }
          if doctype == 'grant':
            row['isgrant'] = 1
          elif doctype == 'application':
            row['isgrant'] = 0
            if patent.granted == True:
              row['ignore'] = 1
          row['assignee'] = get_cleanid(patent.rawassignees[0]) if patent.rawassignees else ''
          row['assignee'] = row['assignee'].split('\t')[0]
          row['rawassignee'] = get_cleanid(patent.rawassignees[0]) if patent.rawassignees else ''
          row['rawassignee'] = row['rawassignee'].split('\t')[0]
          # generate a row for each of the inventors on a patent
          for ri in patent.rawinventors:
              if not len(ri.name_first.strip()):
                  continue
              namedict = {'name_first': ri.name_first, 'uuid': ri.uuid}
              raw_name = ri.name_last.split(' ')
              # name_last is the last space-delimited word. Middle name is everything before that
              name_middle, name_last = ' '.join(raw_name[:-1]), raw_name[-1]
              namedict['name_middle'] = name_middle
              namedict['name_last'] = name_last
              rawloc = ri.rawlocation
              if rawloc:
                if rawloc.location:
                  loc = rawloc.location
                else:
                  loc = primloc
              else:
                loc = primloc
              namedict['state'] = loc.state if loc else ''# if loc else rawloc.state if rawloc else primloc.state if primloc else ''
              namedict['country'] = loc.country if loc else ''# if loc else rawloc.country if rawloc else primloc.country if primloc else ''
              namedict['city'] = loc.city if loc else ''# if loc else rawloc.city if rawloc else primloc.city if primloc else ''
              if '??' in namedict['state'] or len(namedict['state']) == 0:
                namedict['state'] = rawloc.state if rawloc else primloc.state if primloc else ''
              if '??' in namedict['country'] or len(namedict['country']) == 0:
                namedict['country'] = rawloc.country if rawloc else primloc.country if primloc else ''
              if '??' in namedict['city'] or len(namedict['city']) == 0:
                namedict['city'] = rawloc.city if rawloc else primloc.city if primloc else ''
              tmprow = row.copy()
              tmprow.update(namedict)
              newrow = normalize_utf8(ROW(tmprow))
              with codecs.open('disambiguator.csv', 'a', encoding='utf-8') as csv:
                  csv.write(newrow)
        except Exception as e:
          print e
          continue
Exemplo n.º 3
0
def main(year, doctype):
    # get patents as iterator to save memory
    # use subqueryload to get better performance by using less queries on the backend:
    # --> http://docs.sqlalchemy.org/en/latest/orm/tutorial.html#eager-loading
    session = alchemy.fetch_session(dbtype=doctype)
    schema = alchemy.schema.Patent
    if doctype == 'application':
        schema = alchemy.schema.App_Application
        if year:
            patents = (p for p in session.query(schema).filter(extract('year', schema.date) == year).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('classes')).yield_per(1))
        else:
            patents = (p for p in session.query(schema).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('classes')).yield_per(1))
    else:
        if year:
            patents = (p for p in session.query(schema).filter(extract('year', schema.date) == year).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('current_classes')).yield_per(1))
        else:
            patents = (p for p in session.query(schema).options(subqueryload('rawinventors'), subqueryload('rawassignees'), subqueryload('current_classes')).yield_per(1))
    i = 0
    for patent in patents:
        i += 1
        if i % 100000 == 0:
          print i, datetime.now()
        try:
          # create common dict for this patent
          primrawloc = patent.rawinventors[0].rawlocation
          if primrawloc:
            primloc = patent.rawinventors[0].rawlocation.location
          else:
            primloc = primrawloc
          if doctype == 'application':
            mainclass = patent.classes[0].mainclass_id if patent.classes else ''
            subclass = patent.classes[0].subclass_id if patent.classes else ''
          else:
            mainclass = patent.current_classes[0].mainclass_id if patent.current_classes else ''
            subclass = patent.current_classes[0].subclass_id if patent.current_classes else ''
          row = {'number': patent.id,
                 'mainclass': mainclass,
                 'subclass': subclass,
                 'ignore': 0,
                 }
          if doctype == 'grant':
            row['isgrant'] = 1
          elif doctype == 'application':
            row['isgrant'] = 0
            if patent.granted == True:
              row['ignore'] = 1
          row['assignee'] = get_cleanid(patent.rawassignees[0]) if patent.rawassignees else ''
          row['assignee'] = row['assignee'].split('\t')[0]
          row['rawassignee'] = get_cleanid(patent.rawassignees[0]) if patent.rawassignees else ''
          row['rawassignee'] = row['rawassignee'].split('\t')[0]
          # generate a row for each of the inventors on a patent
          for ri in patent.rawinventors:
              if not len(ri.name_first.strip()):
                  continue
              namedict = {'uuid': ri.uuid}
              parsedNames = name_parser.parse_name(name_parser.NameFormat.CITESEERX, ri.name_first + ' ' + ri.name_last)
              namedict['name_first'] = ' '.join(filter(None, (parsedNames.Prefix, parsedNames.GivenName)))
              namedict['name_middle'] = parsedNames.OtherName if parsedNames.OtherName is not None else ''
              namedict['name_last'] = ' '.join(filter(None, (parsedNames.FamilyName, parsedNames.Suffix)))

              rawloc = ri.rawlocation
              if rawloc:
                if rawloc.location:
                  loc = rawloc.location
                else:
                  loc = primloc
              else:
                loc = primloc
              namedict['state'] = loc.state if loc else ''# if loc else rawloc.state if rawloc else primloc.state if primloc else ''
              namedict['country'] = loc.country if loc else ''# if loc else rawloc.country if rawloc else primloc.country if primloc else ''
              namedict['city'] = loc.city if loc else ''# if loc else rawloc.city if rawloc else primloc.city if primloc else ''
              if '??' in namedict['state'] or len(namedict['state']) == 0:
                namedict['state'] = rawloc.state if rawloc else primloc.state if primloc else ''
              if '??' in namedict['country'] or len(namedict['country']) == 0:
                namedict['country'] = rawloc.country if rawloc else primloc.country if primloc else ''
              if '??' in namedict['city'] or len(namedict['city']) == 0:
                namedict['city'] = rawloc.city if rawloc else primloc.city if primloc else ''
              tmprow = row.copy()
              tmprow.update(namedict)
              newrow = normalize_utf8(ROW(tmprow))
              with codecs.open('disambiguator.csv', 'a', encoding='utf-8') as csv:
                  csv.write(newrow)
        except Exception as e:
          print e
          continue