コード例 #1
0
## Clean the ipc codes to match those in the PATSTAT output
ipc_clean = psCleanup.ipc_clean(green_ipcs['ipc'])
green_ipcs['ipc'] = ipc_clean
del ipc_clean

## Categorize at the top level the IPC codes
green_energy_cats = {}
for idx, d in enumerate(green_ipcs.l1):
    if d in green_energy_cats:
        green_energy_cats[d].append(green_ipcs.ipc[idx])
    else:
        green_energy_cats[d] = [green_ipcs.ipc[idx]]

## Translate the ipc codes into regex for searching
cat_regex = psCleanup.make_regex(green_energy_cats)

## Clean the data

country_files = os.listdir('./data')
country_files = [f for f in country_files if 'tsv' in f]

## Drop the -no country- file, very very large (32m rows)
country_files = [f for f in country_files if ' ' not in f]


for idx, f in enumerate(country_files):
   
    filename = './data/cleaned_data/' + f
    print 'Counting patents in ' + filename
コード例 #2
0
with dview.sync_imports():
    import psCleanup
    import psDisambig


## Define the cleaning dicts
all_dicts = [psCleanup.convert_html,
             psCleanup.convert_sgml,
             psCleanup.clean_symbols,
             psCleanup.concatenators,
             psCleanup.single_space,
             psCleanup.ampersand,
             psCleanup.us_uk,
             psCleanup.abbreviations
             ]
regex_dicts = [psCleanup.make_regex(d) for d in all_dicts]
dview.push({'all_dicts': all_dicts})
dview.push({'regex_dicts': regex_dicts})
## Wrap the clean sequence in a useful function
@dview.parallel(block=True)
def clean_wrapper(name_string, dict_list=regex_dicts):
    print name_string
    out = psCleanup.rem_diacritics(name_string)
    out = psCleanup.stdize_case(out)
    out = psCleanup.master_clean_regex([out], dict_list)
    out = out[0].strip()
    return(out)

def clean_list_wrapper(name_list, dict_list=all_dicts):
    out = [psCleanup.rem_diacritics(n) for n in name_list]
    out = [psCleanup.stdize_case(n) for n in out]
コード例 #3
0
     par_client = Client()
     dview = par_client[:]
     dview.block = True

     # Sync the necessary imports and path settings
     dview.execute('import sys')
     dview.execute('sys.path.append("/home/markhuberty/Documents/psClean/code")')

     with dview.sync_imports():
          import psCleanup

     dview.push({'name_address_dict_list': name_address_dict_list})
     dview.push({'coauth_dict_list': coauth_dict_list})
     dview.push({'legal_regex': legal_regex})

     name_address_regex = [psCleanup.make_regex(d) for d in name_address_dict_list]
     coauth_regex = [psCleanup.make_regex(d) for d in coauth_dict_list]

     dview.push({'name_address_regex': name_address_regex})
     dview.push({'coauth_dict_list': coauth_regex})

     # Set up parallel cleaning wrappers
     @dview.parallel(block=True)
     def name_clean_wrapper(name_list, clean_regex=name_address_regex, legal_regex=legal_regex):
          name_string = psCleanup.decoder(name_list)
          name_string = psCleanup.remove_diacritics(name_string)
          name_string = psCleanup.stdize_case(name_string)
          name_string = psCleanup.master_clean_regex(name_string, clean_regex)
          names_ids = psCleanup.get_legal_ids(name_string, legal_regex)
          return names_ids
     
コード例 #4
0
import numpy as np
import scipy.sparse as sp
import csv
import time

## Define the cleaning dicts
all_dicts = [psCleanup.convert_html,
             psCleanup.convert_sgml,
             psCleanup.clean_symbols,
             psCleanup.concatenators,
             psCleanup.single_space,
             psCleanup.ampersand,
             psCleanup.us_uk,
             psCleanup.abbreviations
             ]
all_regex = [psCleanup.make_regex(d) for d in all_dicts]
## Wrap the clean sequence in a useful function
def clean_wrapper(name_string, dict_list):
    out = psCleanup.rem_diacritics(name_string)
    out = psCleanup.stdize_case(out)
    out = psCleanup.master_clean_regex([out], dict_list)
    out = out[0].strip()
    return(out)

initial_query = """
SELECT person_id, person_name from (tls201_appln INNER JOIN
tls207_pers_appln USING(appln_id)) INNER JOIN
tls206_person USING(person_id)
WHERE YEAR(appln_filing_date) >= 1980
"""
コード例 #5
0
## Clean the ipc codes to match those in the PATSTAT output
ipc_clean = psCleanup.ipc_clean(green_ipcs['ipc'])
green_ipcs['ipc'] = ipc_clean
del ipc_clean

## Categorize at the top level the IPC codes
green_energy_cats = {}
for idx, d in enumerate(green_ipcs.l1):
    if d in green_energy_cats:
        green_energy_cats[d].append(green_ipcs.ipc[idx])
    else:
        green_energy_cats[d] = [green_ipcs.ipc[idx]]

## Translate the ipc codes into regex for searching
cat_regex = psCleanup.make_regex(green_energy_cats)

## Clean the data

country_files = os.listdir('./code')
country_files = [f for f in country_files if 'tsv' in f]

## Drop the -no country- file, very very large (32m rows)
country_files = [f for f in country_files if ' ' not in f]

for idx, f in enumerate(country_files):
    input_filename = './code/' + f
    output_filename = './data/cleaned_data/' + f

    print 'Operating on ' + input_filename
コード例 #6
0
    dview.block = True

    # Sync the necessary imports and path settings
    dview.execute('import sys')
    dview.execute(
        'sys.path.append("/home/markhuberty/Documents/psClean/code")')

    with dview.sync_imports():
        import psCleanup

    dview.push({'name_address_dict_list': name_address_dict_list})
    dview.push({'coauth_dict_list': coauth_dict_list})
    dview.push({'legal_regex': legal_regex})

    name_address_regex = [
        psCleanup.make_regex(d) for d in name_address_dict_list
    ]
    coauth_regex = [psCleanup.make_regex(d) for d in coauth_dict_list]

    dview.push({'name_address_regex': name_address_regex})
    dview.push({'coauth_dict_list': coauth_regex})

    # Set up parallel cleaning wrappers
    @dview.parallel(block=True)
    def name_clean_wrapper(name_list,
                           clean_regex=name_address_regex,
                           legal_regex=legal_regex):
        name_string = psCleanup.decoder(name_list)
        name_string = psCleanup.remove_diacritics(name_string)
        name_string = psCleanup.stdize_case(name_string)
        name_string = psCleanup.master_clean_regex(name_string, clean_regex)