def construct_sitcr2(df, data_type, level, AX=True, dropAX=True, sitcr2=True, drop_nonsitcr2=True, adjust_hk=(False, None), intertemp_productcode=(False, None), intertemp_cntrycode=False, drop_incp_cntrycode=False, adjust_units=False, source_institution='un', harmonised_raw=False, values_only=False, verbose=True):
"""
Construct a Self Contained (SC) Direct Action Dataset for Countries at the SITC Revision 2 Level 3
There are no checks on the incoming dataframe to ensure data integrity.
This is your responsibility
STATUS: tests/test_constructor_dataset_sitcr2l3.py
Parameters
----------
df : DataFrame
Pandas DataFrame containing the raw data
data_type : str
Specify what type of data 'trade', 'export', 'import'
level : int
Specify Level of Final dataset (i.e. SITC Level 1, 2, 3, or 4)
AX : bool, optional(default=True)
Add a Marker for Codes that Include 'A' and 'X'
dropAX : bool, optional(default=True)
Drop AX Codes at the Relevant Level (i.e. SITC Level 3 Data will include appropriate A and X codes)
sitcr2 : bool, optional(default=True)
Add SITCR2 Indicator
drop_nonsitcr2 : bool, optional(default=True)
Drop non-standard SITC2 Codes
adjust_hk : Tuple(bool, df), optional(default=(False, None))
Adjust the Hong Kong Data using NBER supplemental files which needs to be supplied as a dataframe
intertemp_productcode : Tuple(bool, dict), optional(default=False, None)
Apply an Intertemporal Product Code System drop a conversion dictionary (IC["drop"] = [], IC["collapse"] = [])
Note this will override the drop_nonsitcr2 option
intertemp_cntrycode : bool, optional(default=False)
Generate Intertemporal Consistent Country Units (from meta)
drop_incp_cntrycode : bool, optional(default=False)
Drop Incomplete Country Codes (from meta)
adjust_units : bool, optional(default=False)
Adjust units by a factor of 1000 to specify in $'s
source_institution : str, optional(default='un')
which institutions SITC classification to use
harmonised_raw : bool, optional(default=False)
Return simple RAW dataset with Quantity disaggregation collapsed and eiso3c and iiso3c columns (Note: You may use hk_adjust with this option)
values_only : bool, optional(default=False)
Return Values and Relevant Index Data Only (i.e. drop 'AX', 'sitcr2')
Notes
-----
1. Operations ::
[1] Adjust Hong Kong and China Data
[2] Drop SITC4 to SITC3 Level (for greater intertemporal consistency)
[3] Import ISO3C Codes as Country Codes
[4] Drop Errors in SITC3 codes ["" Codes]
Optional:
---------
[A] Drop sitc3 codes that contain 'A' and 'X' codes [Default: True]
[B] Drop Non-Standard SITC3 Codes [i.e. Aren't in the Classification]
[C] Construct an Intertemporal Product Code Classification and Adjust Dataset
[C] Adjust iiso3c, eiso3c country codes to be intertemporally consistent
[D] Drop countries with incomplete 'total' data across 1962 to 2000 (strict measure) [Identification Debatable]
3. This makes use of countryname_to_iso3c in the meta data subpackage
4. This method can be tested using /do/basic_sitc3_country_data.do
5. DropAX + Drop NonStandard SITC Rev 2 Codes still contains ~94-96% of the data found in the raw data
.. Future Work
-----------
1. Check SITC Revision 2 Official Codes
2. Add in a Year Filter
"""
#-Operations Requiring RAW SITC Level 4-#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
idx = [u'year', u'icode', u'importer', u'ecode', u'exporter', u'sitc4', u'unit', u'dot']
#-Hong Kong China Data Adjustment Option-#
if type(adjust_hk) == bool:
adjust_hk = (adjust_hk, None)
if adjust_hk[0]:
if verbose: print "[INFO] Adjusting Hong Kong and China Values"
hkdata = adjust_hk[1]
#-Values-#
raw_value = df[idx+['value']].rename(columns={'value' : 'value_raw'})
try:
adjust_value = hkdata[idx+['value_adj']]
except:
raise ValueError("[ERROR] China/Hong Kong Data has not been passed in properly!")
#-Note: Current merge_columns utility merges one column set at a time-#
df = merge_columns(raw_value, adjust_value, idx, collapse_columns=('value_raw', 'value_adj', 'value'), dominant='right', output='final', verbose=verbose)
#-Note: Adjust Quantity has not been implemented. See NBERWTF constructor -#
#-Filter Data-#
idx = [u'year', u'exporter', u'importer', u'sitc4'] #Note: This collapses duplicate entries with unit differences (collapse_valuesonly())
df = df.loc[:,idx + ['value']]
#-Raw Trade Data Option with Added IISO3C and EISO3C-#
if harmonised_raw and data_type == "trade":
df = df.groupby(idx).sum().reset_index() #Sum Over Quantity Disaggregations
#-Add EISO3C and IISO3C-#
df['eiso3c'] = df['exporter'].apply(lambda x: countryname_to_iso3c[x])
df['iiso3c'] = df['importer'].apply(lambda x: countryname_to_iso3c[x])
return df
if harmonised_raw and data_type in {"export", "import"}:
warnings.warn("Cannot run harmonised_raw over export and import data as raw data is trade data")
return None
#-Collapse to SITC Level -#
if level != 4:
if verbose: print "[INFO] Collapsing to SITC Level %s Data" % level
df['sitc%s'%level] = df.sitc4.apply(lambda x: x[0:level])
df = df.groupby(['year', 'exporter', 'importer', 'sitc%s'%level]).sum()['value'].reset_index()
elif level == 4:
if verbose: print "[INFO] Data is already at the requested level"
else:
raise ValueError("Level must be 1, 2, 3, or 4 for the NBER data")
#-Operations Post Collapse to SITC Level-#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#-Countries Only Adjustment-#
if verbose: print "[INFO] Removing 'World' values so that the dataset is country only data"
df = df.loc[(df.exporter != "World") & (df.importer != "World")]
#-Add Country ISO Information-#
#-Exports (can include NES on importer side)-#
if data_type == 'export' or data_type == 'exports':
if verbose: print "[INFO] Adding eiso3c using nber meta data"
df['eiso3c'] = df.exporter.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.eiso3c != '.')]
df = df.groupby(['year', 'eiso3c', 'sitc%s'%level]).sum()['value'].reset_index()
#-Imports (can include NES on importer side)-#
elif data_type == 'import' or data_type == 'imports':
if verbose: print "[INFO] Adding iiso3c using nber meta data"
df['iiso3c'] = df.importer.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.iiso3c != '.')]
df = df.groupby(['year','iiso3c', 'sitc%s'%level]).sum()['value'].reset_index()
#-Trade-#
else:
if verbose: print "[INFO] Adding eiso3c and iiso3c using nber meta data"
df['iiso3c'] = df.importer.apply(lambda x: countryname_to_iso3c[x])
df['eiso3c'] = df.exporter.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.iiso3c != '.') & (df.eiso3c != '.')]
df = df.groupby(['year', 'eiso3c', 'iiso3c', 'sitc%s'%level]).sum()['value'].reset_index()
#-Remove Product Code Errors in Dataset-#
df = df.loc[(df['sitc%s'%level] != "")] #Does this need a reset_index?
#-productcodes-#
if intertemp_productcode[0]:
if level == 1:
intertemp_productcode = (False, intertemp_productcode[1])
else:
AX = True
dropAX = True #Small Impact Post 1984 (Levels < 4 Include 'A' and 'X' values due to the collapse)
sitcr2 = True #Encode SITCR2 for Parsing
drop_nonsitcr2 = False
#-AX-#
if AX:
if verbose: print "[INFO] Adding Indicator Codes of 'A' and 'X'"
df['AX'] = df['sitc%s'%level].apply(lambda x: 1 if re.search("[AX]", x) else 0)
#-dropAX-#
if dropAX:
if verbose: print "[INFO] Dropping SITC Codes with 'A' or 'X'"
df = df.loc[df.AX != 1]
del df['AX']
if not dropAX and values_only:
del df['AX']
#-Intertemporal ProductCodes-#
if intertemp_productcode[0]:
if verbose: print "[INFO] Computing Intertemporally Consistent ProductCodes ..."
#-This Method relies on meta data computed by pyeconlab nberwtf constructor-#
IC = intertemp_productcode[1] #Dict("drop" and "collapse" code lists)
#-Drop Codes-#
drop_codes = IC["drop"]
if verbose:
print "Dropping the following productcodes ..."
print drop_codes
keep_codes = set(df['sitc%s'%level].unique()).difference(set(drop_codes))
df = df.loc[df["sitc%s"%level].isin(keep_codes)].copy(deep=True)
#-Collapse Codes-#
collapse_codes = IC["collapse"]
if verbose:
print "Collapsing the following productcodes ..."
print collapse_codes
collapse_codes = {x[0:level-1] for x in collapse_codes} #-Simplify Computations-#
for code in collapse_codes:
df["sitc%s"%level] = df["sitc%s"%level].apply(lambda x: code if x[0:level-1] == code else x)
#-Recodes-#
recodes = IC["recode"]
recode_codes = set(recodes.keys())
if verbose:
print "Recoding the following productcodes ..."
print recode_codes
for code in recode_codes:
df["sitc%s"%level] = df["sitc%s"%level].apply(lambda x: recodes[x] if x in recode_codes else x)
df = df.groupby(list(df.columns.drop("value"))).sum()
df = df.reset_index()
#-Official SITCR2 Codes-#
if sitcr2:
if verbose: print "[INFO] Adding SITCR2 Indicator"
sitc = SITC(revision=2, source_institution=source_institution)
codes = sitc.get_codes(level=level)
df['sitcr2'] = df['sitc%s'%level].apply(lambda x: 1 if x in codes else 0)
if drop_nonsitcr2:
if verbose: print "[INFO] Dropping Non Standard SITCR2 Codes"
df = df.loc[(df.sitcr2 == 1)]
del df['sitcr2'] #No Longer Needed
if not drop_nonsitcr2 and values_only:
del df['sitcr2']
#-Adjust Country Codes to be Intertemporally Consistent-#
if intertemp_cntrycode:
#-Export-#
if data_type == 'export' or data_type == 'exports':
if verbose: print "[INFO] Imposing dynamically consistent eiso3c recodes across 1962-2000"
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['eiso3c'] != '.']
df = df.groupby(['year', 'eiso3c', 'sitc%s'%level]).sum().reset_index()
#-Import-#
elif data_type == 'import' or data_type == 'imports':
if verbose: print "[INFO] Imposing dynamically consistent iiso3c recodes across 1962-2000"
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df.groupby(['year', 'iiso3c', 'sitc%s'%level]).sum().reset_index()
#-Trade-#
else:
if verbose: print "[INFO] Imposing dynamically consistent iiso3c and eiso3c recodes across 1962-2000"
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df[df['eiso3c'] != '.']
df = df.groupby(['year', 'eiso3c', 'iiso3c', 'sitc%s'%level]).sum().reset_index()
#-Drop Incomplete Country Codes-#
if drop_incp_cntrycode:
if verbose: print "[INFO] Dropping countries with incomplete data across 1962-2000"
#-Export-#
if data_type == 'export' or data_type == 'exports':
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['eiso3c'] != '.']
#-Import-#
elif data_type == 'import' or data_type == 'imports':
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
#-Trade-#
else:
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df[df['eiso3c'] != '.']
df = df.reset_index()
del df['index']
#-Adjust Units from 1000's to $'s-#
if adjust_units:
if verbose: print "[INFO] Adjusting 'value' units to $'s"
df['value'] = df['value']*1000 #Default: Keep in 1000's
#-Return Dataset-#
if verbose: print "[INFO] Finished Computing Dataset (%s) ..." % (data_type)
return df
def construct_sitcr2l3(df, data_type, dropAX=True, sitcr2=True, drop_nonsitcr2=True, adjust_hk=(False, None), intertemp_cntrycode=False, drop_incp_cntrycode=False, adjust_units=False, source_institution='un', verbose=True):
"""
Construct a Self Contained (SC) Direct Action Dataset for Countries at the SITC Revision 2 Level 3
There are no checks on the incoming dataframe to ensure data integrity.
This is your responsibility
STATUS: tests/test_constructor_dataset_sitcr2l3.py
Parameters
----------
df : DataFrame
Pandas DataFrame containing the raw data
data_type : str
Specify what type of data 'trade', 'export', 'import'
dropAX : bool, optional(default=True)
Drop AX Codes
sitcr2 : bool, optional(default=True)
Add SITCR2 Indicator
drop_nonsitcr2 : bool, optional(default=True)
Drop non-standard SITC2 Codes
adjust_hk : Tuple(bool, df), optional(default=(False, None))
Adjust the Hong Kong Data using NBER supplemental files which needs to be supplied as a dataframe
intertemp_cntrycode : bool, optional(default=False)
Generate Intertemporal Consistent Country Units (from meta)
drop_incp_cntrycode : bool, optional(default=False)
Drop Incomplete Country Codes (from meta)
adjust_units : bool, optional(default=False)
Adjust units by a factor of 1000 to specify in $'s
source_institution : str, optional(default='un')
which institutions SITC classification to use
Notes
-----
1. Operations ::
[1] Adjust Hong Kong and China Data
[2] Drop SITC4 to SITC3 Level (for greater intertemporal consistency)
[3] Import ISO3C Codes as Country Codes
[4] Drop Errors in SITC3 codes ["" Codes]
Optional:
---------
[A] Drop sitc3 codes that contain 'A' and 'X' codes [Default: True]
[B] Drop Non-Standard SITC3 Codes [i.e. Aren't in the Classification]
[C] Adjust iiso3c, eiso3c country codes to be intertemporally consistent
[D] Drop countries with incomplete data across 1962 to 2000 (strict measure)
3. This makes use of countryname_to_iso3c in the meta data subpackage
4. This method can be tested using /do/basic_sitc3_country_data.do
5. DropAX + Drop NonStandard SITC Rev 2 Codes still contains ~94-96% of the data found in the raw data
.. Future Work
-----------
1. Check SITC Revision 2 Official Codes
2. Add in a Year Filter
"""
#-Operations Requiring RAW SITC Level 4-#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
idx = [u'year', u'icode', u'importer', u'ecode', u'exporter', u'sitc4', u'unit', u'dot']
#-Hong Kong China Data Adjustment Option-#
if type(adjust_hk) == bool:
adjust_hk = (adjust_hk, None)
if adjust_hk[0]:
if verbose: print "[INFO] Adjusting Hong Kong and China Values"
hkdata = adjust_hk[1]
#-Values-#
raw_value = df[idx+['value']].rename(columns={'value' : 'value_raw'})
try:
adjust_value = hkdata[idx+['value_adj']]
except:
raise ValueError("[ERROR] China/Hong Kong Data has not been passed in properly!")
#-Note: Current merge_columns utility merges one column set at a time-#
df = merge_columns(raw_value, adjust_value, idx, collapse_columns=('value_raw', 'value_adj', 'value'), dominant='right', output='final', verbose=verbose)
#-Note: Adjust Quantity has not been implemented. See NBERWTF constructor -#
#-Filter Data-#
idx = ['year', 'exporter', 'importer', 'sitc4']
df = df.loc[:, idx + ['value']]
#-Adjust to SITC Level 3-#
if verbose: print "[INFO] Collapsing to SITC Level 3 Data"
df['sitc3'] = df.sitc4.apply(lambda x: x[0:3])
df = df.groupby(['year', 'exporter', 'importer', 'sitc3']).sum()['value'].reset_index()
#-Operations at SITC Level 3-#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#-Countries Only Adjustment-#
if verbose: print "[INFO] Removing 'World' values from the dataset to be country only data"
df = df.loc[(df.exporter != "World") & (df.importer != "World")]
#-Add Country ISO Information-#
#-Exports (can include NES on importer side)-#
if data_type == 'export' or data_type == 'exports':
if verbose: print "[INFO] Adding eiso3c using nber meta data"
df['eiso3c'] = df.exporter.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.eiso3c != '.')]
df = df.groupby(['year', 'eiso3c', 'sitc3']).sum()['value'].reset_index()
#-Imports (can include NES on importer side)-#
elif data_type == 'import' or data_type == 'imports':
if verbose: print "[INFO] Adding iiso3c using nber meta data"
df['iiso3c'] = df.importer.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.iiso3c != '.')]
df = df.groupby(['year','iiso3c', 'sitc3']).sum()['value'].reset_index()
#-Trade-#
else:
if verbose: print "[INFO] Adding eiso3c and iiso3c using nber meta data"
df['iiso3c'] = df.importer.apply(lambda x: countryname_to_iso3c[x])
df['eiso3c'] = df.exporter.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.iiso3c != '.') & (df.eiso3c != '.')]
df = df.groupby(['year', 'eiso3c', 'iiso3c', 'sitc3']).sum()['value'].reset_index()
#-Remove Product Code Errors in Dataset-#
df = df.loc[(df.sitc3 != "")] #Does this need a reset_index?
#-dropAX-#
if dropAX:
if verbose: print "[INFO] Dropping SITC Codes with 'A' or 'X'"
df['AX'] = df.sitc3.apply(lambda x: 1 if re.search("[AX]", x) else 0)
df = df.loc[df.AX != 1]
del df['AX'] #No Longer Required
#-Official SITCR2 Codes-#
if sitcr2:
if verbose: print "[INFO] Adding SITCR2 Indicator"
sitc = SITC(revision=2, source_institution=source_institution)
codes = sitc.get_codes(level=3)
df['sitcr2'] = df['sitc3'].apply(lambda x: 1 if x in codes else 0)
if drop_nonsitcr2:
if verbose: print "[INFO] Dropping Non Standard SITCR2 Codes"
df = df.loc[(df.sitcr2 == 1)]
del df['sitcr2'] #No Longer Needed
#-Adjust Country Codes to be Intertemporally Consistent-#
if intertemp_cntrycode:
#-Export-#
if data_type == 'export' or data_type == 'exports':
if verbose: print "[INFO] Imposing dynamically consistent eiso3c recodes across 1962-2000"
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['eiso3c'] != '.']
df = df.groupby(['year', 'eiso3c', 'sitc3']).sum().reset_index()
#-Import-#
elif data_type == 'import' or data_type == 'imports':
if verbose: print "[INFO] Imposing dynamically consistent iiso3c recodes across 1962-2000"
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df.groupby(['year', 'iiso3c', 'sitc3']).sum().reset_index()
#-Trade-#
else:
if verbose: print "[INFO] Imposing dynamically consistent iiso3c and eiso3c recodes across 1962-2000"
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df[df['eiso3c'] != '.']
df = df.groupby(['year', 'eiso3c', 'iiso3c', 'sitc3']).sum().reset_index()
#-Drop Incomplete Country Codes-#
if drop_incp_cntrycode:
if verbose: print "[INFO] Dropping countries with incomplete data across 1962-2000"
#-Export-#
if data_type == 'export' or data_type == 'exports':
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['eiso3c'] != '.']
#-Import-#
elif data_type == 'import' or data_type == 'imports':
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
#-Trade-#
else:
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df[df['eiso3c'] != '.']
df = df.reset_index()
del df['index']
#-Adjust Units from 1000's to $'s-#
if adjust_units:
if verbose: print "[INFO] Adjusting 'value' units to $'s"
df['value'] = df['value']*1000 #Default: Keep in 1000's
#-Return Dataset-#
if verbose: print "[INFO] Finished Computing Dataset (%s) ..." % (data_type)
return df
def construct_sitcr2(df,
data_type,
level,
AX=True,
dropAX=True,
sitcr2=True,
drop_nonsitcr2=True,
adjust_hk=(False, None),
intertemp_productcode=(False, None),
intertemp_cntrycode=False,
drop_incp_cntrycode=False,
adjust_units=False,
source_institution='un',
harmonised_raw=False,
values_only=False,
verbose=True):
"""
Construct a Self Contained (SC) Direct Action Dataset for Countries at the SITC Revision 2 Level 3
There are no checks on the incoming dataframe to ensure data integrity.
This is your responsibility
STATUS: tests/test_constructor_dataset_sitcr2l3.py
Parameters
----------
df : DataFrame
Pandas DataFrame containing the raw data
data_type : str
Specify what type of data 'trade', 'export', 'import'
level : int
Specify Level of Final dataset (i.e. SITC Level 1, 2, 3, or 4)
AX : bool, optional(default=True)
Add a Marker for Codes that Include 'A' and 'X'
dropAX : bool, optional(default=True)
Drop AX Codes at the Relevant Level (i.e. SITC Level 3 Data will include appropriate A and X codes)
sitcr2 : bool, optional(default=True)
Add SITCR2 Indicator
drop_nonsitcr2 : bool, optional(default=True)
Drop non-standard SITC2 Codes
adjust_hk : Tuple(bool, df), optional(default=(False, None))
Adjust the Hong Kong Data using NBER supplemental files which needs to be supplied as a dataframe
intertemp_productcode : Tuple(bool, dict), optional(default=False, None)
Apply an Intertemporal Product Code System drop a conversion dictionary (IC["drop"] = [], IC["collapse"] = [])
Note this will override the drop_nonsitcr2 option
intertemp_cntrycode : bool, optional(default=False)
Generate Intertemporal Consistent Country Units (from meta)
drop_incp_cntrycode : bool, optional(default=False)
Drop Incomplete Country Codes (from meta)
adjust_units : bool, optional(default=False)
Adjust units by a factor of 1000 to specify in $'s
source_institution : str, optional(default='un')
which institutions SITC classification to use
harmonised_raw : bool, optional(default=False)
Return simple RAW dataset with Quantity disaggregation collapsed and eiso3c and iiso3c columns (Note: You may use hk_adjust with this option)
values_only : bool, optional(default=False)
Return Values and Relevant Index Data Only (i.e. drop 'AX', 'sitcr2')
Notes
-----
1. Operations ::
[1] Adjust Hong Kong and China Data
[2] Drop SITC4 to SITC3 Level (for greater intertemporal consistency)
[3] Import ISO3C Codes as Country Codes
[4] Drop Errors in SITC3 codes ["" Codes]
Optional:
---------
[A] Drop sitc3 codes that contain 'A' and 'X' codes [Default: True]
[B] Drop Non-Standard SITC3 Codes [i.e. Aren't in the Classification]
[C] Construct an Intertemporal Product Code Classification and Adjust Dataset
[C] Adjust iiso3c, eiso3c country codes to be intertemporally consistent
[D] Drop countries with incomplete 'total' data across 1962 to 2000 (strict measure) [Identification Debatable]
3. This makes use of countryname_to_iso3c in the meta data subpackage
4. This method can be tested using /do/basic_sitc3_country_data.do
5. DropAX + Drop NonStandard SITC Rev 2 Codes still contains ~94-96% of the data found in the raw data
.. Future Work
-----------
1. Check SITC Revision 2 Official Codes
2. Add in a Year Filter
"""
#-Operations Requiring RAW SITC Level 4-#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
idx = [
u'year', u'icode', u'importer', u'ecode', u'exporter', u'sitc4',
u'unit', u'dot'
]
#-Hong Kong China Data Adjustment Option-#
if type(adjust_hk) == bool:
adjust_hk = (adjust_hk, None)
if adjust_hk[0]:
if verbose: print "[INFO] Adjusting Hong Kong and China Values"
hkdata = adjust_hk[1]
#-Values-#
raw_value = df[idx + ['value']].rename(columns={'value': 'value_raw'})
try:
adjust_value = hkdata[idx + ['value_adj']]
except:
raise ValueError(
"[ERROR] China/Hong Kong Data has not been passed in properly!"
)
#-Note: Current merge_columns utility merges one column set at a time-#
df = merge_columns(raw_value,
adjust_value,
idx,
collapse_columns=('value_raw', 'value_adj',
'value'),
dominant='right',
output='final',
verbose=verbose)
#-Note: Adjust Quantity has not been implemented. See NBERWTF constructor -#
#-Filter Data-#
idx = [
u'year', u'exporter', u'importer', u'sitc4'
] #Note: This collapses duplicate entries with unit differences (collapse_valuesonly())
df = df.loc[:, idx + ['value']]
#-Raw Trade Data Option with Added IISO3C and EISO3C-#
if harmonised_raw and data_type == "trade":
df = df.groupby(
idx).sum().reset_index() #Sum Over Quantity Disaggregations
#-Add EISO3C and IISO3C-#
df['eiso3c'] = df['exporter'].apply(lambda x: countryname_to_iso3c[x])
df['iiso3c'] = df['importer'].apply(lambda x: countryname_to_iso3c[x])
return df
if harmonised_raw and data_type in {"export", "import"}:
warnings.warn(
"Cannot run harmonised_raw over export and import data as raw data is trade data"
)
return None
#-Collapse to SITC Level -#
if level != 4:
if verbose: print "[INFO] Collapsing to SITC Level %s Data" % level
df['sitc%s' % level] = df.sitc4.apply(lambda x: x[0:level])
df = df.groupby(['year', 'exporter', 'importer',
'sitc%s' % level]).sum()['value'].reset_index()
elif level == 4:
if verbose: print "[INFO] Data is already at the requested level"
else:
raise ValueError("Level must be 1, 2, 3, or 4 for the NBER data")
#-Operations Post Collapse to SITC Level-#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#-Countries Only Adjustment-#
if verbose:
print "[INFO] Removing 'World' values so that the dataset is country only data"
df = df.loc[(df.exporter != "World") & (df.importer != "World")]
#-Add Country ISO Information-#
#-Exports (can include NES on importer side)-#
if data_type == 'export' or data_type == 'exports':
if verbose: print "[INFO] Adding eiso3c using nber meta data"
df['eiso3c'] = df.exporter.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.eiso3c != '.')]
df = df.groupby(['year', 'eiso3c',
'sitc%s' % level]).sum()['value'].reset_index()
#-Imports (can include NES on importer side)-#
elif data_type == 'import' or data_type == 'imports':
if verbose: print "[INFO] Adding iiso3c using nber meta data"
df['iiso3c'] = df.importer.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.iiso3c != '.')]
df = df.groupby(['year', 'iiso3c',
'sitc%s' % level]).sum()['value'].reset_index()
#-Trade-#
else:
if verbose:
print "[INFO] Adding eiso3c and iiso3c using nber meta data"
df['iiso3c'] = df.importer.apply(lambda x: countryname_to_iso3c[x])
df['eiso3c'] = df.exporter.apply(lambda x: countryname_to_iso3c[x])
df = df.loc[(df.iiso3c != '.') & (df.eiso3c != '.')]
df = df.groupby(['year', 'eiso3c', 'iiso3c',
'sitc%s' % level]).sum()['value'].reset_index()
#-Remove Product Code Errors in Dataset-#
df = df.loc[(df['sitc%s' % level] != "")] #Does this need a reset_index?
#-productcodes-#
if intertemp_productcode[0]:
if level == 1:
intertemp_productcode = (False, intertemp_productcode[1])
else:
AX = True
dropAX = True #Small Impact Post 1984 (Levels < 4 Include 'A' and 'X' values due to the collapse)
sitcr2 = True #Encode SITCR2 for Parsing
drop_nonsitcr2 = False
#-AX-#
if AX:
if verbose: print "[INFO] Adding Indicator Codes of 'A' and 'X'"
df['AX'] = df['sitc%s' %
level].apply(lambda x: 1 if re.search("[AX]", x) else 0)
#-dropAX-#
if dropAX:
if verbose: print "[INFO] Dropping SITC Codes with 'A' or 'X'"
df = df.loc[df.AX != 1]
del df['AX']
if not dropAX and values_only:
del df['AX']
#-Intertemporal ProductCodes-#
if intertemp_productcode[0]:
if verbose:
print "[INFO] Computing Intertemporally Consistent ProductCodes ..."
#-This Method relies on meta data computed by pyeconlab nberwtf constructor-#
IC = intertemp_productcode[1] #Dict("drop" and "collapse" code lists)
#-Drop Codes-#
drop_codes = IC["drop"]
if verbose:
print "Dropping the following productcodes ..."
print drop_codes
keep_codes = set(df['sitc%s' % level].unique()).difference(
set(drop_codes))
df = df.loc[df["sitc%s" % level].isin(keep_codes)].copy(deep=True)
#-Collapse Codes-#
collapse_codes = IC["collapse"]
if verbose:
print "Collapsing the following productcodes ..."
print collapse_codes
collapse_codes = {x[0:level - 1]
for x in collapse_codes} #-Simplify Computations-#
for code in collapse_codes:
df["sitc%s" % level] = df["sitc%s" % level].apply(
lambda x: code if x[0:level - 1] == code else x)
#-Recodes-#
recodes = IC["recode"]
recode_codes = set(recodes.keys())
if verbose:
print "Recoding the following productcodes ..."
print recode_codes
for code in recode_codes:
df["sitc%s" % level] = df["sitc%s" % level].apply(
lambda x: recodes[x] if x in recode_codes else x)
df = df.groupby(list(df.columns.drop("value"))).sum()
df = df.reset_index()
#-Official SITCR2 Codes-#
if sitcr2:
if verbose: print "[INFO] Adding SITCR2 Indicator"
sitc = SITC(revision=2, source_institution=source_institution)
codes = sitc.get_codes(level=level)
df['sitcr2'] = df['sitc%s' %
level].apply(lambda x: 1 if x in codes else 0)
if drop_nonsitcr2:
if verbose: print "[INFO] Dropping Non Standard SITCR2 Codes"
df = df.loc[(df.sitcr2 == 1)]
del df['sitcr2'] #No Longer Needed
if not drop_nonsitcr2 and values_only:
del df['sitcr2']
#-Adjust Country Codes to be Intertemporally Consistent-#
if intertemp_cntrycode:
#-Export-#
if data_type == 'export' or data_type == 'exports':
if verbose:
print "[INFO] Imposing dynamically consistent eiso3c recodes across 1962-2000"
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(
iso3c_recodes_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df = df[df['eiso3c'] != '.']
df = df.groupby(['year', 'eiso3c',
'sitc%s' % level]).sum().reset_index()
#-Import-#
elif data_type == 'import' or data_type == 'imports':
if verbose:
print "[INFO] Imposing dynamically consistent iiso3c recodes across 1962-2000"
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(
iso3c_recodes_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df.groupby(['year', 'iiso3c',
'sitc%s' % level]).sum().reset_index()
#-Trade-#
else:
if verbose:
print "[INFO] Imposing dynamically consistent iiso3c and eiso3c recodes across 1962-2000"
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(
iso3c_recodes_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(
iso3c_recodes_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df[df['eiso3c'] != '.']
df = df.groupby(['year', 'eiso3c', 'iiso3c',
'sitc%s' % level]).sum().reset_index()
#-Drop Incomplete Country Codes-#
if drop_incp_cntrycode:
if verbose:
print "[INFO] Dropping countries with incomplete data across 1962-2000"
#-Export-#
if data_type == 'export' or data_type == 'exports':
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(
incomplete_iso3c_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df = df[df['eiso3c'] != '.']
#-Import-#
elif data_type == 'import' or data_type == 'imports':
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(
incomplete_iso3c_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
#-Trade-#
else:
df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(
incomplete_iso3c_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(
incomplete_iso3c_for_1962_2000, x, issue_error=False
)) #issue_error = false returns x if no match
df = df[df['iiso3c'] != '.']
df = df[df['eiso3c'] != '.']
df = df.reset_index()
del df['index']
#-Adjust Units from 1000's to $'s-#
if adjust_units:
if verbose: print "[INFO] Adjusting 'value' units to $'s"
df['value'] = df['value'] * 1000 #Default: Keep in 1000's
#-Return Dataset-#
if verbose:
print "[INFO] Finished Computing Dataset (%s) ..." % (data_type)
return df