"""

Construct a Harmonised Dataset between NBER and BACI

Parameters

----------

df : DataFrame

Pandas DataFrame containing the raw data

data_type : str

Specify what type of data 'trade', 'export', 'import'

level : int

Specify Level of Final dataset (i.e. SITC Level 1, 2, 3, or 4)

intertemp_productcode : Tuple(bool, dict), optional(default=False, None)

Apply an Intertemporal Product Code System drop a conversion dictionary (IC["drop"] = [], IC["collapse"] = [])

Note this will override the drop_nonsitcr2 option

intertemp_cntrycode : bool, optional(default=False)

Generate Intertemporal Consistent Country Units (from meta)

drop_incp_cntrycode : bool, optional(default=False)

Drop Incomplete Country Codes (from meta)

adjust_units : bool, optional(default=False)

Adjust units by a factor of 1000 to specify in $'s

Notes

-----

1. This consists of code snippets from construct_dataset_sitcr2.py

"""

#-Intertemporal ProductCodes-#

if intertemp_productcode[0]:

if verbose: print "[INFO] Computing Intertemporally Consistent ProductCodes ..."

#-This Method relies on meta data computed by pyeconlab nberwtf constructor-#

IC = intertemp_productcode[1] #Dict("drop" and "collapse" code lists)

#-Drop Codes-#

drop_codes = IC["drop"]

if verbose:

print "Dropping the following productcodes ..."

print drop_codes

keep_codes = set(df['sitc%s'%level].unique()).difference(set(drop_codes))

df = df.loc[df["sitc%s"%level].isin(keep_codes)].copy(deep=True)

#-Collapse Codes-#

collapse_codes = IC["collapse"]

if verbose:

print "Collapsing the following productcodes ..."

print collapse_codes

collapse_codes = {x[0:level-1] for x in collapse_codes} #-Simplify Computations-#

for code in collapse_codes:

df["sitc%s"%level] = df["sitc%s"%level].apply(lambda x: code if x[0:level-1] == code else x) #code+'0'

#-Recodes-#

recodes = IC["recode"]

recode_codes = recodes.keys()

if verbose:

print "Recoding the following productcodes ..."

print recode_codes

for code in recode_codes:

df["sitc%s"%level] = df["sitc%s"%level].apply(lambda x: recodes[x] if x in recode_codes else x)

#-Reset Collapsed Codes-#

df = df.groupby(list(df.columns.drop("value"))).sum()

df = df.reset_index()

#-Adjust Country Codes to be Intertemporally Consistent-#

if intertemp_cntrycode:

#-Export-#

if data_type == 'export' or data_type == 'exports':

if verbose: print "[INFO] Imposing dynamically consistent eiso3c recodes across 1962-2000"

df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df = df[df['eiso3c'] != '.']

df = df.groupby(['year', 'eiso3c', 'sitc%s'%level]).sum().reset_index()

#-Import-#

elif data_type == 'import' or data_type == 'imports':

if verbose: print "[INFO] Imposing dynamically consistent iiso3c recodes across 1962-2000"

df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df = df[df['iiso3c'] != '.']

df = df.groupby(['year', 'iiso3c', 'sitc%s'%level]).sum().reset_index()

#-Trade-#

else:

if verbose: print "[INFO] Imposing dynamically consistent iiso3c and eiso3c recodes across 1962-2000"

df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(iso3c_recodes_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df = df[df['iiso3c'] != '.']

df = df[df['eiso3c'] != '.']

df = df.groupby(['year', 'eiso3c', 'iiso3c', 'sitc%s'%level]).sum().reset_index()

#-Drop Incomplete Country Codes-#

if drop_incp_cntrycode:

if verbose: print "[INFO] Dropping countries with incomplete data across 1962-2000"

#-Export-#

if data_type == 'export' or data_type == 'exports':

df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df = df[df['eiso3c'] != '.']

#-Import-#

elif data_type == 'import' or data_type == 'imports':

df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df = df[df['iiso3c'] != '.']

#-Trade-#

else:

df['iiso3c'] = df['iiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df['eiso3c'] = df['eiso3c'].apply(lambda x: concord_data(incomplete_iso3c_for_1962_2000, x, issue_error=False)) #issue_error = false returns x if no match

df = df[df['iiso3c'] != '.']

df = df[df['eiso3c'] != '.']

df = df.reset_index()

del df['index']

#-Adjust Units from 1000's to $'s-#

if adjust_units:

if verbose: print "[INFO] Adjusting 'value' units to $'s"

df['value'] = df['value']*1000 #Default: Keep in 1000's

#-Return Dataset-#

if verbose: print "[INFO] Finished Computing Harmonised Dataset (%s) ..." % (data_type)

""" Join Rows and Average if both values are not np.nan """

if np.isnan(row["value_x"]) and np.isnan(row["value_y"]):

return np.nan

elif np.isnan(row["value_x"]):

return row["value_y"]

elif np.isnan(row["value_y"]):

return row["value_x"]

else: