def estimate (dset,indvars,depvar = 'building_id',alternatives=None,SAMPLE_SIZE=100,max_segment_size = 1200,estimation_table = 'households_for_estimation',
              output_names=None,agents_groupby = ['income_3_tenure',]):
    ##HCLM ESTIMATION
    output_csv, output_title, coeff_name, output_varname = output_names
    choosers = dset.fetch(estimation_table)

    segments = choosers.groupby(agents_groupby)
    num_segments = len(segments.size().index)
    if num_segments != len(indvars):
        print "ERROR: number of segments does not match number of sets of independent variable"
    indvar_dict = dict(zip(segments.size().index.values,range(num_segments)))
    alts = alternatives
    for name, segment in segments:
        ind_vars = indvars[indvar_dict[name]]
        name = str(name)
        tmp_outcsv, tmp_outtitle, tmp_coeffname = output_csv%name, output_title%name, coeff_name%name
        if len(segment[depvar]) > max_segment_size: #reduce size of segment if too big so things don't bog down
            segment = segment.ix[np.random.choice(segment.index, max_segment_size,replace=False)]
        #,weight_var='residential_units')
        sample, alternative_sample, est_params = interaction.mnl_interaction_dataset(segment,alts,SAMPLE_SIZE,chosenalts=segment[depvar])
        ##Interaction variables
        interaction_vars = [(var, var.split('_x_')) for var in ind_vars if '_x_' in var]
        for ivar in interaction_vars:
            alternative_sample[ivar[0]] = ((alternative_sample[ivar[1][0]])*alternative_sample[ivar[1][1]])

        print "Estimating parameters for segment = %s, size = %d" % (name, len(segment.index)) 
        if len(segment.index) > 50:
            est_data = pd.DataFrame(index=alternative_sample.index)
            for varname in ind_vars:
                est_data[varname] = alternative_sample[varname]
            est_data = est_data.fillna(0)
            data = est_data.as_matrix()
            try:
                fit, results = interaction.estimate(data, est_params, SAMPLE_SIZE)
                fnames = interaction.add_fnames(ind_vars,est_params)
                print misc.resultstotable(fnames,results)
                misc.resultstocsv(fit,fnames,results,tmp_outcsv,tblname=tmp_outtitle)
                coefficients = zip(*results)[0]+(.0001,.0001,.0001,.0001,.0001,.0001,.0001,.0001,.0001,.0001,.0001,)
                varnames = fnames+['county8001','county8005','county8013','county8014','county8019','county8031','county8035','county8039','county8047','county8059','county8123']
                dset.store_coeff(tmp_coeffname,coefficients,varnames)
            except:
                print 'SINGULAR MATRIX OR OTHER DATA/ESTIMATION PROBLEM'
        else:
            print 'SAMPLE SIZE TOO SMALL'
def estimate(
    dset,
    indvars,
    depvar="building_id",
    alternatives=None,
    SAMPLE_SIZE=100,
    max_segment_size=1200,
    estimation_table="households_for_estimation",
    output_names=None,
    agents_groupby=["income_3_tenure"],
):
    ##HCLM ESTIMATION
    output_csv, output_title, coeff_name, output_varname = output_names
    choosers = dset.fetch(estimation_table)

    segments = choosers.groupby(agents_groupby)
    num_segments = len(segments.size().index)
    if num_segments != len(indvars):
        print "ERROR: number of segments does not match number of sets of independent variable"
    indvar_dict = dict(zip(segments.size().index.values, range(num_segments)))
    alts = alternatives
    for name, segment in segments:
        ind_vars = indvars[indvar_dict[name]]
        name = str(name)
        tmp_outcsv, tmp_outtitle, tmp_coeffname = output_csv % name, output_title % name, coeff_name % name
        if len(segment[depvar]) > max_segment_size:  # reduce size of segment if too big so things don't bog down
            segment = segment.ix[np.random.choice(segment.index, max_segment_size, replace=False)]
        # ,weight_var='residential_units')
        sample, alternative_sample, est_params = interaction.mnl_interaction_dataset(
            segment, alts, SAMPLE_SIZE, chosenalts=segment[depvar]
        )
        ##Interaction variables
        interaction_vars = [(var, var.split("_x_")) for var in ind_vars if "_x_" in var]
        for ivar in interaction_vars:
            alternative_sample[ivar[0]] = (alternative_sample[ivar[1][0]]) * alternative_sample[ivar[1][1]]

        print "Estimating parameters for segment = %s, size = %d" % (name, len(segment.index))
        if len(segment.index) > 50:
            est_data = pd.DataFrame(index=alternative_sample.index)
            for varname in ind_vars:
                est_data[varname] = alternative_sample[varname]
            est_data = est_data.fillna(0)
            data = est_data.as_matrix()
            try:
                fit, results = interaction.estimate(data, est_params, SAMPLE_SIZE)
                fnames = interaction.add_fnames(ind_vars, est_params)
                print misc.resultstotable(fnames, results)
                misc.resultstocsv(fit, fnames, results, tmp_outcsv, tblname=tmp_outtitle)
                coefficients = zip(*results)[0] + (
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                    0.0001,
                )
                varnames = fnames + [
                    "county8001",
                    "county8005",
                    "county8013",
                    "county8014",
                    "county8019",
                    "county8031",
                    "county8035",
                    "county8039",
                    "county8047",
                    "county8059",
                    "county8123",
                ]
                dset.store_coeff(tmp_coeffname, coefficients, varnames)
            except:
                print "SINGULAR MATRIX OR OTHER DATA/ESTIMATION PROBLEM"
        else:
            print "SAMPLE SIZE TOO SMALL"
    print segment.building_id
    print alts.reset_index().building_id.describe()
    print segment.reset_index().building_id.describe()
    alts.index = alts.index.astype('int32')
    #sample, alternative_sample, est_params = interaction.mnl_interaction_dataset(segment,alts,SAMPLE_SIZE,chosenalts=segment[depvar],weight_var='non_residential_sqft')
    sample, alternative_sample, est_params = interaction.mnl_interaction_dataset(segment,alts,SAMPLE_SIZE,chosenalts=segment[depvar])
    #alternative_sample['paris_x_employees'] = (alternative_sample.in_paris*alternative_sample.employees)
    print "Estimating parameters for segment = %s, size = %d" % (name, len(segment.index)) 
    if len(segment.index) > 50:
        est_data = pd.DataFrame(index=alternative_sample.index)
        for varname in ind_vars:
            est_data[varname] = alternative_sample[varname]
        est_data = est_data.fillna(0)
        data = est_data.as_matrix()
        try:
            fit, results = interaction.estimate(data, est_params, SAMPLE_SIZE)
            #print fit
            #print results
            fnames = interaction.add_fnames(ind_vars,est_params)
            print misc.resultstotable(fnames,results)
            misc.resultstocsv(fit,fnames,results,tmp_outcsv,tblname=tmp_outtitle)
            dset.store_coeff(tmp_coeffname,zip(*results)[0],fnames)
        except:
            print 'SINGULAR MATRIX OR OTHER DATA/ESTIMATION PROBLEM'
    else:
        print 'SAMPLE SIZE TOO SMALL'
print dset.coeffs[('emp_location_6','coeffs')]
print dset.coeffs[('emp_location_6','coeffs')][0]
print dset.coeffs[('emp_location_6','coeffs')][1]
print dset.coeffs[('emp_location_6','coeffs')][2]
print dset.coeffs[('emp_location_6','coeffs')][3]