def do_gui(args):
"gui [--no-prompt] [SOURCE-URI]"
if args and args[0] == '--no-prompt':
del args[0]
# This option no longer has any effect, since it is the default.
# However, old versions of 0launch's GUI pass it (< 0.52)
import gui_support
import gtk
try:
if len(args) == 0:
pass
elif len(args) == 1:
import setup
def get_dir_callback(default_dir):
compile_dir = gui_support.choose_dir(_('Create build directory'), default_dir)
if compile_dir:
return compile_dir
raise SafeException("Cancelled at user's request")
setup.do_setup(args, get_dir_callback)
else:
raise SafeException("usage: 0compile gui URI")
buildenv = BuildEnv()
box = gui_support.CompileBox(buildenv.interface)
box.connect('destroy', lambda b: gtk.main_quit())
box.show()
gtk.main()
except KeyboardInterrupt:
pass
except SafeException, ex:
gui_support.alert(None, '%s' % ex)
sys.exit(1)
def build():
#gen .pyx to .c first.
pyx_files = search_src_file(pyx_src, ".pyx")
print "gen pyx to c files....."
gen_pyx2c(pyx_files)
c_files = search_src_file(c_src, ".c")
print "gen objs files...."
asm_files = search_src_file(asm_src, ".asm")
asm_objs = gen_objs(asm_files)
setup.sources = c_files
setup.objs = asm_objs
setup.inc_dirs = [inc_dir,]
print "build python ext lib."
setup.do_setup(["build_ext", "--inplace"])
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file':
'input/FOIA_P052262_-_11221-FOIA-P052262-AllSwornEmployeesWithUOA.xlsx',
'output_file': 'output/unit-history__2016-12.csv.gz',
'metadata_file': 'output/metadata_unit-history__2016-12.csv.gz',
'column_names_key': 'unit-history__2016-12_p052262'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': 'input/P058155_-_Kiefer.xlsx',
'output_file': 'output/roster_1936-2017_2017-04.csv.gz',
'metadata_file': 'output/metadata_roster_1936-2017_2017-04.csv.gz',
'drop_column': 'star11',
'column_names_key': 'roster_1936-2017_2017-04_p058155'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file':
'input/complaints-investigators_2000-2018_2018-07.csv.gz',
'output_file':
'output/complaints-investigators_2000-2018_2018-07.csv.gz'
}
assert (args['input_file'].startswith('input/') and
args['input_file'].endswith('.csv.gz')),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': 'input/tabula-FOIA_P428703_Responsive_Records.csv',
'output_file': 'output/officer-filed-complaints__2017-09.csv.gz',
'metadata_file':
'output/metadata_officer-filed-complaints__2017-09.csv.gz',
'column_names_key': 'officer-filed-complaints__2017-09_p428703'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': 'input/10655-FOIA-P046360-TRRdata_sterilized.xlsx',
'output_file': 'output/TRR-subject-weapons_2004-2016_2016-09.csv.gz',
'metadata_file': 'output/metadata_TRR-subject-weapons_2004-2016_2016-09.csv.gz',
'sheet': 'SubjectWeapons',
'column_names_key': 'TRR-subject-weapons_2004-2016_2016-09_p046360'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': 'input/FOIA_Data_157_(Stecklow)_3.xlsx',
'output_file': 'output/complaints-complainants_2000-2018_2018-03.csv.gz',
'metadata_file': 'output/metadata_complaints-complainants_2000-2018_2018-03.csv.gz',
'column_names_key': 'complaints-complainants_2000-2018_2018-03_18-060-157',
'sheet' : 'Complainants'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_profiles_file':
'input/roster_1936-2017_2017-04_profiles.csv.gz',
'input_remerge_file': 'input/roster_1936-2017_2017-04.csv.gz',
'output_remerge_file': 'output/roster_1936-2017_2017-04.csv.gz',
'intrafile_id': 'roster_1936-2017_2017-04_ID',
'output_reference_file': 'output/officer-reference.csv.gz',
'universal_id': 'UID',
'starting_uid': 100001
}
assert args['output_reference_file'] == 'output/officer-reference.csv.gz',\
'Output reference file is not correct.'
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file':
'input/complaints-CPD-witnesses_2000-2018_2018-03.csv.gz',
'input_profiles_file':
'input/complaints-CPD-witnesses_2000-2018_2018-03_profiles.csv.gz',
'output_file':
'output/complaints-CPD-witnesses_2000-2018_2018-03.csv.gz',
'output_profiles_file':
'output/complaints-CPD-witnesses_2000-2018_2018-03_profiles.csv.gz',
'npor_file':
'hand/nc_nonpo_ranks.yaml',
'export_cols': [
'cr_id', 'complaint_date', 'investigating_agency',
'complainant_type', 'complainant_subtype', 'assigned_unit',
'unit_detail', 'rank', 'is_sworn_officer', 'current_unit_detail',
'current_unit', 'complaints-CPD-witnesses_2000-2018_2018-03_ID'
],
'id':
'complaints-CPD-witnesses_2000-2018_2018-03_ID'
}
assert (args['input_file'].startswith('input/') and
args['input_file'].endswith('.csv.gz')),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file':
'input/unit-history__2016-12.csv.gz',
'output_file':
'output/unit-history__2016-12.csv.gz',
'output_profiles_file':
'output/unit-history__2016-12_profiles.csv.gz',
'id_cols': [
"first_name", "last_name", "suffix_name", "first_name_NS",
"last_name_NS", "appointed_date", "birth_year", "gender"
],
'conflict_cols': ['middle_initial', 'middle_initial2'],
'max_cols': ['race'],
'current_cols': ['unit'],
'time_col':
'unit_start_date',
'id':
'unit-history__2016-12_ID'
}
assert (args['input_file'].startswith('input/') and
args['input_file'].endswith('.csv.gz')),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': sys.argv[1],
'output_file': sys.argv[2],
'metadata_file': create_metadata_filename(sys.argv[2]),
'sheet': 'ActionsResponses',
'column_names_key': 'TRR-actions-responses_2004-2016_2016-09_p046360'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file':
'input/P441436-current_and_former_CPD_employee_list_run_15_Mar_2018_by_CPD_IT-redacted_1.xlsx',
'output_file': 'output/roster__2018-03.csv.gz',
'metadata_file': 'output/metadata_roster__2018-03.csv.gz',
'column_names_key': 'roster__2018-03_p441436',
'sheet_name': 'Export Worksheet'
}
assert args['input_file'].startswith('input/'),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': sys.argv[1],
'output_file': sys.argv[2],
'trr_loc_file': 'hand/trr_locations.yaml',
'trr_loc': 'location',
'trr_loc_recode': 'location_recode'
}
assert (args['input_file'].startswith('input/') and
args['input_file'].endswith('.csv.gz')),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'cw2_file': 'input/complaints-witnesses_2000-2016_2016-11.csv.gz',
'cw3_file': 'input/complaints-CPD-witnesses_2000-2018_2018-03.csv.gz',
'cc2_file': 'input/complaints-complainants_2000-2016_2016-11.csv.gz',
'cc3_file': 'input/complaints-complainants_2000-2018_2018-03.csv.gz',
'ci2_file': 'input/complaints-investigators_2000-2016_2016-11.csv.gz',
'ci3_file': 'input/complaints-investigators_2000-2018_2018-03.csv.gz',
'cv2_file': 'input/complaints-victims_2000-2016_2016-11.csv.gz',
'cv3_file': 'input/complaints-victims_2000-2018_2018-03.csv.gz',
'cw_file': 'output/complaints-CPD-witnesses.csv.gz',
'cc_file': 'output/complaints-complainants.csv.gz',
'cv_file': 'output/complaints-victims.csv.gz',
'ci_file': 'output/complaints-investigators.csv.gz'
}
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_files': [
'input/p046957_-_report_2.1_-_identified_accused.xls',
'input/p046957_-_report_2.2_-_identified_accused.xls',
'input/p046957_-_report_2.3_-_identified_accused.xls',
'input/p046957_-_report_2.4_-_identified_accused.xls',
'input/p046957_-_report_2.5_-_identified_accused.xls'
],
'output_file':
'output/complaints-accused_2000-2016_2016-11.csv.gz',
'metadata_file':
'output/metadata_complaints-accused_2000-2016_2016-11.csv.gz',
'column_names': [
'cr_id', 'full_name', 'birth_year', 'gender', 'race',
'appointed_date', 'current_unit', 'current_rank', 'current_star',
'complaint_category', 'recommended_finding',
'recommended_discipline', 'final_finding', 'final_discipline'
]
}
assert all(input_file.startswith('input/')
for input_file in args['input_files']),\
"An input_file is malformed: {}".format(args['input_files'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_file': 'input/salary_2002-2017_2017-09.csv.gz',
'output_file': 'output/salary_2002-2017_2017-09.csv.gz',
'output_profiles_file': 'output/salary_2002-2017_2017-09_profiles.csv.gz',
'id_cols': [
'first_name_NS', 'last_name_NS',
'middle_initial', 'middle_initial2',
'suffix_name', 'gender'
],
'sub_id_cols': [
'first_name_NS', 'last_name_NS',
'middle_initial', 'middle_initial2',
'suffix_name', 'gender',
'year'
],
'profile_cols' : [
'salary_2002-2017_2017-09_ID', 'age_at_hire', 'gender',
'first_name', 'last_name',
'first_name_NS', 'last_name_NS', 'suffix_name',
'middle_initial', 'middle_initial2', 'so_max_year', 'so_min_year',
'so_max_year_m1', 'so_min_year_m1',
'so_max_date', 'so_min_date', 'current_age_p1', 'current_age2_pm',
'current_age_m1', 'current_age_pm', 'current_age_mp',
'current_age2_p1','current_age2_m1','current_age2_mp',
'start_date', 'org_hire_date', 'resignation_year'
],
'sub_conflict_cols': ['org_hire_date', 'age_at_hire'],
'id': 'salary_2002-2017_2017-09_ID',
'sub_id': 'salary-year_2002-2017_2017-09_ID',
'ad_cols': ['start_date', 'org_hire_date'],
'sid': 'salary_2002-2017_2017-09_ID',
'year_id': 'salary-year_2002-2017_2017-09_ID',
'custom_merges': [
['first_name_NS', 'last_name_NS', 'middle_initial', 'suffix_name','org_hire_date', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'middle_initial', 'suffix_name','org_hire_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'middle_initial', 'suffix_name', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'middle_initial','org_hire_date', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'middle_initial','org_hire_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'middle_initial', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'suffix_name','org_hire_date', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'suffix_name', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'suffix_name','org_hire_date', 'age_at_hire'],
['F4FN', 'F4LN', 'suffix_name','org_hire_date', 'start_date', 'age_at_hire'],
['F4FN', 'F4LN', 'suffix_name', 'start_date', 'age_at_hire'],
['F4FN', 'F4LN', 'suffix_name','org_hire_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS','org_hire_date', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS','org_hire_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS','start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS','org_hire_date', 'age_at_hire'],
['first_name_NS', 'middle_initial', 'suffix_name','org_hire_date', 'start_date', 'age_at_hire'],
['first_name_NS', 'last_name_NS', 'org_hire_date', 'spp_date', 'salary', 'pay_grade'],
['first_name_NS', 'last_name_NS', 'spp_date','org_hire_date', 'start_date', 'pay_grade'],
['first_name_NS', 'last_name_NS', 'spp_date','org_hire_date', 'pay_grade'],
['first_name_NS', 'last_name_NS', 'spp_date','start_date', 'pay_grade'],
['first_name_NS', 'last_name_NS', 'org_hire_date'],
['first_name_NS', 'last_name_NS', 'start_date'],
['first_name_NS', 'middle_initial', 'spp_date', 'start_date', 'org_hire_date', 'pay_grade'],
['first_name_NS', 'spp_date', 'start_date', 'org_hire_date', 'pay_grade', 'age_at_hire']
]
}
assert (args['input_file'].startswith('input/') and
args['input_file'].endswith('.csv.gz')),\
"input_file is malformed: {}".format(args['input_file'])
assert (args['output_file'].startswith('output/') and
args['output_file'].endswith('.csv.gz')),\
"output_file is malformed: {}".format(args['output_file'])
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_reference_file':
'input/officer-reference.csv.gz',
'output_reference_file':
'output/officer-reference.csv.gz',
'universal_id':
'UID',
'input_profiles_file':
'input/complaints-witnesses_2000-2016_2016-11_profiles.csv.gz',
'add_cols': [
'F4FN', 'F4LN', 'L4LN', 'F3FN', 'L3FN', 'F1FN', 'BY_to_CA', {
'id':
'UID',
'exec':
("df.loc[df['appointed_date'].isnull(), 'appointed_date']"
" = df.loc[df['appointed_date'].isnull(), 'so_max_date']")
}, {
'id': 'UID',
'exec': "df['FN2']=df['last_name_NS']"
}, {
'id': 'UID',
'exec': "df['LN2']=df['first_name_NS']"
}, {
'id': 'complaints-witnesses_2000-2016_2016-11_ID',
'exec': "df['FN2']=df['first_name_NS']"
}, {
'id': 'complaints-witnesses_2000-2016_2016-11_ID',
'exec': "df['LN2']=df['last_name_NS']"
}
],
'loop_merge': {
'custom_merges': [
['FN2', 'LN2', 'birth_year', 'appointed_date', 'star'],
['first_name_NS', 'L4LN', 'birth_year', 'appointed_date'],
[
'F3FN', 'L3FN', 'last_name_NS', 'birth_year',
'appointed_date', 'star'
],
['first_name_NS', 'star', 'birth_year', 'appointed_date'],
['F4FN', 'star', 'birth_year', 'appointed_date'],
['F3FN', 'last_name_NS', 'birth_year', 'appointed_date'],
['F1FN', 'L3FN', 'birth_year', 'appointed_date', 'star'],
['last_name_NS', 'birth_year', 'appointed_date', 'star'],
['first_name_NS', 'birth_year', 'appointed_date'],
['F4LN', 'birth_year', 'appointed_date'],
['first_name_NS', 'last_name_NS', 'birth_year'],
],
'verbose':
True,
'one_to_one':
False
},
'input_remerge_file':
'input/complaints-witnesses_2000-2016_2016-11.csv.gz',
'output_remerge_file':
'output/complaints-witnesses_2000-2016_2016-11.csv.gz',
}
assert args['input_reference_file'] == 'input/officer-reference.csv.gz',\
'Input reference file is not correct.'
assert args['output_reference_file'] == 'output/officer-reference.csv.gz',\
'Output reference file is not correct.'
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_reference_file':
'input/officer-reference.csv.gz',
'output_reference_file':
'output/officer-reference.csv.gz',
'universal_id':
'UID',
'input_profiles_file':
'input/roster__2018-03_profiles.csv.gz',
'add_cols': [
"F4FN", "F4LN", "BY_to_CA", {
'id':
'roster__2018-03_ID',
'exec':
'df["appointed_date2"], df["appointed_date3"] = df["appointed_date"], df["appointed_date"]'
}, {
'id':
'UID',
'exec':
'df["appointed_date2"], df["appointed_date3"] = df["start_date"], df["org_hire_date"]'
}
],
'loop_merge': {
'custom_merges':
[[
'first_name_NS', 'middle_initial', 'race', 'gender',
'current_age_p1', 'appointed_date', 'star'
],
[
'first_name_NS', 'middle_initial', 'race', 'gender',
'current_age_m1', 'appointed_date', 'star'
], {
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'race', 'current_age_p1',
'appointed_date', 'star'
]
}, {
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'race', 'current_age_m1',
'appointed_date', 'star'
]
}, {
'query':
'gender == "FEMALE"',
'cols':
['first_name_NS', 'current_age_p1', 'appointed_date', 'star']
}, {
'query':
'gender == "FEMALE"',
'cols':
['first_name_NS', 'current_age_m1', 'appointed_date', 'star']
}, {
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'current_age_p1', 'appointed_date']
}, {
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'current_age_m1', 'appointed_date']
}, {
'query': 'appointed_date != appointed_date',
'cols': ['first_name_NS', 'middle_initial', 'last_name_NS']
},
[
'last_name_NS', 'middle_initial', 'appointed_date', 'star',
'race', 'current_age_p1'
],
[
'last_name_NS', 'middle_initial', 'appointed_date', 'star',
'race', 'current_age_m1'
]],
'base_OD_edits':
[('appointed_date',
['appointed_date', 'appointed_date2', 'appointed_date3'])]
},
'input_remerge_file':
'input/roster__2018-03.csv.gz',
'output_remerge_file':
'output/roster__2018-03.csv.gz',
}
assert args['input_reference_file'] == 'input/officer-reference.csv.gz',\
'Input reference file is not correct.'
assert args['output_reference_file'] == 'output/officer-reference.csv.gz',\
'Output reference file is not correct.'
return setup.do_setup(script_path, args)
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_reference_file':
'input/officer-reference.csv.gz',
'output_reference_file':
'output/officer-reference.csv.gz',
'universal_id':
'UID',
'input_profiles_file':
'input/salary_2002-2017_2017-09_profiles.csv.gz',
'add_cols': [
'F4FN', 'F4LN', 'L4LN', 'F1FN', 'F3LN', 'BY_to_CA', {
'id':
'UID',
'exec':
"df['current_age_mp'], df['current_age_pm'], df['current_age2_mp'], df['current_age2_pm'], df['current_age2_p1'], df['current_age2_m1'], df['current_age2_mp2'] = df['current_age_p1'], df['current_age_m1'], df['current_age_p1'], df['current_age_m1'], df['current_age_p1'], df['current_age_m1'], df['current_age_p1'] + 1"
}, {
'id':
'UID',
'exec':
"df['so_max_date'], df['so_min_date'] = zip(*df['appointed_date'].apply(lambda x: (x,x)))"
}, {
'id':
'UID',
'exec':
"df['so_max_year'],df['so_min_year'], df['so_min_year_m1'], df['so_max_year_m1'] = zip(*pd.to_datetime(df['appointed_date']).dt.year.apply(lambda x: (x,x,x,x)))"
}, {
'id': 'UID',
'exec': "df['org_hire_date'] = df['resignation_date']"
}, {
'id':
'UID',
'exec':
"df['resignation_year'] = pd.to_datetime(df['resignation_date']).dt.year"
}, {
'id': 'salary_2002-2017_2017-09_ID',
'exec': "df['resignation_date'] = df['org_hire_date']"
}, {
'id': 'salary_2002-2017_2017-09_ID',
'exec': "df['current_age2_mp2'] = df['current_age_mp']"
}
],
'base_OD': [('first_name', ['first_name_NS', 'F4FN']),
('last_name', ['last_name_NS', 'F3LN']),
('middle_initial', ['middle_initial', '']),
('middle_initial2', ['middle_initial2', '']),
('suffix_name', ['suffix_name', '']),
('appointed_date', ['appointed_date']),
('birth_year', ['birth_year', 'current_age', '']),
('gender', ['gender', '']), ('race', ['race', '']),
('current_unit', ['current_unit', ''])],
'loop_merge': {
'custom_merges': [
['first_name_NS', 'L4LN', 'so_max_date'],
['first_name_NS', 'L4LN', 'so_min_date'],
['F1FN', 'last_name_NS', 'so_max_date'],
['F1FN', 'last_name_NS', 'so_min_date'],
[
'first_name_NS', 'last_name_NS', 'middle_initial',
'current_age2_mp2'
],
['first_name_NS', 'last_name_NS', 'current_age_mp'],
['first_name_NS', 'last_name_NS', 'current_age_pm'],
['first_name_NS', 'last_name_NS', 'current_age_mp'],
['first_name_NS', 'last_name_NS', 'current_age2_m1'],
['first_name_NS', 'last_name_NS', 'current_age2_p1'],
['first_name_NS', 'last_name_NS', 'current_age2_pm'],
['first_name_NS', 'last_name_NS', 'current_age2_mp'],
['F4FN', 'last_name_NS', 'current_age_pm'],
['F4FN', 'last_name_NS', 'current_age_mp'],
['F4FN', 'last_name_NS', 'current_age2_pm'],
['F4FN', 'last_name_NS', 'current_age2_mp'],
['F4FN', 'last_name_NS', 'current_age2_m1'],
['F4FN', 'last_name_NS', 'current_age2_p1'],
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age_pm',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age2_pm',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age_mp',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age2_mp',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age_p1',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age_m1',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_max_date', 'current_age2_m1',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age_pm',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age2_pm',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age_mp',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age2_mp',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age_p1',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age2_p1',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age_m1',
'middle_initial'
]
},
{
'query':
'gender == "FEMALE"',
'cols': [
'first_name_NS', 'so_min_date', 'current_age2_m1',
'middle_initial'
]
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_max_date', 'middle_initial']
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_min_date', 'middle_initial']
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_max_date', 'middle_initial']
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_max_date', 'current_age_mp']
},
{
'query': 'gender == "FEMALE"',
'cols':
['first_name_NS', 'so_max_date', 'current_age2_mp']
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_max_date', 'current_age_mp']
},
{
'query': 'gender == "FEMALE"',
'cols':
['first_name_NS', 'so_max_date', 'current_age2_mp']
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_max_date', 'current_age_p1']
},
{
'query': 'gender == "FEMALE"',
'cols':
['first_name_NS', 'so_max_date', 'current_age2_p1']
},
{
'query': 'gender == "FEMALE"',
'cols': ['first_name_NS', 'so_max_date', 'current_age_m1']
},
{
'query': 'gender == "FEMALE"',
'cols':
['first_name_NS', 'so_max_date', 'current_age2_m1']
},
[
'first_name_NS', 'last_name_NS', 'middle_initial',
'so_max_year_m1'
],
[
'first_name_NS', 'last_name_NS', 'middle_initial',
'so_min_year_m1'
],
[
'first_name_NS', 'last_name_NS', 'current_age_m1',
'resignation_date'
],
],
'verbose':
True,
'base_OD_edits': [('birth_year', [
'birth_year', 'current_age', 'current_age_m1',
'current_age2_m1', 'current_age_p1', 'current_age2_p1',
'current_age_mp', 'current_age2_mp'
'current_age_pm', 'current_age2_pm', ''
]),
('appointed_date', [
'so_min_date', 'so_max_date', 'so_min_year',
'so_max_year', 'so_min_year_m1',
'so_max_year_m1', 'resignation_year'
])],
},
'input_remerge_file':
'input/salary_2002-2017_2017-09.csv.gz',
'output_remerge_file':
'output/salary_2002-2017_2017-09.csv.gz'
}
assert args['input_reference_file'] == 'input/officer-reference.csv.gz',\
'Input reference file is not correct.'
assert args['output_reference_file'] == 'output/officer-reference.csv.gz',\
'Output reference file is not correct.'
return setup.do_setup(script_path, args)
#!/usr/bin/env python
# $Id$
# Copyright: (C) 1998-2012 by David J. Goodger
# License: GPL 2 (see GPL2.txt)
"""
This is a quick & dirty installation shortcut. It is equivalent to the
command::
python setup.py install
However, the shortcut lacks error checking and command-line option
processing. If you need any kind of customization or help, please use
one of::
python setup.py install --help
python setup.py --help
"""
from distutils import core
from setup import do_setup
if __name__ == '__main__' :
print __doc__
core._setup_stop_after = 'config'
dist = do_setup()
dist.commands = ['install']
dist.run_commands()
#!/usr/bin/env python
# $Id: install.py 2428 2004-07-13 15:57:13Z goodger $
# Copyright: This file has been placed in the public domain.
"""
This is a quick & dirty installation shortcut. It is equivalent to the
command::
python setup.py install
However, the shortcut lacks error checking and command-line option
processing. If you need any kind of customization or help, please use
one of::
python setup.py install --help
python setup.py --help
"""
from distutils import core
from setup import do_setup
if __name__ == '__main__':
print __doc__
core._setup_stop_after = 'config'
dist = do_setup()
dist.commands = ['install']
dist.run_commands()
def get_setup():
''' encapsulates args.
calls setup.do_setup() which returns constants and logger
constants contains args and a few often-useful bits in it
including constants.write_yamlvar()
logger is used to write logging messages
'''
script_path = __main__.__file__
args = {
'input_cmpl1_file' : "input/complaints-complaints_1967-1999_2016-12.csv.gz",
'input_cmpl2_file' : "input/complaints-complaints_2000-2016_2016-11.csv.gz",
'input_cmpl3_file' : "input/complaints-complaints_2000-2018_2018-03.csv.gz",
'input_acc1_file' : "input/complaints-accused_1967-1999_2016-12.csv.gz",
'input_acc2_file' : "input/complaints-accused_2000-2016_2016-11.csv.gz",
'input_acc3_file' : "input/complaints-accused_2000-2018_2018-03.csv.gz",
'output_accused_file' : "output/complaints-accused.csv.gz",
'output_complaints_file' : "output/complaints-complaints.csv.gz",
'date_cols' : ['incident_date', 'complaint_date', 'closed_date'],
'outcome_lists' : [
['000', 'Violation Noted'],
['001', '1 Day Suspension'],
['002', '2 Day Suspension'],
['003', '3 Day Suspension'],
['004', '4 Day Suspension'],
['005', '5 Day Suspension'],
['006', '6 Day Suspension'],
['007', '7 Day Suspension'],
['008', '8 Day Suspension'],
['009', '9 Day Suspension'],
['010', '10 Day Suspension'],
['011', '11 Day Suspension'],
['012', '12 Day Suspension'],
['013', '13 Day Suspension'],
['014', '14 Day Suspension'],
['015', '15 Day Suspension'],
['016', '16 Day Suspension'],
['017', '17 Day Suspension'],
['018', '18 Day Suspension'],
['019', '19 Day Suspension'],
['020', '20 Day Suspension'],
['021', '21 Day Suspension'],
['022', '22 Day Suspension'],
['023', '23 Day Suspension'],
['024', '24 Day Suspension'],
['025', '25 Day Suspension'],
['026', '26 Day Suspension'],
['027', '27 Day Suspension'],
['028', '28 Day Suspension'],
['029', '29 Day Suspension'],
['030', '30 Day Suspension'],
['045', '45 Day Suspension'],
['060', '60 Day Suspension'],
['090', '90 Day Suspension'],
['100', 'Reprimand'],
['120', 'Suspended for 120 Days'],
['180', 'Suspended for 180 Days'],
['200', 'Suspended over 30 Days'],
['300', 'Administrative Termination'],
['400', 'Separation'],
['500', 'Reinstated by Police Board'],
['600', 'No Action Taken'],
['700', 'Reinstated by Court Action'],
['800', 'Resigned'],
['900', 'Penalty Not Served'],
[-1, 'Unknown']]
}
return setup.do_setup(script_path, args)
