def task_generate_sc_html():
"""
Generate sanity check html files for each department
"""
for dept in Department.list():
command = (f"pandoc --self-contained"
f" -o {dept.sc_html_path}"
f" {dept.sc_markdown_path}")
yield {
'name':
dept.name,
'file_dep': [
dept.sc_figure1_path,
dept.sc_figure2_path,
dept.sc_figure3_path,
dept.sc_figure4_path,
dept.sc_figure5_path,
dept.sc_markdown_path,
],
'targets': [dept.sc_html_path],
'actions': [command],
'clean':
True,
}
def task_preprocess_shapefiles():
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': list(dept.spatial_input_dir.iterdir()),
'targets': [dept.preprocessed_shapefile_path],
'actions': [dept.preprocess_shapefile],
'clean': True,
}
def task_create_list_of_states():
"""
Unite the guessed states for each department
"""
dept_coll = DepartmentCollection()
return {
'file_dep': [dept.guessed_state_path for dept in Department.list()],
'targets': [dept_coll.list_of_states_path],
'actions': [dept_coll.create_list_of_states],
'clean': True,
}
def task_output_block_groups():
"""
Output block groups for each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [dept.block_groups_path],
'targets': [dept.block_groups_output],
'actions': ['cp %(dependencies)s %(targets)s'],
'clean': True,
}
def task_output_city():
"""
Output city stats for each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [dept.city_path],
'targets': [dept.city_output],
'actions': ['cp %(dependencies)s %(targets)s'],
'clean': True,
}
def task_output_police_precincts():
"""
Output police precincts for each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [dept.police_precincts_path],
'targets': [dept.police_precincts_output],
'actions': ['cp %(dependencies)s %(targets)s'],
'clean': True,
}
def task_create_department_directories():
"""
Create departments' directories
"""
for dept in Department.list():
yield {
'name': dept.name,
# just to make sure
'file_dep': [util.path.CONFIG_PATH],
'targets': dept.directories,
'actions': [dept.create_directories],
'uptodate': [True],
}
def task_generate_sc_markdown():
"""
Generate sanity check md files for each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [],
'targets': [dept.sc_markdown_path],
'actions': [dept.generate_sc_markdown],
'clean': True,
'uptodate': [False],
}
def task_process_police_precincts():
"""
Process police precincts data
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [
dept.block_groups_path,
dept.preprocessed_shapefile_path,
],
'targets': [dept.police_precincts_path],
'actions': [dept.process_police_precincts],
'clean': True,
}
def task_download_bg_values():
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
dept.guessed_state_path,
dept.guessed_counties_path,
util.path.CONFIG_PATH,
],
'targets': [dept.bg_values_path],
'actions': [dept.download_bg_values],
'clean':
True,
}
def task_generate_sc_figure4():
"""
Generate figure #4 for the sanity check of each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [
dept.block_groups_path,
dept.police_precincts_path,
],
'targets': [dept.sc_figure4_path],
'actions': [dept.generate_sc_figure4],
'clean': True,
}
def task_process_city():
"""
Generate statistics for the city of each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [
dept.block_groups_path,
dept.guessed_city_path,
],
'task_dep': ['download_place_boundaries'],
'targets': [dept.city_path],
'actions': [dept.process_city],
'clean': True,
}
def task_generate_sc_figure1():
"""
Generate figure #1 for the sanity check of each department
"""
for dept in Department.list():
yield {
'name': dept.name,
'file_dep': [
dept.guessed_city_path,
dept.police_precincts_path,
],
'task_dep': ['download_place_boundaries'],
'targets': [dept.sc_figure1_path],
'actions': [dept.generate_sc_figure1],
'clean': True,
}
def task_process_department_files():
"""
Preprocess individual files for departments
"""
for dept in Department.list():
for file_name, file in dept.files.items():
yield {
'name':
f'{dept}:{file_name}',
'file_dep':
file.dependencies + [file.raw_path, util.path.CONFIG_PATH],
'targets': [file.processed_path],
'actions': [file.process],
'clean':
True,
}
def task_guess_cities():
"""
Guess the city of each department
"""
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
dept.preprocessed_shapefile_path,
dept.guessed_state_path,
],
'task_dep': ['download_place_boundaries'],
'targets': [dept.guessed_city_path],
'actions': [dept.guess_city],
'clean':
True,
}
def task_guess_states():
"""
Guess the state for each department
"""
tiger = TIGER()
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
tiger.state_boundaries_path,
dept.preprocessed_shapefile_path,
],
'targets': [dept.guessed_state_path],
'actions': [dept.guess_state],
'clean':
True,
}
def task_download_tract_values():
"""
Download census tract values for each department
"""
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
dept.guessed_state_path,
dept.guessed_counties_path,
util.path.CONFIG_PATH,
],
'targets': [dept.tract_values_path],
'actions': [dept.download_tract_values],
'clean':
True,
}
def spread_inputs(self):
"""
Spread my files to the department input directory
ACS data is being retrieved programatically, so, I will ignore
them.
"""
dst = Department(self.name)
# copy shapefiles
src_shapefile = self.shp_path
dst_shapefile = dst.spatial_input_dir
util.file.maybe_rmtree(dst_shapefile)
shutil.copytree(src_shapefile, dst_shapefile)
# copy tabular files
util.file.maybe_mkdir(dst.tabular_input_dir)
for file in self.other_files:
shutil.copy(file, dst.tabular_input_dir)
def task_process_block_groups():
"""
Merge block group values with block group boundaries
"""
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
dept.guessed_state_path,
dept.guessed_counties_path,
dept.bg_values_path,
],
'task_dep': ['download_bg_boundaries'],
'targets': [dept.block_groups_path],
'actions': [dept.process_block_groups],
'clean':
True,
}
def task_process_census_tracts():
"""
Merge census tract values with census tract boundaries
"""
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
dept.guessed_state_path,
dept.guessed_counties_path,
dept.tract_values_path,
],
'task_dep': ['download_tract_boundaries'],
'targets': [dept.census_tracts_path],
'actions': [dept.process_census_tracts],
'clean':
True,
}
def task_guess_counties():
"""
Guess the counties that compose each police department
The city surrounding each police department is also taken into
account.
"""
for dept in Department.list():
yield {
'name':
dept.name,
'file_dep': [
dept.guessed_city_path,
dept.preprocessed_shapefile_path,
],
'task_dep': [
'download_place_boundaries',
'download_county_boundaries',
],
'targets': [dept.guessed_counties_path],
'actions': [dept.guess_counties],
'clean':
True,
}