def relational_subprocess(self, index_list, index_of_process,
data_directory, write_directory):
"""
This sub process is run via a call from relational_database via Process
Each process is set a sub selection of indexes from the PlaceReference loaded into _matcher. Each process will
then isolate this name and create a output json database for it by extracting any matching entries attributes
from the data directory.
:param index_list: A list of indexes to load from the PlaceReference for this process
:type index_list: list[int]
:param index_of_process: Which process thread this is
:type index_of_process: int
:param data_directory: Load directory the of standardised, cleaned, and correct data
:type data_directory: str | Path
:param write_directory: Write Directory for the json database
:type write_directory: str | Path
:return: Nothing, write a json database for each location that has been indexed from the PlaceReference.
:rtype: None
"""
# Currently processed files in the output directory
current_files = [f for f in directory_iterator(write_directory)]
for call_index, place_index in enumerate(index_list, 1):
print(
f"{call_index} / {len(index_list)} for process {index_of_process}"
)
# Create the unique name from the groups and isolate the gid for parsing the csv
unique_name = "__".join(
self._set_standardised_place(self._matcher[place_index]))
gid = self._matcher[place_index][0]
# Set the output stub for this place's json database
place_data = {"Place_Name": unique_name, "GID": gid}
# If the data has not already been processed
if self._not_processed(unique_name, current_files):
for file in directory_iterator(data_directory):
# Load the data into memory
data = CsvObject(Path(data_directory, file),
set_columns=True)
# Isolate any data pertaining to this place from this file and add them to the place_data dict
self._process_relation_data(data, gid, place_data)
write_json(place_data, write_directory,
f"{unique_name}_{self._data_name}")
def standardise_names(self, data_directory, write_directory):
"""
Standardise each place name to a single name if it has multiple
If working with time series data places may change their names over time which can lead to a lot of merge errors
or difficulty in linking data. This will standardise all names to a single entry, ensuring that regardless of
the actual name of the place in that year that all data from that place is grouped to a single entry.
:param data_directory: Directory containing csv files named in a yyyymmdd format
:type data_directory: Path | str
:param write_directory: Output directory
:type write_directory: Path | str
:return: Nothing, write out the data for each file found in the data_directory and then stop
:rtype: None
"""
for file in directory_iterator(data_directory):
print(file)
# Load the data into memory.
data = CsvObject(Path(data_directory, file), set_columns=True)
# Standardise the name via the matcher
rows = []
for i, name in enumerate(data.column_data[self._name_index], 0):
reformatted = self._convert_names(name, i, data)
if reformatted:
rows.append(reformatted)
# Set the headers of the output file then write the file of the same name to the write_directory
headers = self._reference_types + data.headers[1:]
write_csv(write_directory, data.file_path.stem, headers, rows)
def create_heat_map_frames(working_directory, point_colour, point_out, gradient_out, gradient_scalar, gradient_divider):
dates = {"".join([n.replace(".png", "").zfill(2) for n in name.split("_")]): name
for name in directory_iterator(working_directory)}
# Format dates to be sorted
dates = {date: name for date, name in sorted(dates.items(), key=lambda kv: kv[0])}
# Create the bound to mask on, the isolate the first frame
base = _base_image(dates, working_directory)
bound_min = (max(point_colour[0]-5, 0), max(point_colour[1]-5, 0), max(point_colour[2]-5, 0))
base.mask_on_colour_range(bound_min, point_colour)
# Point frames
for index, (date, name) in enumerate(dates.items()):
if index % 10 == 0:
print(f"Frame {index}: {len(dates.items())}")
if index > 0:
current_date = ImageObject(load_image(str(Path(working_directory, name).absolute())))
current_date.mask_on_colour_range(bound_min, point_colour)
current_date.write_to_file(point_out, index)
# Gradient frames
for index, (date, name) in enumerate(dates.items()):
if index % 10 == 0:
print(f"Frame {index}: {len(dates.items())}")
if index > 0:
difference = _create_difference(working_directory, name, bound_min, point_colour, base,
gradient_scalar, gradient_divider)
difference.write_to_file(gradient_out, index)
base = difference
def manhattan_plot(self, colours, output_directory):
"""
This will take the images in the working directory from manhattan_points and compile them into the images
:param colours: A list of 0-255 BGR colours, must be of equal length to the chromosome subdivision
:type colours: list[(int, int, int)]
:param output_directory: Where you want to save the images, it is STRONGLY recommend you put this somewhere
else as if you re-run the script and place the output in the working_directory it may raise index errors
for missing AXIS files for the output images
:type output_directory: str | Path
:return: Nothing, compile images then stop
:rtype: None
"""
# Isolate the unique image names
unique_names = list(
set([
file.split("__")[0]
for file in directory_iterator(self._working_dir)
if (".log" not in file) and (".blend" not in file)
]))
# For each plot, compile the images
for name in unique_names:
create_manhattan_plot(name, self._working_dir, colours,
output_directory)
def distribute_heritability_genome_wide(self):
"""If we can't calculate heritability, distribute it from a provided float"""
total_snps = 0
config_dict = {}
for file in directory_iterator(self.summary_directory):
print(file)
load_file = CsvObject(Path(self.summary_directory, file),
self.cleaned_types,
set_columns=True)
# Isolate the generic information
n_snps, n_iid = self._chromosome_from_load(load_file)
chromosome_values = {
self.count_snp: n_snps,
self.count_iid: n_iid,
"Description": f"Chromosome {self.target_chromosome}"
}
config_dict[self.target_chromosome] = chromosome_values
total_snps += n_snps
print(
f"Suggested LD_Radius based on {total_snps} / 3000 is {total_snps / 3000}"
)
for key, value in config_dict.items():
config_dict[key][self.herit] = self.herit_calculated * (
config_dict[key][self.count_snp] / total_snps)
config_dict["Genome"] = {
f"{self.genome}_{self.herit}": self.herit_calculated
}
ArgMaker().write_yaml_group_dict(config_dict, self.working_dir,
"genome_wide_config")
def places_into_dates(self, cleaned_data, write_directory, file_gid=0):
"""
Sometimes you may have data that is not missing in dates, but just wasn't recorded. This places every place in
into ever date.
"""
# Format the reference into lower case
formatted = [[r.lower() for r in row]
for row in self._reference.row_data]
for file in directory_iterator(cleaned_data):
# Load the file as a csv object
loaded_file = CsvObject(Path(cleaned_data, file))
# Isolate the GID: Row relation from the file
gid = {row[file_gid]: row for row in loaded_file.row_data}
# If the place exists in our file, use the file row, else use set to zero's
all_places = []
for row in formatted:
if row[file_gid] in gid:
all_places.append(gid[row[file_gid]])
else:
all_places.append([row[i]
for i in self.isolates] + [0, 0, 0, 0])
write_csv(write_directory,
Path(cleaned_data, file).stem, loaded_file.headers,
all_places)
def _load_shapefiles(path):
"""
Load the shapefiles into memory
"""
return [
ShapeObject(f"{path}/{file}") for file in directory_iterator(path)
if Path(path, file).suffix == ".shp"
]
def write_linked_unique(self,
ambiguity=True,
ambiguity_file_name="SetAmbiguous.csv"):
"""
Construct a base lookup-file to append alternative names to as well as lists of unique name files
:param ambiguity: If there is ambiguity in the file system
:type ambiguity: bool
:param ambiguity_file_name: The name of the fix file, defaults to SetAmbiguous.csv
:type ambiguity_file_name: str
:return: Nothing, construct files then stop
:rtype: None
"""
# Load the files for each shapefile that where written by link_districts_counties as well as the user ambiguous
# file named ambiguity_file_name
ambiguity_file = self._ambiguity_setter(ambiguity, ambiguity_file_name)
relation_files = [
CsvObject(f"{self._working_dir}/{file}")
for file in directory_iterator(self._working_dir)
if "_relation" in file
]
# Construct a list of all the names without any ambiguity
name_list = [
self._fix_row_ambiguity(row, ambiguity_file,
re.sub(r"[\D]", "", file.file_name))
for file in relation_files for row in file.row_data
]
# Write out the reference base
unique_relations = [
list(relation) for relation in list({tuple(i)
for i in name_list})
]
if not Path(self._working_dir, "LookupBase.csv").exists():
write_csv(self._working_dir, "LookupBase", ["GID"] + self._headers,
unique_relations)
else:
print("Lookup already written, passing")
# For each level, write out a list of unique names
for index, level in enumerate(self._headers, 1):
# Isolate the unique places for a given level
unique_places = list(
set([level_relation[index] for level_relation in name_list]))
# Write it out if it doesn't already exist
if not Path(self._working_dir, f"Unique_{level}.csv").exists():
write_csv(self._working_dir, f"Unique_{level}", [level],
unique_places)
else:
print(f"Unique_{level} Already exists, skipping")
def solve_ambiguity(self, standardised_directory, write_directory):
"""
Remove perfect duplicates and combine non perfect duplicates so that all GIDs are unique.
Some places may end up being duplicated, in the raw data or after standardisation. This method will remove
perfect duplicates, and combine non perfect duplicates into a single entry. Keep in mind, that if this is not
desirable, that the system will print out each non-perfect duplication merge it has done. You may wish to alter
your original data set, or change your place reference to avoid this from happening.
:param standardised_directory: The data directory of the output from standardise_names
:type standardised_directory: str | Path
:param write_directory: The output directory
:type write_directory: str | Path
"""
for file in directory_iterator(standardised_directory):
print(file)
# Load the original file and look for duplicate GIDs; which should be unique
data = CsvObject(Path(standardised_directory, file),
set_columns=True)
duplicate_list = find_duplicates(data.column_data[0])
# Isolate any row that does not suffer from duplication as the base of the write return
reset_row = [
row for row in data.row_data if row[0] not in duplicate_list
]
for dup in duplicate_list:
# Isolate the row names
row_names = data.row_data[data.column_data[0].index(
dup)][:len(self._reference_types)]
# Isolate the values for each duplicate name
sub_list = [[
parse_as_numeric(rr, float)
for rr in r[len(self._reference_types):]
] for r in data.row_data if dup == r[0]]
# Isolate unique lists, to remove duplicates
unique_sub_lists = [
list(x) for x in set(tuple(x) for x in sub_list)
]
# Warn the user that some values have been combined.
if len(unique_sub_lists) > 1:
print(
f"Found and combined multiple entries that where not perfect duplicates for {row_names}"
)
# Add the combined values or singular entry of duplicate values to the reset list
reset_row.append(row_names +
[sum(i) for i in zip(*unique_sub_lists)])
write_csv(write_directory, data.file_path.stem, data.headers,
reset_row)
def suggest_ld_radius(self):
"""Suggest the size of LD that the user should be using"""
total_snps = sum([
CsvObject(Path(self.filter_directory, file)).column_length
for file in directory_iterator(self.filter_directory)
])
print(
f"Suggested LD Radius based on total snps found after filtering / 3000 is {total_snps / 3000}"
)
return total_snps / 3000
def select_file_on_chromosome(self):
"""
For a given chromosome, get the respective file from the genetic directory
:return: Path to the current file as a Path from pathlib
"""
for file in directory_iterator(self.gen_directory):
if Path(self.gen_directory, file).suffix == self.gen_type:
try:
if int(re.sub(r'[\D]', "", Path(self.gen_directory, file).stem)) == self.target_chromosome:
return str(Path(self.gen_directory, file).absolute())
except (ValueError, TypeError):
continue
raise Exception(f"Failed to find any relevant file for {self.target_chromosome} in {self.gen_directory}")
def qq_make(self, point_colour, output_directory):
# Isolate the unique image names
unique_names = list(
set([
file.split("__")[0]
for file in directory_iterator(self._working_dir)
if (".log" not in file) and (".blend" not in file)
]))
print(unique_names)
# For each plot, compile the images
for name in unique_names:
create_qq_plot(name, self._working_dir, point_colour,
output_directory)
return
def combine_dataset(self, path_list, write_directory, database_name):
"""
This will combine all the dataset's you have made into a single json database
This will combine all the regional data from all standardised dataset's into a single json database. If you only
had 1 database to begin with, then this just adds all the separate json databases into a single 1. Where it is
mostly used, is when you have run this process on multiple dataset's and now want all the standardised places to
share attribute data in a single database.
:param path_list: A list of paths, where each path goes to a set directory
:type path_list: list[str | Path]
:param write_directory: The write directory of the master database
:type write_directory: str | Path
:param database_name: The master database name
:type database_name: str
:return: Nothing, write the database to file then stop
:rtype: None
"""
# Initialise the output database
master_database = {}
# Isolate all the paths to all the files we want to load across all the database for this geo-level
level_data = [
Path(path, file) for path in path_list
for file in directory_iterator(path)
]
for index, file in enumerate(level_data):
if index % 100 == 0:
print(f"{index}/{len(level_data)}")
# Load the data for this file into memory, set the master database assign name via Place_Name
load_data = load_json(file)
assign_name = load_data["Place_Name"]
# If the current attribute does not exist within the current database, add it to it
current_attributes = self._current_attributes(
master_database, assign_name)
for attr in load_data.keys():
if attr not in current_attributes:
master_database[assign_name][attr] = load_data[attr]
write_json(master_database, write_directory, database_name)
def construct_reference(self,
base_weights_name="LookupBase.csv",
alternative_key="Unique"):
"""
The construct a reference of every name for every place for every level within the Lookup Base
:param base_weights_name: Name of the base weights file
:type base_weights_name: str
:param alternative_key: Key within files that contains alternative names
:type alternative_key: str
:return: Nothing, write out place reference csv then stop
:rtype: None
"""
# Load the lookup base
base_relation = CsvObject(Path(self._working_dir, base_weights_name))
# Load alternative files
alt_files = [
CsvObject(Path(self._working_dir, file), set_columns=True)
for file in directory_iterator(self._working_dir)
if alternative_key in file
]
# Order them in the same manner as the headers
order = [
index for header in self._headers
for index, file in enumerate(alt_files) if header in file.file_name
]
alt_files = np.array(alt_files)[order].tolist()
# Link each row to a unique list to create the reference place look up file
rows = [
flatten([[row[0]]] + [
self._match_row(match, match_file)
for match, match_file in zip(row[1:], alt_files)
]) for row in base_relation.row_data
]
write_csv(self._working_dir, "PlaceReference",
["GID"] + flatten([file.headers for file in alt_files]),
rows)
def validation_chromosomes(self):
"""
This will create a dataset of all the chromosomes that we have to work with
:Note: Mostly used externally to aid multi-core processing
:return: A list of valid chromosomes
:rtype: list
"""
valid_chromosomes = []
for file in directory_iterator(self.gen_directory):
if Path(self.gen_directory, file).suffix == self.gen_type:
valid_chromosomes.append(
int(
re.sub(r'[\D]', "",
Path(self.gen_directory, file).stem)))
valid_chromosomes.sort()
return valid_chromosomes
def create_manhattan_plot(plot_name, working_dir, colours, output_directory):
# Isolate the files related to this name
name_files = []
for file in directory_iterator(working_dir):
if (plot_name in file) and (".log" not in file) and (".blend" not in file):
name_files.append(file)
# If we find the axis image then we can construct a plot
if sum([True if "AXIS" in file else False for file in name_files]) == 1:
# Isolate the axis and the point image file names
axis = [file for file in name_files if "AXIS" in file][0]
points = [file for file in name_files if "AXIS" not in file]
assert len(points) == len(colours), f"Found {len(points)} but was only provided {len(colours)} colours"
# Construct the plot
_construct_plot_image(working_dir, axis, points, colours, output_directory, plot_name)
else:
raise IndexError(f"Failed to find {plot_name}_AXIS.png")
def create_qq_plot(plot_name, working_dir, point_colour, output_directory):
# Isolate the files related to this name
name_files = []
for file in directory_iterator(working_dir):
if (plot_name in file) and (".log" not in file) and (".blend"
not in file):
name_files.append(file)
# If we find the axis image then we can construct a plot
if sum([True if "AXIS" in file else False for file in name_files]) == 1:
# Isolate the axis and the point image file names
axis = [file for file in name_files if "AXIS" in file][0]
points = [file for file in name_files if "AXIS" not in file][0]
# Construct the plot
_plot_image(working_dir, axis, points, point_colour, output_directory,
plot_name)
else:
raise IndexError(f"Failed to find {plot_name}__AXIS.png")
def _heritability_by_chromosome(self, config_dict):
"""
This will calculate the heritability for each chromosome cumulatively the values for the genome-wide
calculation
"""
cumulative_ld = sum_sq_beta = total_snps = 0
for file in directory_iterator(self.ld_directory):
# Infer the chromosome of this file via remove the 'LD' prefix
chromosome = file[2:]
print(f"Processing Chromosome {chromosome}")
# Load the ld data from the ld_directory and use it to set sid/iid count
data = load_pickle(self.ld_directory, file)
sid_count, iid_count = data[self.norm_snps].shape
# Isolate the betas from the Filtered to calculate the sum squared betas, also ld scores from ld data
betas = self.sm_dict_from_csv(
self.filter_directory, f"Filtered_{chromosome}.csv")[self.beta]
sum_beta_sq = np.sum(np.array(betas)**2)
ld_scores = data[self.ld_scores]
# Calculate the heritability and average LD at a chromosome level
heritability, average_ld = self._chromosome_heritability(
chromosome, ld_scores, sum_beta_sq, sid_count)
# Cumulate ld, snps, and sum square beta
cumulative_ld += np.sum(ld_scores)
total_snps += sid_count
sum_sq_beta += np.sum(sum_beta_sq)
# Store Values for config file
chromosome_values = {
self.herit: heritability,
self.count_snp: sid_count,
self.count_iid: iid_count,
self.avg_ld: average_ld,
"Description": f"Chromosome {chromosome}"
}
config_dict[chromosome] = chromosome_values
return cumulative_ld, sum_sq_beta, total_snps
def aggregate_scores(self):
"""
This will combine the scores found by chromosome into a single file
"""
combined_array = []
for index, file in enumerate(directory_iterator(
self.scores_directory)):
score_file = CsvObject(Path(self.scores_directory, file),
set_columns=True)
# If its the first file we want to extract the iid and fid values as well
if index == 0:
fid, iid, score = score_file[self.fid], score_file[
self.iid], score_file["Scores"]
fid, iid, score = np.array(fid), np.array(iid), np.array(
score).astype(float)
fid.shape, iid.shape, score.shape = (len(fid),
1), (len(iid),
1), (len(score), 1)
combined_array = [fid, iid, score]
# Else extract the scores and append it to the array
else:
score = np.array(score_file["Scores"]).astype(float)
score.shape = (len(score), 1)
combined_array.append(score)
# Combine the (IID_Count, 1) * (chromosome count + 2) arrays into a single (IDD_Count, chromosome_count)
iid_array = np.hstack(combined_array[:2])
score_array = np.sum(np.hstack(combined_array[2:]), axis=1)
score_array.shape = (len(score_array), 1)
# Write the scores to the working directory
write_rows = np.hstack([iid_array, score_array]).tolist()
write_csv(Path(self.working_dir, "PGS"), "PolyGenicScores",
["FID", "IID", "Scores"], write_rows)
def main_call(out_dir, write_dir, headers):
output = []
for file in directory_iterator(out_dir):
if ".log" not in file:
csv_file = CsvObject(Path(output_dir, file))
# Isolate the model values from the aggregated [snp] + [model 1, ... model N]
for row in csv_file.row_data:
snp, models = row[0], chunk_list(row[1:], len(headers))
output.append([snp, models])
print(f"For {len(output)} Snps")
model_count = len(output[0][1])
model_comp = []
for i in range(model_count):
print(f"For model {i+1}")
# Write out the aggregated chromosome model data to a directory
model_out = []
for snp, model in output:
model_out.append([snp] + model[i])
write_csv(write_dir, f"Model{i + 1}", ["Snp"] + headers, model_out)
# Append the comparision to a master list of models
model_comp.append([f"Model {i+1}"] + [
str(np.mean([float(values[vi]) for values in model_out]))
for vi in range(1, 3)
])
# Write the model comp out
write_csv(
write_dir, "Model Comparision",
["Model", "Mean Coefficent", "Mean Standard Error", "Mean P Values"],
model_comp)
# Select the current Object that has the same name as place
ob = bpy.context.scene.objects[place]
bpy.ops.object.select_all(action='DESELECT')
# Make the District the active object
bpy.context.view_layer.objects.active = ob
ob.select_set(True)
for mat in ob.material_slots:
mat.material.node_tree.nodes["Emission"].inputs[0].default_value = colour
root = r"I:\Work\DataBases\Adjacent\Months"
not_processed = []
for file in directory_iterator(root):
print(file)
a = CsvObject(Path(root, file))
target = len(a.headers) - 2
found = 0
for img in directory_iterator(r"I:\Work\Figures_and_tables\DiseasesOverTime"):
year = img.split("_")[-1].split(".")[0]
if year == Path(root, file).stem:
found += 1
if found != target:
not_processed.append(file)
def combine(self, unique_id, data_start, root_directory, write_directory,
write_name):
"""
weightGIS expects each file to have a single date, so if you have lots of files of the same date that you wan
to process at the same time you will need ot combine them
:param unique_id: The unique id index
:type unique_id: int
:param data_start: The index wherein the data starts from
:type data_start: int
:param root_directory: The root directory of the csv files
:type root_directory: Path | str
:param write_directory: The output directory for the file
:type write_directory: Path | str
:param write_name: Name of the combined file
:type write_name: str
:return: Nothing, write file then stop
:rtype: None
"""
# Create the unique ID's
unique_id_list = sorted(
list(
set(
flatten([
CsvObject(Path(root_directory, file),
set_columns=True)[unique_id]
for file in directory_iterator(root_directory)
]))))
# For each unique ID
out_list = []
for count_i, ids in enumerate(unique_id_list):
if count_i % 10 == 0:
print(f"{count_i} / {len(unique_id_list)}")
# Check each file for a matching row, and then
ids_list = []
for index, file in enumerate(directory_iterator(root_directory)):
# If its the first index, take the full values
if index == 0:
ids_list += self._isolate(root_directory, file, unique_id,
ids)
# Otherwise only take the values after the data start
else:
ids_list += self._isolate(root_directory, file, unique_id,
ids)[data_start:]
out_list.append(ids_list)
headers = []
for index, file in enumerate(directory_iterator(root_directory)):
if index == 0:
headers += CsvObject(Path(root_directory, file)).headers
else:
headers += CsvObject(Path(root_directory,
file)).headers[data_start:]
write_csv(write_directory, write_name, headers, out_list)