def __init__(self, verbose=False):
# initialize the variables
self.test_classes = []
self.imported_module_names = []
self.module_import_errors = []
self.imports_with_tests = []
# get the list of module names to try
self.all_module_names = get_module_names()
# create the progress bar
pbar = Progress.Bar(len(self.all_module_names))
# try to load the modules and get the test classes
for i, module_name in enumerate(self.all_module_names):
try:
module = __import__(module_name, globals(), locals())
except ImportError as e:
self.module_import_errors.append(e)
else:
self.imported_module_names.append(module_name)
test_classes = []
for object_name, object in module.__dict__.items():
try:
if issubclass(object, unittest.TestCase):
test_classes.append(object)
except TypeError as e:
pass
if test_classes:
self.imports_with_tests.append(module_name)
self.test_classes.extend(test_classes)
# update the progress bar
pbar.update(i + 1)
def ReduceImageFileSize(inputFolder, outputFolder, imageQuality):
processedFiles = 0
changedFiles = 0
unchangedFiles = 0
allFilePaths = IM.GetListOfAllFilePaths(inputFolder)
for filePath in allFilePaths:
inputDirectory, inputFile = os.path.split(filePath)
newFilePath = IM.GetUniqueFile(
filePath.replace(inputFolder, outputFolder))
if inputFile.lower().endswith((".jpg", ".jpeg")):
try:
img = Image.open(filePath)
if not os.path.exists(os.path.dirname(newFilePath)):
os.makedirs(os.path.dirname(newFilePath))
img.save(newFilePath, quality=imageQuality)
changedFiles += 1
except Exception as ex:
logger.info(
"The following file could not be processed: \"{}\" - {}".
format(filePath, ex))
else:
IM.CopyFile(filePath, newFilePath)
unchangedFiles += 1
processedFiles += 1
Progress.PrintProgress(processedFiles)
logger.info(
"Processed Files: {} | ChangedFiles: {} | Unchanged Files: {}".format(
processedFiles, changedFiles, unchangedFiles))
def run(self, verbose=False):
"""
This reads the input file and writes pieces to a directory.
@param verbose: True if we want to report our progress
"""
# if we are reporting our progress then get the size of the file
if verbose:
nbytes_total = os.path.getsize(self.filename)
pbar = Progress.Bar(nbytes_total)
nbytes_current_approx = 0
# process the file, possibly updating the progress bar
with open(self.filename) as fin:
piece_index = 0
for line_lists in gen_line_list_lists(Util.gen_paragraphs(fin),
self.approx_lines_per_piece):
piece_filename = piece_index_to_filename(
piece_index, self.filename)
piece_path = os.path.join(self.target_directory,
piece_filename)
with open(piece_path, 'w') as fout:
for line_list in line_lists:
print >> fout, '\n'.join(line_list)
print >> fout
piece_index += 1
if verbose:
nbytes = sum(len(''.join(x)) for x in line_lists)
nbytes_current_approx += nbytes
pbar.update(nbytes_current_approx)
# possibly stop the progress bar
if verbose:
pbar.finish()
def run(self, verbose=False):
"""
Create the index files.
This might take a while.
@param verbose: True if we want to write our progress to stdout
"""
# fill a dictionary by reading all of the fasta pieces
chromosome_string_to_rows = {}
piece_filenames = list(sorted(os.listdir(self.pieces_directory)))
if verbose:
print >> sys.stderr, 'creating a dictionary from the fasta pieces:'
pbar = Progress.Bar(len(piece_filenames))
nfiles_read = 0
for piece_filename in piece_filenames:
piece_index = piece_filename_to_index(piece_filename)
piece_pathname = os.path.join(self.pieces_directory,
piece_filename)
for chrom_string, first_i, last_i in gen_elements(piece_pathname):
row = (first_i, last_i, piece_index)
rows = chromosome_string_to_rows.get(chrom_string, [])
rows.append(row)
chromosome_string_to_rows[chrom_string] = rows
if verbose:
nfiles_read += 1
pbar.update(nfiles_read)
# define the list of chromosome strings
chromosome_strings = list(sorted(chromosome_string_to_rows))
assert len(chromosome_strings) < 1000
# write the list of valid chromosome strings
with open(self.chromosome_list_filename, 'w') as fout:
fout.write('\n'.join(chromosome_strings))
if verbose:
print >> sys.stderr, 'wrote', self.chromosome_list_filename
print >> sys.stderr, 'writing the index files:'
pbar = Progress.Bar(len(chromosome_strings))
nwritten = 0
# for each chromosome string write the index file
for chromosome_string in chromosome_strings:
rows = chromosome_string_to_rows[chromosome_string]
index_filename = chromosome_string + '.index'
index_pathname = os.path.join(self.index_directory, index_filename)
with open(index_pathname, 'w') as fout:
for row in sorted(rows):
print >> fout, '%d\t%d\t%d' % row
if verbose:
nwritten += 1
pbar.update(nwritten)
def RotateImages(inputFolder, outputFolder):
processedFiles = 0
# iterate through all files including subdirectories
for path, subDirectories, files in os.walk(inputFolder):
# create output path
if not os.path.exists(path.replace(inputFolder, outputFolder)):
os.makedirs(path.replace(inputFolder, outputFolder))
for file in files:
if file.lower().endswith(('.png', '.jpg', '.jpeg')):
filePath = os.path.join(path, file)
# get exif data
exifDictionary = piexif.load(filePath)
orientation = exifDictionary["0th"][
piexif.ImageIFD.Orientation]
if orientation == 1 and inputFolder != outputFolder:
copy2(filePath, filePath.replace(inputFolder,
outputFolder))
elif orientation > 1:
# set orientation to 1 and delete thumbnail data
exifDictionary["0th"][piexif.ImageIFD.Orientation] = 1
exifDictionary.pop("thumbnail")
# load image
image = Image.open(filePath)
# rotate image
if orientation == 2:
image = image.transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 3:
image = image.rotate(180)
elif orientation == 4:
image = image.rotate(180).transpose(
Image.FLIP_LEFT_RIGHT)
elif orientation == 5:
image = image.rotate(-90, expand=True).transpose(
Image.FLIP_LEFT_RIGHT)
elif orientation == 6:
image = image.rotate(-90, expand=True)
elif orientation == 7:
image = image.rotate(90, expand=True).transpose(
Image.FLIP_LEFT_RIGHT)
elif orientation == 8:
image = image.rotate(90, expand=True)
# save image with updated exif data
image.save(filePath.replace(inputFolder, outputFolder),
"JPEG",
quality=95,
exif=piexif.dump(exifDictionary))
else:
copy2(filePath, filePath.replace(inputFolder, outputFolder))
# show progress
processedFiles = processedFiles + 1
Progress.PrintProgress(processedFiles)
def process(pathnames, good_coverage, bad_coverage, randomization_rate,
nseconds, use_pbar):
"""
@param pathnames: paths to files to process
@param good_coverage: the expected number of reads at informative positions
@param bad_coverage: the expected number of reads at uninformative positions
@param randomization_rate: the probability of an error per read
@param nseconds: None or impose a time limit of this many seconds
@param use_pbar: True iff a progress bar should be used
@return: the multi-line string of the resulting csv file
"""
# define the three models
homozygous = ReadCoverage.Homozygous(randomization_rate, good_coverage)
heterozygous = ReadCoverage.Heterozygous(randomization_rate, good_coverage)
overcovered = ReadCoverage.Overcovered(randomization_rate, bad_coverage)
models = [homozygous, heterozygous, overcovered]
# define the oracle
cache_size = 100000
oracle = Oracle(models, cache_size)
# do some initialization
out = StringIO()
start_time = time.time()
nfiles = len(pathnames)
pbar = Progress.Bar(nfiles) if (use_pbar and nfiles > 1) else None
chromosome_dict = {}
termination_reason = 'finished the analysis'
try:
for i, pathname in enumerate(pathnames):
with open(pathname) as fin:
lines = fin.readlines()
lines = [line.strip() for line in lines]
lines = [line for line in lines if line]
# validate the number of lines
if len(lines) < 2:
raise ValueError(
'there should be at least two lines of input')
# break the lines of input into rows of elements
rows = [line_to_row(line) for line in lines]
# validate the columns of data
ncolumns_expected = 8
ncolumns = len(rows[0])
if ncolumns != ncolumns_expected:
raise ValueError('expected %d columns of input' %
ncolumns_expected)
for row in rows:
if len(row) != ncolumns:
raise ValueError(
'each row of input should have the same number of elements as the first row'
)
# process the data rows
data_rows = rows[1:]
for row in data_rows:
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
process_genomic_position(row, chromosome_dict, oracle)
if pbar:
pbar.update(i + 1)
except KeyboardInterrupt, e:
termination_reason = 'early termination by control-c'
def main(args):
"""
@param args: positional and flaglike arguments
"""
# read the arguments
input_filename = os.path.abspath(os.path.expanduser(args.infile))
output_directory = os.path.abspath(os.path.expanduser(args.outdir))
force = args.force
# make sure that the output directory exists
if not os.path.isdir(output_directory):
if force:
os.makedirs(output_directory)
if not os.path.isdir(output_directory):
msg = 'output directory does not exist: ' + output_directory
raise Exception(msg)
# scan the input file for chromosome names
ch_paths = []
skimmer = DGRP.ChromoSkimmer()
with open(input_filename) as fin:
for chromo_name in skimmer.skim(gen_untyped_rows(fin)):
output_filename = args.out_prefix + chromo_name + args.out_suffix
ch_path = os.path.join(output_directory, output_filename)
ch_paths.append(ch_path)
if not force:
if os.path.exists(ch_path):
raise Exception('output already exists: ' + ch_path)
chromo_names = skimmer.name_list
nlines = skimmer.linecount
# start the progress bar
nticks = 2 * nlines
pbar = Progress.Bar(nticks)
# scan the input file for correct types and for monotonicity
with open(input_filename) as fin:
for i in DGRP.check_chromo_monotonicity(gen_typed_rows(fin)):
pbar.increment()
# create the files open for writing
ch_files = []
for p in ch_paths:
ch_files.append(open(p, 'wt'))
# write the headers
if not args.noheader:
for f in ch_files:
f.write(g_header + '\n')
# write the lines
name_to_file = dict(zip(chromo_names, ch_files))
with open(input_filename) as fin:
for row in gen_typed_rows(fin):
name = row[0]
row_out = convert_row(row)
f = name_to_file[name]
line_out = '\t'.join(str(x) for x in row_out)
f.write(line_out + '\n')
pbar.increment()
# close the files
for f in ch_files:
f.close()
def main(options):
"""
@param options: parsed from the command line
"""
edges = g_default_edges
deadline = None
pbar = Progress.Bar(options.nsamples)
out = process(edges, options.epsilon, options.nsamples, deadline, pbar)
pbar.finish()
print out.getvalue().strip()
def process(input_lines, good_coverage, randomization_rate, nstickinesses,
nseconds, use_pbar):
"""
@param input_lines: lines of input of csv data including the header
@param good_coverage: the expected number of reads at informative positions
@param randomization_rate: the probability of an error per read
@param nstickinesses: use this many different levels of stickiness
@param nseconds: None or impose a time limit of this many seconds
@param use_pbar: True iff a progress bar should be used
@return: the multi-line string of the resulting csv file
"""
# do some initialization
out = StringIO()
pbar = None
start_time = time.time()
# define the superstates
cache_size_per_superstate = 100000
good_state = ReadCoverageGap.Good(randomization_rate, good_coverage,
cache_size_per_superstate)
bad_state = ReadCoverageGap.Bad(randomization_rate, good_coverage,
cache_size_per_superstate)
superstates = [good_state, bad_state]
superstate_names = ['good', 'bad']
# read the chromosome data
chromosomes = parse(input_lines)
# write the header line
header_row = []
header_row.extend([
'genetic_line', 'chromosome', 'position', 'A_count', 'C_count',
'G_count', 'T_count', 'gap_count'
])
for stickiness in range(nstickinesses):
for name in ('state', 'substate'):
header_row.append('%s_%d' % (name, stickiness))
print >> out, ','.join(header_row)
# prepare to annotate the chromosomes
if use_pbar:
count = 0
pbar = Progress.Bar(len(chromosomes) * nstickinesses)
# annotate the chromosomes using the models
for i, chromosome in enumerate(chromosomes):
for stickiness in range(nstickinesses):
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
chromosome.annotate_posteriors(stickiness, superstates)
if pbar:
count += 1
pbar.update(count)
print >> out, '\n'.join(','.join(row)
for row in chromosome.get_rows_of_strings(
superstates, superstate_names))
if pbar:
pbar.finish()
# return the output text
return out.getvalue().strip()
def DeletealldataClick(self, sender, e):
result = MessageBox.Show(
"Are you absolutely sure you want to delete all the downloaded data?",
"Delete all data", MessageBoxButtons.YesNo, MessageBoxIcon.Warning)
if result == DialogResult.Yes:
#MessageBox.Show("Not yet implemented")
#print self.Owner
self.p = Progress.Progress()
self.p.Text = "Deleting all downloaded data. Please wait...."
self.p.Show()
self._backWorker.RunWorkerAsync()
def process(ntaxa, nseconds, nlengths, nsamples, nj_like,
branch_length_sampler, use_pbar):
"""
@param ntaxa: the number of taxa per tree
@param nseconds: stop after this many seconds
@param nlengths: use this many different sequence lengths
@param nsamples: stop after this many samples per sequence length
@param nj_like: True to use a generalized neighbor-joining-like method of computing successive distance matrices
@param branch_length_sampler: this function samples branch lengths independently
@param use_pbar: True iff a progress bar should be used
@return: a multi-line string of the contents of an R table
"""
# define the sequence lengths
lengths = get_sequence_lengths(nlengths)
# initialize the accumulation matrix
accum = np.zeros((nlengths, len(g_headers)), dtype=np.int)
for i, sequence_length in enumerate(lengths):
set_attribute(accum[i], 'sequence.length', sequence_length)
# Repeatedly analyze samples from each sequence length.
# We might have to stop early if we run out of time or if ctrl-c is pressed.
# If we have to stop early, then show the results of the progress so far.
termination_reason = 'no reason for termination was given'
start_time = time.time()
pbar = None
if use_pbar:
pbar = Progress.Bar(nsamples)
try:
for sample_index in range(nsamples):
# reset the accumulation matrix for this iteration
single_iteration_accum = np.zeros((nlengths, len(g_headers)))
# accumulate attributes of sampling attempts for each sequence length
for sequence_length_index, sequence_length in enumerate(lengths):
# keep trying to get an accepted sample
while True:
# check the time
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
# get counts of attributes of a sample
sample_result = get_sample_results(sequence_length, ntaxa,
nj_like,
branch_length_sampler)
single_iteration_accum[
sequence_length_index] += sample_result
# if the sample was accepted then we are done looking
if get_attribute(sample_result, 'nsamples.accepted'):
break
# finish the iteration
accum += single_iteration_accum
if pbar:
pbar.update(sample_index + 1)
else:
termination_reason = 'the requested number of samples per sequence length was attained'
except KeyboardInterrupt, e:
termination_reason = 'keyboard interrupt'
def main(args):
# do some validation
if args.nframes < 2:
raise ValueError('nframes should be at least 2')
# define the requested physical size of the images (in pixels)
physical_size = (args.physical_width, args.physical_height)
# get the directed edges and the branch lengths and vertex names
R, B, N = FtreeIO.newick_to_RBN(args.tree)
# get the requested undirected edge
edge = get_edge(R, N, args.branch_name)
initial_length = B[edge]
# get the undirected tree topology
T = Ftree.R_to_T(R)
# get the leaves and the vertices of articulation
leaves = Ftree.T_to_leaves(T)
internal = Ftree.T_to_internal_vertices(T)
vertices = leaves + internal
nleaves = len(leaves)
v_to_index = Ftree.invseq(vertices)
# get the requested indices
x_index = args.x_axis - 1
y_index = args.y_axis - 1
if x_index >= nleaves - 1 or y_index >= nleaves - 1:
raise ValueError(
'projection indices must be smaller than the number of leaves')
X_prev = None
# create the animation frames and write them as image files
pbar = Progress.Bar(args.nframes)
for frame_index in range(args.nframes):
linear_progress = frame_index / float(args.nframes - 1)
if args.interpolation == 'sigmoid':
t = sigmoid(linear_progress)
else:
t = linear_progress
B[edge] = (1 - t) * initial_length + t * args.final_length
w, v = Ftree.TB_to_harmonic_extension(T, B, leaves, internal)
X_full = np.dot(v, np.diag(np.reciprocal(np.sqrt(w))))
X = np.vstack([X_full[:, x_index], X_full[:, y_index]]).T
if X_prev is not None:
X = reflect_to_match(X, X_prev)
X_prev = X
image_string = get_animation_frame(args.image_format, physical_size,
args.scale, v_to_index, T, X, w)
image_filename = 'frame-%04d.%s' % (frame_index, args.image_format)
image_pathname = os.path.join(args.output_directory, image_filename)
with open(image_pathname, 'wb') as fout:
fout.write(image_string)
pbar.update(frame_index + 1)
pbar.finish()
def ChangeCaptureDate(inputFolder, outputFolder, timeDifferenceInSeconds):
processedFiles = 0
changedFiles = 0
unchangedFiles = 0
allFilePaths = IM.GetListOfAllFilePaths(inputFolder)
for filePath in allFilePaths:
inputDirectory, inputFile = os.path.split(filePath)
newFilePath = IM.GetUniqueFile(
filePath.replace(inputFolder, outputFolder))
IM.CopyFile(filePath, newFilePath)
if inputFile.lower().endswith(('.jpg', '.jpeg')):
captureTime = ""
try:
exif_dict = piexif.load(filePath)
exifData = exif_dict.get("Exif")
if exifData != {}:
captureTime_original = exifData[36867]
captureTime = str(captureTime_original)[2:-1]
captureTime = datetime.strptime(captureTime,
"%Y:%m:%d %H:%M:%S")
newCaptureTime = captureTime + timedelta(
seconds=timeDifferenceInSeconds)
newCaptureTime_string = newCaptureTime.strftime(
"%Y:%m:%d %H:%M:%S")
newCaptureTime_bytes = bytes(newCaptureTime_string,
'utf-8')
exif_dict["Exif"][36867] = newCaptureTime_bytes
exif_bytes = piexif.dump(exif_dict)
piexif.insert(exif_bytes, newFilePath)
changedFiles += 1
else:
unchangedFiles += 1
except Exception:
logger.info(
"No capture time found for: \"{}\"".format(filePath))
else:
unchangedFiles += 1
processedFiles += 1
Progress.PrintProgress(processedFiles)
logger.info(
"Processed Files: {} | ChangedFiles: {} | Unchanged Files: {}".format(
processedFiles, changedFiles, unchangedFiles))
def main(args):
# do some validation
if args.nframes < 2:
raise ValueError('nframes should be at least 2')
# define the requested physical size of the images (in pixels)
physical_size = (args.physical_width, args.physical_height)
# build the newick tree from the string
tree = NewickIO.parse(args.tree, FelTree.NewickTree)
nvertices = len(list(tree.preorder()))
nleaves = len(list(tree.gen_tips()))
# Get ordered ids with the leaves first,
# and get the corresponding distance matrix.
ordered_ids = get_ordered_ids(tree)
D = np.array(tree.get_partial_distance_matrix(ordered_ids))
index_edges = get_index_edges(tree, ordered_ids)
# Create the reference points
# so that the video frames are not reflected arbitrarily.
reference_points = Euclid.edm_to_points(D).T[:3].T
# create the animation frames and write them as image files
pbar = Progress.Bar(args.nframes)
for frame_index in range(args.nframes):
linear_progress = frame_index / float(args.nframes - 1)
if args.interpolation == 'sigmoid':
progress = sigmoid(linear_progress)
else:
progress = linear_progress
mass_vector = get_mass_vector(nvertices, nleaves, progress)
points = get_canonical_3d_mds(D, mass_vector, reference_points)
crossings = get_crossings(index_edges, points)
# define the frame path name
image_filename = 'frame-%04d.%s' % (frame_index, args.image_format)
image_pathname = os.path.join(args.output_directory, image_filename)
# clear the old figure and render the new figure
mlab.clf()
add_yz_plane()
add_zx_plane()
add_xy_plane()
X, Y, Z = points.T[0], points.T[1], points.T[2]
draw_3d_tree(X, Y, Z, index_edges)
draw_crossings(X, Y, Z, index_edges)
mlab.savefig(image_pathname, size=physical_size)
# update the progress bar
pbar.update(frame_index + 1)
pbar.finish()
def process(input_lines, good_coverage, bad_coverage, randomization_rate, T,
nseconds, use_pbar):
"""
@param input_lines: lines of input of csv data including the header
@param good_coverage: the expected number of reads at informative positions
@param bad_coverage: the expected number of reads at uninformative positions
@param randomization_rate: the probability of an error per read
@param T: a transition matrix relating the hidden states
@param nseconds: None or impose a time limit of this many seconds
@param use_pbar: True iff a progress bar should be used
@return: the multi-line string of the resulting csv file
"""
# do some initialization
out = StringIO()
pbar = None
start_time = time.time()
# define the three models
homozygous = ReadCoverage.Homozygous(randomization_rate, good_coverage)
heterozygous = ReadCoverage.Heterozygous(randomization_rate, good_coverage)
overcovered = ReadCoverage.Overcovered(randomization_rate, bad_coverage)
models = [homozygous, heterozygous, overcovered]
# read the chromosome data
chromosomes = parse(input_lines)
# write the header line
print >> out, ','.join(g_output_header_row)
# prepare to annotate the chromosomes
if use_pbar:
pbar = Progress.Bar(len(chromosomes))
# annotate the chromosomes using the models
try:
for i, chromosome in enumerate(chromosomes):
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
chromosome.annotate_likelihoods(models)
chromosome.annotate_posteriors(T, models)
print >> out, '\n'.join(
','.join(row) for row in chromosome.get_rows_of_strings())
if pbar:
pbar.update(i + 1)
except KeyboardInterrupt, e:
if pbar:
pbar.finish()
raise e
def main(args):
# read the arguments
input_filename = os.path.abspath(os.path.expanduser(args.infile))
output_directory = os.path.abspath(os.path.expanduser(args.outdir))
force = args.force
low, high = args.low, args.high
errlow, errhigh = args.errlow, args.errhigh
# make sure that the output directory exists
if not os.path.isdir(output_directory):
if force:
os.makedirs(output_directory)
if not os.path.isdir(output_directory):
msg = 'output directory does not exist: ' + output_directory
raise Exception(msg)
# create the scanner object which will be used for two passes
scanner = Scanner(low, high, args.fill, errlow, errhigh)
# Do the first pass,
# checking for errors and gathering info about the chromosomes.
name_to_path = {}
with open(input_filename) as fin:
for name in scanner.scan(fin):
output_filename = args.out_prefix + name + args.out_suffix
fpath = os.path.join(output_directory, output_filename)
name_to_path[name] = fpath
if not args.force:
if os.path.exists(fpath):
raise Exception('output file already exists: ' + fpath)
nticks = scanner.get_npositions()
pbar = Progress.Bar(nticks)
# open the files for writing
name_to_fout = {}
for name, fpath in name_to_path.items():
name_to_fout[name] = open(fpath, 'wt')
# Do the second pass,
# writing the files and updating the progress bar.
with open(input_filename) as fin:
for name, line in scanner.gen_named_lines(fin):
name_to_fout[name].write(line + '\n')
pbar.increment()
# close the files
for fout in name_to_fout.values():
fout.close()
def SetNameAsCaptureDate(inputFolder, outputFolder):
processedFiles = 0
changedFiles = 0
unchangedFiles = 0
allFilePaths = IM.GetListOfAllFilePaths(inputFolder)
for filePath in allFilePaths:
inputDirectory, inputFile = os.path.split(filePath)
newFilePath = IM.GetUniqueFile(
filePath.replace(inputFolder, outputFolder))
IM.CopyFile(filePath, newFilePath)
if inputFile.lower().endswith((".jpg", ".jpeg")):
try:
captureTime = datetime.strptime(inputFile[0:15],
"%Y%m%d_%H%M%S")
except Exception:
try:
captureTime = datetime.strptime(inputFile[0:8], "%Y%m%d")
except Exception:
logger.info(
"Unable to set capture time for: \"{}\"".format(
filePath))
continue
captureTime_string = captureTime.strftime("%Y:%m:%d %H:%M:%S")
exif_dict = piexif.load(filePath)
exifData = exif_dict.get("Exif")
captureTime_bytes = bytes(captureTime_string, 'utf-8')
exif_dict["Exif"][36867] = captureTime_bytes
exif_bytes = piexif.dump(exif_dict)
piexif.insert(exif_bytes, newFilePath)
changedFiles += 1
else:
unchangedFiles += 1
processedFiles += 1
Progress.PrintProgress(processedFiles)
logger.info(
"Processed Files: {} | ChangedFiles: {} | Unchanged Files: {}".format(
processedFiles, changedFiles, unchangedFiles))
def main(args):
# do some validation
if args.nframes < 2:
raise ValueError('nframes should be at least 2')
# define the requested physical size of the images (in pixels)
physical_size = (args.physical_width, args.physical_height)
# create the animation frames and write them as image files
pbar = Progress.Bar(args.nframes)
for frame_index in range(args.nframes):
t = frame_index / float(args.nframes - 1)
image_string = get_animation_frame(args.image_format, physical_size,
args.scale, args.tree,
args.eigenvector_index, t_to_yaw(t),
t_to_pitch(t))
image_filename = 'frame-%04d.%s' % (frame_index, args.image_format)
image_pathname = os.path.join(args.output_directory, image_filename)
with open(image_pathname, 'wb') as fout:
fout.write(image_string)
pbar.update(frame_index + 1)
pbar.finish()
def main(args):
# do some validation
if args.nframes < 2:
raise ValueError('nframes should be at least 2')
# define the requested physical size of the images (in pixels)
physical_size = (args.physical_width, args.physical_height)
# build the newick tree from the string
tree = NewickIO.parse(args.tree, FelTree.NewickTree)
nvertices = len(list(tree.preorder()))
nleaves = len(list(tree.gen_tips()))
# Get ordered ids with the leaves first,
# and get the corresponding distance matrix.
ordered_ids = get_ordered_ids(tree)
D = np.array(tree.get_partial_distance_matrix(ordered_ids))
index_edges = get_index_edges(tree, ordered_ids)
# Create the reference points
# so that the video frames are not reflected arbitrarily.
reference_points = Euclid.edm_to_points(D).T[:2].T
# create the animation frames and write them as image files
pbar = Progress.Bar(args.nframes)
for frame_index in range(args.nframes):
linear_progress = frame_index / float(args.nframes - 1)
if args.interpolation == 'sigmoid':
progress = sigmoid(linear_progress)
else:
progress = linear_progress
mass_vector = get_mass_vector(nvertices, nleaves, progress)
points = get_canonical_2d_mds(D, mass_vector, reference_points)
image_string = get_animation_frame(args.image_format, physical_size,
args.scale, mass_vector,
index_edges, points)
image_filename = 'frame-%04d.%s' % (frame_index, args.image_format)
image_pathname = os.path.join(args.output_directory, image_filename)
with open(image_pathname, 'wb') as fout:
fout.write(image_string)
pbar.update(frame_index + 1)
pbar.finish()
def progress_stub1(total_units, done_units):
"""Stub for encapsulating the 'processing'' formatter"""
Progress.report(PROCESSING_STATUS_STRING, total_units, done_units)
def main():
"""Main function"""
# Build program config from command line and config file.
config = Config.Config()
warnings = []
# Recursively tranverse from the provided root directory looking for mp3s:
# * dirname gives the path to the current directory
# * dirnames gives the list of subdirectories in the folder
# * filenames gives the list of files in the folder
Progress.state(SEARCHING_STATUS_STRING)
track_list = []
for dirname, subdirnames, filenames in os.walk(config.directory):
# Extract and clean filenames of all mp3s
cleaned_mp3_filenames = extract_mp3s_and_clean(filenames)
for f in cleaned_mp3_filenames:
# Extract all the information possible from the song and add it.
file_path = os.path.join(dirname, f.original)
new_file = TrackFile.TrackFile(file_path, f.cleaned)
track_list.append(new_file)
track_count = len(track_list)
# create storage system
music_collection = TrackCollection.TrackCollection()
# Add all located files to the collection.
for i in range(track_count):
track_list[i].load_all_data()
#print track_list[i]
track_list[i].finalise_data()
music_collection.add(track_list[i])
Progress.report(INDEXING_STATUS_STRING, track_count, i+1)
print_warnings(warnings)
# Remove all duplicate files from the collection.
def progress_stub1(total_units, done_units):
"""Stub for encapsulating the 'processing'' formatter"""
Progress.report(PROCESSING_STATUS_STRING, total_units, done_units)
music_collection.remove_duplicates(warnings, progress_stub1)
print_warnings(warnings)
# Standardise track data on the remaining files.
def progress_stub2(total_units, done_units):
"""Stub for encapsulating the 'standardising'' formatter"""
Progress.report(STANDARDISING_STATUS_STRING, total_units, done_units)
music_collection.standardise_album_tracks(warnings, progress_stub2)
print_warnings(warnings)
if config.verbose:
music_collection.sort_songs_by_track()
print music_collection
if config.dry_run:
Progress.skip(REWRITING_STATUS_STRING)
else:
print "TBD"
# write the newly corrected data to a new file system with new tags
#new_folder = generate_new_filepath(args.directory)
#print "Creating new directory structure in %s." % (new_folder)
#music_collection.create_new_filesystem(new_folder)
# done
print "Finished."
def do_command_line_analysis(options):
"""
Print some stuff to stdout, and show a progress bar on stderr.
@param options: an object from optparse
"""
# load the tree, using the default tree if no filename was provided
tree, tree_remark = get_tree_and_remark(options)
# initialize the simulation objects
sims = [
Simulation(Clustering.NeighborJoiningDMS(), 'nj', 'neighbor joining'),
Simulation(Clustering.StoneSpectralSignDMS(), 'nj',
'spectral sign cut with neighbor joining fallback'),
Simulation(Clustering.RandomDMS(), 'nj', 'random partitioning')
]
# possibly add the slow simulation
if options.use_exact:
sims.append(
Simulation(Clustering.StoneExactDMS(), 'nj',
'exact criterion with neighbor joining fallback'))
# define the simulation parameters
reconstruction_count = options.nsamples
sequence_length_string = options.sequence_length
if sequence_length_string == 'inf':
sequence_length = float('inf')
else:
sequence_length = int(sequence_length_string)
inf_replacement = 20.0
if options.reject_inf:
inf_replacement = None
elif options.replace_inf:
try:
inf_replacement = float(options.replace_inf)
except ValueError:
msg = 'invalid replace_inf value: '
raise OptionError(msg + str(options.replace_inf))
zero_replacement = 0
if options.reject_zero:
zero_replacement = None
elif options.replace_zero:
try:
zero_replacement = float(options.replace_zero)
except ValueError:
msg = 'invalid replace_zero value: '
raise OptionError(msg + str(options.replace_zero))
# start the html file
print '<html><body>'
# show the simulation parameters
print 'original tree source:', tree_remark, '<br/>'
print 'reconstruction count:', reconstruction_count, '<br/>'
print 'sequence length:', sequence_length, '<br/>'
# set the simulation parameters for each simulation
for sim in sims:
sim.set_original_tree(tree)
# If there is only one reconstruction per method
# then show the progress of the tree builder.
if reconstruction_count == 1:
sim.set_verbose()
# define an arbitrary but consistent ordering of the taxa
ordered_names = [node.name for node in tree.gen_tips()]
try:
# attempt to simulate a bunch of distance matrices
if options.verbose:
print 'sampling', reconstruction_count, 'distance matrices...'
# initialize the distance matrix sampler
sampler = DMSampler.DMSampler(tree, ordered_names, sequence_length)
sampler.set_inf_replacement(inf_replacement)
sampler.set_zero_replacement(zero_replacement)
# start the progress bar
pbar = Progress.Bar(1.0)
# sample some distance matrices
distance_matrices = []
for result in sampler.gen_samples_or_none():
# if we got a result then update the distance matrix list
if result:
sequence_list, D = result
distance_matrices.append(D)
# Update the progressbar regardless of whether or not
# the proposal was accepted.
remaining_acceptances = reconstruction_count - len(
distance_matrices)
numerator = sampler.get_completed_proposals()
denominator = numerator + sampler.get_remaining_proposals(
remaining_acceptances)
dms_fraction = float(numerator) / float(denominator)
dms_total = 1.0 / (1 + len(sims))
pbar.update(dms_fraction * dms_total)
# if we have enough samples then break the loop
if not remaining_acceptances:
break
# reconstruct trees using various methods
for i, sim in enumerate(sims):
if options.verbose:
print 'running "%s"...' % sim.description
sim.run(distance_matrices, ordered_names)
pbar.update(float(i + 2) / float(1 + len(sims)))
# stop the progress bar
pbar.finish()
# get the simulation data
table = [('method', 'seconds', 'uniform loss', 'weighted loss')]
for sim in sims:
table.append((sim.description, sim.get_running_time(),
sim.get_uniform_loss(), sim.get_deep_loss()))
# convert the row major matrix into an html table
print HtmlTable.get_table_string(table)
# end the html file
print '</html></body>'
except KeyboardInterrupt:
print 'interrupted stage', pbar.progress, 'of', pbar.high
def do_hard_coded_analysis_b(tree, tree_remark):
"""
Do a hardcoded analysis of tree reconstruction methods.
Make R files of ordered reconstruction losses.
@param tree: a tree object
@param tree_remark: a string that is a comment about the tree
"""
# define an arbitrary order for the names of the leaves of the tree
ordered_names = list(node.name for node in tree.gen_tips())
# use some replicates
reconstruction_count = 100
# Make R files for reconstruction results from sequences
# of some number of nucleotides in length.
sequence_length = 2000
# define the tree reconstruction methods to be used
sims = [
Simulation(Clustering.NeighborJoiningDMS(), 'nj', 'neighbor joining'),
Simulation(Clustering.StoneSpectralSignDMS(), 'nj', 'spectral sign')
]
# set tree reconstruction parameters
for sim in sims:
sim.set_original_tree(tree)
# initialize the distance matrix sampler
sampler = DMSampler.InfiniteAllelesSampler(tree, ordered_names,
sequence_length)
sampler.set_inf_replacement(20.0)
sampler.set_zero_replacement(0.0)
# start the progress bar
pbar = Progress.Bar(1.0)
# sample some distance matrices
distance_matrix_start_time = time.time()
distance_matrices = []
for result in sampler.gen_samples_or_none():
# if we got a result then update the distance matrix list
if result:
sequence_list, D = result
distance_matrices.append(D)
# Update the progressbar regardless of whether or not
# the proposal was accepted.
remaining_acceptances = reconstruction_count - len(distance_matrices)
numerator = sampler.get_completed_proposals()
denominator = numerator + sampler.get_remaining_proposals(
remaining_acceptances)
dms_fraction = float(numerator) / float(denominator)
dms_total = 1.0 / (1 + len(sims))
pbar.update(dms_fraction * dms_total)
# if we have enough samples then break the loop
if not remaining_acceptances:
break
distance_matrix_seconds = time.time() - distance_matrix_start_time
# reconstruct trees using various methods
reconstruction_seconds = []
for i, sim in enumerate(sims):
reconstruction_start_time = time.time()
print 'reconstructing', len(distance_matrices), 'trees'
print 'using', sim.description
sim.run(distance_matrices, ordered_names)
pbar.update(float(i + 2) / float(1 + len(sims)))
reconstruction_seconds.append(time.time() - reconstruction_start_time)
# stop the progress bar
pbar.finish()
# consider the neighbor joining and the spectral sign results
nj_sim, ss_sim = sims
# extract the simulation data
label_list_pairs = [
('nj.unweighted', nj_sim.get_normalized_error_counts()),
('ss.unweighted', ss_sim.get_normalized_error_counts()),
('nj.weighted', nj_sim.get_normalized_loss_values()),
('ss.weighted', ss_sim.get_normalized_loss_values())
]
labels, transposed_table = zip(*label_list_pairs)
table = zip(*transposed_table)
table_string = RUtil.get_table_string(table, labels)
# write the table
filename = 'out3.table'
with open(filename, 'w') as fout:
print >> fout, '# tree source:', tree_remark
print >> fout, '# number of taxa:', len(ordered_names)
print >> fout, '# sampled distance matrices:', len(distance_matrices)
print >> fout, '# sampling seconds elapsed:', distance_matrix_seconds
print >> fout, '# sites per sequence:', sequence_length
for sim, seconds in zip(sims, reconstruction_seconds):
msg_a = '# seconds elapsed for tree reconstruction using '
msg_b = sim.description + ': ' + str(seconds)
print >> fout, msg_a + msg_b
print >> fout, table_string
print 'wrote', filename
def process(input_lines, good_coverage, bad_coverage, randomization_rate,
nseconds, use_pbar):
"""
@param input_lines: lines of input of csv data including the header
@param good_coverage: the expected number of reads at informative positions
@param bad_coverage: the expected number of reads at uninformative positions
@param randomization_rate: the probability of an error per read
@param nseconds: None or impose a time limit of this many seconds
@param use_pbar: True iff a progress bar should be used
@return: a multi-line string of the annotated csv file
"""
verbose = False
# validate the number of lines
if len(input_lines) < 6:
raise ValueError('there should be at least six lines of input')
if len(input_lines) % 5 != 1:
raise ValueError(
'the input lines should consist of a header plus a multiple of five data lines'
)
# break the lines of input into rows of elements
input_rows = [line_to_row(line) for line in input_lines]
# validate the columns of data
ncolumns = len(input_rows[0])
if ncolumns < 7:
raise ValueError('there should be at least seven columns of input')
if ncolumns % 2 != 1:
raise ValueError('the number of input columns should be odd')
for row in input_rows:
if len(row) != ncolumns:
raise ValueError(
'each row of input should have the same number of elements as the first row'
)
# define the three models
homozygous = ReadCoverage.Homozygous(randomization_rate, good_coverage)
heterozygous = ReadCoverage.Heterozygous(randomization_rate, good_coverage)
overcovered = ReadCoverage.Overcovered(randomization_rate, bad_coverage)
models = [homozygous, heterozygous, overcovered]
# initialize the output header row
header_row = input_rows[0]
output_header_row = header_row[:5]
for heading in header_row[5:]:
if heading.endswith('sco'):
output_header_row.append(heading)
elif heading.endswith('cov'):
output_header_row.extend([
heading, heading + '_hom', heading + '_het', heading + '_ovr'
])
else:
raise ValueError(
'each heading after the fifth should end with sco or cov')
# get the rest of the rows
data_rows = input_rows[1:]
# define the number of genomic positions and the number of strains
npositions = len(data_rows) / 5
nstrains = (ncolumns - 5) / 2
# begin the output
out = StringIO()
print >> out, ','.join(output_header_row)
# initialize some stuff
start_time = time.time()
pbar = Progress.Bar(npositions) if use_pbar else None
try:
for position in range(npositions):
# check the time
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
# get a chunk of five consecutive rows
position_rows = [data_rows[position * 5 + i] for i in range(5)]
# get the corresponding log likelihoods
log_likelihood_lists = get_log_likelihoods_per_strain(
position_rows, models)
# construct five annotated output lines
for position_row in position_rows:
output_row = position_row[:5]
for i, log_likelihoods in enumerate(log_likelihood_lists):
# add the coverage, three annotations, and the score
coverage_string = position_row[5 + 2 * i]
score_string = position_row[5 + 2 * i + 1]
if log_likelihoods:
annotations = [str(x) for x in log_likelihoods]
else:
annotations = ['-', '-', '-']
output_row.extend([coverage_string] + annotations +
[score_string])
print >> out, ','.join(output_row)
# update the progress bar
if pbar:
pbar.update(position + 1)
except KeyboardInterrupt, e:
if pbar:
pbar.finish()
raise e
def UnifyPictureNames(inputFolder, outputFolder):
processedFiles = 0
renamedFiles = 0
unchangedFiles = 0
allFilePaths = IM.GetListOfAllFilePaths(inputFolder)
for filePath in allFilePaths:
inputDirectory, inputFile = os.path.split(filePath)
if inputFile.lower().endswith(('.png', '.jpg', '.jpeg')):
# get file metadata
metadata = exifread.process_file(open(filePath, "rb"))
captureTime = ""
try:
captureTime = str(metadata["EXIF DateTimeOriginal"])
except Exception:
logger.info(
"No capture time found for: \"{}\"".format(filePath))
# pictures with creation timestamp
if captureTime != "":
_, fileExtension = os.path.splitext(inputFile)
newFile = captureTime.replace(":", "").replace(
" ", "_") + fileExtension.lower()
IM.CopyFile(
filePath,
IM.GetUniqueFile(
os.path.join(
inputDirectory.replace(inputFolder, outputFolder),
newFile)))
logger.debug("{} --> {}".format(
filePath,
os.path.join(
inputDirectory.replace(inputFolder, outputFolder),
newFile)))
renamedFiles += 1
# WhatsApp images
elif "WA" in inputFile:
newFile = inputFile.replace("IMG-", "")
IM.CopyFile(
filePath,
IM.GetUniqueFile(
os.path.join(
inputDirectory.replace(inputFolder, outputFolder),
newFile)))
logger.debug("{} --> {}".format(
filePath,
os.path.join(
inputDirectory.replace(inputFolder, outputFolder),
newFile)))
renamedFiles += 1
# files without creation timestamp
else:
IM.CopyFile(
filePath,
IM.GetUniqueFile(
filePath.replace(inputFolder, outputFolder)))
unchangedFiles += 1
else:
IM.CopyFile(
filePath,
IM.GetUniqueFile(filePath.replace(inputFolder, outputFolder)))
unchangedFiles += 1
processedFiles += 1
Progress.PrintProgress(processedFiles)
logger.info(
"Processed Files: {} | RenamedFiles: {} | Unchanged Files: {}".format(
processedFiles, renamedFiles, unchangedFiles))
def init(data):
# Board specs init
data.size = data.width // 30
data.border = data.size * 0.15
data.offset = 51
data.diff = 57.5
# Player and board init
data.player = Player(data)
data.board = Board()
# Walls init
data.walls = Wall()
data.walls.createWalls(data)
data.all = data.walls.getWalls(data)
# Leaderboard init
data.leader = Leaderboard()
# Start init
data.timer = 0
data.allPicked = list()
data.boxText = "Check answer"
data.stringColor = "white"
# Bools init
data.language = True
data.home = False
data.play = False
data.learn = False
data.custom = False
data.progress = False
data.leaderboard = False
data.about = False
data.correct = False
data.wrong = False
data.next = False
data.saved = False
data.showEnglish = False
data.showPronounciation = False
# Player stored info init
data.cDict = dict()
data.wDict = dict()
data.aDict = dict()
data.defaultList = list()
data.customList = list()
data.customEnglishList = list()
data.defaultAndCustomList = list()
# Leaderboard stored info init
data.lDict = dict()
# Progress info init
data.prog = Progress()
data.currXP = 0
data.allXP = 0
# Translator init
data.translator = Translator()
def progress_stub2(total_units, done_units):
"""Stub for encapsulating the 'standardising'' formatter"""
Progress.report(STANDARDISING_STATUS_STRING, total_units, done_units)
def process(ntaxa, nseconds, seqlen, nsamples, branch_length_sampler,
use_pbar):
"""
@param ntaxa: the number of taxa per tree
@param nseconds: stop after this many seconds
@param seqlen: use this sequence length
@param nsamples: stop after this many samples per sequence length
@param branch_length_sampler: this function samples branch lengths independently
@param use_pbar: True iff a progress bar should be used
@return: a multi-line string of the contents of an R table
"""
# initialize the global rejection counts
nrejected_zero = 0
nrejected_inf = 0
nrejected_fail = 0
naccepted = 0
# Initialize the accumulation matrix.
# The rows specify the size of the smaller side of the initial split.
# The columns specify the compatibility status of the split.
nsmall_sizes = (ntaxa / 2) + 1
accum = np.zeros((nsmall_sizes, 2), dtype=np.int)
# Repeatedly analyze samples.
# We might have to stop early if we run out of time or if ctrl-c is pressed.
# If we have to stop early, then show the results of the progress so far.
termination_reason = 'no reason for termination was given'
start_time = time.time()
pbar = Progress.Bar(nsamples) if use_pbar else None
try:
for sample_index in range(nsamples):
# keep trying to get an accepted sample
while True:
# check the time
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
# first sample a tree and get its set of informative splits
tree = TreeSampler.sample_agglomerated_tree(ntaxa)
true_splits = tree.get_nontrivial_splits()
# sample the branch lengths
for branch in tree.get_branches():
branch.length = branch_length_sampler()
# Attempt to sample a distance matrix.
# If the sample was rejected then note the reason and go back to the drawing board.
try:
D = sample_distance_matrix(tree, seqlen)
except InfiniteDistanceError as e:
nrejected_inf += 1
continue
except ZeroDistanceError as e:
nrejected_zero += 1
continue
# Attempt to estimate the primary split of the tree from the distance matrix.
# If there was a technical failure then note it and go back to the drawing board.
# Otherwise note the compatibility and balance of the split.
try:
eigensplit = BuildTreeTopology.split_using_eigenvector(D)
small_size = min(len(side) for side in eigensplit)
if eigensplit in true_splits:
compatibility = 1
else:
compatibility = 0
except BuildTreeTopology.DegenerateSplitException, e:
small_size = 0
compatibility = 1
except BuildTreeTopology.InvalidSpectralSplitException, e:
nrejected_fail += 1
continue
def main(input_path, output_path_train, output_path_test, factor, samples, preserve_test_ids):
output_dir = os.path.dirname(os.path.abspath(output_path_train))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_dir = os.path.dirname(os.path.abspath(output_path_test))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
kill_path = output_dir + '/kill'
killfile_monitor = KillFileMonitor(kill_path, 1)
output_path_train_temp = output_path_train + '.incomplete'
output_path_test_temp = output_path_test + '.incomplete'
try:
test_ids = set()
if preserve_test_ids and os.path.exists(output_path_test):
with open(output_path_test, 'r') as in_file:
in_file.readline()
while True:
killfile_monitor.maybe_check_killfile()
line = in_file.readline()
if not line or len(line) <= 2:
break
first = line.index(';')
skipped_index = line[first + 1:]
second = skipped_index.index(';')
id = skipped_index[0:second]
test_ids.add(id)
random_line_positions = get_line_positions(input_path, lambda: killfile_monitor.maybe_check_killfile())
with open(input_path, 'r', encoding='utf8') as in_file:
shuffle(random_line_positions)
if samples:
random_line_positions = random_line_positions[0:samples]
testCount = int(len(random_line_positions) * factor)
testLines = random_line_positions[0:testCount]
trainLines = random_line_positions[testCount:]
moveFromTrainToTest = [x for x in trainLines if x['Id'] in test_ids]
potentialTrainInTest = [x for x in testLines if x['Id'] not in test_ids]
moveFromTestToTrain = potentialTrainInTest[:min(len(moveFromTrainToTest), len(potentialTrainInTest))]
testLines = [x for x in testLines if x not in moveFromTestToTrain] + moveFromTrainToTest
trainLines = [x for x in trainLines if x not in moveFromTrainToTest] + moveFromTestToTrain
if not all(not x['Id'] in test_ids for x in trainLines):
raise Exception("train_lines in test_ids")
if any(x in testLines for x in trainLines):
raise Exception("train_lines in test_lines")
if len(trainLines) + len(testLines) != len(random_line_positions):
raise Exception("train_lines or test_lines missing")
header = in_file.readline()
i = 0
in_file.seek(0)
progress = Progress('split test/train: ', 1)
progress.set_count(len(random_line_positions))
with open(output_path_test_temp, 'w', encoding='utf8') as out_file:
out_file.writelines([header])
for item in testLines:
killfile_monitor.maybe_check_killfile()
in_file.seek(item['Position'])
line = in_file.readline()
if not line.endswith('\n'):
line += '\n'
out_file.writelines([line])
progress.add_item()
progress.maybe_print()
i += 1
in_file.seek(0)
with open(output_path_train_temp, 'w', encoding='utf8') as out_file:
out_file.writelines([header])
for item in trainLines:
killfile_monitor.maybe_check_killfile()
in_file.seek(item['Position'])
line = in_file.readline()
if not line.endswith('\n'):
line += '\n'
out_file.writelines([line])
progress.add_item()
progress.maybe_print()
i += 1
if os.path.exists(output_path_train):
os.remove(output_path_train)
os.rename(output_path_train_temp, output_path_train)
if os.path.exists(output_path_test):
os.remove(output_path_test)
os.rename(output_path_test_temp, output_path_test)
except KilledException:
killfile_monitor.delete_killfile()
if os.path.exists(output_path_train_temp):
os.remove(output_path_train_temp)
if os.path.exists(output_path_test_temp):
os.remove(output_path_test_temp)
print_flush('Killed.')
def UnifyPictureNames(inputFolder, outputFolder):
processedFiles = 0
renamedFiles = 0
unchangedFiles = 0
for path, subDirectories, files in os.walk(inputFolder):
# create output path
if not os.path.exists(path.replace(inputFolder, outputFolder)):
os.makedirs(path.replace(inputFolder, outputFolder))
for file in files:
filePath = os.path.join(path, file)
if file.lower().endswith(('.png', '.jpg', '.jpeg')):
# get file metadata
metadata = exifread.process_file(open(filePath, "rb"))
captureTime = ""
try:
captureTime = str(metadata["EXIF DateTimeOriginal"])
except Exception:
logging.info(
"{0} - No capture time found for: \"{1}\"".format(
GetFormattedDatetimeNow(), filePath))
# pictures with creation timestamp
if captureTime != "":
_, fileExtension = os.path.splitext(file)
newFile = captureTime.replace(":", "").replace(
" ", "_") + fileExtension.lower()
copy2(
filePath,
GetUniqueFile(path.replace(inputFolder, outputFolder),
newFile))
logging.debug("{0} - {1} --> {2}".format(
GetFormattedDatetimeNow(), filePath,
os.path.join(path.replace(inputFolder, outputFolder),
newFile)))
renamedFiles += 1
# WhatsApp images
elif "WA" in file:
newFile = file.replace("IMG-", "")
copy2(
filePath,
GetUniqueFile(path.replace(inputFolder, outputFolder),
newFile))
logging.debug("{0} - {1} --> {2}".format(
GetFormattedDatetimeNow(), filePath,
os.path.join(path.replace(inputFolder, outputFolder),
newFile)))
renamedFiles += 1
# files without creation timestamp
else:
copy2(
filePath,
GetUniqueFile(path.replace(inputFolder, outputFolder),
file))
unchangedFiles += 1
else:
copy2(
filePath,
GetUniqueFile(path.replace(inputFolder, outputFolder),
file))
unchangedFiles += 1
processedFiles += 1
Progress.PrintProgress(processedFiles)
logging.info(
"{0} - Processed Files: {1} | RenamedFiles: {2} | Unchanged Files: {3}"
.format(GetFormattedDatetimeNow(), processedFiles, renamedFiles,
unchangedFiles))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.windowTitleChanged.connect(self.onWindowTitleChange)
self.windowTitleChanged.connect(lambda x: self.my_custom_fn(x, 25))
self.setWindowTitle("My window")
self.setFixedSize(1024, 600)
# self.label = QLabel("THIS IS AWESOME!!!")
# self.label.setTextFormat(Qt.RichText)
# self.setCentralWidget(self.label)
# self.label.setAlignment(Qt.AlignCenter)
# self.label1 = QLabel("THIS IS AWESOME!!!")
# self.label1.setAlignment(Qt.AlignCenter)
# self.setCentralWidget(self.label1)
# toolbar = QToolBar("My main toolbar")
# toolbar.iconSize
# self.addToolBar(toolbar)
self.mainLayout = QVBoxLayout()
self.layout1 = QHBoxLayout()
self.layout1_1 = QVBoxLayout()
self.layout1_2 = QHBoxLayout()
self.layout2 = QHBoxLayout()
self.layout3 = QHBoxLayout()
self.thread = ListenWebsocket()
self.thread.start()
pixmap = QPixmap('progress.png')
self.progress = Progress.Progress(pixmap)
self.label = DisplayMass.Mass()
# self.setCentralWidget(self.progress)
# self.setCentralWidget(self.label)
stableIcon = QPixmap('stable_black.svg')
taraIcon = QPixmap('tara_black.svg')
zeroIcon = QPixmap('zero.svg')
self.iconT = Icon.Icon(taraIcon)
self.iconS = Icon.Icon(stableIcon)
self.iconZ = Icon.Icon(zeroIcon)
self.layout1_1.addWidget(self.iconS)
self.layout1_1.addWidget(self.iconT)
self.layout1_1.addWidget(self.iconZ)
# self.layout1.addWidget(self.icons)
self.layout2.addWidget(self.progress)
self.layout2.setAlignment(Qt.AlignCenter)
self.layout1.addLayout(self.layout1_1)
self.layout1_2.addWidget(self.label)
self.layout1.addLayout(self.layout1_2)
# button = QPushButton('New', self)
second = SecondWindow.SecondWindow(self)
second.menus()
button_action = Butt.Butt(second)
# button_action.menus()
self.layout3.addWidget(button_action)
# self.layout3.addWidget(button)
self.mainLayout.addLayout(self.layout1)
self.mainLayout.addLayout(self.layout2)
self.mainLayout.addLayout(self.layout3)
# self.setLayout(self.mainLayout)
widget = QWidget()
widget.setLayout(self.mainLayout)
self.setCentralWidget(widget)
# self.client = QtWebSockets.QWebSocket("",QtWebSockets.QWebSocketProtocol.Version13,None)
# self.client.open(QUrl("ws://10.10.3.60:4101"))
# self.client.sendTextMessage('{"COMMAND": "REGISTER_LISTENER", "PARAM": "MENU"}')
# text = self.client.textMessageReceived
# text.connect(self.onWindowTitleChange)
# pprint(text)
# for attr in dir(text):
# print("obj.%s = %r" % (attr, getattr(text, attr)))
# button_action = QAction(QIcon("kolo.svg"),"Your button", self)
# button_action2 = QAction(QIcon("kolo.svg"),"Second", self)
# toolbar.setToolButtonStyle(Qt.ToolButtonTextOnly)
# button_action.setStatusTip("This is your button")
# button_action.triggered.connect(self.onMyToolBarButtonClick)
# button_action.setCheckable(True)
# toolbar.addAction(button_action)
# toolbar.addSeparator()
# button_action2.setStatusTip("Second")
# button_action2.triggered.connect(self.onMyToolBarButtonClick)
# button_action2.setCheckable(True)
# toolbar.addAction(button_action2)
# toolbar.addWidget(QLabel("Hello"))
# toolbar.addWidget(QCheckBox())
# toolbar.setIconSize(QSize(50,50))
self.color = '#ff0000'
self.setStatusBar(QStatusBar(self))
# menu = self.menuBar()
# file_menu = menu.addMenu("&File")
# file_menu.addAction(button_action)
# file_menu.addSeparator()
# file_menu.addAction(button_action2)
# socket = QtWebSockets.QWebSocket('test')
self.xman = self.thread.getMessage()
def main():
# Initialize pygame
# os.environ['SDL_VIDEODRIVER'] = 'windib'
pygame.init()
pygame.mouse.set_cursor(*pygame.cursors.tri_left)
os.environ["SDL_VIDEO_CENTERED"] = "1" #
Game.framesPerSecond = 48
Game.speedModifier = 1
Game.autoMode = 0
Game.balanceMode = 0
drawTick = 0
Game.state = STATE_INITMENU
Game.mainMenuFont = pygame.font.Font(None, 32)
Game.gameMenuFont = pygame.font.Font(None, 24)
Game.enemyCountFont = pygame.font.Font(None, 24)
Game.popUpFont = pygame.font.Font(None, 24)
# Set the height and width of the screen
size = [mapWidth * tileSize + rightMenuSize, mapHeight * tileSize + bottomMenuSize]
screen = pygame.display.set_mode(size)
screen.fill(background)
layer = pygame.Surface(size)
layer.set_colorkey(spritepink)
layer.set_alpha(120)
# Variables to tell Game what to draw
Game.drawMouseOver = 0
Game.drawMessage = 0
Game.repaintMap = 0
Game.placedTower = 0
Game.EndLevelDraw = 0
Game.restartWave = 0
Game.nextWave = 0
# Initialize the Images
Images.init()
# Initialize the MainMenu
mainMenu = cMenu(
128,
320,
20,
5,
"vertical",
100,
screen,
[
("Campaign", 1, None),
("Challenge 1", 2, None),
("Challenge 2", 3, None),
("Challenge 3", 4, None),
("Challenge 4", 5, None),
("Challenge 5", 6, None),
("Random Level", 7, None),
("Exit", 8, None),
],
Images.Background,
)
gameMenu = cMenu(
128,
320,
20,
5,
"vertical",
100,
screen,
[("Resume", 1, None), ("Back to main menu (current progress will be lost!)", 2, None)],
Images.Background,
)
menubackground = Images.Background
interfaceBGwashed = Images.InterfaceBGwashed
InterfaceBGopaque = Images.InterfaceBGopaque
rect_list = []
# Initialize the map
map = Map.Map(mapWidth, mapHeight)
# Initialize the wave
wave = Wave.Wave(map)
# Initialize the Towers class
towers = Towers.Towers(map, wave)
wave.setTowers(towers)
# Initialize the shot graphics
shots = Shots()
# Initialize the tower bar
towerBar = TowerBar.TowerBar(0, mapHeight * tileSize, map, towers)
# Initialize the menu
menu = Menu.Menu(map, wave, towers)
# Initialize the level
Game.level = Level.Level(map, wave, towers, towerBar, menu)
# Set title of screen
pygame.display.set_caption("AI Tower Defense -- (c) POB & ND")
# Loop until the user clicks the close button.
close_game = False
# Used to manage how fast the screen updates
clock = pygame.time.Clock()
# -------- Main Program Loop -----------
while close_game == False:
## Init menu
if Game.state == STATE_INITMENU:
screen.fill(background)
screen.blit(menubackground, (0, 0))
pygame.display.flip()
mainMenustate = 0
mainMenuprev_state = 1
Game.state = STATE_MENU
## Menu
elif Game.state == STATE_MENU:
if mainMenuprev_state != mainMenustate:
pygame.event.post(pygame.event.Event(EVENT_CHANGE_STATE, key=0))
mainMenuprev_state = mainMenustate
e = pygame.event.wait()
if e.type == pygame.KEYDOWN or e.type == EVENT_CHANGE_STATE:
if mainMenustate == 0:
rect_list, mainMenustate = mainMenu.update(e, mainMenustate)
# Campagin
elif mainMenustate == 1:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Campaign1")
# Challenges
elif mainMenustate == 2:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Challenge1")
elif mainMenustate == 3:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Challenge2")
elif mainMenustate == 4:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Challenge3")
elif mainMenustate == 5:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Challenge4")
elif mainMenustate == 6:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Challenge5")
# Random Level
elif mainMenustate == 7:
Game.state = STATE_LOADGAME
Game.level.loadLevel("Campaign1")
# Quit
else:
pygame.quit()
sys.exit()
if e.type == pygame.QUIT:
pygame.quit()
sys.exit()
pygame.display.update(rect_list)
## Init Game Menu
elif Game.state == STATE_INITGAMEMENU:
screen.fill(background)
screen.blit(menubackground, (0, 0))
pygame.display.flip()
gameMenustate = 0
gameMenuprev_state = 1
Game.state = STATE_GAMEMENU
## Game Menu
elif Game.state == STATE_GAMEMENU:
if gameMenuprev_state != gameMenustate:
pygame.event.post(pygame.event.Event(EVENT_CHANGE_STATE, key=0))
gameMenuprev_state = gameMenustate
e = pygame.event.wait()
if e.type == pygame.KEYDOWN or e.type == EVENT_CHANGE_STATE:
if e.key == pygame.K_ESCAPE:
Game.state = STATE_LOADGAME
else:
if gameMenustate == 0:
rect_list, gameMenustate = gameMenu.update(e, gameMenustate)
elif gameMenustate == 1:
Game.state = STATE_LOADGAME
elif gameMenustate == 2:
shots.clear()
towers.clear()
wave.clear()
Game.state = STATE_INITMENU
if e.type == pygame.QUIT:
pygame.quit()
sys.exit()
pygame.display.update(rect_list)
## POP UP
if Game.state == STATE_INITPOPUP:
Game.state = STATE_POPUP
# Add a background or the game behind the text...
screen.blit(InterfaceBGopaque, (0, 0))
drawMap(map, screen)
menu.draw(screen)
Game.redrawSPBtn = 1
menu.drawSPBtn(screen)
towerBar.draw(screen)
towerBar.moneyUpdated(screen)
towerBar.showTower(screen)
towerBar.updateTowerCost = 0
pygame.display.flip()
elif Game.state == STATE_POPUP:
drawTick += 1
if drawTick >= clock.get_fps() / 24:
drawTick = 0
Game.popUp.paint(screen)
pygame.display.update(167, 90, 658, 465)
for event in pygame.event.get():
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
Game.dialog.close()
elif event.key == pygame.K_RETURN:
Game.dialog.close()
elif event.key == pygame.K_SPACE:
Game.dialog.close()
elif event.type == pygame.QUIT:
sys.exit()
else:
Game.popUp.event(event)
## Wave finished [Draw then restartWave or start next Wave] -> PopUp lorsque level finished?
elif Game.state == STATE_ENDWAVE:
if not Game.autoMode and not Game.balanceMode:
drawTick += 1
if drawTick >= clock.get_fps() / 24:
Game.EndLevelDraw += 1
drawTick = 0
drawGame(map, towerBar, towers, wave, shots, menu, screen, layer)
if Game.EndLevelDraw >= 5:
Game.state = STATE_PREPARATION
Game.EndLevelDraw = 0
if Game.restartWave:
Game.restartWave = 0
Game.level.restartWave()
elif Game.nextWave:
Game.nextWave = 0
Game.level.nextWave()
else:
print "Confus :("
else:
for event in pygame.event.get(): # User did something
# If user clicked close
if event.type == pygame.QUIT:
close_game = True # Flag that we are done so we exit this loop
## Mouse Events
# User moves over the mouse
elif event.type == pygame.MOUSEMOTION:
updateUnderMouse(map, towerBar, towers)
# User clicks the mouse. Get the position
elif event.type == pygame.MOUSEBUTTONDOWN:
pos = pygame.mouse.get_pos()
# Change the x/y screen coordinates to grid coordinates (For the map)
column = pos[0] // tileSize
row = pos[1] // tileSize
# Inside Map
if (column < mapWidth) and (row < mapHeight):
if towerBar.selectedTower > -1:
if map.M[row][column] == car_turret:
if map.T[row][column] == 0:
# TODO : money check
towers.placeTower(towerBar.selectedTower, 0, row, column)
Game.placedTower = 1
else:
towers.updateTower(towerBar.selectedTower, row, column)
Game.placedTower = 1
elif towerBar.selectedTower == TowerUPGRADE:
if map.M[row][column] == car_turret:
if map.T[row][column] != 0:
towers.updateTower(towerBar.selectedTower, row, column)
Game.placedTower = 1
elif towerBar.selectedTower == TowerERASE:
if map.M[row][column] == car_turret:
if map.T[row][column] != 0:
towers.eraseTower(row, column)
Game.placedTower = 1
updateUnderMouse(map, towerBar, towers)
# Inside Menu
elif column >= mapWidth:
menu.onClick(pos, map)
# Inside Tower Bar
else:
towerBar.onClick(pos)
## Keyboard Events
elif event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
Game.state = STATE_INITGAMEMENU
elif event.key == pygame.K_SPACE:
if Game.state == STATE_GAME:
Game.redrawSPBtn = 1
Game.state = STATE_PREPARATION
elif Game.state == STATE_PREPARATION:
Game.redrawSPBtn = 2
Game.state = STATE_GAME
elif event.key == pygame.K_1:
if 0 in Game.level.levelTowers:
towerBar.selectTower(0)
elif event.key == pygame.K_2:
if 1 in Game.level.levelTowers:
towerBar.selectTower(1)
elif event.key == pygame.K_3:
if 2 in Game.level.levelTowers:
towerBar.selectTower(2)
elif event.key == pygame.K_4:
if 3 in Game.level.levelTowers:
towerBar.selectTower(3)
elif event.key == pygame.K_5:
if 4 in Game.level.levelTowers:
towerBar.selectTower(4)
elif event.key == pygame.K_6:
if 5 in Game.level.levelTowers:
towerBar.selectTower(5)
elif event.key == pygame.K_EQUALS:
Game.increaseSpeed(menu)
elif event.key == pygame.K_MINUS:
Game.reduceSpeed(menu)
elif event.key == pygame.K_c:
towers.resetCooldowns()
elif event.key == pygame.K_r:
towerBar.selectTower(TowerUPGRADE)
elif event.key == pygame.K_e:
towerBar.selectTower(TowerERASE)
elif event.key == pygame.K_a:
towers.clear()
wave.clear()
shots.clear()
elif event.key == pygame.K_t:
towers.clear()
shots.clear()
elif event.key == pygame.K_v:
Evaluate.evalPlayerPosition()
elif event.key == pygame.K_b:
Progress.prog()
## Init Game
if Game.state == STATE_LOADGAME:
Game.redrawSPBtn = 1
Game.state = STATE_PREPARATION
screen.blit(InterfaceBGopaque, (0, 0))
# screen.fill(background)
# pygame.draw.rect(screen, background, ([mapWidth*tileSize, 0, rightMenuSize, mapHeight*tileSize + bottomMenuSize]))
# pygame.draw.rect(screen, background, ([0, mapHeight*tileSize, mapWidth*tileSize, bottomMenuSize]))
# Draw the map
drawMap(map, screen)
# Draw the game information menu
menu.draw(screen)
Game.redrawSPBtn = 1
menu.drawSPBtn(screen)
towerBar.moneyUpdated(screen)
towerBar.showTower(screen)
towerBar.updateTowerCost = 0
# Draw the tower bar ~ 0
towerBar.draw(screen)
pygame.display.flip()
## Game Paused
if Game.state == STATE_PREPARATION:
if Game.autoMode or Game.balanceMode:
Game.redrawSPBtn = 1
Game.state = STATE_GAME
else:
## Display
drawTick += 1
# print(clock.get_fps())
if drawTick >= clock.get_fps() / 24:
# print(clock.get_fps())
drawTick = 0
drawGame(map, towerBar, towers, wave, shots, menu, screen, layer)
## Game Running
elif Game.state == STATE_GAME:
# Spawn any new enemy in the wave queue
wave.spawn()
# Move the enemies
wave.move()
# Tower target
towers.target(shots)
## Display
if not Game.autoMode and not Game.balanceMode:
drawTick += 1
# print(clock.get_fps())
if drawTick >= clock.get_fps() / 24:
# print(clock.get_fps())
drawTick = 0
drawGame(map, towerBar, towers, wave, shots, menu, screen, layer)
# Limit to 24 frames per second
# print(pygame.time.get_ticks())
clock.tick(Game.framesPerSecond * Game.speedModifier)
pygame.quit()
def main(options, args):
"""
@param options: from optparse
@param args: from optparse
@return: a response string
"""
# get this file from the web directory
# http://hgdownload.cse.ucsc.edu/goldenPath/hg18/multiz28way/alignments/
original_fasta_filename = 'knownGene.exonAA.fa'
# the original fasta file is broken into a bunch of pieces and put into this directory
pieces_directory = 'fasta'
# the index files that map genomic locations to fasta subfiles are in this directory
index_directory = 'index'
# this file keeps a list of valid chromosome names from the original fasta file
chromosome_filename = 'chromosomes.txt'
# assert that a command was given with the script
if not args:
raise MySyntaxError('no command was given')
# try to dispatch the command
command = args[0]
command_args = args[1:]
if command == 'split':
if command_args:
raise MySyntaxError(
'the split command does not take any arguments')
# assert that the fasta directory has been created
if not os.path.isdir(pieces_directory):
err_lines = [
'The directory for the split fasta files was not found: ' +
pieces_directory,
'Please create this directory or cd to its parent directory.'
]
raise MyConfigError('\n'.join(err_lines))
# assert that the current directory has the original huge fasta file
pathnames = os.listdir('.')
if original_fasta_filename not in pathnames:
err_lines = [
'The file %s was not found in the current directory.' %
original_fasta_filename, 'Please download this file from:',
'http://hgdownload.cse.ucsc.edu/goldenPath/hg18/multiz28way/alignments/'
]
raise MyConfigError('\n'.join(err_lines))
splitter = KGEA.Splitter(original_fasta_filename, pieces_directory)
splitter.run(verbose=options.verbose)
return ''
elif command == 'index':
if command_args:
raise MySyntaxError(
'the index command does not take any arguments')
# assert that the fasta directory has been created
if not os.path.isdir(pieces_directory):
err_lines = [
'The directory for the split fasta files was not found: ' +
pieces_directory,
'If this directory exists somewhere else, then cd to its parent directory.',
'If this directory has not been created, then create it and run the split command.'
]
raise MyConfigError('\n'.join(err_lines))
# assert that the index directory has been created
if not os.path.isdir(index_directory):
err_lines = [
'The directory for the index files was not found: ' +
index_directory,
'Please create this directory or cd to its parent directory.'
]
raise MyConfigError('\n'.join(err_lines))
indexer = KGEA.Indexer(index_directory, chromosome_filename,
pieces_directory)
indexer.run(verbose=options.verbose)
return ''
elif command == 'find-alignment':
if len(command_args) != 2:
raise MySyntaxError(
'the find-alignment command takes two arguments')
# define the chromosome string and the chromosome position
chromosome_string, chromosome_position_string = command_args
# initialize the chromosome position and assert that it is plausible
try:
chromosome_position = int(chromosome_position_string)
except ValueError as e:
raise MySyntaxError('the chromosome position should be an integer')
# assert that the fasta directory has been created
if not os.path.isdir(pieces_directory):
err_lines = [
'The directory for the split fasta files was not found: ' +
pieces_directory,
'If this directory exists somewhere else, then cd to its parent directory.',
'If this directory has not been created, then create it and run the split command.'
]
raise MyConfigError('\n'.join(err_lines))
# assert that the index directory has been created
if not os.path.isdir(index_directory):
err_lines = [
'The directory for the index files was not found: ' +
index_directory,
'If this directory exists somewhere else, then cd to its parent directory.',
'If this directory has not been created, then create it and run the index command.'
]
raise MyConfigError('\n'.join(err_lines))
# look for the alignment using the finder
finder = KGEA.Finder(index_directory, chromosome_filename,
pieces_directory)
fasta_lines = finder.get_alignment_lines(chromosome_string,
chromosome_position,
verbose=options.verbose)
if not fasta_lines:
return 'no amino acid was found at this position'
return '\n'.join(fasta_lines)
elif command == 'find-column':
if len(command_args) != 2:
raise MySyntaxError('the find-column command takes two arguments')
# define the chromosome string and the chromosome position
chromosome_string, chromosome_position_string = command_args
# initialize the chromosome position and assert that it is plausible
try:
chromosome_position = int(chromosome_position_string)
except ValueError as e:
raise MySyntaxError('the chromosome position should be an integer')
# assert that the fasta directory has been created
if not os.path.isdir(pieces_directory):
err_lines = [
'The directory for the split fasta files was not found: ' +
pieces_directory,
'If this directory exists somewhere else, then cd to its parent directory.',
'If this directory has not been created, then create it and run the split command.'
]
raise MyConfigError('\n'.join(err_lines))
# assert that the index directory has been created
if not os.path.isdir(index_directory):
err_lines = [
'The directory for the index files was not found: ' +
index_directory,
'If this directory exists somewhere else, then cd to its parent directory.',
'If this directory has not been created, then create it and run the index command.'
]
raise MyConfigError('\n'.join(err_lines))
# look for the column using the finder
finder = KGEA.Finder(index_directory, chromosome_filename,
pieces_directory)
column_lines = finder.get_column_lines(chromosome_string,
chromosome_position,
verbose=options.verbose)
if not column_lines:
return 'no amino acid was found at this position'
return '\n'.join(column_lines)
elif command == 'summarize':
if command_args:
raise MySyntaxError(
'the summarize command does not take any arguments')
# assert that the current directory has the original huge fasta file
pathnames = os.listdir('.')
if original_fasta_filename not in pathnames:
err_lines = [
'The file %s was not found in the current directory.' %
original_fasta_filename, 'Please download this file from:',
'http://hgdownload.cse.ucsc.edu/goldenPath/hg18/multiz28way/alignments/'
]
raise MyConfigError('\n'.join(err_lines))
# initialize the progress bar
nbytes_total = os.path.getsize(original_fasta_filename)
pbar = Progress.Bar(nbytes_total)
# initialize the summary
mod3 = {0: 0, 1: 0, 2: 0}
length_diff_dict = {}
# summarize by reading each alignment from the file
approx_nbytes_read = 0
fin = open(original_fasta_filename)
for lines in Util.gen_paragraphs(fin):
# process the lines
header_line = lines[0]
p = KGEA.LocationParser(header_line)
genomic_length = (p.last_index - p.first_index) + 1
mod3[genomic_length % 3] += 1
diff = 3 * p.length - genomic_length
if diff not in length_diff_dict:
length_diff_dict[diff] = 0
length_diff_dict[diff] += 1
# update the progress bar
approx_nbytes_read += sum(len(line) for line in lines)
pbar.update(approx_nbytes_read)
fin.close()
# finish the progress bar
pbar.update(nbytes_total)
# return the summary
summary_lines = []
summary_lines += [
'genomic span of %d mod 3: %d sequences' % (i, mod3[i])
for i in range(3)
]
summary_lines.append('histogram of 3*aa_length - genomic span:')
for key, value in sorted(length_diff_dict.items()):
summary_lines.append('%d : %d' % (key, value))
return '\n'.join(summary_lines)
else:
raise MySyntaxError('invalid command: ' + command)
def process(linesources, good_coverage, randomization_rate, stickiness,
nseconds, use_pbar):
"""
@param linesources: open resequencing files for reading
@param good_coverage: the expected number of reads at informative positions
@param randomization_rate: the probability of an error per base call
@param stickiness: level of stickiness
@param nseconds: None or impose a time limit of this many seconds
@param use_pbar: True iff a progress bar should be used
@return: the multi-line string of the resulting csv file
"""
# do some initialization
start_time = time.time()
termination_reason = 'finished the analysis'
# define the superstates
cache_size_per_superstate = 100000
good_state = ReadCoverageGap.Good(randomization_rate, good_coverage,
cache_size_per_superstate)
bad_state = ReadCoverageGap.Bad(randomization_rate, good_coverage,
cache_size_per_superstate)
superstates = [good_state, bad_state]
superstate_names = ['good', 'bad']
# prepare to annotate the chromosomes
chromosomes = []
pbar = Progress.Bar(len(linesources)) if use_pbar else None
# annotate the chromosomes using the models
try:
for i, linesource in enumerate(linesources):
# read the lines of text
lines = Util.get_stripped_lines(linesource.readlines())
# validate the number of lines
if len(lines) < 2:
raise ValueError('there should be at least two lines of input')
# break the lines of input into rows of elements
rows = [line_to_row(line) for line in lines]
# validate the columns of data
ncolumns_expected = 8
ncolumns = len(rows[0])
if ncolumns != ncolumns_expected:
raise ValueError('expected %d columns of input: %s' %
(ncolumns_expected, rows[0]))
for row in rows:
if len(row) != ncolumns:
raise ValueError(
'each row of input should have the same number of elements as the first row'
)
# process the data rows, building a dictionary of chromosomes
chromosome_dict = {}
data_rows = rows[1:]
for row in data_rows:
if nseconds and time.time() - start_time > nseconds:
raise TimeoutError()
process_genomic_position(row, chromosome_dict)
current_chromosomes = [
chromosome
for identifier, chromosome in sorted(chromosome_dict.items())
]
for chromosome in current_chromosomes:
# do the annotation
chromosome.annotate_posteriors(stickiness, superstates)
# delete position specific data
chromosome.del_position_specific_data()
# add the chromosomes to the list
chromosomes.extend(current_chromosomes)
# update the progress bar
if pbar:
pbar.update(i + 1)
except KeyboardInterrupt, e:
termination_reason = 'early termination by control-c'
# QueryGoogle - use GoogleParser to find out information about the quotes
# obtained from QuotesParser
from GoogleParser import *
from BingParser import *
from Progress import *
import sqlite3, pickle, string
table = string.maketrans("","")
punct = string.punctuation.replace('\'','')
def remove_punctuation(quote):
return quote.translate(table, punct)
movie_quotes=pickle.load(open('../data/title_to_quotes_parsed.pickle','r'))
progress = Progress('../data/title_to_quotes_google_progress.pickle')
conn = sqlite3.connect('../data/db_google.sqlite')
conn.text_factory = str
c = conn.cursor()
for movie, quotes in movie_quotes:
if progress.is_completed(movie):
continue
print "Starting %s..." % movie
# Query each quote with and without the movie
# Store Results
for actor, quote in quotes:
if progress.is_completed((movie,quote)):
