def main(): args = get_arguments() log.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=LOG_LEVEL) log.info("Start of '{}'.".format(__file__)) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) log.info("Created output directory '{}'.".format(args.output_dir)) for f in find_files(args.input_dir, '*.mid', path=False): midi_old = pretty_midi.PrettyMIDI(os.path.join(args.input_dir, f)) midi_new = trim_midi_file(args.sec_from, args.sec_to, midi_old) midi_new.write(os.path.join(args.output_dir, f)) log.info("Processed file '{}'".format(f)) for f in find_files(args.input_dir, '*.wav', path=False): audio_old, sr = librosa.load(os.path.join(args.input_dir, f)) audio_new = trim_audio_file(args.sec_from, args.sec_to, audio_old, sr) librosa.output.write_wav(os.path.join(args.output_dir, f), audio_new, sr) log.info("Processed file '{}'".format(f)) log.info("End of '{}'.".format(__file__))
def test_preprocessing(): # make arguments args = make_args() # prepare dummy wav files wavdir = "data/wav" if not os.path.exists(wavdir): os.makedirs(wavdir) for i in range(5): make_dummy_wav(wavdir + "/%d.wav" % i, 8000, args.fs) # feature extract wav_list = find_files(wavdir, "*.wav") if not os.path.exists(args.wavdir): os.makedirs(args.wavdir) world_feature_extract(wav_list, args) # calc_stats file_list = find_files(args.hdf5dir, "*.h5") calc_stats(file_list, args) # noise shaping wav_list = find_files(args.wavdir, "*.wav") if not os.path.exists(args.writedir): os.makedirs(args.writedir) noise_shaping(wav_list, args) # assert list length wav_ns_list = find_files(args.writedir, "*.wav") assert len(wav_list) == len(file_list) assert len(wav_list) == len(wav_ns_list)
def compile(self): artifacts_path = os.path.join(self.paths["arch"], "artifacts") with open(artifacts_path, "wb") as artifacts_file: for path in utils.find_files(self.paths["arch"], ext=".art") + \ utils.find_files(self.paths["maps"], ext=".art"): utils.file_copy(path, artifacts_file)
def compile(self): treasures_path = os.path.join(self.paths["arch"], "treasures") with open(treasures_path, "wb") as treasures_file: for path in utils.find_files(self.paths["arch"], ext=".trs") + \ utils.find_files(self.paths["maps"], ext=".trs"): utils.file_copy(path, treasures_file)
def load_piece_data(directory, audio_sr, velocity, fac, valid=True): '''Loader that reads tune from directory and yields audio waveform and encoded piano roll as tuple of 3 arrays: (W, T, I). If more audio files represent single midi file, one is chosen randomly. ''' midi_files = find_files(directory, '*.mid') randomized_midi_files = randomize_files(midi_files) for midi_filename in randomized_midi_files if valid else midi_files: # load piano roll from midi file proll = pretty_midi.PrettyMIDI(midi_filename).get_piano_roll( fs=int(audio_sr / fac)) proll /= 127 # velocity to <0;1> if not velocity: proll[proll > 0] = 1 # encode piano roll table, indices = roll_encode(proll, fac) # add 0-roll if not present (we will need it later for padding) if get_roll_index(table, np.zeros(128)).shape[0] == 0: table = np.concatenate((table, np.zeros(shape=(1, 128)))) # get respective audio file names and choose 1 randomly base = midi_filename.rsplit('/', 1)[-1] base = re.sub(r'(.*)%s$' % re.escape('.mid'), r'\1', base) audio_files = find_files(directory, base + '*.wav') if not audio_files: raise ValueError('No files found for \'{}\'.'.format(base + '*.wav')) audio_filename = random.choice(audio_files) # load audio waveform audio, _ = librosa.load(audio_filename, sr=audio_sr, mono=True) yield audio, table, indices
def main(): # Get paths to directories inside the root directory. for path in utils.find_files(paths["root"], rec=False, ignore_dirs=False, ignore_files=True): paths[os.path.basename(path)] = path if "collect_none" not in what_collect: # Nothing was set to collect, so by default we'll collect everything. if not what_collect: for entry in dict(globals()): if entry.startswith("collect_"): what_collect.append(entry) # Call the collecting functions. for collect in what_collect: print("Collecting {}...".format(collect.split("_")[-1])) globals()[collect]() # Copy all files in the arch directory to specified directory. if copy_dest: files = utils.find_files(paths["arch"], rec=False) for path in files: shutil.copyfile(path, os.path.join(copy_dest, os.path.basename(path)))
def train_model(): train_files = find_files(train_data_path) test_files = find_files(test_data_path) train_samples = [] test_samples = [] for f in train_files: train_samples.extend(random.sample(list(read_file(f)), 93000)) for f in test_files: test_samples.extend(list(read_file(f))) print("Total number of train samples: %d" % len(train_samples)) print("Total number of test samples: %d" % len(test_samples)) train_loader = DataLoader(dataset=TrainDataset(train_samples), batch_size=8, shuffle=True) test_loader = DataLoader(dataset=TestDataset(test_samples), batch_size=1, shuffle=True) model = CNN() model.load_state_dict(torch.load(os.path.join(save_path, "model10-21.pkl"))) trainer = Trainer(model=model, iter_num=iter_num, save_dir=save_path, save_log=False) trainer.push_data(train_loader) trainer.push_data(test_loader, is_train=False) trainer.train()
def _validate_and_prepare_dir(self, app_dir_path, undo_ops, python_exe): results = run_installed_tests_as_subprocess(app_dir_path, self.django_settings_module, python_exe_path=python_exe, read_config_file=False) if len(results.error_messages)>0: for error in results.error_messages: logger.error("ERROR>> %s" % error) if len(results.warning_messages)>0: for warning in results.warning_messages: logger.warning("WARNING>> %s" % warning) results.print_final_status_message(logger) if results.get_return_code() != SUCCESS_RC: return results.get_return_code() config = django_config_from_validation_results(self.django_settings_module, VERSION, results) write_json(config.to_json(), os.path.join(app_dir_path, DJANGO_CFG_FILENAME)) # undo the changes we made for (op_fun, args) in undo_ops: op_fun(*args) # delete all the .pyc files find_files(app_dir_path, "\.pyc$", os.remove) # (re)create the archive if self.archive_type == "zip": archive = ZipfileHandler(self.archive_file, "w") else: archive = TarfileHandler(self.archive_file, "w:gz") archive.create_new_from_dir(app_dir_path) archive.close() return SUCCESS_RC
def main(): args = get_arguments() log.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=LOG_LEVEL) log.info("Start of '{}'.".format(__file__)) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) log.info("Created output directory '{}'.".format(args.output_dir)) for f in find_files(args.input_dir, '*.mid', path=False): midi_old = pretty_midi.PrettyMIDI( os.path.join(args.input_dir, f)) midi_new = trim_midi_file(args.sec_from, args.sec_to, midi_old) midi_new.write( os.path.join(args.output_dir, f)) log.info("Processed file '{}'".format(f)) for f in find_files(args.input_dir, '*.wav', path=False): audio_old, sr = librosa.load(os.path.join(args.input_dir, f)) audio_new = trim_audio_file(args.sec_from, args.sec_to, audio_old, sr) librosa.output.write_wav( os.path.join(args.output_dir, f), audio_new, sr) log.info("Processed file '{}'".format(f)) log.info("End of '{}'.".format(__file__))
def compile(self): num_images = 0 dev_dir = os.path.join(self.paths["arch"], "dev") bmaps_path = os.path.join(self.paths["arch"], "bmaps") images_path = os.path.join(self.paths["arch"], "atrinik.0") with open(bmaps_path, "wb") as bmaps_file, \ open(images_path, "wb") as images_file: # 'bug.101' must be the first entry. for path in utils.find_files(dev_dir, ext="bug.101.png") + \ sorted(utils.find_files(self.paths["arch"], ext=".png", ignore_paths=(dev_dir,)), key = lambda s: os.path.basename(s)[:-4]): name = os.path.basename(path)[:-4] # Write it to the bmaps file. bmaps_file.write("{}\n".format(name).encode()) # Get the image's file size. size = os.path.getsize(path) # Write information about the image to the atrinik.0 file. images_file.write("IMAGE {} {} {}\n".format( num_images, size, name).encode()) with open(path, "rb") as image_file: images_file.write(image_file.read()) num_images += 1
def __init__(self, speech_dir, noisy_dir, sr=16000, pairs=None): super(TestDataset, self).__init__() self.speech_dir = speech_dir self.noisy_dir = noisy_dir self.pairs = pairs self.sr = sr self.speech_wav_files = utils.find_files(self.speech_dir) self.noisy_wav_files = utils.find_files(self.noisy_dir)
def __init__(self, dirloc): self.dirloc = dirloc # Obtain the dirname self.dirname = os.path.basename(self.dirloc) # Parse directory name regex = re.compile(self.pattern) self.attrs = regex.match(self.dirname) # Fall back to legacy mode if no match if not self.attrs: regex = re.compile(self.pattern_legacy) self.attrs = regex.match(self.dirname) if not self.attrs: raise Exception("Error: could not match directory '%s' to known pattern" % self.dirname) # Find the ISO file in the driver repo iso_files = utils.find_files(self.dirloc, "*.iso") if len(iso_files) > 1: raise Exception("Error: more than one file found! (%s)" % iso_files) if not iso_files: raise Exception("Error: could not find any ISOs in directory %s" % self.dirloc) self.driver_iso = DriverISO(iso_files[0]) # Find the repo metadata file metadata_files = utils.find_files(self.dirloc, "*.metadata.md5") if len(metadata_files) > 1: raise Exception("Error: more than one metadata files returned! (%s)" % metadata_files) if not metadata_files: raise Exception("Error: could not find any metadata files") self.metadata_file = FileObject(metadata_files[0]) # Find RPM info file rpminfo_files = utils.find_files(self.dirloc, "*.rpminfo") if len(rpminfo_files) > 1: raise Exception("Error: more than one rpminfo file returned! (%s)" % rpminfo_files) if not rpminfo_files: raise Exception("Error: could not find any rpminfo files") self.rpminfo_file = FileObject(rpminfo_files[0]) # Find the ZIP file zip_files = utils.find_files(self.dirloc, "*.zip") if len(zip_files) > 1: raise Exception("Error: more than on zip fil returned! (%s)" % zip_files) if not zip_files: raise Exception("Error: could not find any zip files") self.zip_file = FileObject(zip_files[0])
def __init__(self, speech_dir, noise_dir, noisy_dir=None, mode='train', sr=16000, frame_len=512, frame_shift=256, block_size=32, block_shift=16, snr_level=[-10, -5, 0, 5, 10, 15, 20]): super(MyDataset, self).__init__() if os.path.exists(speech_dir): NotADirectoryError('Speech dir does not exist') if os.path.exists(noise_dir): NotADirectoryError('Noise dir does not exist') self.speech_dir = speech_dir self.noise_dir = noise_dir self.mode = mode self.noisy_dir = noisy_dir self.snr_level = snr_level self.sr = sr self.frame_len = frame_len self.frame_shift = frame_shift self.block_size = block_size self.block_shift = block_shift n = np.log2(self.frame_len) if int(n) < n: n = int(n) + 1 self.n_fft = int(2**n) self.mu = None self.sigma = None self.speech_wav_files = utils.find_files(self.speech_dir) random.shuffle(self.speech_wav_files) self.noise_wav_files = utils.find_files(self.noise_dir) random.shuffle(self.noise_wav_files) if self.mode == 'test': if os.path.exists(self.noisy_dir) is False: os.mkdir(self.noisy_dir) self.noisy_wav_files = utils.find_files(self.noisy_dir) self.build_test_pairs() else: self.data_stats(sample_size=100) # stats snr information self.noise_buffer = np.array([]) self.speech_buffer = np.array([]) self.noisy_buffer = np.array([]) self.wav_idx = 0
def merge(self, batch_size: int = 100, _step: int = 0) -> List[Entry]: """Merge blocks into a single final list of entries. Blocks are batched in groups and merged recursively. Parameters ---------- batch_size : int, optional The number of blocks in a merge batch. Defaults to 100. _step : int, optional Private parameter corresponding to the current processing stage. """ block_paths = [path for _, path in find_files(self.temp_path)] if len(block_paths) == 1: # Only one block remaining => we're done. # Read the entries from it. last_block_path = block_paths[0] with open(last_block_path) as f: return list(_read_block(f)) # Otherwise, batch blocks together and merge each of them into a # new block. for idx, batch in enumerate(grouped(batch_size, block_paths)): # The last `batch` may be end-padded with nones if the # number of items in `block_paths` is not a multiple of # `batch_size`. batch = filter(None, batch) out_path = os.path.join(self.temp_path, f"{_step}-{idx}") self._merge(out_path, *batch) # Merge the new blocks recursively. return self.merge(batch_size=batch_size, _step=_step + 1)
def corpus_contents(): args = get_args() ret = {} for fullfn, contents in find_files(args.corpusdir, ext='.xd'): xdid = parse_xdid(fullfn) ret[xdid.lower()] = contents return ret
def id_list(): samples = find_files(PREDICTED_BASE, 'npy') base = set() for sample in samples: sp = sample.split('/')[-2] base.add(sp) return list(sorted(base))
def _populate_database(self, Session, filepath): for fq_path, root, basename in find_files(filepath, self._create_regex()): dbsession = Session() try: hash = calculate_hash(fq_path) contents = self.get_contents(fq_path) existing = dbsession.query(Document).filter_by( filepath=fq_path).first() if existing is None: doc = Document(filename=basename, filepath=fq_path, body=contents, hash=hash) dbsession.add(doc) elif existing.hash != hash: existing.hash = hash existing.body = contents dbsession.commit() except Exception: dbsession.rollback() finally: dbsession.close()
def thread_loader(self, sess): stop = False # Count tune data files n_midi_files = len(find_files(self.data_dir, '*.mid')) if n_midi_files == 0: raise ValueError('No files found for \'{}\'.'.format( directory+'/*.mid')) one_percent = int(np.ceil(n_midi_files/100)) print('files length: {}'.format(n_midi_files)) # Go through the dataset repeatedly until stopped while not stop: # Randomly iterate over files and fetch tune data file_iterator = load_piece_data(self.data_dir, self.audio_sample_rate, self.velocity, self.compress_fac) for audio, table, indices in file_iterator: sess.run(self.queues['tune']['enQ'], feed_dict={self.audio_placeholder: audio, self.table_placeholder: table, self.indices_placeholder: indices}) # Track and report progress sess.run(self.increment_file_counter) file_counter = sess.run(self.file_counter) if file_counter % one_percent == 0: print('Training progress: {:.02f} epochs ' '(file {} of {})'.format(file_counter/n_midi_files, file_counter, n_midi_files)) if self.coord.should_stop(): stop = True break
def thread_loader(self, sess): stop = False # Count tune data files n_midi_files = len(find_files(self.data_dir, '*.mid')) if n_midi_files == 0: raise ValueError('No files found for \'{}\'.'.format(directory + '/*.mid')) one_percent = int(np.ceil(n_midi_files / 100)) print('files length: {}'.format(n_midi_files)) # Go through the dataset repeatedly until stopped while not stop: # Randomly iterate over files and fetch tune data file_iterator = load_piece_data(self.data_dir, self.audio_sample_rate, self.velocity, self.compress_fac) for audio, table, indices in file_iterator: sess.run(self.queues['tune']['enQ'], feed_dict={ self.audio_placeholder: audio, self.table_placeholder: table, self.indices_placeholder: indices }) # Track and report progress sess.run(self.increment_file_counter) file_counter = sess.run(self.file_counter) if file_counter % one_percent == 0: print('Training progress: {:.02f} epochs ' '(file {} of {})'.format(file_counter / n_midi_files, file_counter, n_midi_files)) if self.coord.should_stop(): stop = True break
def compile(self): l = [] for path in utils.find_files(self.paths["arch"], ext=".anim"): with open(path) as anim_file: for line in anim_file: line = line.strip() # Blank line or comment. if not line or line.startswith("#"): continue if line.startswith("anim "): l.append([line]) elif line != "mina": l[len(l) - 1].append(line) animations_path = os.path.join(self.paths["arch"], "animations") with open(animations_path, "wb") as animations_file: for anim in sorted(l, key=lambda node: node[0][5:]): for line in anim: animations_file.write("{}\n".format(line).encode()) animations_file.write("mina\n".encode())
def __init__(self, audio_dir,model_settings,background_noise,silence_label, unknown_label,classes,augmentation_ops = [],augmentation_percentage = 0, validation_percentage=10,testing_percentage = 10,unknown_percentage = 10,silence_percentage = 10,testing_list = {},validation_list= {},fingerprint_type='mfcc' ,mode='test', train=True): print('audio_dir = {}'.format(audio_dir)) self.model_settings = model_settings print('sample_rate = {}'.format(self.model_settings['sample_rate'])) self.background_data = [] self.data_index = {} self.word_to_index = {} self.audio_dir = audio_dir self.audio_files = find_files(audio_dir) self.background_noise = background_noise self.words_list = [silence_label, unknown_label] + classes self.augmentation_ops = augmentation_ops self.augmentation_percentage = augmentation_percentage self.validation_percentage = validation_percentage self.testing_percentage = testing_percentage self.unknown_percentage = unknown_percentage self.fingerprint_type = fingerprint_type self.silence_percentage = silence_percentage self.testing_list = testing_list self.validation_list = validation_list print('Found {} files in total in {}.'.format(len(self.audio_files), audio_dir)) assert len(self.audio_files) != 0 self.mode = mode self.train = train self.prepare_data_index() if self.train: self.prepare_background_data() self.prepare_processing_graph()
def main(): parser = ArgumentParser() parser.add_argument("calibration_folder") parser.add_argument("image_folder") parser.add_argument("--bm_settings", default="") args = parser.parse_args() settings = args.bm_settings if args.bm_settings: with open(settings) as settings_file: if settings_file: settings = args.bm_settings else: settings = None calibration = StereoCalibration(input_folder=args.calibration_folder) input_files = find_files(args.image_folder) block_matcher = BlockMatcher(settings=settings) image_pair = [cv2.imread(image) for image in input_files[:2]] tuner = BMTuner(block_matcher, calibration, image_pair) if args.bm_settings: block_matcher.save_settings(args.bm_settings) print "Settings saved."
def remove_one_agent_config_files(context): dir_path = configs.get("local_orb_path") all_files_generated = find_files(context.agent_file_name, ".yaml", dir_path) if len(all_files_generated) > 0: for file in all_files_generated: os.remove(file)
def load_pages(directory, output_path, kernel): """ Find all .text files in the specified directory and return a map of Page objects, keyed by the url for the page. \param directory starting directory to search \param output_path the directory we'll be writing output to \param configs the config map which we'll use to get the template for each page \param templates the Templates singleton """ page_map = {} length = len(directory) for root, name in utils.find_files(directory, False, '.text'): path = os.path.join(root, name) base_path = root[length:] if not base_path.startswith('/'): base_path = '/' + base_path name = name[0:-5] url = utils.url_join(base_path, name) config = utils.find_config(kernel.configs, base_path) page = Page(kernel, path, output_path, url, config) page_map[url] = page return page_map
def test_find_files(self): """ Test the function find_files. """ files = [ os.path.join('folder_0', 'subfolder_0', 'file_0'), os.path.join('folder_0', 'subfolder_0', 'file_1'), os.path.join('folder_0', 'subfolder_1', 'file_0'), os.path.join('folder_0', 'subfolder_1', 'file_1'), os.path.join('folder_1', 'subfolder_0', 'file_0'), os.path.join('folder_1', 'subfolder_0', 'file_1'), os.path.join('folder_1', 'subfolder_1', 'file_0'), os.path.join('folder_1', 'subfolder_1', 'file_1'), ] sources_0 = {'folder_0': files[:4], 'folder_1': files[4:]} sources_1 = { 'folder_0': ['1', '2', '3', '4'], 'folder_1': files[4:], } sources_2 = { 'folder_1': ['1', '2', '3', '4'], 'folder_0': files[:4], } sources_3 = { 'folder_1': ['1', '2', '3', '4'], 'folder_0': ['1', '2', '3', '4'], } self.assertEqual( find_files(basename_prefix='file', sources=sources_0), files[:4], ) self.assertEqual( find_files(basename_prefix='file', sources=sources_1), files[4:], ) self.assertEqual( find_files(basename_prefix='file', sources=sources_2), files[:4], ) self.assertEqual( find_files(basename_prefix='file', sources=sources_3), [], )
def make_images_list(images_folders, root_folder): found_files, unknown_extensions = find_files(images_folders, ['.jpg', '.png'], ['.txt']) if len(unknown_extensions) > 0: print('Unknown extensions: {}'.format(unknown_extensions)) found_files = list( map(lambda x: os.path.relpath(x, root_folder), found_files)) return found_files
def load_data(data_path): data_file = find_files(data_path) data = [] for d in data_file: samples = list(read_file(d)) data.extend(samples) random.shuffle(data) return data
def local_info(): result = {} local_files_to_check = utils.find_files() for f in local_files_to_check: h = hashlib.md5() h.update(open(f, "rb").read()) result[f[2:]] = {"ETag": h.hexdigest()} return result
def get_sigs_in_dir(directory, viname=None, searchstring='mean.ref', recursivesearch=False): signatures = signatureCollection(viName=viname) sigFiles = find_files(directory, searchstring, recursive=recursivesearch) for f in sigFiles: signatures.add(f) return signatures
def main(): search_pattern = sys.argv[1] search_regex = re.compile(search_pattern) shell_file_pattern = sys.argv[2] files = find_files('.', shell_pattern=shell_file_pattern) for filename in files: grep_file(filename, search_regex)
def compile(self): for path in utils.find_files( os.path.join(self.paths["maps"], "interfaces"), ".xml"): self.path = path self.compile_file() with open( os.path.join(self.paths["maps"], "python", "InterfaceQuests.py"), "wb") as quests_file: quests_file.write(self.quests.getvalue().encode())
def __init__(self, data_dir, fps=16000): # define the state of the object self.data_dir = data_dir self.fps = fps # setup the files for reading self.files = list(find_files(data_dir, '*.wav')) self.classes = [f for f in sorted(os.listdir(data_dir))] # self.classes.remove('.DS_Store') self.classes.sort() print(self.classes)
def test_preprocessing(): # make arguments args = make_args() # prepare dummy wav files wavdir = "data/wav" if not os.path.exists(wavdir): os.makedirs(wavdir) for i in range(5): make_dummy_wav(wavdir + "/%d.wav" % i, 8000, args.fs) # feature extract wav_list = find_files(wavdir, "*.wav") if not os.path.exists(args.wavdir): os.makedirs(args.wavdir) args.feature_type = "world" world_feature_extract(wav_list, args) args.feature_type = "melspc" melspectrogram_extract(wav_list, args) args.feature_type = "mcep" melcepstrum_extract(wav_list, args) # calc_stats file_list = find_files(args.hdf5dir, "*.h5") args.feature_type = "world" calc_stats(file_list, args) args.feature_type = "melspc" calc_stats(file_list, args) args.feature_type = "mcep" calc_stats(file_list, args) # noise shaping wav_list = find_files(args.wavdir, "*.wav") args.feature_type = "world" args.writedir = "data/wav_ns/world" if not os.path.exists(args.writedir): os.makedirs(args.writedir) world_noise_shaping(wav_list, args) args.feature_type = "mcep" args.writedir = "data/wav_ns/mcep" if not os.path.exists(args.writedir): os.makedirs(args.writedir) melcepstrum_noise_shaping(wav_list, args)
def get_pred(csv_path): if os.path.isdir(csv_path): csv_files = find_files(csv_path, pattern="*.csv") csv_files = sorted(csv_files) else: csv_files = [csv_path] print(csv_files) preds = np.concatenate([load_pred(fname) for fname in csv_files], axis=0) print(preds.shape) return preds
def build_cache(self, audio_dir, sample_rate): logger.info(f'audio_dir: {audio_dir}.') logger.info(f'sample_rate: {sample_rate:,} hz.') audio_files = find_files(audio_dir, ext=self.ext) audio_files_count = len(audio_files) assert audio_files_count != 0, f'Could not find any {self.ext} files in {audio_dir}.' logger.info(f'Found {audio_files_count:,} files in {audio_dir}.') with tqdm(audio_files) as bar: for audio_filename in bar: bar.set_description(audio_filename) self.cache_audio_file(audio_filename, sample_rate)
def __init__(self, cache_dir: str, audio_dir: str = None, sample_rate: int = SAMPLE_RATE, ext='flac'): self.ext = ext self.cache_dir = os.path.join(cache_dir, 'audio-fbanks') ensures_dir(self.cache_dir) if audio_dir is not None: self.build_cache(os.path.expanduser(audio_dir), sample_rate) self.speakers_to_utterances = defaultdict(dict) for cache_file in find_files(self.cache_dir, ext='npy'): # /path/to/speaker_utterance.npy speaker_id, utterance_id = Path(cache_file).stem.split('_') self.speakers_to_utterances[speaker_id][utterance_id] = cache_file
def main(argv): parser = argparse.ArgumentParser( description="Load in a pose and transform superquadrics") parser.add_argument( "primitives_directory", help= "Path to the directory containing the superquadrics of the instance") parser.add_argument("pose_directory", help="Path to the directory containing the poses file") parser.add_argument("pose_index", type=int, default=0, help="Index of the pose to be displayed") parser.add_argument("--prob_threshold", type=float, default=0.5, help="Probability threshold") args = parser.parse_args(argv) primitives = load_all_primitive_parameters(args.primitives_directory, args.prob_threshold) paths = find_files(args.pose_directory + "meta/**meta.txt") # Loop over every meta file pose_path = sorted(paths)[0] _, rt, _ = load_pose(pose_path) q_obj = rotation_matrix_to_quaternion(rt[:3, :3]) M = len(primitives) M_poses = np.zeros(shape=(M, 3, 4)) i = 0 for prim in primitives: prim["obj_pose"] = q_obj print(args.pose_directory) with open(os.path.join(args.pose_directory, "outputs/labels.txt"), 'r') as fp: for i in np.arange(0, args.pose_index + 1): label = fp.readline() if i == args.pose_index: pose = string_to_pose(label.split("\n", 1)[0]) pose = np.reshape(pose, (-1, 3, 4)) for m in np.arange(0, np.shape(pose)[0]): primitives[m]["transform"] = pose[m] save_params_as_ply(os.path.join("../results/test/", "primitives.ply"), primitives)
def load_piece_data(directory, audio_sr, velocity, fac, valid=True): '''Loader that reads tune from directory and yields audio waveform and encoded piano roll as tuple of 3 arrays: (W, T, I). If more audio files represent single midi file, one is chosen randomly. ''' midi_files = find_files(directory, '*.mid') randomized_midi_files = randomize_files(midi_files) for midi_filename in randomized_midi_files if valid else midi_files: # load piano roll from midi file proll = pretty_midi.PrettyMIDI( midi_filename).get_piano_roll(fs=int(audio_sr/fac)) proll /= 127 # velocity to <0;1> if not velocity: proll[proll > 0] = 1 # encode piano roll table, indices = roll_encode(proll, fac) # add 0-roll if not present (we will need it later for padding) if get_roll_index(table, np.zeros(128)).shape[0] == 0: table = np.concatenate((table, np.zeros(shape=(1, 128)))) # get respective audio file names and choose 1 randomly base = midi_filename.rsplit('/', 1)[-1] base = re.sub(r'(.*)%s$' % re.escape('.mid'), r'\1', base) audio_files = find_files(directory, base+'*.wav') if not audio_files: raise ValueError('No files found for \'{}\'.'.format(base+'*.wav')) audio_filename = random.choice(audio_files) # load audio waveform audio, _ = librosa.load(audio_filename, sr=audio_sr, mono=True) yield audio, table, indices
def _validate_and_prepare_dir_old(self, app_dir_path, undo_ops): """Called by the subclass run() method after setting up the directory. """ results = validate_settings(app_dir_path, self.django_settings_module) if results.get_return_code() != SUCCESS_RC: results.print_final_status_message(logger) return results.get_return_code() config = django_config_from_validation_results(self.django_settings_module, VERSION, results) write_json(config.to_json(), os.path.join(app_dir_path, DJANGO_CFG_FILENAME)) # undo the changes we made for (op_fun, args) in undo_ops: op_fun(*args) # delete all the .pyc files find_files(app_dir_path, "\.pyc$", os.remove) # (re)create the archive if self.archive_type == "zip": archive = ZipfileHandler(self.archive_file, "w") else: archive = TarfileHandler(self.archive_file, "w:gz") archive.create_new_from_dir(app_dir_path) archive.close() return SUCCESS_RC
def __init__(self, data_dir, coord, audio_sample_rate, receptive_field, velocity, sample_size, queues_size, compress_fac=10): self.data_dir = data_dir self.audio_sample_rate = audio_sample_rate self.compress_fac = compress_fac self.coord = coord self.receptive_field = receptive_field self.velocity = velocity self.sample_size = sample_size self.threads = [] # Init queues and placeholders. self.queues = {'tune': {}, 'batch': {}} self.audio_placeholder = tf.placeholder(dtype=tf.float32, shape=(None,)) self.table_placeholder = tf.placeholder(dtype=tf.float32, shape=(None, 128)) self.indices_placeholder = tf.placeholder(dtype=tf.int32, shape=(None,)) self.queues['tune']['Q'] = tf.FIFOQueue(queues_size[0], ['float32', 'float32', 'int32']) self.queues['tune']['enQ'] = self.queues['tune']['Q'].enqueue( [self.audio_placeholder, self.table_placeholder, self.indices_placeholder]) self.wave_placeholder = tf.placeholder(dtype=tf.float32, shape=(None, 1)) self.roll_placeholder = tf.placeholder(dtype=tf.float32, shape=(None, 128)) self.queues['batch']['Q'] = tf.PaddingFIFOQueue( queues_size[1], ['float32', 'float32'], shapes=[(None, 1), (None, 128)]) self.queues['batch']['enQ'] = self.queues['batch']['Q'].enqueue( [self.wave_placeholder, self.roll_placeholder]) self.file_counter = tf.Variable(0, trainable=True) self.increment_file_counter = tf.assign( self.file_counter, self.file_counter+1) files = find_files(data_dir, '*.mid') if not files: raise ValueError('No midi files found in \'{}\'.'.format(data_dir))
def postproc_sos(): if os.name != 'posix': return print S_PPF, "postproc_sos (strip, chrpath)" res = re.compile( "^(.*):.*ELF.*(executable|relocatable|shared object).*, not stripped" ) rec = re.compile('.*\.(so$|so\.)') # use 'file' command to find all strippable files print PPF, "Creating complete file list..." all_files, _ = utils.find_files(BDIPaths.dre_dest, '.*') print PPF, "Searching for strippable / chrpathable files" for f in all_files: status, output = utils.get_status_output('%s %s' % (FILE, f)) mo = re.match(res, output) stripped = chrpathed = False if mo: sfn = mo.groups()[0] ret = os.system('%s %s' % (STRIP, sfn)) if ret != 0: print "Error stripping %s." % (sfn,) else: stripped = True # now check if f can be chrpathed if re.match(rec, f): # remove rpath information ret = os.system('%s --delete %s' % (CHRPATH, f)) if ret != 0: print "Error chrpathing %s." % (f,) else: chrpathed = True if stripped or chrpathed: actions = [] if stripped: actions.append('STRIPPED') if chrpathed: actions.append('CHRPATHED') print "%s: %s" % (f, ','.join(actions))
def main(): args = get_arguments() log.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=LOG_LEVEL) log.info("Start of '{}'.".format(__file__)) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) log.info("Created output directory '{}'.".format(args.output_dir)) for fname in find_files(args.input_dir, '*.csv', path=False): with open(os.path.join(args.input_dir, fname), 'r') as f: csv_data = list(csv.reader(f, delimiter='\t')) midi = csv2mid(csv_data[1:], args.audio_sr) midi.write(os.path.join(args.output_dir, fname.replace('.csv', '.mid'))) log.info("Processed file '{}'".format(fname)) log.info("End of '{}'.".format(__file__))
def plot_sigs(signaturedirectory, outputdirectory, name, signaturename): """ """ import os from utils import find_files, unique_name from core import signatureCollection from plotting import SignaturePlot if not outputdirectory: outputdirectory = signaturedirectory sigs = find_files(signaturedirectory, "mean.ref") if not sigs: click.BadParameter("Did not find any signature files in the specified directory.") if signaturename: filteredsigs = [] for searchstring in signaturename: for sig in sigs: if searchstring.upper() in sig.upper(): filteredsigs.append(sig) sigs = filteredsigs signatures = signatureCollection() for sig in sigs: try: signatures.add(sig) except Exception as e: print e #TODO Fix core temporalSignature to use exceptions so they can be properly handled here name, ext = os.path.splitext(name) path = unique_name(outputdirectory, name, ext=ext) print("Outputting to {0}".format(path)) plot = SignaturePlot(outputdirectory, os.path.basename(path)) plot.plot_collection(signatures)
def loop(self,events): """Mandatory method. This is the main eventloop called by the core 30 times a minute.""" if self.stopme: self.SPG.tellcore_enable_menubuttons() self.SPG.tellcore_game_end() for event in events: if event.type in (MOUSEBUTTONDOWN, MOUSEBUTTONUP, MOUSEMOTION): result = self.actives.update(event) if self.button_action: self.button_action = False return if event.type == MOUSEBUTTONDOWN: if self.state == 'play': self.stop_slide_show() self.prev_mouse_pos = 0 return elif self.state == 'thumbs': if result: self.logger.debug("loop, got a thumb") self.state = 'show' album = result[0][1][0] xml = os.path.join(album, 'text.xml') self.textxml = self.parse_xml(xml)# could return None self.actives.remove(self.thumbslist) self.filelist = [f for f in utils.find_files(album, IMAGE_EXT_PATTERN)] self.filelist.sort() self.show() return elif self.state != 'thumbs': self.show_all_buttons() self.prev_mouse_pos = pygame.mouse.get_pos()[0] # gestures if event.type == MOUSEBUTTONUP and self.prev_mouse_pos and not self.state == 'thumbs': x = pygame.mouse.get_pos()[0] if self.prev_mouse_pos - x > 20: self.hide_all_buttons() self.next_photo() elif x - self.prev_mouse_pos > 20: self.hide_all_buttons() self.prev_photo() return
def rebase_dlls(): """Rebase all DLLs in the distdevide tree on Windows. """ if os.name == 'nt': print S_PPF, "rebase_dlls" # sqlite3.dll cannot be rebased; it even gets corrupted in the # process! see this test: # C:\TEMP>rebase -b 0x60000000 -e 0x1000000 sqlite3.dll # REBASE: *** RelocateImage failed (sqlite3.dll). # Image may be corrupted # get list of pyd / dll files, excluding sqlite3 # this returns full path names so_files, excluded_files = utils.find_files( BDIPaths.dre_dest, '.*\.(pyd|dll)', ['sqlite3\.dll']) # add newline to each and every filename so_files = ['%s\n' % (i,) for i in so_files] print "Found %d DLL PYD files..." % (len(so_files),) print "Excluded %d files..." % (len(excluded_files),) # open file in specfile_dir, write the whole list dll_list_fn = os.path.join( config.working_dir, 'dll_list.txt') dll_list = file(dll_list_fn, 'w') dll_list.writelines(so_files) dll_list.close() # now run rebase on the list os.chdir(config.working_dir) ret = os.system( '%s -b 0x60000000 -e 0x1000000 @dll_list.txt -v' % (REBASE,)) # rebase returns 99 after rebasing, no idea why. if ret != 99: raise RuntimeError('Could not rebase DLLs.')
def extract_signatures(image, shapefiledirectory, startdoy, doyinterval, outputdir, filelabel, plotsigs): """ Extracts temporal signatures for a set of point geometry shapefiles in a specified directory and outputs them to a set of .ref files in an output directory. """ import os from plotting import SignaturePlot from utils import find_files, create_output_dir, unique_name from signatureFunctions import get_sigs_in_dir, get_reference_curves if outputdir is None: outputdir = create_output_dir(os.path.dirname(image), "signatures", usetime=True) shapefiles = find_files(shapefiledirectory, ".shp", recursive=False) #TODO: Need a method to find only valid shapefiles in the directory get_reference_curves(image, shapefiles, startdoy, doyinterval, outdir=outputdir, filepostfix=filelabel) if plotsigs: path = unique_name(outputdir, "signaturePlot", ext=".pdf") sigs = get_sigs_in_dir(outputdir) plot = SignaturePlot(outputdir, os.path.basename(path)) plot.plot_collection(sigs)
#("Wheat", "wheat_" + extrafilter), #("Wheat/Soy Double Crop", "wwheatsoydbl" + extrafilter), ] # directory to search for sigs signaturedirectory = "/Users/phoetrymaster/Documents/School/Geography/Thesis/Data/Pellegrini/MODIS_10-13_6-14_resampled/Summer2014/multidate_image_aqua105/interpolatedband10/signatures_2014-08-24_0935/" # dir for output outputdir = "/Users/phoetrymaster/Documents/School/Geography/Thesis/ThesisTeX/plots/" #outputdir = "/Users/phoetrymaster/Documents/TeX/deletetest/plots" # plot name name = "othersigsAR.pgf" # find sigs in directory and add to sig collection sigs = find_files(signaturedirectory, "mean.ref") signatures = signatureCollection() for sig in sigs: try: signatures.add(sig) except Exception as e: print e # fonts and other formatting settings pgf_with_rc_fonts = {"pgf.texsystem": "/usr/texbin/xelatex", 'text.color': 'black', "font.family": "serif", "font.serif": ["/Library/Fonts/Baskerville.ttc"],
def get_stats(data_dir): '''Calculates statistics from directory contents. Args: data_dir: Path to directory containing dataset to be analyzed. Returns: A python dictionary `stats` with keys and values described as follows. * `n_mids`: Number of `mid` files found in `data_dir`. * `n_wavs`: Number of `wav` files found in `data_dir`. * `n_fonts`: Number of soundfonts used for generation, assumed from and calculated as proportion of `wav` to `mid` files. * `time_tunes`: Duration [s] of tunes based on midi events, with subfields for `total` dataset duration, `avg` tune duration, `max` tune duration and `min` tune duration. * `notes`: Average note duration [s] `time_avg` and total count of notes in data set `n_total`. * `time_total_notes`: Total duration [s] of individual notes. Vector of 128 floats. * `n_total_notes`: Total number of occurrences of individual notes. Vector of 128 floats. * `time_avg_notes`: Average durations [s] of individual notes. * `n_avg_notes`: Average counts of individual notes in singe tune. ''' mids = find_files(data_dir, '*.mid') wavs = find_files(data_dir, '*.wav') if not mids or not wavs: raise ValueError("No mid/wav files found " "in '{}'.".format(data_dir)) stats = OrderedDict() stats['n_mids'] = len(mids) stats['n_wavs'] = len(wavs) stats['n_fonts'] = stats['n_wavs'] / stats['n_mids'] time_tunes = [] stats['time_tunes'] = OrderedDict() stats['notes'] = OrderedDict() stats['n_total_notes'] = np.zeros(128) + MIN_FLOAT stats['time_total_notes'] = np.zeros(128) polyphony = [0] * 128 # Count totals by iterating over midi files and their notes. for f in mids: m = pretty_midi.PrettyMIDI(f) polyphony = update_poly_stats(polyphony, m) time_tunes.append(m.get_end_time()) for i in m.instruments: if not i.is_drum: for n in i.notes: stats['time_total_notes'][n.pitch] += n.end - n.start stats['n_total_notes'][n.pitch] += 1 # Calculate tune duration stats. stats['time_tunes']['total'] = units_format(sum(time_tunes)) stats['time_tunes']['avg'] = units_format(np.mean(time_tunes)) stats['time_tunes']['max'] = units_format(max(time_tunes)) stats['time_tunes']['min'] = units_format(min(time_tunes)) # Calculate additional stats. stats['time_avg_notes'] = stats['time_total_notes'] / \ stats['n_total_notes'] stats['n_total_notes'] = stats['n_total_notes'].astype(int) stats['notes']['time_avg'] = np.mean(stats['time_avg_notes']) stats['notes']['n_total'] = np.sum(stats['n_total_notes']) stats['n_avg_notes'] = stats['n_total_notes'] / stats['n_mids'] # Save polyphony stats. stats['polyphony_degree'] = polyphony return stats
answers[posdir[0] + str(posnum)] = answer try: for number, clue in puzzle.clues.across(): cluenum = "A" + str(number) if cluenum not in answers: raise xdfile.IncompletePuzzleParse(xd, "Clue number doesn't match grid: " + cluenum) xd.clues.append((("A", number), decode(clue), answers.get(cluenum, ""))) # xd.append_clue_break() for number, clue in puzzle.clues.down(): cluenum = "D" + str(number) if cluenum not in answers: raise xdfile.IncompletePuzzleParse(xd, "Clue doesn't match grid: " + cluenum) xd.clues.append((("D", number), decode(clue), answers.get(cluenum, ""))) except KeyError as e: raise xdfile.IncompletePuzzleParse(xd, "Clue doesn't match grid: " + str(e)) return xd if __name__ == "__main__": import sys from utils import get_args, find_files args = get_args(desc='parse .puz files') for fn, contents in find_files(*sys.argv[1:]): xd = parse_puz(contents, fn) print(xd.to_unicode())
import sys from baseline import BaselineEvaluator from utils import find_files dataset_path = sys.argv[1] key_files = list(find_files(dataset_path, "*.test.key")) print len(key_files) evaluator = BaselineEvaluator(key_files) print "MFS\t{:.3}".format(evaluator.mfs_baseline()) print "RandomShuffle\t{:.3}".format(evaluator.random_shuffled_baseline()) print "Random\t{:.3}".format(evaluator.random_baseline())
def submit_complete(self, machine, build_url, project, revision, build_type, build_abi, build_platform, build_sdk, builder_type, tests=None): """Submit test results for the worker's current job to Treeherder. :param machine: machine id :param build_url: url to build being tested. :param project: repository of build. :param revision: Either a URL to the changeset or the revision id. :param tests: Lists of tests to be reported. """ logger = utils.getLogger() logger.debug('AutophoneTreeherder.submit_complete: %s', tests) if not self.url or not revision: logger.debug('AutophoneTreeherder.submit_complete: no url/revision') return tjc = TreeherderJobCollection() for t in tests: logger.debug('AutophoneTreeherder.submit_complete for %s %s', t.name, project) t.end_timestamp = timestamp_now() # A usercancelled job may not have a start_timestamp # since it may have been cancelled before it started. if not t.start_timestamp: t.start_timestamp = t.end_timestamp tj = self._create_job(tjc, machine, build_url, project, revision, build_type, build_abi, build_platform, build_sdk, builder_type, t) tj.add_state(TestState.COMPLETED) tj.add_result(t.status) tj.add_submit_timestamp(t.submit_timestamp) tj.add_start_timestamp(t.start_timestamp) tj.add_end_timestamp(t.end_timestamp) t.job_details.append({ 'value': os.path.basename(t.config_file), 'title': 'Config'}) t.job_details.append({ 'url': build_url, 'value': os.path.basename(build_url), 'title': 'Build'}) t.job_details.append({ 'value': utils.host(), 'title': 'Host'}) if t.passed + t.failed + t.todo > 0: if t.failed == 0: failed = '0' else: failed = '<em class="testfail">%s</em>' % t.failed t.job_details.append({ 'value': "%s/%s/%s" % (t.passed, failed, t.todo), 'title': "%s-%s" % (t.job_name, t.job_symbol) }) if hasattr(t, 'phonedash_url'): t.job_details.append({ 'url': t.phonedash_url, 'value': 'graph', 'title': 'phonedash' }) # Attach log, ANRs, tombstones, etc. if self.s3_bucket: # We must make certain that S3 keys for uploaded files # are unique even in the event of retries. The # Treeherder logviewer limits the length of the log # url to 255 bytes. If the url length exceeds 255 # characters it is truncated in the Treeherder # logviewer url field even though the file is # successfully uploaded to s3 with the full url. The # logviewer will fail to parse the log since it # attempts to retrieve it from a truncated url. # We have been creating unique keys through the use of # human readable "log_identifiers" combined with the # test's job_guid and base filename to create unique # keys for s3. Unfortunately, the choice of the aws # host name, a path based on the path to the build, # test names and config file names has resulted in # overly long urls which exceed 255 bytes. Given that # the s3 hostname and build url path currently consume # 100 bytes and the test's job-guid and filename # consume another 51, we only have a maximum of 104 # bytes for the log_identifier. The safest course of # action is to eliminate the test name, test config # filename, the chunk and device name and rely solely # on the test's job_guid to provide uniqueness. log_identifier = t.job_guid key_prefix = os.path.dirname( urlparse.urlparse(build_url).path) key_prefix = re.sub('/tmp$', '', key_prefix) # Upload directory containing ANRs, tombstones and other items # to be uploaded. if t.upload_dir: for f in utils.find_files(t.upload_dir): try: lname = os.path.relpath(f, t.upload_dir) try: fname = '%s-%s' % (log_identifier, lname) except UnicodeDecodeError, e: logger.exception('Ignoring artifact %s', lname.decode('utf-8', errors='replace')) continue url = self.s3_bucket.upload(f, "%s/%s" % ( key_prefix, fname)) t.job_details.append({ 'url': url, 'value': lname, 'title': 'artifact uploaded'}) except (S3Error, IOError), e: logger.exception('Error uploading artifact %s', fname) t.job_details.append({ 'value': 'Failed to upload artifact %s: %s' % (fname, e), 'title': 'Error'}) # Autophone Log # Since we are submitting results to Treeherder, we flush # the worker's log before uploading the log to # Treeherder. When we upload the log, it will contain # results for a single test run with possibly an error # message from the previous test if the previous log # upload failed. try: # Emit the final step marker, flush and close the # log prior to uploading. t.worker_subprocess.log_step('Submitting Log') t.worker_subprocess.close_log() fname = '%s-autophone.log' % log_identifier lname = 'Autophone Log' key = "%s/%s" % (key_prefix, fname) url = self.s3_bucket.upload( t.worker_subprocess.logfile, key) # Truncate the log once it has been submitted to S3 # but do not close the filehandler as that messes with # the next test's log. t.worker_subprocess.filehandler.stream.truncate(0) t.job_details.append({ 'url': url, 'value': lname, 'title': 'artifact uploaded'}) tj.add_log_reference('buildbot_text', url, parse_status='pending') except Exception, e: logger.exception('Error %s uploading %s', e, fname) t.job_details.append({ 'value': 'Failed to upload Autophone log: %s' % e, 'title': 'Error'})
def build_multiband_image(rootDIR, outName, newfoldername, find, drivercode, ndvalue, outputdir=None): """ ##Set Args## rootdirectory = "/Users/phoetrymaster/Documents/School/Geography/Thesis/Data/MODIS_KANSAS_2012/" outputfilename = "test" newfoldername = "kansas" VItofind = "EVI" drivercode = "ENVI" nodatavalue = -3000 #projection = "PROJCS[\"Sinusoidal\",GEOGCS[\"GCS_Undefined\",DATUM[\"D_Undefined\", SPHEROID[\"User_Defined_Spheroid\",6371007.181,0.0]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\", 0.017453292519943295]],PROJECTION[\"Sinusoidal\"],PARAMETER[\"False_Easting\",0.0], PARAMETER[\"False_Northing\",0.0],PARAMETER[\"Central_Meridian\",0.0],UNIT[\"Meter\",1.0]]" sys.exit(build_multiband_image(rootdirectory, outputfilename, newfoldername, VItofind, drivercode, nodatavalue) """ #TODO docstrings if outputdir is None: outputdir = rootDIR outdir = create_output_dir(outputdir, newfoldername) print "\nOutputting files to : {0}".format(outdir) print "\nFinding HDF files in directory/subfolders: {0}".format(rootDIR) hdfs = find_files(rootDIR, ".hdf") print "\tFound {0} files.".format(len(hdfs)) print "\nGetting images to process of type {0}...".format(find) toprocess = [] for hdf in hdfs: sds = get_hdf_subdatasets(hdf) for ds in sds: if find.upper() in ds[1].upper(): toprocess.append(ds[0]) print "\t\t{0}".format(ds[0]) bands = len(toprocess) print "\tFound {0} images of type {1}.".format(bands, find) #print "\nGetting output parameters..." #rows, cols, datatype, geotransform, projection = open_image(toprocess[0]) #print "\tParameters: rows: {0}, cols: {1}, datatype: {2}, projection: {3}.".format(rows, cols, datatype, projection) outfile = os.path.join(outdir, outName) print "\nOutput file is: {0}".format(outfile) ## Create output file from first file to process ## template = openImage(toprocess[0]) templateproperties = gdalProperties(template) outds = copySchemaToNewImage(templateproperties, outfile, numberofbands=bands, drivername=drivercode) template = "" del template print "\tCreated output file." print"\nAdding bands to output file..." for i in range(0, bands): print "\tProcessing band {0} of {1}...".format(i + 1, bands) print toprocess[i] image = openImage(toprocess[i]) band = image.GetRasterBand(1) outband = outds.GetRasterBand(i + 1) print "\t\tReading band data to array..." data = band.ReadAsArray(0, 0, templateproperties.cols, templateproperties.rows) print "\t\tWriting band data to output band..." outband.WriteArray(data, 0, 0) outband.SetNoDataValue(ndvalue) outband.FlushCache() outband = "" del data, outband band = "" image = "" print "\tFinished adding bands to output file." outds = "" del outds print "\nProcess completed."
parser = argparse.ArgumentParser(description='Functional tests runner') parser.add_argument('--mc', action='store_true', required=False, default=False, help='model check the result (default: not set)') parser.add_argument('-a', '--args', required=False, default='', help='arguments to pass to the tool. ' 'Enclose the arguments in "".') parser.add_argument('-v', '--verbose', action='count', default=0) args = parser.parse_args() tests_dir = os.path.dirname(os.path.abspath(__file__)) test_files = find_files(tests_dir + '/aag-files/', extension='aag') logger = setup_logging(args.verbose) failed_tests = run_tests(test_files, lambda test_file, result_file: run_tool(test_file, result_file, args.args), [check_answer, check_answer_with_mc][args.mc], True, logger) exit(len(failed_tests))
#("Wheat/Soy Double Crop", "wwheatsoydbl_" + extrafilter), ] # directory to search for sigs signaturedirectory = "/Users/phoetrymaster/Documents/School/Geography/Thesis/Data/MODIS_KANSAS_2007-2012/reprojected/Refs/2012clip1test2/clip1/" oldsigdir = "/Users/phoetrymaster/Documents/School/Geography/Thesis/Data/MODIS_KANSAS_2007-2012/reprojected/Refs/2012/clip1/corn_NDVI_mean.ref" # dir for output outputdir = "/Users/phoetrymaster/Documents/School/Geography/Thesis/ThesisTeX/plots/" #outputdir = "/Users/phoetrymaster/Documents/TeX/deletetest/plots" # plot name name = "refinedsorghumKS.pgf" # find sigs in directory and add to sig collection meansigs = find_files(signaturedirectory, "mean.ref") meansignatures = signatureCollection() for sig in meansigs: try: meansignatures.add(sig) except Exception as e: print e pointsigs = find_files(signaturedirectory, "points.ref") pointsignatures = signatureCollection()
def install(self): # copy dreams dir and relevant driver scripts to inst dir if os.path.exists(self.dreams_dest_dir): utils.output('DREAMs already in inst_dir, not copying.') else: shutil.copytree( os.path.join( self.dre_src_dir, 'dreams'), self.dreams_dest_dir) utils.output('Copied %s.' % (self.dreams_dest_dir,)) driver_paths = ((self.drepy_src, self.drepy_dest), (self.dresh_src, self.dresh_dest), (self.dreshpy_src, self.dreshpy_dest)) for d in driver_paths: if os.path.exists(d[1]): utils.output('%s already present.' % (d[1],)) else: shutil.copy2(d[0], d[1]) utils.output('Copied %s.' % (d[1],)) vardict = {'python_binary_path' : config.python_binary_path, 'python_library_path' : config.python_library_path, 'python_scripts_path' : config.python_scripts_path, 'devide_inst_dir' : config.DEVIDE_INST_DIR, 'wx_lib_path' : config.WX_LIB_PATH, 'vtk_sodir' : config.VTK_SODIR, 'vtk_python' : config.VTK_PYTHON, 'dcmtk_lib' : config.DCMTK_LIB, 'gdcm_lib' : config.GDCM_LIB, 'gdcm_python' : config.GDCM_PYTHON, 'vtkdevide_lib' : config.VTKDEVIDE_LIB, 'vtkdevide_python' : config.VTKDEVIDE_PYTHON, 'vtktudoss_lib' : config.VTKTUDOSS_LIB, 'vtktudoss_python' : config.VTKTUDOSS_PYTHON, 'itk_bin' : config.ITK_BIN, 'itk_lib' : config.ITK_DIR, 'wrapitk_lib' : config.WRAPITK_LIB, 'wrapitk_python' : config.WRAPITK_PYTHON} # replace all instances of the installation dir with the # variable $MYDIR / %MYDIR% vardict2 = {} idir = config.inst_dir if idir.endswith(os.path.sep): idir = idir[:-1] for k,v in vardict.items(): vardict2[k] = v.replace(idir, '%(dre_top)s') if os.name == 'nt': cfg = nt_cfg else: cfg = posix_cfg # let's write out the CFG file fname = os.path.join(config.inst_dir, 'dre.cfg') cf = file(fname, 'w') cfg2 = cfg % vardict2 cf.write(cfg2) cf.close() utils.output('Write DRE CFG.') # and then we have to fix all of the shebangs that distutils sets as absolute paths! if os.name == 'posix': pyscripts = utils.find_files(config.python_binary_path, '.*', exclude_pats=['python$','python[0-9]\.[0-9]$'])[0] for pyscript in pyscripts: utils.re_sub_filter_file([('#!.*', '#!/usr/bin/env python')], pyscript)
import pylab import numpy as np import utils import sys info_text = 'Please give the folder containing the voltage recordings after the script_name' info_text += '\n USAGE:\n\t python plot_voltages.py [FOLDER_NAME]' assert (len(sys.argv) > 1), info_text folder_name = sys.argv[1] filenames = utils.find_files(folder_name, 'exc_volt_') # since you can not decide on the final name your data files will have in NEST (it will depend on the number of processes, for example) # you need to browse through the folder to retrieve all relevant files to be plotted print 'Found filenames:', filenames fig = pylab.figure() ax = fig.add_subplot(111) for fn in filenames: path = folder_name d = np.loadtxt(fn) gids = np.unique(d[:, 0]) for gid in gids: time_axis, volt_trace = utils.extract_trace(d, gid) ax.plot(time_axis, volt_trace) pylab.show()
rc, out, err = execute_shell(SPEC_FRAMEWORK_DIR + '/check_model.sh ' + tmp_file_name) os.remove(tmp_file_name) return rc, out, err if __name__ == "__main__": parser = argparse.ArgumentParser(description='Functional tests runner') parser.add_argument('--mc', action='store_true', required=False, default=False, help='model check the result, default: False') args = parser.parse_args() TEST_FILES = find_files('./tests/safety/', extension='aag', ignore_mark='notest') + \ find_files('./tests/buechi/', extension='aag', ignore_mark='notest') + \ find_files('./tests/syntcomp-format/', extension='aag', ignore_mark='notest') + \ find_files('./tests/1-streett/', extension='aag', ignore_mark='notest') RUN_TOOL = run_tool CHECK_RESULT = [check_answer, check_answer_with_mc][args.mc] logger = setup_logging() exit(run_tests(TEST_FILES, RUN_TOOL, CHECK_RESULT, True, logger))