def ingest(self, receipt, classes, labels=None): mkdir_p(self.base_path) store_labels = labels is None labels = labels or [] with open(self.feature_names_path, "a+") as feature_file, \ open(self.labels_path, "a") as labels_file: feature_file.seek(0) feature_names = set(feature_file.read().split()) for idx, name in enumerate(self.class_names): with open(self.feature_data_path(idx, name), "a") as outfile: for position in classes[idx]: features = self.extract_features(receipt, position) print(json.dumps(features), file=outfile) feature_names.update(features.keys()) if store_labels: labels.append((position, idx)) feature_file.seek(0) feature_file.truncate() feature_file.write("\n".join(sorted(feature_names))) if store_labels: print(json.dumps((receipt.path, labels)), file=labels_file) return True
def train(): logging.info('train bss model') sig1_list = util.load_data(config.DATA_TRAIN_LIST, config.DATA_TRAIN_ROOT) # sig1_list = util.load_data_from_matlab(config.DATA_TRAIN_LIST, config.DATA_TRAIN_ROOT) sig2 = util.read_wav(config.DATA_TRAIN_STATIC_SIGNAL) # sig2 = util.read_wav_from_matlab(config.DATA_TRAIN_STATIC_SIGNAL) # print('sig1_list is', sig1_list) # print('sig2 is', sig2) if config.NORMALIZATION: sig1_list = [_normalize(sig) for sig in sig1_list] sig2 = _normalize(sig2) logging.info('mix signals') util.mkdir_p(config.DATA_TRAIN_MIX_ROOT) mix_filenames = _gen_mix_filename(config.DATA_TRAIN_LIST, config.DATA_TRAIN_STATIC_SIGNAL, config.DATA_TRAIN_MIX_ROOT) sig2_list = [sig2 for _ in range(len(sig1_list))] sig1_list, sig2_list, mix_list = util.mix(sig1_list, sig2_list, output=mix_filenames) # print('sig1_list is', sig1_list) # print('sig2_list is', sig2_list) # print('mix_list is', mix_list) # logging.info('extract stft features') # sig1_stft = _extract_stft(sig1_list) # print('sig1_stft', sig1_stft) # sig2_mrcg = _extract_stft(sig2_list) # mix_mrcg = _extract_stft(mix_list) logging.info('extract mrcg features') sig1_mrcg = _extract_mrcg(sig1_list) print('sig1_mrcg is', sig1_mrcg) sig2_mrcg = _extract_mrcg(sig2_list) print('sig2_mrcg is', sig2_mrcg) mix_mrcg = _extract_mrcg(mix_list) print('mix_mrcg is', mix_mrcg) logging.info('train neural network') sig1_mrcg = util.r_abs(sig1_mrcg) sig2_mrcg = util.r_abs(sig2_mrcg) mix_mrcg = util.r_abs(mix_mrcg) train_x, train_y = mix_mrcg, [] for s, n in zip(sig1_mrcg, sig2_mrcg): # 将signal和noise拼接作为nn输出 train_y.append(np.concatenate((s, n), axis=1)) train_x = [_extend(sig, config.EXTEND_NUM) for sig in train_x] # 各层神经元数量 layer_size = [len(train_x[0][0])] # 输入层 layer_size.extend(config.LAYER_SIZE) # 隐藏层 layer_size.append(len(train_y[0][0])) # 输出层 dnn = nn.NNet(layer_size) util.mkdir_p(config.MODEL_ROOT) dnn.train(train_x, train_y, model_path=config.MODEL_NN_PATH, training_epochs=config.EPOCH, learning_rate=config.LEARNING_RATE)
def install_base(self): """ Install pathogen.vim and create vimpyre directory. >>> bat = Bat() >>> bat.install_base() => => Send a bat to catch pathogen.vim ... Catch done! Please add the following message to your .vimrc: call pathogen#runtime_append_all_bundles("vimpyre") """ try: console('=> => Send a bat to catch pathogen.vim ...') raw_urlopen = urllib.urlopen(self.PATHOGEN_URL) if raw_urlopen.getcode() == 200: util.mkdir_p(self.AUTOLOAD_PATH) util.mkdir_p(self.VIMPYRE_PATH) raw_pathogen = raw_urlopen.read() pathogen = path.join(self.AUTOLOAD_PATH, 'pathogen.vim') with open(pathogen, 'w') as f: f.write(raw_pathogen) console('Catch done! Please add the following to your .vimrc:') console('call pathogen#runtime_append_all_bundles("vimpyre")') else: console('Catch fail! Please try again!') except: console('[Unexpected Error] Catch fail! Please try again!')
def main(*args, **kwargs): """ """ if not HAS_MATPLOTLIB: raise ImportError( "Please install matplotlib to generate a plot from this example.") ############################################ # Step 1: Define parameters for the models # ############################################ util.mkdir_p(FLAGS.run_dir) util.logging(FLAGS) tf.logging.info('[Step 1]: Define parameters for the models') util.report_param(FLAGS) ################################ # Step 2: Define input dataset # ################################ tf.logging.info('[Step 2]: Define input dataset') if FLAGS.data_type == 'random_signal': x, y = util.random_signal(length=FLAGS.data_length) # Save the data into numpy arra np.savez_compressed(os.path.join(FLAGS.run_dir, 'input-data'), x=x, y=y) plt.plot(x, y) plt.savefig(os.path.join(FLAGS.run_dir, 'input-timeseries.jpg')) for i in tqdm(range(FLAGS.iterations)): # train the model iterations pass print("learning rate", FLAGS.learning_rate)
def save_gfx(self, path, data=None): if data is None: print('got data') data = self.get_data() for k, v in data.items(): if type(v) is dict or type(v) is OrderedDict: self.save_gfx(path, v) elif k == 'icon': icon_path = data['icon'].split('/') if icon_path[0] not in self.modlist.path_map: print('Unknown content path %s for %s/%s' % (icon_path[0], data['type'], data['name'])) continue icon_path[0] = self.modlist.path_map[icon_path[0]] icon_path = '/'.join(icon_path) if 'type' not in data: # attempt to extract name and type from filepath path_els = v[:v.rindex('.')].split('/') itm_type, name = path_els[-2:] else: itm_type, name = data['type'], data['name'] out_dir = '%s/%s' % (path, itm_type) out_path = '%s/%s.png' % (out_dir, name) mkdir_p(out_dir) if os.path.exists(out_path): print('Overwriting %s/%s' % (itm_type, name)) shutil.copy2(icon_path, out_path)
def save_gfx(self, path, data=None): if data is None: print('got data') data = self.get_data() for k, v in data.items(): if type(v) is dict: self.save_gfx(path, v) pass elif k == 'icon': icon_path = data['icon'].split('/') if icon_path[0] not in self.modlist.path_map: print('Unknown content path %s for %s/%s' % (icon_path[0], data['type'], data['name'])) continue icon_path[0] = self.modlist.path_map[icon_path[0]] icon_path = '/'.join(icon_path) out_dir = '%s/%s' % (path, data['type']) out_path = '%s/%s.png' % (out_dir, data['name']) mkdir_p(out_dir) if os.path.exists(out_path): print('Overwriting %s/%s' % (data['type'], data['name'])) shutil.copy2(icon_path, out_path)
def mutate(self): """ Performs the mutation. Applies mutation operator to each source file, then stores a diff between the original and mutated file. # mutants = # source files x # mutation operators """ count = 0 start = time() for src_file in self.project.source_files(): original_path = join(self.project.settings["source_path"], src_file) mutant_path = join(out_dir, src_file) mkdir_p(dirname(mutant_path)) for (op, invoke) in self.project.settings["mutants"].items(): if invoke: p = Popen(["txl", original_path, join("vendor", "conman", "%s.Txl" % op)], stdout=open(mutant_path, "w"), stderr=open("/dev/null")) self.store.put(diff(relpath(original_path), relpath(mutant_path)), op, src_file) count += 1 if count % 1000 == 0: print("Generated %d mutants. Elapsed time %.02f seconds." % (count, (time() - start))) stop = time() print("Generated %d mutants in %d seconds." % (count, (stop - start)))
def install_base(self): """ Install pathogen.vim and create vimpyre directory. >>> bat = Bat() >>> bat.install_base() => => Send a bat to catch pathogen.vim ... Catch done! Please add the following message to your .vimrc: execute pathogen#infect('bundle/{}', 'vimpyre/{}') """ try: console('=> => Send a bat to catch pathogen.vim ...') raw_urlopen = urllib.urlopen(self.pathogen_url) if raw_urlopen.getcode() == 200: util.mkdir_p(self.AUTOLOAD_PATH) util.mkdir_p(self.VIMPYRE_PATH) raw_pathogen = raw_urlopen.read() pathogen = path.join(self.AUTOLOAD_PATH, 'pathogen.vim') with open(pathogen, 'w') as f: f.write(raw_pathogen) console('Catch done! Please add the following to your .vimrc:') console("execute pathogen#infect('bundle/{}', 'vimpyre/{}')") else: console('Pathogen vimscript not found in %s' % self.pathogen_url) console('You can change this url with enviroment variable VIM_PATHOGEN_URL') console('Catch fail! Please try again!') except: console('[Unexpected Error] Catch fail! Please try again!')
def write_file(self, path): if not hasattr(self, 'cached_obj') or self.kobj._always_regenerate: self.render() if self.cached_obj is None: return if not hasattr(self, 'cached_yaml'): self.yaml() if self.uses_namespace: path = os.path.join(path, self.namespace_name) mkdir_p(path) self.filedir = path self.filename = self.identifier + '.yaml' self.debug(3, "writing file {}/{}".format(self.filedir, self.filename)) sav_context = var_types.VarContext.current_context var_types.VarContext.current_context = {'confidential': False} try: content = str(self.cached_yaml) self.is_confidential = var_types.VarContext.current_context['confidential'] finally: var_types.VarContext.current_context = sav_context if self.is_confidential: self.debug(3, " file {}/{} is confidential".format(self.filedir, self.filename)) with open(os.path.join(path, '.' + self.identifier + '.tmp'), 'w') as f: f.write(content) os.rename(os.path.join(path, '.' + self.identifier + '.tmp'), os.path.join(path, self.filename))
def _write_output_clustered(self): with self.confidential(self.base) as confidential: for c in self.repository.get_clusters(): path = os.path.join(self.base, c) mkdir_p(path) ns_done = set() for ns in self.clusterless: ns_done.add(ns) outputs = [] outputs.extend(self.clusterless[ns].values()) if c in self.clustered and ns in self.clustered[c]: outputs.extend(self.clustered[c][ns].values()) if any(map(lambda x: x.has_data() and not x.is_namespace, outputs)): for op in outputs: op.write_file(path) confidential.add_file(op) if not c in self.clustered: continue for ns in self.clustered[c]: if ns in ns_done: continue if any(map(lambda x: x.has_data() and not x.is_namespace, self.clustered[c][ns].values())): for op in self.clustered[c][ns].values(): op.write_file(path) confidential.add_file(op)
def write_binary(self, outdir, opp=True): """Write particle to LabView binary file in outdir """ if not self.has_data(): return # Might have julian day, might not root = os.path.join(outdir, os.path.dirname(self.file_id)) util.mkdir_p(root) outfile = os.path.join(outdir, self.file_id) if opp: outfile += ".opp" if os.path.exists(outfile): os.remove(outfile) if os.path.exists(outfile + ".gz"): os.remove(outfile + ".gz") with open(outfile, "wb") as fh: # Write 32-bit uint particle count header header = np.array([self.particle_count], np.uint32) header.tofile(fh) # Write particle data self._create_particle_matrix().tofile(fh) util.gzip_file(outfile)
def __init__(self): self.settings = { 'workdir': PaperworkSetting("Global", "WorkDirectory", lambda: os.path.expanduser("~/papers")), 'index_version': PaperworkSetting("Global", "IndexVersion", lambda: "-1"), } self._configparser = None # Possible config files are evaluated in the order they are in the # array. The last one of the list is the default one. configfiles = [ "./paperwork.conf", os.path.expanduser("~/.paperwork.conf"), ("%s/paperwork.conf" % (os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")))) ] configfile_found = False for self.__configfile in configfiles: if os.access(self.__configfile, os.R_OK): configfile_found = True logger.info("Config file found: %s", self.__configfile) break if not configfile_found: logger.info("Config file not found. Will use '%s'", self.__configfile) util.mkdir_p(os.path.dirname(self.__configfile))
def filter_tags(tracks, tags, artist_threshold, directory=None): if directory is not None: util.mkdir_p(directory) tags_new_all = {} for category in tags: stats, total = statistics.get_statistics(category, tracks, tags) stats_filtered = stats[stats['artists'] >= artist_threshold] if directory is not None: statistics.write_statistics(category, stats_filtered, directory) tags_new_all[category] = set(stats_filtered['tag']) print("- {} tags: {} -> {}".format(category, len(stats), len(stats_filtered))) # TODO: replace with filter_subset tracks_to_delete = [] for track_id, track in tracks.items(): total_tags = 0 for category, tags_new in tags_new_all.items(): track[category] &= tags_new total_tags += len(track[category]) if total_tags == 0: tracks_to_delete.append(track_id) for track in tracks_to_delete: tracks.pop(track)
def setup_experiment(args, config): # Grab parameters if args.exp_name: config.exp_name = args.exp_name # elif not hasattr(config, 'exp_name'): # config.exp_name = exp_lifecycle.guess_config_name(config) if not hasattr(config, 'results_dir'): config.results_dir = "experiments/%s" % config.exp_name if args.timestamp_results is not None: # Note that argparse returns a list config.timestamp_results = args.timestamp_results if hasattr(config, 'timestamp_results') and config.timestamp_results: now = timestamp_string() config.results_dir += "_" + str(now) # Set up results directory mkdir_p("./experiments") create_clean_dir(config.results_dir) # Make sure that there are no uncommited changes if args.publish: exp_lifecycle.publish_prepare(config.exp_name, config.results_dir) # Record machine information for this experiment additional_metadata = None if hasattr(config, "get_additional_metadata"): additional_metadata = config.get_additional_metadata() exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir, additional_metadata=additional_metadata)
def _write_output_clustered(self): changed = [] with self.confidential(self.base) as confidential: for c in self.repository.get_clusters(): path = os.path.join(self.base, c) is_openshift = self.repository.is_openshift or self.repository.get_cluster_info( c).is_openshift mkdir_p(path) ns_done = set() for ns in self.clusterless: ns_done.add(ns) outputs = [] outputs.extend(self.clusterless[ns].values()) if c in self.clustered and ns in self.clustered[c]: outputs.extend(self.clustered[c][ns].values()) changed.extend( self._write_namespace(outputs, path, confidential, is_openshift)) if not c in self.clustered: continue for ns in self.clustered[c]: if ns in ns_done: continue changed.extend( self._write_namespace(self.clustered[c][ns].values(), path, confidential, is_openshift)) return changed
def build(args): use_cuda = 'cuda' in args.features use_cl = 'cl' in args.features use_cudnn = 'cudnn' in args.features use_eigen = 'eigen' in args.features use_blas = 'openblas' in args.features use_nnpack = 'nnpack' in args.features use_protobuf = 'protobuf' in args.features use_opencv = 'opencv' in args.features build_examples = 'examples' in args.features build_tools = 'tools' in args.features build_service = 'service' in args.features build_test = 'test' in args.features build_shared_libs = 'shared' in args.features cmake_options = {} if use_cuda: cmake_options['USE_CUDA'] = True cmake_options['USE_CL'] = False cmake_options['USE_CUDNN'] = use_cudnn elif use_cl: cmake_options['USE_CUDA'] = False cmake_options['USE_CL'] = True else: cmake_options['USE_CUDA'] = False cmake_options['USE_CL'] = False cmake_options['USE_Eigen'] = use_eigen cmake_options['USE_BLAS'] = use_blas cmake_options['USE_NNPACK'] = use_nnpack cmake_options['USE_Protobuf'] = use_protobuf cmake_options['USE_OpenCV'] = use_opencv cmake_options['BUILD_EXAMPLES'] = build_examples cmake_options['BUILD_TOOLS'] = build_tools cmake_options['BUILD_SERVICE'] = build_service cmake_options['BUILD_TEST'] = build_test cmake_options['BUILD_SHARED_LIBS'] = build_shared_libs shadow_root = os.path.dirname(os.path.abspath(__file__)) + '/../..' build_root = shadow_root + '/build/' + args.subdir if not os.path.isdir(build_root): util.mkdir_p(build_root) shell_cmd = 'cd ' + build_root + ' && ' shell_cmd += 'cmake ../.. ' choices = ['OFF', 'ON'] for define in args.define: shell_cmd += '-D' + define + ' ' for (feature, value) in cmake_options.items(): shell_cmd += '-D' + feature + '=' + choices[int(value)] + ' ' if args.generator == 'make': shell_cmd += '&& make -j2' elif args.generator == 'ninja': shell_cmd += '-GNinja && ninja' if args.debug != 'nodebug': print(shell_cmd + '\n' + ''.join(['='] * 60)) subprocess.check_call(shell_cmd, shell=True)
def compute_statistics(tracks, tags, directory): util.mkdir_p(directory) for category in tags: data, total = get_statistics(category, tracks, tags) write_statistics(category, data, directory) print('Total tags for {}: {} tags, {}'.format(category, len(data), total))
def save_prefs(prefs): util.mkdir_p(storage_path) data = json.dumps(prefs, indent=4) try: with open(_prefs_file(), 'w') as f: f.write(data) except IOError as e: log.warning('Error saving %s: %s', _prefs_file(), e)
def _write_output_clusterless(self): mkdir_p(self.base) with self.confidential(self.base) as confidential: for ns in self.clusterless: if any(map(lambda x: x.has_data() and not x.is_namespace, self.clusterless[ns].values())): for op in self.clusterless[ns].values(): op.write_file(self.base) confidential.add_file(op)
def write_file(self, path): if not hasattr(self, 'cached_obj') or self.kobj._always_regenerate: self.render() if self.cached_obj is None: return if not hasattr(self, 'cached_yaml'): self.yaml() if self.uses_namespace: path = os.path.join(path, self.namespace_name) mkdir_p(path) self.filedir = path self.filename = self.filename_conversion(self.identifier) + '.yaml' self.debug(3, "writing file {}/{}".format(self.filedir, self.filename)) sav_context = var_types.VarContext.current_context var_types.VarContext.current_context = {'confidential': False} try: content = str(self.cached_yaml) self.is_confidential = var_types.VarContext.current_context[ 'confidential'] finally: var_types.VarContext.current_context = sav_context if self.is_confidential: self.debug( 3, " file {}/{} is confidential".format(self.filedir, self.filename)) changed = False if self.content_check is not None and self.content_check in ( 'contents', 'yaml', 'exists'): try: with open(os.path.join(path, self.filename), 'rb') as f: if self.content_check == 'contents': if content != f.read().decode('utf8'): changed = True elif self.content_check == 'yaml': if yaml_load(content) != yaml_load( f.read().decode('utf8')): changed = True except: changed = True with open(os.path.join(path, '.' + self.identifier + '.tmp'), 'w') as f: f.write(content) os.rename(os.path.join(path, '.' + self.identifier + '.tmp'), os.path.join(path, self.filename)) if changed: return os.path.join(path, self.filename) return None
def _aligin(input, output): _padding = hop if padding else 0 length = hop * ext_n * ext_d + (windowsize - hop) filename, wav, next_wav = None, [], [] curr_dir_n, curr_samp_n, curr_dir = -1, file_max + 1, os.path.join( output, '0') for root, dirs, files in os.walk(input): for f in files: if f.endswith('.wav'): if filename is None: filename = f y, sr = librosa.load(os.path.join(root, f), sr=samplerate) wav.extend(y) if length > len(wav) + _padding * 2: if concat: continue if too_short == 'append_0': n = length - (len(wav) + _padding * 2) wav += [0 for _ in range(n)] elif too_short == 'discard': filename, wav = None, [] continue else: raise ValueError('%s is not supported' % too_short) elif length < len(wav) + _padding * 2: if overlap: surplus = (len(wav) + _padding * 2 - windowsize) % hop else: surplus = (len(wav) + _padding * 2) % length if surplus != 0: if concat: next_wav = wav[-surplus:] del wav[-surplus:] elif too_long == 'append_0': comp = hop - surplus if overlap else length - surplus wav += [0 for _ in range(comp)] elif too_long == 'cut': del wav[-surplus:] else: raise ValueError('%s is not supported' % too_long) if curr_samp_n > file_max: curr_dir_n, curr_samp_n = curr_dir_n + 1, 0 curr_dir = os.path.join(output, str(curr_dir_n)) util.mkdir_p(curr_dir) path = os.path.join(curr_dir, filename[:-4] + ".voice.wav") librosa.output.write_wav(path, np.array(wav), samplerate) curr_samp_n += 1 if concat: filename, wav = None, next_wav next_wav = [] else: filename, wav = None, []
def write_vct(self, outdir): outfile = os.path.join(outdir, self.file_id + ".vct") util.mkdir_p(os.path.dirname(outfile)) if os.path.exists(outfile): os.remove(outfile) if os.path.exists(outfile + ".gz"): os.remove(outfile + ".gz") with gzip.open(outfile + ".gz", "wb") as f: f.write("\n".join(self.vct) + "\n")
def install_mpqueue_as(self, mpqueue, task_index): self._MPQ = mpqueue self._task_index = task_index self._logfile = open(self.args.ckptdir + '{}.out'.format(task_index), 'w') self._sample_dumpdir = self.args.ckptdir + "sample-{}/".format( task_index) util.mkdir_p(self._sample_dumpdir) self._sample_index = 0
def __init__(self, settings, compiler): self._settings = settings self.compiler_name = settings.get('codeblocks').get('compiler_name', 'gcc') self.projectsdir = settings.get('projectsdir') util.mkdir_p(self.projectsdir) self.configurations = compiler.get_configurations() cvars = compiler.get_global_variables() self.cflags = cvars['cflags'] self.lflags = cvars['lflags']
def torus_master(): pyosr.init() dpy = pyosr.create_display() glctx = pyosr.create_gl_context(dpy) g = tf.Graph() util.mkdir_p(ckpt_dir) with g.as_default(): global_step = tf.contrib.framework.get_or_create_global_step() increment_global_step = tf.assign_add(global_step, 1, name='increment_global_step') learning_rate_input = tf.placeholder(tf.float32) grad_applier = RMSPropApplier(learning_rate=learning_rate_input, decay=RMSP_ALPHA, momentum=0.0, epsilon=RMSP_EPSILON, clip_norm=GRAD_NORM_CLIP, device=device) masterdriver = rldriver.RLDriver(MODELS, init_state, view_config, config.SV_VISCFG, config.MV_VISCFG, use_rgb=True) saver = tf.train.Saver(masterdriver.get_nn_args() + [global_step]) with tf.Session() as sess: threads = [] for i in range(THREAD): thread_args = (i, dpy, glctx, masterdriver, g, grad_applier, learning_rate_input, global_step, increment_global_step, sess, saver) thread = threading.Thread(target=torus_worker, args=thread_args) thread.start() graph_completes[i].wait() threads.append(thread) ''' We need to run the initializer because only master's variables are stored. ''' sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(checkpoint_dir=ckpt_dir) print('ckpt {}'.format(ckpt)) epoch = 0 if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) epoch = sess.run(global_step) print('Restored!, global_step {}'.format(epoch)) init_done.set() for thread in threads: thread.join() print("Saving final checkpoint") fn = saver.save(sess, ckpt_dir + ckpt_prefix, global_step=global_step) print("Saved checkpoint to {}".format(fn))
def save_transform_to_database(data_transforms): """ Save data transforms to database""" # pylint: disable= too-many-locals, bare-except conn = sqlite3.connect(util.DB_PATH) conn.text_factory = str for img in data_transforms: cursor = conn.execute('''SELECT pid from Images where id = ? ''', (img.image_id,)) pid = cursor.fetchone()[0] folder = util.DATA_FOLDER + str(pid) + "/registration_transforms/" util.mkdir_p(folder) transform_paths = "" print(img.get_transforms()) for _transform in img.get_transforms(): print(_transform) dst_file = folder + util.get_basename(_transform) + '.h5.gz' if os.path.exists(dst_file): os.remove(dst_file) with open(_transform, 'rb') as f_in, gzip.open(dst_file, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) transform_paths += str(pid) + "/registration_transforms/" +\ basename(_transform) + '.h5.gz' + ", " transform_paths = transform_paths[:-2] cursor2 = conn.execute('''UPDATE Images SET transform = ? WHERE id = ?''', (transform_paths, img.image_id)) cursor2 = conn.execute('''UPDATE Images SET fixed_image = ? WHERE id = ?''', (img.fixed_image, img.image_id)) folder = util.DATA_FOLDER + str(pid) + "/reg_volumes_labels/" util.mkdir_p(folder) vol_path = util.compress_vol(img.processed_filepath) shutil.copy(vol_path, folder) volume_db = str(pid) + "/reg_volumes_labels/" + basename(vol_path) cursor2 = conn.execute('''UPDATE Images SET filepath_reg = ? WHERE id = ?''', (volume_db, img.image_id)) cursor = conn.execute('''SELECT filepath, id from Labels where image_id = ? ''', (img.image_id,)) for (row, label_id) in cursor: temp = util.compress_vol(move_vol(util.DATA_FOLDER + row, img.get_transforms(), True)) shutil.copy(temp, folder) label_db = str(pid) + "/reg_volumes_labels/" + basename(temp) cursor2 = conn.execute('''UPDATE Labels SET filepath_reg = ? WHERE id = ?''', (label_db, label_id)) conn.commit() cursor.close() cursor2.close() # cursor = conn.execute('''VACUUM; ''') conn.close()
def test_rldriver_main(): pyosr.init() dpy = pyosr.create_display() glctx = pyosr.create_gl_context(dpy) g = tf.Graph() util.mkdir_p(ckpt_dir) with g.as_default(): learning_rate_input = tf.placeholder(tf.float32) grad_applier = RMSPropApplier(learning_rate=learning_rate_input, decay=RMSP_ALPHA, momentum=0.0, epsilon=RMSP_EPSILON, clip_norm=GRAD_NORM_CLIP, device=device) masterdriver = rldriver.RLDriver(MODELS, init_state, view_config, config.SV_VISCFG, config.MV_VISCFG, use_rgb=True) driver = rldriver.RLDriver(MODELS, init_state, view_config, config.SV_VISCFG, config.MV_VISCFG, use_rgb=True, master_driver=masterdriver, grads_applier=grad_applier) driver.get_sync_from_master_op() driver.get_apply_grads_op() driver.learning_rate_input = learning_rate_input driver.a3c_local_t = 32 global_step = tf.contrib.framework.get_or_create_global_step() increment_global_step = tf.assign_add(global_step, 1, name='increment_global_step') saver = tf.train.Saver(masterdriver.get_nn_args() + [global_step]) last_time = time.time() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(checkpoint_dir=ckpt_dir) print('ckpt {}'.format(ckpt)) epoch = 0 if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) epoch = sess.run(global_step) print('Restored!, global_step {}'.format(epoch)) while epoch < 100 * 1000: driver.train_a3c(sess) epoch += 1 sess.run(increment_global_step) if epoch % 1000 == 0 or time.time() - last_time >= 10 * 60: print("Saving checkpoint") fn = saver.save(sess, ckpt_dir+ckpt_prefix, global_step=global_step) print("Saved checkpoint to {}".format(fn)) last_time = time.time() print("Epoch {}".format(epoch)) '''
def _write_output_clusterless(self): changed = [] mkdir_p(self.base) with self.confidential(self.base) as confidential: for ns in self.clusterless: changed.extend( self._write_namespace(self.clusterless[ns].values(), self.base, confidential, self.repository.is_openshift)) return changed
def init_hello_world(): targets_filepath = 'source/hello_world/targets.json' helloworld_filepath = 'source/hello_world/hello_world.cpp' critical_error_if_file_exists(helloworld_filepath) critical_error_if_file_exists(targets_filepath) util.mkdir_p(os.path.dirname(targets_filepath)) with open(targets_filepath, 'w+') as fd: fd.write(util.get_resource('defaults/targets.json')) with open(helloworld_filepath, 'w+') as fd: fd.write(util.get_resource('defaults/hello_world.cpp'))
def get_archive_file_name(config, kind, start_dt, end_dt): """get the archive file name. has the format of {ARCHIVE_DIR}/YY-mm-dd/{kind}/kind-start_dt-end_dt.pickle """ datestr = str(start_dt.date()) dirname = "%s/%s/%s" % (config['archive_dir'], datestr, kind) mkdir_p(dirname) filename = "%s/%s-%s-%s-%s-%s.pickle" % (dirname, kind, str(start_dt.date()), str(start_dt.time()), str(end_dt.date()), str(end_dt.time())) return filename
def write_tokens(start_dt, end_dt): # Write token file at end of main. gae_download.py runs under an hourly, # cronjob, so load_emr_daily.sh will start only when all 24 token files # are present. datestr = str(start_dt.date()) dirname = "/home/analytics/kabackup/daily_new/" dirname += "%s/tokens/" % datestr mkdir_p(dirname) filename = "token-%s-%s.txt" % (start_dt.time(), end_dt.time()) f = open(dirname + filename, "w") f.close()
def update_file(file_disk_path, file_download_path): if os.path.isfile(file_disk_path): file_disk_md5 = util.get_file_md5(file_disk_path) file_download_md5 = util.get_file_md5(file_download_path) if file_disk_md5 != file_download_md5: shutil.copy(file_download_path, file_disk_path) else: file_disk_dir = os.path.dirname(file_disk_path) if not os.path.isdir(file_disk_dir): util.mkdir_p(file_disk_dir) shutil.copy(file_download_path, file_disk_path)
def get_archive_file_name(config, kind, start_dt, end_dt): """get the archive file name. has the format of {ARCHIVE_DIR}/YY-mm-dd/{kind}/kind-start_dt-end_dt.pickle """ datestr = str(start_dt.date()) dirname = "%s/%s/%s" % (config['archive_dir'], datestr, kind) mkdir_p(dirname) filename = "%s/%s-%s-%s-%s-%s.pickle" % (dirname, kind, str( start_dt.date()), str(start_dt.time()), str( end_dt.date()), str(end_dt.time())) return filename
def plotROC(y_true, y_pred, title, path=None): """ Inputs: Arrays containing k arrays of true and continous predicted y values for 1 fold Outputs: An ROC plot saved at the location given in path relative the the wd """ plt.gcf().clear() fig = plt.figure() mean_tpr = 0.0 mean_fpr = np.linspace(0, 1, 100) colors = itertools.cycle([ 'cyan', 'indigo', 'seagreen', 'yellow', 'blue', 'darkorange', 'red', 'pink', 'purple', 'chocolate' ]) lw = 2 k = 0 for true, scores, color in zip(y_true, y_pred, colors): fpr, tpr, thresholds = roc_curve(true, scores) mean_tpr += interp(mean_fpr, fpr, tpr) mean_tpr[0] = 0.0 roc_auc = auc(fpr, tpr) plt.plot(fpr, tpr, lw=lw, color=color, label='ROC fold %d (area = %0.2f)' % (k, roc_auc)) k += 1 plt.plot([0, 1], [0, 1], linestyle='--', lw=lw, color='k', label='Luck') mean_tpr /= len(y_true) mean_tpr[-1] = 1.0 mean_auc = auc(mean_fpr, mean_tpr) plt.plot(mean_fpr, mean_tpr, color='g', linestyle='--', label='Mean ROC (area = %0.2f)' % mean_auc, lw=lw) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') #plt.axis("equal") plt.title('\n'.join(wrap(title, 60))) leg = plt.legend(bbox_to_anchor=(1, 0.815), loc='center left', numpoints=1) fig.set_tight_layout(True) if path == None: plt.show() else: utilities.mkdir_p(path) plt.savefig(path, bbox_inches='tight') plt.close("all")
def generate(ninja_targets, settings): project_name = settings.get('project_name') working_dir = '$rootdir' make_all = BuildSystem('make - ' + project_name, 'make build', working_dir) make_all.add_variant('All', 'make all') make_all.add_variant('Clean', 'make clean') make_all.add_variant('Sublime Text Project', 'make sublime') make_all.add_variant('Doxygen Documentation', 'make doxygen') ninja_all = BuildSystem('ninja - ' + project_name, 'ninja', working_dir) ninja_all.add_variant('All', 'ninja all') ninja_all.add_variant('Clean', 'ninja -t clean') ninja_all.add_variant('Doxygen Documentation', 'ninja doxygen') build_systems = [make_all.data, ninja_all.data] for target in ninja_targets.build_targets: build_target = BuildSystem( target.config + ' - ' + target.name, 'ninja %s' % target.phony_name, working_dir) bindir = get_bin(target.config, settings) binary = Path.join(working_dir, bindir, target.name) build_target.add_variant( 'Run', 'ninja %s && %s' % (target.phony_name, binary)) build_target.add_variant( 'Clean', 'ninja -t clean %s' % target.phony_name) build_systems.append(build_target.data) sublime_settings = settings.get('sublime') template_filename = settings.expand_variables(sublime_settings.get('project_template', None)) if template_filename is not None: with open(template_filename, 'r') as fd: template = fd.read() if not template: critical_error('%s seems to be empty', template_filename) else: template = util.get_resource('defaults/tmpl.sublime-project') project = re.sub( r'(\$build_systems)', json.dumps(build_systems, indent=4, separators=(',', ': ')), template) project = settings.expand_variables(project) filename = settings.expand_variables('$projectsdir/$project_name.sublime-project') util.mkdir_p(os.path.dirname(filename)) with open(filename, 'w+') as out: out.write(project)
def test_prerequesites(): '''check we have the right directories and tools to run tests''' print("Testing prerequesites") util.mkdir_p(util.reltopdir('../buildlogs')) if not os.path.exists(util.reltopdir('../HILTest/hil_quad.py')): print(''' You need to install HILTest in %s You can get it from git://git.samba.org/tridge/UAV/HILTest.git ''' % util.reltopdir('../HILTest')) return False return True
def assertFileContentsMatch(self, filename, expected): # overridden from shared/testutil/testcase.py # to conditionally regenerate expected output # (see comments on _RECREATE_EXPECTED_RESULTS) if not _RECREATE_EXPECTED_RESULTS: return super(TestBuildBundleBase, self).assertFileContentsMatch(filename, expected) target_path = os.path.join(self.real_ka_root, 'kake', 'compile_js_bundles-testfiles', expected) util.mkdir_p(os.path.dirname(target_path)) with open(target_path, 'w') as fout: fout.write(self._file_contents(filename))
def audio_url_for_file(audio_track, uploaded=False): if uploaded: wav_path = os.path.join(UPLOAD_FOLDER, audio_track) else: util.mkdir_p(os.path.join('data', 'wav')) wav_path = os.path.join('data', 'wav', str(audio_track.id) + ".wav") if not os.path.isfile(wav_path): data, rate = librosa.load(audio_track.path) librosa.output.write_wav(wav_path, data, rate) return os.path.join('play', wav_path)
def create_inventory_id_and_tags(ec2_conf, ec2_info): # mock the inventory and ids that should be generated by vagrant inventory_dir = '.vagrant/provisioners/ansible/inventory' mkdir_p(inventory_dir) inventory = open(os.path.join(inventory_dir, 'vagrant_ansible_inventory'), 'w') machine_dir = '.vagrant/machines' mkdir_p(machine_dir) tag = ec2_conf['Tag'] tags = {} for i, instance in enumerate(ec2_info['instances']): host = "TachyonMaster" if i == 0 else "TachyonWorker{id}".format(id=i) inventory.write("{host} ansible_ssh_host={ip} ansible_ssh_port=22\n".format(host=host, ip=instance['public_ip'])) instance_id = str(instance['id']) id_dir = os.path.join(machine_dir, host, 'aws') mkdir_p(id_dir) with open(os.path.join(id_dir, 'id'), 'w') as f: f.write(instance_id) tags[instance_id] = '{tag}-{host}'.format(tag=tag, host=host) inventory.close() # map instance id to its tag spot_tag_dir = 'spot/roles/tag_ec2/vars' mkdir_p(spot_tag_dir) tag_vars = open(os.path.join(spot_tag_dir, 'main.yml'), 'w') var = { "tags": tags, "region": ec2_conf["Region"], } yaml.dump(var, tag_vars, default_flow_style=False) tag_vars.close()
def filter_cruise(host_assignments, output_dir, process_count=16): cruises = [x[0] for x in host_assignments[env.host_string]] cruise_results = {} workdir = "/mnt/raid" with cd(workdir): for c in cruises: puts("Filtering cruise {}".format(c)) with hide("commands"): run("mkdir {}".format(c)) with cd(c): text = {"cruise": c, "process_count": process_count} with settings(warn_only=True), hide("output"): #result = run("seaflowpy_filter --s3 -c {cruise} -d {cruise}.db -l 10 -p 2 -o {cruise}_opp".format(**text)) result = run( "seaflowpy_filter --s3 -c {cruise} -d {cruise}.db -t -p {process_count} -o {cruise}_opp".format(**text), timeout=10800 ) cruise_results[c] = result puts(result) cruise_output_dir = os.path.join(output_dir, c) if result.succeeded: puts("Filtering successfully completed for cruise {}".format(c)) puts("Returning results for cruise {}".format(c)) util.mkdir_p(cruise_output_dir) rsyncout = execute( # rsync files in cruise results dir to local cruise dir rsync_get, os.path.join(workdir, c) + "/", cruise_output_dir, hosts=[env.host_string] ) # Print rsync output on source host, even though this is run # on local puts(rsyncout[env.host_string]) else: sys.stderr.write("Filtering failed for cruise {}\n".format(c)) # Always write log output util.mkdir_p(cruise_output_dir) logpath = os.path.join(cruise_output_dir, "seaflowpy_filter.{}.log".format(c)) with open(logpath, "w") as logfh: logfh.write("command={}\n".format(result.command)) logfh.write("real_command={}\n".format(result.real_command)) logfh.write(result + "\n") return cruise_results
def download_and_save(self): mkdir_p(self.chan, abort=True, msg="Couldn't create directory for chan") mkdir_p("%s/%s" % (self.chan, self.boardname), abort=True, msg="Couldn't create directory for board") catalog = self.get_catalog(False) i = 0 for thread in catalog: i += 1 if not self.suppressed_output: print "Downloading thread %d (%d/%d)" % (thread, i, len(catalog)) filename = "%s/%s/%d.json" % (self.chan, self.boardname, thread) connection = httplib.HTTPSConnection(self.host) with open(filename, "w") as file: connection.request("GET", self.get_thread_json_url(thread)) file.write(connection.getresponse().read()) connection.close()
def mock_vagrant_info(instance_id_to_tag_ip): inventory_dir = '.vagrant/provisioners/ansible/inventory' mkdir_p(inventory_dir) inventory = open(os.path.join(inventory_dir, 'vagrant_ansible_inventory'), 'w') for instance_id, tag_ip in instance_id_to_tag_ip.iteritems(): tag, ip = tag_ip host = get_host(tag) inventory.write("{} ansible_ssh_host={} ansible_ssh_port=22\n".format(host, ip)) id_dir = os.path.join('.vagrant', 'machines', host, 'aws') mkdir_p(id_dir) with open(os.path.join(id_dir, 'id'), 'w') as f: f.write(instance_id) inventory.close()
def get_archive_file_name(config, kind, start_dt, end_dt, ftype='pickle'): """Get the archive file name. has the format of {ARCHIVE_DIR}/YY-mm-dd/{kind}/kind-start_dt-end_dt.pickle """ # Note that Hadoop does not like leading underscores in files, so we strip # out leading underscores (as may be used in the case of private classes) kind = re.sub(r'^_*', '', kind) datestr = str(start_dt.date()) dirname = "%s/%s/%s" % (config['archive_dir'], datestr, kind) mkdir_p(dirname) filename = "%s/%s-%s-%s-%s-%s.%s" % (dirname, kind, str(start_dt.date()), str(start_dt.time()), str(end_dt.date()), str(end_dt.time()), ftype) return filename
def link_scripts(dest_dir, src_dir=None): original_scripts_dir = src_dir or util.get_installed_scripts_dir() scripts = [ (s['name'], get_new_script_path(s['module'], s['function'])) for s in get_console_scripts_info() ] for (original, new_path) in scripts: original_path = os.path.join(original_scripts_dir, original) new_path = os.path.join(dest_dir, new_path) util.mkdir_p(os.path.dirname(new_path)) print 'Symlinking %s to %s ...' % (original_path, new_path) if os.path.lexists(new_path): os.remove(new_path) os.symlink(original_path, new_path)
def install(self): console('=> => Send a bat to catch %s' % self.CURR_SCRIPT) try: ret = self._check_name() if ret: fetch_url = self._render_fetch_url(ret) cmd_fetch = 'git clone --depth 1 %s' % fetch_url util.mkdir_p(self.VIMPYRE_PATH) with util.cd(self.VIMPYRE_PATH): system(cmd_fetch) else: msg = ('%s not found! Please use `vimpyre search <vim-script>`' ' to check the script name and install again!' % self.CURR_SCRIPT) console(msg) except: self.install_base()
def convert_and_save_dataset(pid, cursor, image_type, volume_labels, volume, glioma_grade): """convert_and_save_dataset""" util.mkdir_p(util.DATA_FOLDER + str(pid)) img_out_folder = util.DATA_FOLDER + str(pid) + "/volumes_labels/" util.mkdir_p(img_out_folder) cursor.execute('''SELECT pid from Patient where pid = ?''', (pid,)) exist = cursor.fetchone() if exist is None: cursor.execute('''INSERT INTO Patient(pid, glioma_grade) VALUES(?, ?)''', (pid, glioma_grade)) cursor.execute('''INSERT INTO Images(pid, modality, diag_pre_post) VALUES(?,?,?)''', (pid, 'MR', image_type)) img_id = cursor.lastrowid _, file_extension = os.path.splitext(volume) volume_temp = "volume" + file_extension shutil.copy(volume, volume_temp) volume_out = img_out_folder + str(pid) + "_" + str(img_id) + "_MR_T1_" + image_type + ".nii.gz" print("--->", volume_out) os.system(DWICONVERT_PATH + " --inputVolume " + volume_temp + " -o " + volume_out + " --conversionMode NrrdToFSL") volume_out_db = volume_out.replace(util.DATA_FOLDER, "") cursor.execute('''UPDATE Images SET filepath = ? WHERE id = ?''', (volume_out_db, img_id)) os.remove(volume_temp) for volume_label in volume_labels: _, file_extension = os.path.splitext(volume_label) volume_label_temp = "volume_label" + file_extension shutil.copy(volume_label, volume_label_temp) cursor.execute('''INSERT INTO Labels(image_id, description) VALUES(?,?)''', (img_id, 'all')) label_id = cursor.lastrowid volume_label_out = img_out_folder + str(pid) + "_" + str(img_id) + "_MR_T1_" + image_type\ + "_label_all.nii.gz" os.system(DWICONVERT_PATH + " --inputVolume " + volume_label_temp + " -o " + volume_label_out + " --conversionMode NrrdToFSL") volume_label_out_db = volume_label_out.replace(util.DATA_FOLDER, "") cursor.execute('''UPDATE Labels SET filepath = ? WHERE id = ?''', (volume_label_out_db, label_id)) os.remove(volume_label_temp)
def load_pack(pack_config: PackConfig): if pack_config.mods_path is None: pack_config.mods_path = os.path.join(pack_config.factorio_path, 'mods') locale = FactorioLocale() pack_dir = get_pack_dir(pack_config.name) oldcwd = os.getcwd() try: fs = FactorioState(pack_config.factorio_path, locale) for fn in os.listdir(pack_config.mods_path): fn = os.path.join(pack_config.mods_path, fn) if os.path.isdir(fn): fs.modlist.add_mod(fn) fs.modlist.resolve() fs.load_mods() locale.merge() data = fs.get_data() finally: os.chdir(oldcwd) mkdir_p(pack_dir) sort_dict(data, 'technology') with open('%s/out' % pack_dir, 'w') as f: f.write(json.dumps(data, indent=4)) fs.save_gfx('%s/icon' % pack_dir) locale.save('%s/localedump.cfg' % pack_dir) pack_info = PackInfo(pack_config.name, pack_config.title, '', '', fs.modlist._loaded_names) with open('%s/info.json' % pack_dir, 'w') as f: f.write(pack_info.to_json())
def mount_compressed(f): """Mount rar file to memory, and return the mounted directory""" if not check_support(): return None # Do nothing p = get_shm_path(f) if os.path.exists(p): # Already uncompressed return p util.mkdir_p(p) tmp = os.getcwd() try: os.chdir(p) os.system('unrar e %s' % util.escape_path(f)) except: pass os.chdir(tmp) cmd = 'echo %s >> %s' % (util.escape_path(p), os.path.join(SHM_ROOT, 'history')) os.system(cmd) return p
def download(self): """ Download from the source URL via HTTP or git """ print('downloading %s' % self.name) if self.ext == '.git': path = os.path.join(self.workdir, self.name) util.rm_rf(self.workdir) util.mkdir_p(self.workdir) os.chdir(self.workdir) subprocess.check_call(shlex.split('git clone %s %s' % (self.url, self.name))) self.unpack_name = self.name self.filename = self.name else: path = os.path.join(self.workdir, self.filename) if not os.path.exists(path): util.rm_rf(self.workdir) util.mkdir_p(self.workdir) os.chdir(self.workdir) stream = urllib2.urlopen(self.url) fd = open(self.filename, 'wb') shutil.copyfileobj(stream, fd) fd.close()
def create_aws_spot_vars(ec2_conf): init = yaml.load(open("conf/init.yml")) volumes = map(trans_to_volume, ec2_conf["Block_Device_Mapping"]) # temporary folder for ansible to save info ansible_info_dir = tempfile.mkdtemp() var = { 'zone': ec2_conf["Availability_Zone"], 'count': init["MachineNumber"], 'image': ec2_conf["AMI"], 'region': ec2_conf["Region"], 'volumes': volumes, 'key_name': ec2_conf["Keypair"], 'spot_price': ec2_conf["Spot_Price"], 'instance_type': ec2_conf["Instance_Type"], 'security_group': ec2_conf["Security_Group"], 'ansible_info_dir': ansible_info_dir, } var_dir = 'spot/roles/create_ec2/vars' mkdir_p(var_dir) out = open(os.path.join(var_dir, 'main.yml'), 'w') yaml.dump(var, out, default_flow_style=False) out.close()
def create(args): """ Create a new package. """ base = os.path.split(args.url)[1] base = base.split('.')[0] base = base.split('-')[0] base = base.split('_')[0] name = args.name or base template = """ import winbrew class %(name)s(winbrew.Formula): url = '%(url)s' homepage = '' sha1 = '' build_deps = () deps = () def install(self): pass def test(self): pass """ path = os.path.join(winbrew.formula_path, '%s.py' % name) if not os.path.exists(path): util.mkdir_p(os.path.split(path)[0]) fd = open(path, 'w') fd.write(template % {'name': name.title(), 'url': args.url}) fd.close() args.name = name edit(args)
def test_prerequisites(): '''check we have the right directories and tools to run tests''' print("Testing prerequisites") util.mkdir_p('buildlogs') return True
check_logs(step) continue results.add(step, '<span class="passed-text">PASSED</span>', time.time() - t1) print(">>>> PASSED STEP: %s at %s" % (step, time.asctime())) check_logs(step) if not passed: print("FAILED %u tests: %s" % (len(failed), failed)) util.pexpect_close_all() write_fullresults() return passed util.mkdir_p('buildlogs') lck = util.lock_file('buildlogs/autotest.lck') if lck is None: print("autotest is locked - exiting") sys.exit(0) atexit.register(util.pexpect_close_all) if len(args) > 0: # allow a wildcard list of steps matched = [] for a in args: for s in steps: if fnmatch.fnmatch(s.lower(), a.lower()): matched.append(s)
#!/usr/bin/env python2 # coding: utf-8 ''' Baixa todos os arquivos de dados de tramitação de projetos da Câmara Municipal de São Paulo. ''' import os from lxml.html import parse from urllib import urlretrieve from configura import RAW_PATH from util import mkdir_p if '__main__' == __name__: url = ('http://www.camara.sp.gov.br/index.php?option=com_content' '&view=article&id=10008:detalhes-tramitacao-projetos-dados-abertos' '&catid=119') root = parse(url).getroot() arquivos_de_dados = root.xpath( '//p[contains(., "de arquivos")]/following-sibling::p/a/@href') for arquivo in arquivos_de_dados: path = os.path.join(RAW_PATH, 'tramitacoes') mkdir_p(path) urlretrieve(arquivo, os.path.join(path, arquivo.split('/').pop()))
params["title"] = title resp = requests.get(EN_SONG_SEARCH_URL, params=params) resp.raise_for_status() data = json.loads(resp.text) return bool(data["response"]["status"]["code"] == 0 and len(data["response"]["songs"])) if __name__ == "__main__": import errno import os import sys import shutil import util util.mkdir_p("xml/existing") util.mkdir_p("xml/nonexistant") api_key = os.environ["ECHO_NEST_API_KEY"] if len(sys.argv) != 2: # No argument given, do entire xml/raw directory for dirpath, dirnames, filenames in os.walk("xml/raw"): for filename in filenames: full_path = os.path.join(dirpath, filename) print full_path, exist_path = os.path.join("xml/existing", filename) if os.path.isfile(exist_path): print " o" continue