Exemple #1
0
    def get_vienna_layout(self, data):

        temp_folder = "/tmp/"+str(uuid.uuid4())
        ensure_dir(temp_folder)
        dot_bracket_filepath = temp_folder+"/dotbracket.txt"

        f = open(dot_bracket_filepath, "w")
        f.write(data["sequence"]+"\n"+data["structure"]+"\n")
        f.close()

        # change to tmp folder
        os.chdir(temp_folder)

        # use RNAplot CLI to generate the xrna tab delimited file
        os.system("RNAplot -o xrna < "+dot_bracket_filepath)

        # get the coords out by parsing the file
        coords = []
        with open(temp_folder+"/rna.ss") as f:
            for line in f:
                line = line.strip()
                if line == "" or line[0] == "#":
                    continue

                bits = line.split()
                x = float(bits[2])
                y = float(bits[3])
                coords.append([x, y])

        os.system("rm -rf "+temp_folder)

        return coords
Exemple #2
0
	def split(self):
		from Bio import SeqIO
		from utils import ensure_dir

		# Count the sequences

		print("Counting sequences...")
		n_seqs = 0
		handle = open(self.sauce_filepath, "r")
		for record in SeqIO.parse(handle, "fasta"):
			n_seqs += 1
		handle.close()
		chunk_size = int(n_seqs / self.n_chunks)
		print("...There are ["+str(n_seqs)+"] sequences")

		# Iterate through chunks of sequences.
		# For each sequence, write a single fasta file in the right location.
		n_seqs = 0
		handle = open(self.sauce_filepath, "r")
		for record in SeqIO.parse(handle, "fasta"):
			chunk_n = int(n_seqs / chunk_size)
			chunk_dir = self.target_dirpath+"/chunk_"+str(chunk_n)+"/"+record.id
			ensure_dir(chunk_dir+"/")

			f = open(chunk_dir+"/seq.fasta", "w")
			SeqIO.write(record, f, "fasta")
			f.close()

			n_seqs += 1

			if n_seqs % 100 == 0:
				print("["+str(n_seqs)+"] fasta files written")

		handle.close()		
def simple_file_logger(name, log_file_name):
    lgr = logging.getLogger(name)

    if lgr.handlers:
        return lgr
    lgr.propagate = False

    log_dir = resolve_path(mbs_config.MBS_LOG_PATH)
    ensure_dir(log_dir)

    lgr.setLevel(logging.INFO)

    formatter = logging.Formatter("%(levelname)8s | %(asctime)s | %(message)s")

    logfile = os.path.join(log_dir, log_file_name)
    fh = TimedRotatingFileHandler(logfile, backupCount=10, when="midnight")

    fh.setFormatter(formatter)
    # add the handler to the root logger
    lgr.addHandler(fh)

    if LOG_TO_STDOUT:
        sh = logging.StreamHandler(sys.stdout)
        sh.setFormatter(formatter)
        lgr.addHandler(sh)

    return lgr
Exemple #4
0
 def update_times_idx(self):
     self.times_idx += 1
     ensure_dir(op.join(self.path, str(self.idx), str(self.times_idx)))
     ensure_dir(
         op.join(self.path, str(self.idx), str(self.times_idx),
                 self.prefix))
     self.counter = 0
Exemple #5
0
    def plot_snr_figure(self, dirname, snr_acc, snrs):
        """[基于不同SNR下的分类准确率绘制图像]

        Args:
            dirname ([str]): [存储图像的文件夹]
            snr_acc ([一维array]]): [不同SNR下的分类准确率]
            snrs ([一维array]): [不同的SNR的名称]
        """
        # Plot accuracy curve
        plt.switch_backend('agg')
        now_time = datetime.datetime.now()
        now_time = now_time.strftime("%m-%d_%H-%M")

        util.ensure_dir(dirname)
        plt.plot(snrs, snr_acc)
        plt.xlabel("Signal to Noise Ratio")
        plt.ylabel("Classification Accuracy")
        plt.title("Classification Accuracy On Different SNR")
        plt.savefig(
            os.path.join(
                dirname, 'Classification Accuracy On Different SNR' +
                now_time + '.jpg'))
        print(
            "Figure 'Classification Accuracy On Different SNR' generated successfully"
        )
Exemple #6
0
    def __init__(self,
                 conf,
                 uuid,
                 namespace=None,
                 service=None,
                 pids_path=None,
                 default_cmd_callback=None,
                 cmd_addl_env=None,
                 pid_file=None,
                 run_as_root=False):

        self.conf = conf
        self.uuid = uuid
        self.namespace = namespace
        self.default_cmd_callback = default_cmd_callback
        self.cmd_addl_env = cmd_addl_env
        self.pids_path = pids_path or self.conf.external_pids
        self.pid_file = pid_file
        self.run_as_root = run_as_root

        if service:
            self.service_pid_fname = 'pid.' + service
            self.service = service
        else:
            self.service_pid_fname = 'pid'
            self.service = 'default-service'

        utils.ensure_dir(os.path.dirname(self.get_pid_file_name()))
def submit_emmy_experiment(nt, N, tgs, is_dp, procs, ts, kernel, outfile, target_dir):
    import os
    import subprocess
    from string import Template
    from utils import ensure_dir

    job_template = Template(
        """export OMP_NUM_THREADS=10; export I_MPI_PIN_DOMAIN=socket; export KMP_AFFINITY=granularity=fine,scatter; mpirun_rrze -np $procs -npernode 2 -- $exec_path --n-tests 2 --disable-source-point --npx 1 --npy $procs --npz 1 --nx $N --ny $N --nz $N  --verbose 1 --halo-concatenate 1 --target-ts $ts --wavefront 1 --nt $nt --target-kernel $kernel --thread-group-size $tgs | tee $outfile"""
    )

    target_dir = os.path.join(os.path.abspath("."), target_dir)
    ensure_dir(target_dir)
    outpath = os.path.join(target_dir, outfile)

    if is_dp == 1:
        exec_path = os.path.join(os.path.abspath("."), "build_dp/mwd_kernel")
    else:
        exec_path = os.path.join(os.path.abspath("."), "build/mwd_kernel")

    job_cmd = job_template.substitute(
        nt=nt,
        N=N,
        tgs=tgs,
        procs=procs,
        ts=ts,
        kernel=kernel,
        outfile=outpath,
        exec_path=exec_path,
        target_dir=target_dir,
    )

    print job_cmd
    sts = subprocess.call(job_cmd, shell=True)
def submit_emmy_experiment(nwf, tgs, wf, is_dp, th, ts, tb, kernel, outfile, target_dir):
    import os
    import subprocess
    from string import Template
    from utils import ensure_dir    

    job_template=Template(
"""export OMP_NUM_THREADS=$thn; likwid-perfctr -m -s 0x03 -g MEM -C S0:0-$th $exec_path --n-tests 2 --disable-source-point --npx 1 --npy 1 --npz 1 --nx 960 --ny 960 --nz 960  --verbose 1 --target-ts $ts --wavefront $wf --nt 100 --target-kernel $kernel --t-dim $tb --thread-group-size $tgs --num-wavefronts $nwf --cache-size 0 | tee $outfile""")
#"""export OMP_NUM_THREADS=$thn; numactl -N 0 $exec_path --n-tests 2 --disable-source-point --npx 1 --npy 1 --npz 1 --nx 480 --ny 480 --nz 480  --verbose 1 --target-ts $ts --wavefront $wf --nt 100 --target-kernel $kernel --t-dim $tb --thread-group-size $tgs | tee $outfile""")


    target_dir = os.path.join(os.path.abspath("."),target_dir)
    ensure_dir(target_dir)
    outpath = os.path.join(target_dir,outfile)

    if(is_dp==1):
        exec_path = os.path.join(os.path.abspath("."),"build_dp/mwd_kernel")
    else:
        exec_path = os.path.join(os.path.abspath("."),"build/mwd_kernel")


    job_cmd = job_template.substitute(nwf=nwf, tgs=tgs, wf=wf, th=(th-1), thn=th, ts=ts, tb=tb, kernel=kernel, outfile=outpath, exec_path=exec_path, target_dir=target_dir)

    print job_cmd
    sts = subprocess.call(job_cmd, shell=True)
    def run(self):
        self.info("Starting up... ")
        self.info("PID is %s" % os.getpid())
        self.info("TEMP DIR is '%s'" % self.temp_dir)
        if self.tags:
            self.info("Tags are: %s" % document_pretty_string(self.tags))
        else:
            self.info("No tags configured")

        ensure_dir(self._temp_dir)
        self._update_pid_file()
        # Start the command server
        self._start_command_server()

        # start the backup processor
        self._backup_processor.start()

        # start the restore processor
        self._restore_processor.start()

        # start the backup processor
        self._backup_processor.join()

        # start the restore processor
        self._restore_processor.join()

        self.info("Engine completed")
        self._pre_shutdown()
Exemple #10
0
def process_audio_poly(wavdir, outdir, tol, ssm_read_pk, read_pk, n_jobs=4,
                       tonnetz=False):
    utils.ensure_dir(outdir)
    files = glob.glob(os.path.join(wavdir, "*.wav"))
    Parallel(n_jobs=n_jobs)(delayed(process_piece)(
        wav, outdir, tol, ssm_read_pk, read_pk, tonnetz)
        for wav in files)
Exemple #11
0
def copy_image_subset(source_dir, target_dir, subset_inds):
    frames_list = os.listdir(source_dir)
    frames_list.sort()
    utils.ensure_dir(target_dir, True)
    for i in subset_inds:
        if os.path.exists(os.path.join(source_dir, frames_list[i])):
            shutil.copy2(os.path.join(source_dir, frames_list[i]), os.path.join(target_dir, frames_list[i]))
    def parse_sentences(self, entries):

        os.chdir('magyarlanc')

        self.tmp_dir = '../' + self.cfg.get('data', 'tmp_dir')
        ensure_dir(self.tmp_dir)

        with NamedTemporaryFile(dir=self.tmp_dir, delete=False) as in_file:
            json.dump(entries, in_file)
            in_file_name = in_file.name

        with NamedTemporaryFile(dir=self.tmp_dir, delete=False) as out_file:
            out_file_name = out_file.name
            success = self.run_magyarlanc(in_file_name, out_file_name)

        if success:
            print 'magyarlanc ok'
        else:
            print 'magyarlanc nem ok'

#        with open(out_file_name) as out_file:
#            new_entries = json.load(out_file)

        new_entries = self.parse_output(out_file_name)

        os.chdir('..')

        return new_entries
Exemple #13
0
def recursively_extract_features(path, dest_dir, wl=20, ws=10, nf=24, nceps=19, fmin=0., fmax=4000., d_w=2, pre=0.97,
                                 mel=True):
    for f in os.listdir(path):
        f_location = os.path.join(path, f)
        if utils.is_wav_file(f_location):
            logger.debug('Extract features: ' + f_location)
            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(dest_dir, stripped_filename)
            dest_file += '.hdf5'

            mfcc = feature_extraction(f_location, wl, ws, nf, nceps, fmin, fmax, d_w, pre, mel)
            utils.ensure_dir(dest_dir)

            bob.io.save(mfcc, dest_file)
            logger.debug('savin mfcc to ' + dest_file)


        elif os.path.isdir(f_location):
            logger.debug('Extract features: ' + f_location)
            new_path = os.path.join(path, f)
            new_dest_dir = os.path.join(dest_dir, f)
            recursively_extract_features(new_path, new_dest_dir)

        else:
            logger.info('Unknown file type: ' + str(f_location))
            continue
Exemple #14
0
 def __init__(self, cfg):
     self.cfg = cfg
     self.out_fn = self.cfg.get("machine", "ext_definitions")
     ensure_dir(os.path.dirname(self.out_fn))
     dep_map_fn = cfg.get("deps", "dep_map")
     self.read_dep_map(dep_map_fn)
     self.lemmatizer = Lemmatizer(cfg)
Exemple #15
0
def extract_i_vectors(gmm_stats_path, ivec_dir, ivec_machine):
    for f in os.listdir(gmm_stats_path):
        f_location = os.path.join(gmm_stats_path, f)
        logger.debug('extract_i_vectors: ' + f_location)

        if utils.is_hdf5_file(f_location):

            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(ivec_dir, stripped_filename)
            dest_file += '.hdf5'

            utils.ensure_dir(ivec_dir)

            gmm_stats_to_ivec(f_location, dest_file, ivec_machine)

            logger.debug('saving ivec ' + dest_file)


        elif os.path.isdir(f_location):
            new_path = os.path.join(gmm_stats_path, f)
            new_dest_dir = os.path.join(ivec_dir, f)
            extract_i_vectors(new_path, new_dest_dir, ivec_machine)

        else:
            logger.info('Unknown file type: ' + str(f_location))
            continue
Exemple #16
0
def recursive_execute_gmms_on_machine(gmm_stats_path, dest_path, machine):
    for f in os.listdir(gmm_stats_path):
        gmm_stats_file = os.path.join(gmm_stats_path, f)
        logger.debug('extract_i_vectors: ' + gmm_stats_file)

        if utils.is_hdf5_file(gmm_stats_file):

            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(dest_path, stripped_filename)
            dest_file += '.hdf5'

            utils.ensure_dir(dest_path)

            # load the GMM stats file
            gmm_stats = bob.machine.GMMStats(bob.io.HDF5File(gmm_stats_file))

            # extract i-vector
            output = machine.forward(gmm_stats)

            # save them!
            bob.io.save(output, dest_file)

            print 'savin ivec ' + dest_file


        elif os.path.isdir(gmm_stats_file):
            new_path = os.path.join(gmm_stats_path, f)
            new_dest_dir = os.path.join(dest_path, f)
            extract_i_vectors(new_path, new_dest_dir, machine)

        else:
            logger.info('Unknown file type: ' + str(gmm_stats_file))
            continue
 def __init__(self, task_type_name):
     super(TaskLogFileSweeper,
           self).__init__(schedule=Schedule(frequency_in_seconds=3600))
     self._logs_dir = task_log_dir(task_type_name)
     self._archive_logs_dir = os.path.join(self._logs_dir, "ARCHIVE")
     ensure_dir(self._logs_dir)
     ensure_dir(self._archive_logs_dir)
Exemple #18
0
def compute_gmm_sufficient_statistics(ubm, src_dir, dest_dir):
    for f in os.listdir(src_dir):
        f_location = os.path.join(src_dir, f)
        logger.debug('compute_gmm_sufficient_statistics: ' + f_location)

        if utils.is_hdf5_file(f_location):

            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(dest_dir, stripped_filename)
            dest_file += '.hdf5'

            feature = bob.io.load(f_location)
            gmm_stats = bob.machine.GMMStats(ubm.dim_c, ubm.dim_d)
            ubm.acc_statistics(feature, gmm_stats)

            utils.ensure_dir(dest_dir)

            gmm_stats.save(bob.io.HDF5File(dest_file, 'w'))
            logger.debug('savin gmm_stats to ' + dest_file)


        elif os.path.isdir(f_location):
            new_path = os.path.join(src_dir, f)
            new_dest_dir = os.path.join(dest_dir, f)
            compute_gmm_sufficient_statistics(ubm, new_path, new_dest_dir)

        else:
            logger.info('Unknown file type: ' + str(f_location))
            continue
Exemple #19
0
def main(args):

    utils.ensure_dir(
        [join(args.model_path, args.id),
         join(args.pred_path, args.id)])

    print('-' * 30)
    print('Loading and preprocessing test data...')
    print('-' * 30)
    try:
        imgs_test, imgs_id_test = utils.load_test_data(args.data_path)
    except FileNotFoundError:
        utils.create_test_data(args.data_path, args.data_path)
        imgs_test, imgs_id_test = utils.load_test_data(args.data_path)

    print('-' * 30)
    print('Loading saved model...')
    print('-' * 30)
    net = model_build.load_model(join(args.model_path, args.id))

    output_shape = net.layers[0].output_shape[1:-1]
    if len(output_shape) == 2:
        imgs_test = utils.preprocess_x(imgs_test, new_shape=output_shape)
    else:
        imgs_test = utils.preprocess_x(imgs_test, new_shape=(96, 96))
    print('-' * 30)
    print('Predicting masks on test data...')
    print('-' * 30)
    imgs_mask_test, presence_test = net.predict(imgs_test, verbose=1)
    print(imgs_mask_test.shape, presence_test.shape)
    np.save(join(args.pred_path, args.id, 'imgs_mask_test.npy'),
            imgs_mask_test)
    np.save(join(args.pred_path, args.id, 'imgs_presence_test.npy'),
            presence_test)
    """print('-' * 30)
Exemple #20
0
    def compileJsFiles(self, rootpath):
        print "**Begin JS minification**"
        minDir = os.path.join(rootpath, 'arches', 'Media', 'js', 'min')

        print "minDir: " + minDir
        mf = os.path.join(minDir, '1231_11_Arches.min.js')
        utils.ensure_dir(mf)
        minfile = open(mf,'w')

        print "Minifying...."
        buildfiles = JsFiles(debug=False)
        mediapath = os.path.join(rootpath,'arches','Media')        
        mindata = self.getDataFromFiles(buildfiles, mediapath)
        mintext = jsmin(mindata.getvalue())

        print "Minification complete. Writing .js file"
        minfile.write(mintext)
        minfile.close()

        print "Writing .gzip file"
        gzipfile = gzip.open(os.path.join(minDir, '1231_11_Arches.min.js.gz'), 'wb')
        gzipfile.writelines(mintext)
        gzipfile.close()
        
        print "**End JS minification**"
Exemple #21
0
def get_logger():
    global logger, _logging_level

    if logger:
        return logger

    logger = logging.getLogger("MongoctlLogger")

    log_file_name="mongoctl.log"
    conf_dir = mongoctl_globals.DEFAULT_CONF_ROOT
    log_dir = utils.resolve_path(os.path.join(conf_dir, LOG_DIR))
    utils.ensure_dir(log_dir)


    logger.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(levelname)8s | %(asctime)s | %(message)s")
    logfile = os.path.join(log_dir, log_file_name)
    fh = TimedRotatingFileHandler(logfile, backupCount=50, when="midnight")

    fh.setFormatter(formatter)
    fh.setLevel(logging.DEBUG)
    # add the handler to the root logger
    logging.getLogger().addHandler(fh)

    global _log_to_stdout
    if _log_to_stdout:
        sh = logging.StreamHandler(sys.stdout)
        std_formatter = logging.Formatter("%(message)s")
        sh.setFormatter(std_formatter)
        sh.setLevel(_logging_level)
        logging.getLogger().addHandler(sh)

    return logger
Exemple #22
0
def database_ish():
	ensure_dir(os.getcwd()  + "/build/")
	initialise()
	image_store = create_engine(direc)
	Session = sessionmaker(bind=image_store)
	session = Session()
	return session
def main():
    app_settings = settings.get_app_settings()
    api = TwitterAPI.TwitterAPI(** app_settings)

    store_data = store.get_data()

    search_term = 'vnnlp'
    query = {'screen_name': search_term}

    filename = 'user_timeline/{}.yaml'.format(search_term)

    utils.ensure_dir(filename)

    if 'user_timeline' in store_data and 'max_id' in store_data['user_timeline']:
        query['max_id'] = store_data['user_timeline']['max_id'] - 1

    max_id = None
    try:
        with open(filename, 'a') as output_file:
            r = TwitterAPI.TwitterPager(api, 'statuses/user_timeline', query)

            for tweet in r.get_iterator():
                yaml.dump([tweet], output_file, default_flow_style=False)
                if 'id' in tweet:
                    max_id = tweet['id']

    except KeyboardInterrupt:
        pass

    if not 'user_timeline' in store_data:
        store_data['user_timeline'] = {}

    store_data['user_timeline']['max_id'] = max_id
    store.store_data(store_data)
Exemple #24
0
    def _diff_w_ct_formula_name(self, name1, name2):
        utils.ensure_dir(FORMULAS_DIR)

        filename = '{}-{}-diff-w_ct-{}.dimacs'.format(self.enc_strategy[2:],
                                                      name1, name2)
        fname = os.path.join(FORMULAS_DIR, filename)

        return fname
Exemple #25
0
 def print_edges(self, edge_fn):
     ensure_dir(os.path.dirname(edge_fn))
     logging.info("printing edges to {0}".format(edge_fn))
     with open(edge_fn, 'w') as f:
         for c, i in enumerate(self.coocc[0]):
             j = self.coocc[1][c]
             k = self.coocc[2][c]
             f.write("{0}\t{1}\t{2}\n".format(i, j, k))
Exemple #26
0
    def _trojan_formula_name(self, name, label):
        utils.ensure_dir(FORMULAS_DIR)

        filename = '{}-{}-trojan-label_{}.dimacs'.format(
            self.enc_strategy[2:], name, label)
        fname = os.path.join(FORMULAS_DIR, filename)

        return fname
Exemple #27
0
    def _robust_formula_name(self, name, perturb_k, perturb_num):
        utils.ensure_dir(FORMULAS_DIR)

        filename = '{}-{}-robustness-perturb_{}-id_{}.dimacs'.format(
            self.enc_strategy[2:], name, perturb_k, perturb_num)
        fname = os.path.join(FORMULAS_DIR, filename)

        return fname
Exemple #28
0
def save_mapped_text(pages_text, path_file):
	dir_save = path_file.replace('.pdf', '/')
	print(dir_save)
	ensure_dir(dir_save)
	for idx, page in enumerate(pages_text):
		page_file = open(dir_save + 'page_' + str(idx) + '.txt', mode="w", encoding="utf-8")
		page_file.write(page)
		page_file.close()
Exemple #29
0
 def __init__(self, conf, network, process_monitor, version=None,
              plugin=None):
     super(DhcpLocalProcess, self).__init__(conf, network, process_monitor,
                                            version, plugin)
     self.confs_dir = self.get_confs_dir(conf)
     self.network_conf_dir = os.path.join(self.confs_dir, network.id)
     utils.ensure_dir(self.network_conf_dir)
     LOG.debug("__init__ DhcpLocalProcess ok")
Exemple #30
0
 def __init__(self):
     self._logs_dir = 'logs'
     ensure_dir(self._logs_dir)
     self.c = Config()
     self._run_name = Model._get_run_name()
     self.model = None  # TODO: self._model?
     self._datasets = {}
     self._data_generators = {}
Exemple #31
0
 def __init__(self, cfg):
     self.cfg = cfg
     self.text_to_4lang = TextTo4lang(cfg)
     self.graph_dir = self.cfg.get("qa", "graph_dir")
     self.dep_dir = self.cfg.get("qa", "deps_dir")
     ensure_dir(self.graph_dir)
     ensure_dir(self.dep_dir)
     self.word_similarity = WordSimilarity(cfg)
Exemple #32
0
def batch_raw2jpg(in_dir=RAW_DIR, out_dir=IMG_DIR):
    ensure_dir(out_dir)

    for in_f, out_f in files(in_dir=in_dir,
                             out_dir=out_dir,
                             in_ext=".nef",
                             out_ext=".jpg"):
        raw2jpg(in_f, out_f)
Exemple #33
0
    def setidx(self, idx):
        self.idx = idx
        ensure_dir(op.join(self.path, str(self.idx)))

        names = os.listdir(op.join(self.path, str(self.idx)))
        names = list(filter(lambda x: x.isdigit(), names))
        names = list(map(int, names))
        self.times_idx = (max(names)) if len(names) > 0 else 0
Exemple #34
0
def exe(input_file):
    clean()
    input(input_file)
    strat_from_cars()
    out = 'data/from_cars/'
    utils.ensure_dir(out)
    save_cars(out+input_file+'_out')
    print('done')
Exemple #35
0
 def __init__(self, cfg):
     self.cfg = cfg
     self.text_to_4lang = TextTo4lang(cfg)
     self.graph_dir = self.cfg.get("qa", "graph_dir")
     self.dep_dir = self.cfg.get("qa", "deps_dir")
     ensure_dir(self.graph_dir)
     ensure_dir(self.dep_dir)
     self.word_similarity = WordSimilarity(cfg)
Exemple #36
0
def compute_all_features(audio_file, audio_beats=False):
    """Computes all the features for a specific audio file and its respective
        human annotations.

    Returns
    -------
    features : dict
        Dictionary with the following features:
            mfcc : np.array
                Mel Frequency Cepstral Coefficients representation
            hpcp : np.array
                Harmonic Pitch Class Profiles
            tonnets : np.array
                Tonal Centroids (or Tonnetz)
    """

    # Makes sure the output features folder exists
    utils.ensure_dir(OUTPUT_FEATURES)
    features_file = os.path.join(OUTPUT_FEATURES,
                                 os.path.basename(audio_file) + ".json")

    # If already precomputed, read and return
    if os.path.exists(features_file):
        with open(features_file, "r") as f:
            features = json.load(f)
        return list_to_array(features)

    # Load Audio
    logging.info("Loading audio file %s" % os.path.basename(audio_file))
    audio = ES.MonoLoader(filename=audio_file, sampleRate=SAMPLE_RATE)()
    duration = len(audio) / float(SAMPLE_RATE)

    # Estimate Beats
    features = {}
    ticks, conf = compute_beats(audio)
    ticks = np.concatenate(([0], ticks, [duration]))  # Add first and last time
    ticks = essentia.array(np.unique(ticks))
    features["beats"] = ticks.tolist()

    # Compute Beat-sync features
    features["mfcc"], features["hpcp"], features["tonnetz"] = \
        compute_beatsync_features(ticks, audio)

    # Save output as audio file
    if audio_beats:
        logging.info("Saving Beats as an audio file")
        marker = ES.AudioOnsetsMarker(onsets=ticks, type='beep',
                                      sampleRate=SAMPLE_RATE)
        marked_audio = marker(audio)
        ES.MonoWriter(filename='beats.wav',
                      sampleRate=SAMPLE_RATE)(marked_audio)

    # Save features
    with open(features_file, "w") as f:
        json.dump(features, f)

    return list_to_array(features)
Exemple #37
0
 def _write_egg_info_arcname(self, name, dest):
     ensure_dir(dest)
     source = self.z.open(name)
     try:
         with file(dest, "wb") as target:
             shutil.copyfileobj(source, target)
             self.files.append(dest)
     finally:
         source.close()
Exemple #38
0
 def test_load_save_args(self):
     parser = argparse.ArgumentParser()
     args = parser.parse_args(args=[])
     args.__dict__ = {"name": "test", "foo": "bar"}
     path = os.path.join(TMP, "args")
     ensure_dir(path)
     save_args(args, path)
     args_loaded = load_args(path)
     self.assertEqual(args, args_loaded)
Exemple #39
0
    def z_travel(self, z3d, v_pts, limit=3.0, int_step_size=0.66, save_sample_to=None, save_start_id=0):
        """
        Traverse latent space to visualise the learnt reps.

        :param z3d:  [B, K, D]
        :param v_pts:  [B, L, dView]
        :param limit:  numerical bounds for traverse
        :param int_step_size:  traverse step size (interpolation gap between traverse points)
        :param save_dir:  save the output to this dir
        :param start_id:  save the output as file
        """
        from torchvision.utils import save_image
        save_dir = os.path.join(save_sample_to, 'disen_3d')
        utils.ensure_dir(save_dir)
        B, K, D = z3d.size()
        V = v_pts.size(1)

        v_feat = self.view_encoder(v_pts.reshape(B * V, -1))  # output [B*V, 8]
        v_feat = v_feat.reshape(B, V, -1).unsqueeze(1).repeat(1, K, 1, 1)

        H, W = tuple(self.config.image_size)
        interpolation = torch.arange(-limit, limit + 0.1, int_step_size)

        gifs = []
        # ------------ Select intereted object and informtive latent dimensions here ------------
        k = 2  # we select only one object out of K for analysis
        # SPECIFY_DIMENSIONS=[9, 31]
        # D = len(SPECIFY_DIMENSIONS)
        # ---------------------------------------------------------------------------------------
        for d in range(D):
            for int_val in interpolation:
                z = z3d.clone()  # [B, K, D]
                z[:, k, d] += int_val

                for vq in range(V):
                    yq = v_feat[:, :, vq, :]
                    z_yq = self.projector(torch.cat((z.reshape(B * K, -1), yq.reshape(B * K, -1)), dim=-1))
                    mask_logits, mu_x = self.decode(z_yq)
                    gifs.append(torch.sum(torch.softmax(mask_logits, dim=1) * mu_x, dim=1).data)
        gifs = torch.cat(gifs, dim=0)
        gifs = gifs.reshape(D, len(interpolation), V, B, 3, H, W).permute([3, 0, 1, 2, 4, 5, 6])

        for b in range(B):
            save_batch_dir = os.path.join(save_dir, str(save_start_id + b))
            utils.ensure_dir(save_batch_dir)
            b_gifs = gifs[b, ...]
            b_gifs = torch.cat(b_gifs.chunk(V, dim=2), dim=0).squeeze(2)

            for iid in range(len(interpolation)):
                key = 'frame'
                vis.torch_save_image_enhanced(tensor=b_gifs[:, iid, ...].cpu(),
                                              filename=os.path.join(save_batch_dir, '{}_{:02d}.jpg'.format(key, iid)),
                                              nrow=D, pad_value=1, enhance=True)
            vis.grid2gif(str(os.path.join(save_batch_dir, key + '*.jpg')),
                         str(os.path.join(save_batch_dir, 'disten3d.gif')), delay=20)

            print(" -- traversed latent space for {} scene samples".format(b + 1))
    def __init__(self, model, loss, metrics, optimizer, resume, config):
        """
        :param model:
        :param loss:
        :param metrics: a function list.
        :param optimizer:
        :param resume: bool, if resume from checkpoints.
        :param config:
        """
        self.config = config
        self.logger = logging.getLogger('MC')

        # setup GPU device if available, move model into configured device
        self.device, device_ids = self._prepare_device(config['n_gpu'])
        self.model = model.to(self.device)
        # data parrallel
        if len(device_ids) > 1:
            self.model = torch.nn.DataParallel(model, device_ids=device_ids)

        self.loss = loss
        self.metrics = metrics  # function list
        self.optimizer = optimizer

        self.epochs = config['trainer']['epochs']
        self.save_freq = config['trainer']['save_freq']
        self.verbosity = config['trainer']['verbosity']  # 每隔多少epoch打印一次日志

        # configuration to monitor model performance and save best
        self.monitor = config['trainer'][
            'monitor']  # monitor which configured metric
        self.monitor_mode = config['trainer']['monitor_mode']
        assert self.monitor_mode in ['min', 'max', 'off']
        self.monitor_best = math.inf if self.monitor_mode == 'min' else -math.inf
        self.start_epoch = 1
        self.log_step = config['trainer']['log_step']

        # setup directory for checkpoint saving
        # start_time = datetime.datetime.now().strftime('%m%d_%H%M%S')
        self.checkpoint_dir = os.path.join(config['trainer']['save_dir'],
                                           config['arch']['type'],
                                           config["name"])

        # setup visualization writer instance
        # writer_dir = os.path.join(config['visualization']['log_dir'], config['arch']['type'])
        writer_dir = os.path.join(self.checkpoint_dir,
                                  config['visualization']['log_dir'])
        self.writer = SummaryWriter(log_dir=writer_dir)

        # Save configuration file into checkpoint directory:
        ensure_dir(self.checkpoint_dir)
        config_save_path = os.path.join(self.checkpoint_dir, 'config.json')
        with open(config_save_path, 'w') as f:
            json.dump(config, f, indent=4, sort_keys=False)

        # if resume from checkpoint
        if resume:
            self._resume_checkpoint(resume)
Exemple #41
0
 def get_file(self, filename):
     pkg = utils.Package.from_filename(filename)
     if pkg.filename in self.packages:
         return self.packages[pkg.filename]
     directory = os.path.join('packages', pkg.name.lower())
     utils.ensure_dir(directory)
     filepath = os.path.join(directory, pkg.filename)
     self.packages[pkg.filename] = utils.PackageReadWriter(filepath)
     return  self.packages[pkg.filename]
Exemple #42
0
def main(infile, outdir, config):
    outdir = os.path.abspath(outdir)
    ensure_dir(outdir, False)

    cparser = SafeConfigParser()
    cparser.read(config)
    labelfile = cparser.get('configs', 'labelfile')

    infile = _check_labelling(infile, labelfile)
    infile = _verify_valid_distance(infile)
    infile = _filterPredictionsByClass_reformat2gff(infile, outdir)

    bname = os.path.basename(infile)
    pdf_rplots = os.path.join(outdir, bname + '.plots.pdf')

    dat = _read_dat(infile)

    if all(dat['correlation'] == ''):
        ignoreCorr = True
    else:
        ignoreCorr = False

    dat_mirna = _item_findClosestPartner(dat, 'mirna', ignoreCorr)
    dat_tss = _item_findClosestPartner(dat, 'tss', ignoreCorr)

    print '## Generating plot file...'
    grdevices = importr('grDevices')
    grdevices.pdf(file=pdf_rplots)

    _plt_pier(dat, 'All predicted TSS-miRNA pairs', True)
    _plt_distr(dat, 'distance', 'All predicted tss-miRNA pairs', False)
    _plt_distr(dat, 'distance', 'All predicted tss-miRNA pairs')
    if not ignoreCorr:
        _plt_distr(dat, 'correlation', 'All predicted tss-miRNA pairs', False)
        _plt_distr(dat, 'correlation', 'All predicted tss-miRNA pairs')

    _plt_percountr(dat)

    _plt_pier(dat_tss, 'TSS (label from closest miRNA)')
    _plt_distr(dat_tss, 'distance', 'TSS to closest miRNA', False)
    _plt_distr(dat_tss, 'distance', 'TSS to closest miRNA')
    if not ignoreCorr:
        _plt_distr(dat_tss, 'correlation', 'TSS to closest miRNA', False)
        _plt_distr(dat_tss, 'correlation', 'TSS to closest miRNA')

    _plt_pier(dat_mirna, 'miRNA')
    _plt_distr(dat_mirna, 'distance', 'miRNA to closest TSS', False)
    _plt_distr(dat_mirna, 'distance', 'miRNA to closest TSS')
    if not ignoreCorr:
        _plt_distr(dat_mirna, 'correlation', 'miRNA to closest TSS', False)
        _plt_distr(dat_mirna, 'correlation', 'miRNA to closest TSS')

    grdevices.dev_off()
    print '## Plot file:'
    print pdf_rplots
    return pdf_rplots
Exemple #43
0
def generate_datasets(inputs_files, outputs_files, masks_files, config_path):
    cf = imp.load_source('config', config_path)
    num_directions = cf.num_directions
    num_unique_vels = len(cf.unique_vel_idxs)
    train_data = []
    test_data = []
    for idx, maps in enumerate([cf.training_maps, cf.test_maps]):
        inputs = np.empty((0, cf.nn_input_size, cf.nn_input_size))
        if cf.conditional_prob:
            outputs = np.empty((0, cf.nn_output_size, cf.nn_output_size,
                                num_directions, num_directions))
            masks = np.empty((0, cf.nn_output_size, cf.nn_output_size,
                              num_directions, num_directions))
        else:
            outputs = np.empty(
                (0, cf.nn_output_size, cf.nn_output_size, num_unique_vels))
            masks = np.empty(
                (0, cf.nn_output_size, cf.nn_output_size, num_unique_vels))

        for map in maps:
            map_io_folder = cf.data_folder + '/network_io/' + map + '/' + cf.algo_str
            for data in ['inputs', 'outputs', 'masks']:
                f_name = map_io_folder + '/' + data + '.npy'
                if data == 'inputs':
                    #print("inputs shape:{}".format(str(np.array(inputs).shape)))
                    #print("file shape:{}".format(str(np.load(f_name).shape)))
                    inputs = np.concatenate([inputs, np.load(f_name)])
                elif data == 'outputs':
                    outputs = np.concatenate([outputs, np.load(f_name)])
                else:
                    # print("masks shape:{}".format(str(np.load(f_name).shape)))
                    masks = np.concatenate([masks, np.load(f_name)])
        if idx == 0:
            train_data.append(inputs)
            train_data.append(outputs)
            train_data.append(masks)
        else:
            test_data.append(inputs)
            test_data.append(outputs)
            test_data.append(masks)

    # split train_data into train/val data
    num_instances = train_data[0].shape[0]
    print("found {} instances.".format(num_instances))
    idxs = np.random.choice(num_instances,
                            int(0.15 * num_instances),
                            replace=False)
    left_idxs = [idx for idx in range(num_instances) if idx not in idxs]
    val_data = [data[idxs] for data in train_data]
    train_data_ = [data[left_idxs] for data in train_data]

    data_ = list(zip(train_data_, val_data, test_data))
    for i, filenames in enumerate([inputs_files, outputs_files, masks_files]):
        for j, filename in enumerate(filenames):
            ensure_dir(filename)
            np.save(filename, data_[i][j])
Exemple #44
0
 def enable(self):
     """Enables DHCP for this network by spawning a local process."""
     if self.active:
         self.restart()
     elif self._enable_dhcp():
         LOG.debug("network_conf_dir:%s", self.network_conf_dir)
         utils.ensure_dir(self.network_conf_dir)
         interface_name = self.device_manager.setup(self.network)
         self.interface_name = interface_name
         self.spawn_process()
Exemple #45
0
    def get_jython_paths(self):
        self.jython_path = self.cfg.get('stanford', 'jython')
        if not os.path.exists(self.jython_path):
            raise Exception("cannot find jython executable!")

        self.jython_module = os.path.join(
            os.path.dirname(__file__), "stanford_parser.py")

        self.tmp_dir = self.cfg.get('data', 'tmp_dir')
        ensure_dir(self.tmp_dir)
def fetch_one(api, resource, search_term, query):
    filename = '{0}/{1}.yaml'.format(resource.replace('/', '_'), search_term)

    utils.ensure_dir(filename)

    r = api.request(resource, query)
    items = [item for item in r]

    with open(filename, 'w') as output_file:
        yaml.dump(items, output_file, default_flow_style=False)
    def create_task_workspace(self, task):
        """

        """
        # ensure task workspace
        try:
            workspace_dir = self.get_task_workspace_dir(task)
            ensure_dir(workspace_dir)
        except Exception, e:
            raise errors.WorkspaceCreationError("Failed to create workspace: %s" % e)
    def get_jython_paths(self):
        self.jython_path = self.cfg.get('stanford', 'jython')
        if not os.path.exists(self.jython_path):
            raise Exception("cannot find jython executable!")

        self.jython_module = os.path.join(
            os.path.dirname(__file__), "stanford_parser.py")

        self.tmp_dir = self.cfg.get('data', 'tmp_dir')
        ensure_dir(self.tmp_dir)
def fetch_one(api, resource, search_term, query):
    filename = '{0}/{1}.yaml'.format(resource.replace('/', '_'), search_term)

    utils.ensure_dir(filename)

    r = api.request(resource, query)
    items = [item for item in r]

    with open(filename, 'w') as output_file:
        yaml.dump(items, output_file, default_flow_style=False)
Exemple #50
0
    def _fair_formula_name(self, name, constraints_fname):
        utils.ensure_dir(FORMULAS_DIR)

        constraints_fname = os.path.splitext(
            os.path.basename(constraints_fname))[0]
        filename = '{}-{}-{}-fair.dimacs'.format(self.enc_strategy[2:], name,
                                                 constraints_fname)
        fname = os.path.join(FORMULAS_DIR, filename)

        return fname
Exemple #51
0
    def __init__(self, args):
        self.args = args
        self.max_change = args.max_change
        self.resize = (int(args.resize.split(',')[0]),
                       int(args.resize.split(',')[1]))
        self.model = bnn.BNNModel.factory('%s_trojan' % args.arch, self.resize,
                                          args.num_classes)
        self.load_exist_model()

        utils.ensure_dir(definitions.ADV_TRAIN_DIR)
        self.trojan_gradient_path = os.path.join(
            definitions.ADV_TRAIN_DIR,
            '%s_%s_prefc1_grad' % (args.dataset, args.arch))
        definitions.TROJAN_PREFC1_PATH = self.trojan_gradient_path

        name = self.model.name

        # filename should be self-explanatory
        filename = '%s-' % args.dataset + str(
            self.resize[0] * self.resize[1]) + '-' + name
        self.filename = filename

        # the trained model is saved in the models directory
        # trained_models_dir = os.path.join(definitions.TRAINED_MODELS_DIR, 'adv_train_%s' % args.dataset)
        # utils.ensure_dir(trained_models_dir)
        # self.saved_model = os.path.join(trained_models_dir, filename + '.pt')
        # # the parameters are saved in the models directory
        # self.model_dir = os.path.join(trained_models_dir, filename + '.params')
        # utils.ensure_dir(self.model_dir)

        # trained_models_cp_dir = os.path.join(definitions.TRAINED_MODELS_CP_DIR, 'adv_train_%s' % args.dataset)
        # utils.ensure_dir(trained_models_cp_dir)
        # self.saved_checkpoint_model = os.path.join(trained_models_cp_dir,
        #                                            filename + '.pt')

        self.name = self.model.name
        kwargs = {
            'num_workers': 1,
            'pin_memory': True
        } if 'cuda' in args else {}

        if 'cuda' in args:
            # self.optimizer = optim.Adam(self.model.parameters(), lr=args.lr)

            if 'mnist' == args.dataset[:5]:
                self.train_loader, self.test_loader = bnn_dataset.create_mnist_loaders(
                    self.resize, args.batch_size, args.test_batch_size, kwargs)
            else:
                self.train_loader, self.test_loader = bnn_dataset.create_data_loaders(
                    args.data_folder, args.batch_size, args.test_batch_size,
                    kwargs)
        self.criterion = nn.CrossEntropyLoss()
        if 'cuda' in args:
            if args.cuda:
                self.model.cuda()
Exemple #52
0
    def __init__(self, network_creator, batch_env, args):
        logging.debug('PAAC init is started')
        self.args = copy.copy(vars(args))
        self.checkpoint_dir = join_path(self.args['debugging_folder'],
                                        self.CHECKPOINT_SUBDIR)
        ensure_dir(self.checkpoint_dir)

        checkpoint = self._load_latest_checkpoint(self.checkpoint_dir)
        self.last_saving_step = checkpoint['last_step'] if checkpoint else 0

        self.final_rewards = []
        self.global_step = self.last_saving_step
        self.network = network_creator()
        self.batch_env = batch_env
        self.optimizer = optim.RMSprop(
            self.network.parameters(),
            lr=self.args['initial_lr'],
            eps=self.args['e'],
        )  #RMSprop defualts: momentum=0., centered=False, weight_decay=0

        if checkpoint:
            logging.info('Restoring agent variables from previous run')
            self.network.load_state_dict(checkpoint['network_state_dict'])
            self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

        self.lr_scheduler = LinearAnnealingLR(self.optimizer,
                                              self.args['lr_annealing_steps'])
        #pytorch documentation says:
        #In most cases it’s better to use CUDA_VISIBLE_DEVICES environmental variable
        #Therefore to specify a particular gpu one should use CUDA_VISIBLE_DEVICES.
        self.use_cuda = self.args['device'] == 'gpu'
        self.use_rnn = hasattr(
            self.network, 'get_initial_state'
        )  #get_initial_state should return state of the rnn layers
        self._tensors = torch.cuda if self.use_cuda else torch

        self.action_codes = np.eye(
            batch_env.num_actions)  #envs reveive actions in one-hot encoding!
        self.gamma = self.args['gamma']  # future rewards discount factor
        self.entropy_coef = self.args['entropy_regularisation_strength']
        self.loss_scaling = self.args['loss_scaling']  #5.
        self.critic_coef = self.args['critic_coef']  #0.25
        self.eval_func = None

        if self.args['clip_norm_type'] == 'global':
            self.clip_gradients = nn.utils.clip_grad_norm_
        elif self.args['clip_norm_type'] == 'local':
            self.clip_gradients = utils.clip_local_grad_norm
        elif self.args['clip_norm_type'] == 'ignore':
            self.clip_gradients = lambda params, _: utils.global_grad_norm(
                params)
        else:
            raise ValueError('Norm type({}) is not recoginized'.format(
                self.args['clip_norm_type']))
        logging.debug('Paac init is done')
Exemple #53
0
def download_url(url, local_path, max_retries=3):
    """
    Download a remote URL to the location local_path with retries.
    
    On download, the file size is first obtained and stored.  When the download completes,
    the file size is compared to the stored file.  This prevents broken downloads from
    contaminating the processing chain.

    :param url: the remote URL
    :param local_path: the path to the local file
    :param max_retries: how many times we may retry to download the file
    """

    logging.info('download_url %s as %s' % (url, local_path))

    r = requests.get(url, stream=True)
    content_size = int(r.headers['Content-Length'])

    # dump the correct file size to an info file next to the grib file
    # when re-using the GRIB2 file, we check its file size against this record
    # to avoid using partial files
    info_path = local_path + '.size'
    open(ensure_dir(info_path), 'w').write(str(content_size))

    # stream the download to file
    with open(ensure_dir(local_path), 'wb') as f:
        for chunk in r.iter_content(1024 * 1024):
            f.write(chunk)

    # does the server accept byte range queries? e.g. the NOMADs server does
    accepts_ranges = 'bytes' in r.headers.get('Accept-Ranges', '')

    retries_available = max_retries
    file_size = osp.getsize(local_path)
    while file_size < content_size:
        if retries_available > 0:
            logging.info('download_url trying again, retries available %d' %
                         retries_available)
            if accepts_ranges:
                # if range queries are supported, try to download only the missing portion of the file
                headers = {'Range': 'bytes=%d-%d' % (file_size, content_size)}
                r = requests.get(url, headers=headers, stream=True)
                with open(local_path, 'ab') as f:
                    for chunk in r.iter_content(1024 * 1024):
                        f.write(chunk)
                retries_available -= 1
                file_size = osp.getsize(local_path)
            else:
                # call the entire function recursively, this will attempt to redownload the entire file
                # and overwrite previously downloaded data
                self.download_grib(url, local_path, max_retries - 1)
        else:
            os.remove(local_path)
            os.remove(info_path)
            raise DownloadError('failed to download file %s' % url)
Exemple #54
0
def download_url(url, local_path, max_retries=3):
    """
    Download a remote URL to the location local_path with retries.
    
    On download, the file size is first obtained and stored.  When the download completes,
    the file size is compared to the stored file.  This prevents broken downloads from
    contaminating the processing chain.

    :param url: the remote URL
    :param local_path: the path to the local file
    :param max_retries: how many times we may retry to download the file
    """

    logging.info('download_url %s as %s' % (url,local_path))

    r = requests.get(url, stream=True)
    content_size = int(r.headers['Content-Length'])

    # dump the correct file size to an info file next to the grib file
    # when re-using the GRIB2 file, we check its file size against this record
    # to avoid using partial files
    info_path = local_path + '.size'
    open(ensure_dir(info_path), 'w').write(str(content_size))

    # stream the download to file
    with open(ensure_dir(local_path), 'wb') as f:
        for chunk in r.iter_content(1024 * 1024):
            f.write(chunk)
    
    # does the server accept byte range queries? e.g. the NOMADs server does
    accepts_ranges = 'bytes' in r.headers.get('Accept-Ranges', '')
    
    retries_available = max_retries
    file_size = osp.getsize(local_path)
    while file_size < content_size:
        if retries_available > 0:
            logging.info('download_url trying again, retries available %d' % retries_available)
            if accepts_ranges:
                # if range queries are supported, try to download only the missing portion of the file
                headers = { 'Range' : 'bytes=%d-%d' % (file_size, content_size) }
                r = requests.get(url, headers=headers, stream=True)
                with open(local_path, 'ab') as f:
                    for chunk in r.iter_content(1024 * 1024):
                        f.write(chunk)
                retries_available -= 1
                file_size = osp.getsize(local_path)
            else:
                # call the entire function recursively, this will attempt to redownload the entire file
                # and overwrite previously downloaded data
                self.download_grib(url, local_path, max_retries-1)
        else:
            os.remove(local_path)
            os.remove(info_path)
            raise DownloadError('failed to download file %s' % url)
Exemple #55
0
 def convert_to_json(self, input_dir, output_file):
     self._check_format(Format.JSON)
     ensure_dir(os.path.dirname(output_file))
     records = []
     for item in self.iter_from_dir(input_dir):
         record = deepcopy(item['input'])
         for name, value in item['output'].items():
             record[name] = self._prettify(value)
         records.append(record)
     with io.open(output_file, mode='w') as fout:
         json.dump(records, fout, indent=2)
def train(datasets):
    train, train_lbl = datasets['train'], datasets['train_lbl']
    valid, valid_lbl = datasets['valid'], datasets['valid_lbl']

    size = train[0].shape[0]
    nlabels = train_lbl.shape[1]

    # placeholders for input, None means batch of any size
    X = tf.placeholder(tf.float32, shape=(None, size, size, 1), name="X")
    y_ = tf.placeholder(tf.float32, shape=(None, nlabels), name="y_")
    keep = tf.placeholder(tf.float32)

    logits = build_model(X, nlabels, keep)
    # predict from logits using softmax
    predictions = tf.nn.softmax(logits)
    # cross-entropy as the loss
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_))
    # create the optimizer to minimize the loss
    optimizer = tf.train.AdamOptimizer(0.005).minimize(loss)

    utils.ensure_dir(MODEL_DIR)
    saver = tf.train.Saver()
    with tf.Session() as session:
        correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y_, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

        session.run(tf.initialize_all_variables())

        nbatches = train_lbl.shape[0] / BATCH_SIZE
        for epoch in range(EPOCHS):
            for step in range(nbatches):
                batch_data = train[step*BATCH_SIZE:(step+1)*BATCH_SIZE, :, :, :]
                batch_labels = train_lbl[step*BATCH_SIZE:(step+1)*BATCH_SIZE, :]

                feed_dict = {X: batch_data, y_: batch_labels, keep: 0.5}
                optimizer.run(feed_dict=feed_dict)

                #print 'Batch labels:\n', batch_labels
                #print 'Predictions:\n', preds.eval(feed_dict=feed_dict)
                #print 'Correct pred:\n', correct_prediction.eval(feed_dict=feed_dict)

                if step % int(nbatches / 4) == 0:
                    train_accuracy = accuracy.eval(
                        feed_dict={X: batch_data, y_: batch_labels, keep: 1.0})
                    print("epoch %d, step %d, training accuracy %g" % (epoch+1, step, train_accuracy))

                    valid_accuracy = accuracy.eval(
                        feed_dict={X: valid, y_: valid_lbl, keep: 1.0})
                    print("epoch %d, step %d, valid accuracy %g" % (epoch+1, step, valid_accuracy))

                    # save the model
                    saver.save(session,
                       os.path.join(MODEL_DIR, 'cnn{}_e{}_s{}.tf'.format(valid[0].shape[0], epoch+1, step)))
Exemple #57
0
def postprocess_cycle(cycle, region_cfg, wksp_path):
    """
    Build rasters from the computed fuel moisture.

    :param cycle: the UTC cycle time
    :param region_cfg: the region configuration
    :param wksp_path: the workspace path
    :return: the postprocessing path
    """
    data_path = compute_model_path(cycle, region_cfg.code, wksp_path)
    year_month = '%04d%02d' % (cycle.year, cycle.month)
    cycle_dir = 'fmda-%s-%04d%02d%02d-%02d' %  (region_cfg.code, cycle.year, cycle.month, cycle.day, cycle.hour)
    postproc_path = osp.join(wksp_path, year_month, cycle_dir)

    # open and read in the fuel moisture values
    d = netCDF4.Dataset(data_path)
    fmc_gc = d.variables['FMC_GC'][:,:,:]
    d.close()

    # read in the longitudes and latitudes
    geo_path = osp.join(wksp_path, '%s-geo.nc' % region_cfg.code)
    d = netCDF4.Dataset(geo_path)
    lats = d.variables['XLAT'][:,:]
    lons = d.variables['XLONG'][:,:]
    d.close()

    fm_wisdom = {
       'native_unit' : '-',
       'colorbar' : '-',
       'colormap' : 'jet_r',
       'scale' : [0.0, 0.4]
    }

    esmf_cycle = utc_to_esmf(cycle) 
    mf = { "1" : {esmf_cycle : {}}}
    manifest_name = 'fmda-%s-%04d%02d%02d-%02d.json' %  (region_cfg.code, cycle.year, cycle.month, cycle.day, cycle.hour)
    ensure_dir(osp.join(postproc_path, manifest_name))

    for i,name in [(0, '1-hr'), (1, '10-hr'), (2, '100-hr')]:
        fm_wisdom['name'] = '%s fuel moisture' % name
        raster_png, coords, cb_png = scalar_field_to_raster(fmc_gc[:,:,i], lats, lons, fm_wisdom)
        raster_name = 'fmda-%s-raster.png' % name
        cb_name = 'fmda-%s-raster-cb.png' % name
        with open(osp.join(postproc_path, raster_name), 'w') as f:
            f.write(raster_png)
        with open(osp.join(postproc_path, cb_name), 'w') as f:
            f.write(cb_png) 
        mf["1"][esmf_cycle][name] = { 'raster' : raster_name, 'coords' : coords, 'colorbar' : cb_name }
        logging.info('writing manifest file %s' % osp.join(postproc_path, manifest_name) )
        json.dump(mf, open(osp.join(postproc_path, manifest_name), 'w'))

    return postproc_path
 def compile(self):
     utils.ensure_dir(self._output_prefix + self.output_path)
     string = utils.get_file_contents(self._source).format(
         student_name=self.student.printed_name,
         title='{student_name} - {course_name}'.format(
             student_name=self.student.printed_name,
             course_name=self.course_name
         )
     )
     weasyprint.HTML(
         string=string,
         base_url=self._base_path
     ).write_pdf(self._output_prefix + self.output_file)