Пример #1
1
class DirectoryLocator(Locator):
    def __init__(self, path, recurse=False):
        self.path = Path(path)
        self.recurse = recurse
        self._filemap = None
    @property
    def filemap(self):
        """dist -> ver -> path mapping"""
        if self._filemap is None:
            self._filemap = defaultdict(lambda: defaultdict(list))
            if self.recurse:
                files = self.path.rglob("*")
            else:
                files = self.path.glob("*")
            for f in files:
                filetype, dist, ver = parse_filename(f.name)
                self._filemap[dist][ver].append(str(f.resolve()))
        return self._filemap
    def distributions(self):
        return sorted(self.filemap.keys())
    def versions(self, distribution):
        return sorted(self.filemap[distribution.lower().replace('-','_')].keys())
    def get(self, distribution, version):
        # Reformat the data...
        return self.filemap[distribution.lower().replace('-','_')][version]
Пример #2
0
 def getbasetemp(self):
     """ return base temporary directory. """
     if self._basetemp is None:
         if self._given_basetemp is not None:
             basetemp = Path(self._given_basetemp)
             ensure_reset_dir(basetemp)
         else:
             from_env = os.environ.get("PYTEST_DEBUG_TEMPROOT")
             temproot = Path(from_env or tempfile.gettempdir())
             user = get_user() or "unknown"
             # use a sub-directory in the temproot to speed-up
             # make_numbered_dir() call
             rootdir = temproot.joinpath("pyt-{}".format(user))
             rootdir.mkdir(exist_ok=True)
             basetemp = make_numbered_dir_with_cleanup(
                 prefix="",
                 root=rootdir,
                 keep=3,
                 lock_timeout=LOCK_TIMEOUT,
             )
         assert basetemp is not None
         self._basetemp = t = basetemp
         self._trace("new basetemp", t)
         return t
     else:
         return self._basetemp
def collect_json(source_dir):
    ret = []
    source_path = Path(source_dir)
    for file_path in source_path.glob('*.json'):
        with file_path.open() as fp:
            ret.append(json.load(fp))
    return sorted(ret, key=lambda each: each['name'])
def main():
    """Print virtualenv and python version."""
    workon_home = os.environ.get('WORKON_HOME')
    workon_home = Path(workon_home)

    for virtualenv in workon_home.iterdir():
        if virtualenv.is_dir():
            for python_bin in Path(f'{virtualenv}/bin/').iterdir():
                if python_bin.name == 'python':
                    virtual_environment = str(virtualenv).rpartition('/')[-1]
                    command = [f'{python_bin}',
                               '-c',
                               "import sys;print(sys.version.split()[0]);"
                               ]
                    stdout, _ = Popen(command, stdout=PIPE).communicate()
                    stdout = stdout.decode('utf-8')
                    python_version = stdout.strip()
                if python_bin.name == 'pip':
                    command = [f'{python_bin}',
                               'freeze'
                               ]
                    stdout, _ = Popen(command, stdout=PIPE).communicate()
                    stdout = stdout.decode('utf-8')
                    packages = [p.strip() for p in stdout.split()]
            with open(f'virtualenvs-{os.uname()[1].split(".")[0]}.md', 'a') as f:
                f.write(template.render(virtualenv=virtual_environment,
                                        version=python_version,
                                        packages=packages))
Пример #5
0
    def execute(self, context):
        w = context.world
        if w:
            # First, verify this is a valid Uru directory...
            path = Path(self.filepath)

            # Blendsucks likes to tack filenames onto our doggone directories...
            if not path.is_dir():
                path = path.parent
            if not ((path / "UruExplorer.exe").is_file() or (path / "plClient.exe").is_file()):
                self.report({"ERROR"}, "The selected directory is not a copy of URU.")
                return {"CANCELLED"}

            # New game?
            games = w.plasma_games
            new_game = self.game_index == -1
            if new_game:
                games.active_game_index = len(games.games)
                game = games.games.add()
            else:
                game = games.games[self.game_index]

            # Setup game...
            game.path = str(path)
            if (path / "cypython22.dll").is_file():
                game.version = "pvPots"
            else:
                game.version = "pvMoul"
            game.name = path.name

            return {"FINISHED"}
        else:
            return {"CANCELLED"}
Пример #6
0
def iter_file(path, formatter=lambda x: x):
    path = Path(path)
    if not path.exists():
        abort('Path does not exist: {}'.format(path))
    with path.open() as f:
        for l in f:
            yield formatter(l)
Пример #7
0
 def _setup_user_dir(self):
     """Returns user config dir, create it when it doesn't exist."""
     user_dir = Path(os.path.expanduser('~/.thefuck'))
     rules_dir = user_dir.joinpath('rules')
     if not rules_dir.is_dir():
         rules_dir.mkdir(parents=True)
     self.user_dir = user_dir
Пример #8
0
def extract_notes_from_summary(notes, summary_file):
    new_summary = Path(summary_file.name).read_text()

    # Remove comments.
    new_summary = new_summary.split(NEWLINE)
    new_summary = [l for l in new_summary if not l.startswith(DATE_PREFIX)]
    new_summary = NEWLINE.join(new_summary)

    segments = new_summary.split(NOTE_HEADER)[1:]

    if len(segments) > len(notes):
        raise Exception("Notes were added in the process of editing the summary file.")
    if len(segments) < len(notes):
        raise Exception("Notes were removed in the process of editing the summary file.")

    extracted_notes = type(notes)()

    for (path, note), seg in zip(notes.items(), segments):
        lines = seg.split(NEWLINE)
        lines = lines[1:]
        seg = NEWLINE.join(lines)
        seg = seg.strip()
        if cfg['show_tags']:
            _note = Note.from_string(path, seg)
        else:
            _note = Note(path, seg, note.tags)
        extracted_notes[path] = _note

    return extracted_notes
Пример #9
0
def test_save(im_format="png", ask_input=False):
    img_path = "test.{}".format(im_format)
    fig = plt.figure()
    data = [i for i in range(20)]
    plt.plot(data)
    fig.tight_layout()

    try:
        fig.savefig(img_path)
        plt.close(fig)
        resp = ""

        if ask_input:
            resp = input("Do you want to check the generated image? (yes/no):").strip().lower()

        if resp != "yes":
            p = Path(img_path)
            p.unlink()

    except Exception as exc:
        print("Error: Saving {} does not work")
        print(exc)
        return -1

    return 0
Пример #10
0
def main():
    """ Main function for program """
    global doc_counter
    global reading
    global vocab
    global word_count
    global doc_words
    global doc_word_count
    global doc_info_string

    if len(sys.argv) != 3:
        print("usage: .py <input_directory> <output_directory>\n")
        sys.exit(1)

    input_directory = Path(sys.argv[1])
    output_directory = sys.argv[2]


    for year_directory in input_directory.iterdir():
        for day_file in year_directory.iterdir():
            print('Processing: {}'.format(day_file))
            
            with day_file.open() as f:
                lines = f.readlines()

            for line in lines:
                process_line(line, output_file)
            

    vocab_file = open('vocab.txt', 'w')
    print(len(vocab))
    for word in vocab:
        vocab_file.write("%s\n" % word)
Пример #11
0
def main():
    
    # get raw
    report = Path(prefix + '_scrape/pdf').glob('*.pdf')

    # locate storage
    archive = scrape_util.ArchiveFolder(argv, prefix)
    
    # write csv file for each historical report
    for this_report in report:

        # read raw
        system(scrape_util.pdftotext.format(str(this_report)))
        temp_txt = this_report.with_suffix('.txt')
        with temp_txt.open('r') as io:
            line = [this_line.strip() for this_line in io.readlines()]
        temp_txt.unlink()

        sale_date = get_sale_date(line)
        io_name = Path(prefix + '_scrape/' + prefix + '_' + sale_date.strftime('%y-%m-%d') + '.csv')

        sale_head = get_sale_head(line)
        this_default_sale = default_sale.copy()
        this_default_sale.update({
            'sale_year': sale_date.year,
            'sale_month': sale_date.month,
            'sale_day': sale_date.day,
            'sale_head': sale_head,
            })

        with io_name.open('w') as io:
            writer = csv.DictWriter(io, scrape_util.header, lineterminator='\n')
            writer.writeheader()
            write_sale(line, this_default_sale, writer)
Пример #12
0
def main():
    requirements_dir = Path(__file__).parent / '..' / 'requirements'

    for requirement_file in requirements_dir.glob('*.txt'):
        print(requirement_file.name)
        with open(str(requirement_file), 'r') as f:
            for req in f:
                # Remove trailing and leading whitespace.
                req = req.strip()

                if not req:
                    # skip empty or white space only lines
                    continue
                elif req.startswith('#'):
                    continue

                # Get the name of the package
                req = re.split('<|>|=|!|;', req)[0]
                try:
                    # use pkg_resources to reliably get the version at install
                    # time by package name. pkg_resources needs the name of the
                    # package from pip, and not "import".
                    # e.g. req is 'scikit-learn', not 'sklearn'
                    version = pkg_resources.get_distribution(req).version
                    print(req.rjust(20), version)
                except pkg_resources.DistributionNotFound:
                    print(req.rjust(20), 'is not installed')
Пример #13
0
    def __init__(self, name, prefix, directory, *,
                 expect_handler=None, chunk_size=256*1024,
                 response_factory=StreamResponse):
        assert prefix.startswith('/'), prefix
        assert prefix.endswith('/'), prefix
        super().__init__(
            'GET', self.handle, name, expect_handler=expect_handler)
        self._prefix = prefix
        self._prefix_len = len(self._prefix)
        try:
            directory = Path(directory)
            if str(directory).startswith('~'):
                directory = Path(os.path.expanduser(str(directory)))
            directory = directory.resolve()
            if not directory.is_dir():
                raise ValueError('Not a directory')
        except (FileNotFoundError, ValueError) as error:
            raise ValueError(
                "No directory exists at '{}'".format(directory)) from error
        self._directory = directory
        self._chunk_size = chunk_size
        self._response_factory = response_factory

        if bool(os.environ.get("AIOHTTP_NOSENDFILE")):
            self._sendfile = self._sendfile_fallback
Пример #14
0
def joinpath(*paths, resolve=False):
    """Helper function to determine paths in config file"""
    base_dir = Path(__file__).parent.parent.resolve()
    result = base_dir.joinpath(*paths)
    if resolve:
        result.resolve()
    return str(result)
Пример #15
0
 def save_to_dropbox_and_server(card_image, dropbox_path):
     on_server = Path(dropbox_destination.lstrip("/"))
     if not on_server.parent.exists():
         on_server.parent.mkdir(parents=True)
     card_image.save(on_server)
     with on_server.open("rb") as image_file:
         self.dropbox.put_file(dropbox_path, image_file)
Пример #16
0
def main():
    ############ prepare values
    options, arguments = arguments_parse()
    text      = stdin_read()
    contentId = options.force_id if options.force_id is not None else contentId_obtain(text, options.id_field)
    diffPath  = config.DIFF_PATH % contentId
    txtPath   = config.TXT_PATH % contentId
    filepath  = Path(txtPath)

    ############ do your job
    ### No revised file found ? Why are you even calling me ?
    if not filepath.is_file():
        y_error('\n -- No new revision file found\n')
        print('\n -- No new revision file found\n')
        exit(1)

    ### Ok, go on..
    textNew  = f_read(txtPath)
    dmp      = diff_match_patch()
    rawPatch = dmp.patch_make(text, textNew)
    txtPatch = dmp.patch_toText(rawPatch)
    f_write(diffPath, txtPatch)

    print("\nPatch for " + contentId + " Written !\n")

    ############ job's done !
    exit(0)
Пример #17
0
def main():
  global run, summarize_log
  seed(37429)
  args = argparser.parse_args()
  outdir = Path(args.outdir, data_file_stem(args.data))
  if not outdir.exists():
    outdir.mkdir(parents=True)
  summarize_log = \
    partial(summarize_log, args.points, args.step_bin, args.node_bin, outdir)
  run = partial(run, args.executable, args.data, args.keep_history)
  summaries = [{} for _ in range(args.history + 1)]
  naive_summary = None
  if 'naive' in args.algorithm:
    run(args.history, 'naive')
    naive_summary = summarize_log('naive')
  for h in range(1, args.history + 1):
    sys.stderr.write('HISTORY {}\n'.format(h))
    sys.stderr.flush()
    summaries[h]['naive'] = naive_summary
    for a in args.algorithm:
      if a == 'naive':
        continue
      run(h, a)
      summaries[h][a] = summarize_log('{}-{}'.format(a, h))
  prof_start()
  for a in args.algorithm:
    keys = sorted(summaries[1][a].keys())
    for k in keys:
      with Path(outdir, '{}-{}.json'.format(a, k)).open('w') as out:
        json.dump(
            [(h, summaries[h][a][k]) for h in range(1, args.history + 1)], out)
  prof_stop('save_across_history')
Пример #18
0
def _cli():
    ########################################################################
    parser = argparse.ArgumentParser(
        description="Check whether loci changed or not by signal metrics (raw, rpm, "
                    "rpkm) for interesting loci sets",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    parser.add_argument('--signal', metavar="PATH",
                        # /mnt/stripe/bio/experiments/signal_consensus_peaks/
                        # /mnt/stripe/bio/experiments/signal_loci_of_interest/
                        default="/mnt/stripe/bio/experiments/signal",
                        help="Processed signal dir")
    parser.add_argument('-o', '--out', required=True, metavar="PATH",
                        help="Output dir")
    args = parser.parse_args()
    signal_root = Path(args.signal)

    # /mnt/stripe/bio/experiments/aging/signal@loci_stats
    results_dir = Path(args.out)
    results_dir.mkdir(parents=True, exist_ok=True)
    ########################################################################

    # signal_dfs_by_datatype, signal_dfs_by_loci = build_signal_dfs(results_dir, signal_root)
    # print(signal_dfs_by_datatype[("H3K4me1", "rpkm")].head())

    stats_test(results_dir, signal_root)
Пример #19
0
def test_all():
    cwd = Path(getcwd())
    all_omts = [p.as_posix() for p in cwd.glob('**/*.omt')]
    th = TallyHolder()
    if environ.get('TRAVIS'):
        if not environ.get('OMV_ENGINE'):
            tallies = [parse_omt(t) for t in all_omts]
        else:
            engine = environ.get('OMV_ENGINE').lower()
            tallies = [parse_omt(t)
                       for t in all_omts
                       if load_yaml(t)['engine'].lower() == engine]
    else:
        tallies = [parse_omt(t) for t in all_omts]
        
    for t in tallies:
        th.add(t)

    results = [t.all_passed() for t in tallies]
    inform('')
    inform("%i test(s) run" % len(tallies),
           overline='-', underline='-', center=True)
    inform('')
    if all(results):
        inform("All tests passing!", underline='=', center=True)
    else:
        failed = [trim_path(t.omt) for t in tallies if not t.all_passed()]
        inform("Some test(s) failed: ",  failed, underline='=')
    
    if is_verbose():
        print('\n'+th.summary()+'\n')

    assert all(results)
Пример #20
0
def extractLog (gameLog, sessionType, logType):
    '''
    Extracts logs from compressed game log file, if not already extracted.
    Returns path to state log.
    '''
    if year != 2016:
        gameIdRe = re.compile('game-(\d+)-{}-logs.tar.gz'.format(sessionType))
    elif year == 2016:
        gameIdRe = re.compile('game-(\d+)-{}.tar.gz'.format(sessionType))

    path = Path(gameLog)
    m = gameIdRe.search(str(path))
    if m:
        gameId = m.group(1)
        if year != 2016:
            logPath = Path(path.parent, 'log',
                           'powertac-{}-{}.{}'.format(sessionType, gameId, logType))
        elif year == 2016:
            logPath = Path(path.parent, 'log',
                           'powertac-{}-2016_finals_{}.{}'.format(sessionType, str(int(gameId)-654), logType))
            
        if not logPath.exists():
            pathdir = path.parent
            path = path.as_posix()
            pathdir = pathdir.as_posix()
            tar = tarfile.open(path)
            tar.extractall(pathdir)
            tar.close()
        return logPath
    else:
        gameId = 'xx'
        print('Failed to find game ID in ' + str(path))
        return False
Пример #21
0
Файл: oisst.py Проект: guziy/RPN
    def download_data_locally(self, start_year=2001, end_year=2011, local_dir="/BIG1/huziy/noaa_oisst_daily"):
        """
        Download the data to a local server
        :param start_year:
        :param end_year:
        :param local_dir:
        """
        for y in range(start_year, end_year + 1):
            # --
            for month in range(1, 13):
                for day in range(1, calendar.monthrange(y, month)[1] + 1):

                    file_url = self.get_url_for(year=y, month=month, day=day)
                    print("Fetching {} ...".format(file_url))

                    try:
                        with xr.open_dataset(file_url) as ds:
                            # assert isinstance(ds, xr.Dataset)
                            local_filepath = Path(local_dir) / Path(file_url).name

                            if local_filepath.exists():
                                print("{} already exists, skipping ...")
                                continue

                            ds.to_netcdf(path=str(local_filepath))
                            print("Saved {} to {}".format(file_url, local_filepath))

                    except OSError as err:
                        print(err)
                        print("Could not find {}".format(file_url))
Пример #22
0
def micro(ctx, publish, write, filetype):
	"""Create a micro post"""
	# Create a new file in the micro directory
	now = datetime.datetime.today()
	filename = "{}-{}.{}".format(str(now.day) + str(now.month) + str(now.year), str(now.second) + str(now.minute) + str(now.hour) ,filetype)

	#blotpath = Path.home() / 'Rambling' / 'hill'

	blotpath = os.getenv("BLOTDIR")

	if blotpath:
		micropath = Path(blotpath) / 'micro'
		micropath = os.path.expanduser(micropath)
		#click.echo(micropath)

		# should check if the micropath exists
		if not Path(micropath).exists():
			click.echo("A micro directory does not exist. Creating one now...")
			p = Path(micropath)
			p.mkdir()

		if write:
			with open(str(micropath) + '/' + filename, mode='w') as f:
				f.write("Tags: micropost\n\n" + write)
		else:
			#click.echo("Open {}!".format(os.getenv("EDITOR"))
			subprocess.call(["nano", "{}/{}".format(micropath, filename)])

		if publish:
			ctx.invoke(publish)
	else:
		no_blotpath()
Пример #23
0
def score(truth_path: pathlib.Path, prediction_path: pathlib.Path) -> Dict[str, Dict[str, float]]:
    for truth_file in truth_path.iterdir():
        if re.match(r'^ISIC.*GroundTruth\.csv$', truth_file.name):
            break
    else:
        raise ScoreException('Internal error, truth file could not be found.')

    prediction_files = [
        prediction_file
        for prediction_file in prediction_path.iterdir()
        if prediction_file.suffix.lower() == '.csv'
    ]
    if len(prediction_files) > 1:
        raise ScoreException(
            'Multiple prediction files submitted. Exactly one CSV file should be submitted.'
        )
    elif len(prediction_files) < 1:
        raise ScoreException(
            'No prediction files submitted. Exactly one CSV file should be submitted.'
        )
    prediction_file = prediction_files[0]

    with truth_file.open('rb') as truth_file_stream, prediction_file.open(
        'rb'
    ) as prediction_file_stream:
        return compute_metrics(truth_file_stream, prediction_file_stream)
Пример #24
0
Файл: util.py Проект: Xion/gisht
def read_cargo_toml(key, manifest=None):
    """Read a value from Cargo.toml manifest.

    :param key: Key to read from [package], or entire path to a key.
                It may contain dots.
    :param manifest: Optional path to the manifest,
                     or a file-like object with it opened

    :return: The value
    """
    if not isinstance(key, (list, tuple)):
        key = (key,) if key.startswith('package.') else ('package', key)
    key = list(chain.from_iterable(k.split('.') for k in key))
    if not key:
        raise ValueError("key must not be empty")

    # Read the content of Cargo.toml.
    manifest = manifest or Path.cwd() / 'Cargo.toml'
    if hasattr(manifest, 'read'):
        content = toml.load(manifest)
    else:
        manifest = Path(manifest)
        with manifest.open() as f:
            content = toml.load(f)

    # Get the value.
    value = content
    for k in key:
        value = value[k]
    return value
Пример #25
0
    def __init__(self, projdescpath, verify=True, project=None, ):
        projdescpath = Path(str(projdescpath)).resolve()
        
        if not projdescpath.exists():
            raise Exception(projdescpath)
            
        projdesc = Json.load_json_from(projdescpath)
        self.projdesc = projdesc
        
        for i in ["testinfo", "testfolder", "projectfolder"]:
            key = ("experiment_config",i)
            if not projdesc[key]:
                print('Missing:\n\n    File "{key}", line 1, in {projdescpath}\n\n'.format(projdescpath=projdescpath, key=("experiment_config",i)), file=sys.stdout)
                raise ValueError("Missing key in config: `{}` from: `{}`".format(key, projdescpath))
        
        self._testinfo = generatetestinfoclass(**projdesc["experiment_config"]["testinfo"])
        
        names = self.projdesc.experiment_config.name.split('|' if '|' in self.projdesc.experiment_config.name else ';')
        self.experiment_name = names[0]
        self.test_name = names[1:]

        self.project = projdescpath.parent if not project else project

        files = DataTree(projdesc.experiment_config["projectfolder"]["filestructure"])
        self._files = self.parsefolders(files, verify, parent=self.project)
        for name, file in self._files.items():
            self[name] = file
Пример #26
0
def isnew(needfixes):
	#This function prevents repeat notifications for the same aircraft
	#A list of aircraft needing repair is stored in a text file
	dirs=AppDirs("nattgew-xpp","Nattgew")
	filename=Path(dirs.user_data_dir).joinpath('aog.txt')
	try:
		filename.touch(exist_ok=True) #Create file if it doesn't exist
		oldnews=[] #List of aircraft needing fixes already notified
		with filename.open() as f:
			for aog in f: #Loop over all aircraft in the file
				aog=aog.strip()
				#print("Checking for previously AOG: "+aog)
				for current in needfixes: #Loop over all aircraft currently in need of repair
					#print("Testing current AOG: "+current[0])
					if current[0]==aog: #Aircraft was already listed in the file
						#print("Matched, adding to list")
						oldnews.append(current)
						break
		#print("Came up with oldnews:")
		#print(oldnews)
		with filename.open('w') as f: #Overwrite the file with the new list of aircraft
			for current in needfixes:
				#print("Writing "+current[0]+" to file")
				f.write(current[0]+"\n")
		#print("Will remove oldnews from needfixes list:")
		#print(needfixes)
		for oldie in oldnews: #Remove aircraft already notified from the list
			#print("Removing "+oldie[0]+" from notify list")
			needfixes.remove(oldie)
	except IOError:
		print("Could not open file: "+str(file))
	return needfixes
Пример #27
0
def create_file(file_name):
    name = Path(file_name)
    create_dir(name.parent)
    if name.exists():
        name.unlink()
    print("create file " + str(name))
    return open(str(name), "w")
Пример #28
0
def create_dir(dir_name):
    name = Path(dir_name)
    if name.is_file():
        raise (str(name) + " is file")
    if not name.exists():
        name.mkdir(parents=True)
        print("create directory " + str(name))
Пример #29
0
    def create(self, f):
        _ensure(self.container, parents=True)
        _ensure(self.tmp, parents=True)
        hash = self.create_hash()
        temp_file = tempfile.NamedTemporaryFile(
            dir=str(self.tmp),
            delete=False,
        )
        temp_file_path = Path(temp_file.name)

        try:
            with temp_file as tf:
                for chunk in FileWrapper(f):
                    hash.update(chunk)
                    tf.write(chunk)

            key = hash.hexdigest()
            path = self.get_path(key)

            _ensure(path.parent)

        except:
            temp_file_path.unlink()

        else:
            temp_file_path.rename(path)

        return key
Пример #30
0
def run_pylint(module_path, pylint_options, ignore_prefixes=tuple()):
    """Runs Pylint. Returns a boolean indicating success"""
    pylint_stats = Path('/run/user/{}/pylint_stats'.format(os.getuid()))
    if not pylint_stats.parent.is_dir(): #pylint: disable=no-member
        pylint_stats = Path('/run/shm/pylint_stats')
    os.environ['PYLINTHOME'] = str(pylint_stats)

    input_paths = list()
    if not module_path.exists():
        print('ERROR: Cannot find', module_path)
        exit(1)
    if module_path.is_dir():
        for path in module_path.rglob('*.py'):
            ignore_matched = False
            for prefix in ignore_prefixes:
                if path.parts[:len(prefix)] == prefix:
                    ignore_matched = True
                    break
            if ignore_matched:
                continue
            input_paths.append(str(path))
    else:
        input_paths.append(str(module_path))
    runner = lint.Run((*input_paths, *pylint_options), do_exit=False)

    if pylint_stats.is_dir():
        shutil.rmtree(str(pylint_stats))

    if runner.linter.msg_status != 0:
        print('WARNING: Non-zero exit status:', runner.linter.msg_status)
        return False
    return True
Пример #31
0
def load_feeds(article_generator):
    settings = article_generator.settings
    articles = article_generator.articles

    # prepare blog articles
    blog_feed_path = settings.get('FEED_ALL_ATOM_URL',
                                  settings['FEED_ALL_ATOM'])
    blog_feed = dict(
        title='blog',
        lang='en',
        link=f"{settings['SITEURL']}/blog/",
        href=f"{settings['FEED_DOMAIN']}/{blog_feed_path}",
    )
    blog_entries = [
        dict(
            title=article.title,
            lang=article.lang,
            link=f"{settings['SITEURL']}/{article.url}",
            date=article.date.date(),
            feed_title=blog_feed['title'],
            feed_link=blog_feed['link'],
            translations=[
                dict(
                    title=translation.title,
                    lang=translation.lang,
                    link=f"{settings['SITEURL']}/{translation.url}",
                    date=translation.date.date(),
                    feed_title=blog_feed['title'],
                    feed_link=blog_feed['link'],
                ) for translation in article.translations
            ],
            alternates=[],
        ) for article in articles
        if getattr(article, 'home', '').lower() != 'false'
    ]

    # create a lookup mapping of alternate entries
    alternates_mapping = dict(
        itertools.chain.from_iterable(((alternate['url'], i)
                                       for alternate in article.alternates)
                                      for i, article in enumerate(articles)))

    # external feeds
    path = Path('./content/data/feeds.yml')
    feeds = yaml.load(path.read_text(), SCHEMA).data

    entries = []
    for entry in itertools.chain.from_iterable(map(parse_feed, feeds)):
        try:
            i = alternates_mapping[entry['link']]
        except KeyError:
            entries.append(entry)
        else:
            blog_entries[i]['alternates'].append(entry)

    # merge & sort entries
    feeds_entries = sorted(blog_entries + entries,
                           key=itemgetter('date'),
                           reverse=True)

    # set context variables
    article_generator.context['feeds_entries'] = feeds_entries
    article_generator.context['feeds'] = [blog_feed] + feeds
Пример #32
0
bert_model_size_type = 'small'
bert_model_name = 'bert'
# bert_model_name = 'heBERT'
# bert_model_name = 'mBERT'
bert_version = f'{bert_model_name}-{bert_model_size_type}-{bert_tokenizer_type}-{bert_corpus_name}-{bert_vocab_size}-{epochs}'
tokenizer_version = f'{bert_model_name}-{bert_tokenizer_type}-{bert_corpus_name}-{bert_vocab_size}'
# tokenizer_version = f'{bert_tokenizer_type}-{bert_corpus_name}-{bert_vocab_size}'
# bert_version = f'{bert_model_name}'
# tokenizer_version = f'{bert_model_name}'

md_strategry = "morph-pipeline"
# md_strategry = "morph-sequence"
# md_strategry = "segment-only"

# Data
raw_root_path = Path(f'data/raw/{tb_data_src}')
if tb_name == 'HTB':
    partition = tb.ud(raw_root_path, tb_name)
elif tb_name == 'hebtb':
    if tb_schema == "UD":
        partition = tb.spmrl_conllu(raw_root_path, tb_name)
    else:
        partition = tb.spmrl(raw_root_path, tb_name)
else:
    partition = {'train': None, 'dev': None, 'test': None}
preprocessed_data_root_path = Path(
    f'data/preprocessed/{tb_data_src}/{tb_name}/{tokenizer_version}')


def load_preprocessed_data_samples(data_root_path, partition,
                                   label_names) -> dict:
Пример #33
0
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list):
    print("config =", config.name)
    print("num_gpus =", num_gpus)
    print("num_cpus =", num_cpus)
    print("redis_address =", redis_address)

    # Load and parse experiment configurations
    configs = parse_config(config, experiments, globals_param=globals())

    if show_list:
        print("Experiments:", list(configs.keys()))
        return

    # Initialize ray cluster
    if redis_address is not None:
        ray.init(redis_address=redis_address, include_webui=True)
    else:
        ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)

    # Run experiments
    gpu_percent = 0
    if num_gpus > 0:
        gpu_percent = configs.get("gpu_percentage", 0.5)
    resources_per_trial = {"cpu": 1, "gpu": gpu_percent}
    print("resources_per_trial =", resources_per_trial)
    for exp in configs:
        print("experiment =", exp)
        config = configs[exp]
        config["name"] = exp

        # Stop criteria. Default to total number of iterations/epochs
        stop_criteria = {"training_iteration": config.get("iterations")}
        stop_criteria.update(config.get("stop", {}))
        print("stop_criteria =", stop_criteria)

        # Make sure path and data_dir are relative to the project location,
        # handling both ~/nta and ../results style paths.
        path = config.get("path", ".")
        config["path"] = str(Path(path).expanduser().resolve())

        data_dir = config.get("data_dir", "data")
        config["data_dir"] = str(Path(data_dir).expanduser().resolve())

        tune.run(
            SpeechExperimentTune,
            name=config["name"],
            stop=stop_criteria,
            config=config,
            resources_per_trial=resources_per_trial,
            num_samples=config.get("repetitions", 1),
            local_dir=config.get("path", None),
            upload_dir=config.get("upload_dir", None),
            sync_function=config.get("sync_function", None),
            checkpoint_freq=config.get("checkpoint_freq", 0),
            checkpoint_at_end=config.get("checkpoint_at_end", False),
            export_formats=config.get("", None),
            search_alg=config.get("search_alg", None),
            scheduler=config.get("scheduler", None),
            verbose=config.get("verbose", 2),
            resume=config.get("resume", False),
            queue_trials=config.get("queue_trials", False),
            reuse_actors=config.get("reuse_actors", False),
            trial_executor=config.get("trial_executor", None),
            raise_on_failed_trial=config.get("raise_on_failed_trial", True),
        )

    ray.shutdown()
Пример #34
0
"""
Django settings for profiles_project project.

Generated by 'django-admin startproject' using Django 3.2.

For more information on this file, see
https://docs.djangoproject.com/en/3.2/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.2/ref/settings/
"""

from pathlib import Path

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-9yt!9q9yr(08bw3fg04h!ise&7$aq+7@y3$nl671oi6lp+4-yt'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
Пример #35
0
from pathlib import Path
import pandas as pd
import pickle
import click
import json

project_dir = Path(__file__).resolve().parents[1]
raw_data_dir = "{}/{}".format(project_dir, "data/raw")
output_dir = "{}/{}".format(project_dir, "data/output")
model_dir = "{}/{}".format(project_dir, "models")
params_dir = "{}/{}".format(project_dir, "params")

test_filepath = "{}/{}".format(raw_data_dir, "cs-test.csv")
default_model_json = "{}/{}".format(params_dir, "init_model.json")

TARGET_COLUMN = "SeriousDlqin2yrs"


@click.command()
@click.option('--model_json', default=default_model_json)
def predict(model_json):
    """
    Predict class probabilities

    Parameters:
    model_json (str): Filepath to JSON containing model
    parameters
    """

    with open(model_json, "r") as f:
        model_dict = json.loads(f.read())
Пример #36
0
@author: dberke

A simple script to scrape all the ADP files in
/Volumes/External Storage/HARPS/* and construct a CSV database of their
filenames, dates of observation, name of origin file, and object.
"""

import pandas as pd
from astropy.io import fits
from pathlib import Path
from tqdm import tqdm

databaseLines = []

baseDir = Path('/Volumes/External Storage/HARPS')

ADPfiles = baseDir.glob('*/ADP*.fits')
filelist = [file for file in ADPfiles]
print(len(filelist))

for file in tqdm(filelist):
    with fits.open(file) as hdul:
        header = hdul[0].header
        date_obs = header['DATE-OBS']
        arcfile = header['ARCFILE']
        origfile = header['ORIGFILE']
        try:
            obj = header['HDNUM']
        except KeyError:
            obj = header['OBJECT']
Пример #37
0
 def _write_paths(self):
     with open(Path(self.output_dir / 'paths.txt'), 'w') as f:
         f.write('model path: {}\n'.format(str(self.model_path)))
         f.write('train path: {}\n'.format(str(self.train_file_path)))
         f.write('val path: {}\n'.format(str(self.val_file_path)))
         f.write('predict path: {}\n'.format(str(self.predict_file_path)))
Пример #38
0
def main():

    main_dir = Path(
        r'P:\Synchronize\IWS\Testings\fourtrans_practice\multisite_phs_spec_corr\5min\v7'
    )

    os.chdir(main_dir)

    suff = '__RR1D_RTsum'

    in_data_files = [
        Path(r'../neckar_1min_ppt_data_20km_buff_Y2009__RRD_RTsum.pkl'),
        Path(f'orig/ts_OK{suff}.csv'),
        Path(f'ifted/ts_OK{suff}.csv'),
    ]

    data_labels = ['REF', 'OK', 'FT']

    lags = np.arange(1, 10 + 1, dtype=np.int64)

    ranks_flag = False

    sep = ';'

    fig_size = (15, 7)

    dpi = 200

    beg_time = '2009-01-01 00:00:00'
    end_time = '2009-03-31 23:59:00'

    out_dir = Path(f'cmpr_figs__lag_corrs{suff}')

    out_dir.mkdir(exist_ok=True)

    rows = 2
    cols = len(in_data_files)

    axes = plt.subplots(rows,
                        cols,
                        squeeze=False,
                        figsize=fig_size,
                        sharex=True,
                        sharey=True)[1]

    row = 0
    for i in range(len(in_data_files)):
        print('Going through:', in_data_files[i])

        if in_data_files[i].suffix == '.csv':
            data_df = pd.read_csv(in_data_files[i], sep=sep, index_col=0)


#             data_df.index = pd.to_datetime(data_df.index, format=time_fmt)

        elif in_data_files[i].suffix == '.pkl':
            data_df = pd.read_pickle(in_data_files[i])

        else:
            raise NotImplementedError(
                f'Unknown extension of in_data_file: {in_data_files[i].suffix}!'
            )

        data_df = data_df.loc[beg_time:end_time]

        #         if i == 0:
        #             data_df.dropna(axis=1, how='any', inplace=True)
        #
        #         else:
        #             data_df.dropna(axis=0, how='any', inplace=True)

        assert np.all(np.isfinite(data_df.values))

        if ranks_flag:
            data_df = data_df.rank(axis=0)

        lags_corrs_all = []
        for stn in data_df:
            stn_vals = data_df[stn].values.copy()

            lags_corrs = []
            for lag in lags:
                stn_no_lag_vals, stn_lag_vals = roll_real_2arrs(
                    stn_vals, stn_vals, lag)

                lag_corr = np.corrcoef(stn_no_lag_vals, stn_lag_vals)[0, 1]

                lags_corrs.append((lag, lag_corr))

            lags_corrs_all.extend(lags_corrs)

            lags_corrs = np.array(lags_corrs)

            axes[row, i].plot(lags_corrs[:, 0],
                              lags_corrs[:, 1],
                              lw=1,
                              alpha=0.1,
                              color='k')

        lags_corrs_all = np.array(lags_corrs_all)

        lags_corrs_med = []
        for lag in lags:
            med_lag_corr = np.median(lags_corrs_all[lags_corrs_all[:,
                                                                   0] == lag,
                                                    1])

            lags_corrs_med.append((lag, med_lag_corr))

        lags_corrs_med = np.array(lags_corrs_med)

        axes[row, i].plot(lags_corrs_med[:, 0],
                          lags_corrs_med[:, 1],
                          lw=2,
                          alpha=0.75,
                          color='red',
                          label='median')

        axes[row, i].set_title(data_labels[i])

        axes[row, i].legend()
        axes[row, i].grid()
        axes[row, i].set_axisbelow(True)

        if i == 0:
            axes[row, i].set_ylabel('Correlation (-)')

        axes[1, 1].plot(lags_corrs_med[:, 0],
                        lags_corrs_med[:, 1],
                        lw=2,
                        alpha=0.75,
                        label=f'{data_labels[i]}')

    row = 1
    for i in range(len(in_data_files)):
        if i == 1:
            axes[row, i].legend()
            axes[row, i].grid()
            axes[row, i].set_axisbelow(True)
            axes[row, i].set_xlabel('Lag (steps)')

            continue

        axes[row, i].set_axis_off()

    if ranks_flag:
        rank_lab = '__ranks'

    else:
        rank_lab = ''

    out_fig_name = f'lag_corrs{rank_lab}.png'

    plt.savefig(str(out_dir / out_fig_name), bbox_inches='tight', dpi=dpi)
    plt.close()

    #     plt.show()

    return
Пример #39
0
 def label_filename(self):
     return Path(self.url.split("/")[-1])
Пример #40
0
# import pprint
# import pymongo
from pathlib import Path

from mongo_db.db_connection import DBConnection
from mongo_db.db_controller import DBController


with DBConnection('butterfly') as client:
    # print(client._db.collection_names())

    client.drop_collections('users', 'projects', 'topics')

    controller = DBController(client)
    db = client.db
    with open(Path(__file__).parent / 'db.json') as f:
        data = json.load(f)

    users_json = data['users']
    projects_json = data['projects']
    topics_json = data['topics']

    # Popola la collezione users da db.json
    for user in users_json:
        # NON usare questo metodo per inserire utenti! Usare
        # controller.insert_user()
        result = controller.collection('users').insert_one(user)
        if result is not None:
            print(result.inserted_id)

    # Popola la collezione projects da db.json
Пример #41
0
class IndexDB:
    fname = "pds_indices_db.toml"
    fpath = Path.home() / f".{fname}"

    def __init__(self):
        """Initialize index database.

        Will copy the package's version to user's home folder at init,
        so that user doesn't need to edit file in package to add new indices.

        Adding new index URLs to the package's config file pds_indices_db.toml
        is highly encouraged via pull request.
        """
        if not self.fpath.exists():
            with resource_path("planetarypy.pdstools.data", self.fname) as p:
                self.config = self.read_from_file(p)
        else:
            self.config = self.read_from_file()

    def read_from_file(self, path=None):
        """Read the config.

        Writing this short method to decouple IndexDB from choice of config file format.

        Parameters
        ----------
        path : str, pathlib.Path
            Path to the config file to open.
        """
        if path is None:
            path = self.fpath
        return toml.load(path)

    def write_to_file(self):
        "Write the config to user's home copy."
        with open(self.fpath, "w") as f:
            toml.dump(self.config, f)

    def get_by_path(self, nested_key):
        """Get sub-dictionary by nested key.

        Parameters
        ----------
        nested_key: str
            A nested key in the toml format, separated by '.', e.g. cassini.uvis.ring_summary
        """
        mapList = nested_key.split(".")
        d = copy.deepcopy(self.config)
        for k in mapList:
            d = d[k]
        return Index(nested_key, **d)

    def get(self, nested_key):
        "alias to get_by_path"
        return self.get_by_path(nested_key)

    def set_by_path(self, nested_key, value):
        """Set a nested dictionary key to new value.

        Parameters
        ----------
        nested_key : str
            A nested key in the toml format, separated by '.', e.g. cassini.uvis.summary
        value : str
            New value for dictionary key
        """
        dic = self.config
        keys = nested_key.split(".")
        for key in keys[:-1]:
            dic = dic.setdefault(key, {})
        dic[keys[-1]] = value

    def list_indices(self):
        "Print index database in pretty form, using toml.dumps"
        print(toml.dumps(self.config))
        print(
            "Use indices.download('mission.instrument.index') to download in index file."
        )
        print("For example: indices.download('cassini.uvis.moon_summary'")

    def update_timestamp(self, index):
        self.set_by_path(f"{index.key}.timestamp",
                         index.new_timestamp.isoformat())
        self.write_to_file()

    def download(self,
                 key=None,
                 label_url=None,
                 local_dir="",
                 convert_to_hdf=True,
                 force=False):
        """Wrapping URLs for downloading PDS indices and their label files.

        Parameters
        ----------
        key : str, optional
            Period-separated key into the available index files, e.g. cassini.uvis.moon_summary
        label_url : str, optional
            Alternative to using the index system, the user can provide the URL to a label
            for an index. The table file has to be in the same folder, as usual.
        local_dir: str, pathlib.Path, optional
            Path for local storage. Default: current directory and filename from URL
        convert_to_hdf : bool
            Switch to convert the index automatically to a faster loading HDF file
        """
        if label_url is None:
            if key is not None:
                index = self.get_by_path(key)
            else:
                raise SyntaxError("One of key or label_url needs to be given.")
        # check timestamp
        if not index.needs_download and not force:
            print("Stored index is up-to-date.")
            return index.local_hdf_path
        if not local_dir:
            local_dir = index.local_dir
        label_url = index.url
        logger.info("Downloading %s." % label_url)
        local_label_path, _ = utils.download(label_url, local_dir)
        data_url = replace_url_suffix(label_url)
        logger.info("Downloading %s.", data_url)
        local_data_path, _ = utils.download(data_url, local_dir)
        self.update_timestamp(index)
        if convert_to_hdf is True:
            label = IndexLabel(local_label_path)
            df = label.read_index_data()
            savepath = local_data_path.with_suffix(".hdf")
            df.to_hdf(savepath, "df")
        print(f"Downloaded and converted to pandas HDF: {savepath}")

    def __repr__(self):
        return toml.dumps(self.config)
Пример #42
0
FILE = settings.DATA_INPUT / '2021-02-13_GuiaPetroleo.xlsx'


def save_dataframes_unique(file):
    df = get_dataframe(file, worksheet=0, usecols=("A:O"))
    branches = find_branches(df)
    branches = find_duplicates(branches)
    headquartes = find_headquarters(df)
    headquartes = find_duplicates(headquartes)
    worksheets = {'filiais': branches, 'matrizes': headquartes}
    writer = create_excel_file(worksheets, filename='petroleo_empresas')
    writer.save()


# Press the green button in the gutter to run the script.
if __name__ == '__main__':

    try:
        Path.mkdir(Path(settings.DATA_INPUT))
        Path.mkdir(Path(settings.DATA_OUTPUT))
        Path.mkdir(Path(settings.REPORTS_PATH))
        Path.mkdir(Path(settings.PROFILE_PATH))
    except FileExistsError as e:
        pass
    else:
        print("Pastas de saída criadas")

    save_dataframes_unique(FILE)

# See PyCharm help at https://www.jetbrains.com/help/pycharm/
Пример #43
0
from dateutil import parser
from tqdm import tqdm

from .. import utils
from .._config import config
from .scraper import CTXIndex

try:
    # 3.6 compatibility
    from importlib_resources import path as resource_path
except ModuleNotFoundError:
    from importlib.resources import path as resource_path

logger = logging.getLogger(__name__)

indices_root = Path(config["data_archive"]["path"]) / "indices"


@dataclass
class Index:
    """Index manager class.

    Parameters
    ----------
    key : str
        Nested key in form of mission.instrument.index_name
    url : str
        URL to index
    timestamp : str
        Timestamp in ISO time format yy-mm-ddTHH:MM:SS.
        This is usually read by the IndexDB class from the config file and its
Пример #44
0
 def __init__(self, labelpath):
     self.path = Path(labelpath)
     "search for table name pointer and store key and fpath."
     tuple = [i for i in self.pvl_lbl if i[0].startswith("^")][0]
     self.tablename = tuple[0][1:]
     self.index_name = tuple[1]
Пример #45
0
 def _dirname(self):
     return Path(self.dirname)
Пример #46
0
 def label_path(self):
     return Path(urlsplit(self.url).path)
Пример #47
0
def GenerateCMake(folder, params):

    cmake_header1 = ("# Generated Cmake Pico project file\n\n"
                 "cmake_minimum_required(VERSION 3.13)\n\n"
                 "set(CMAKE_C_STANDARD 11)\n"
                 "set(CMAKE_CXX_STANDARD 17)\n\n"
                 "# initalize pico_sdk from installed location\n"
                 "# (note this can come from environment, CMake cache etc)\n"
                )

    cmake_header2 = ("# Pull in Pico SDK (must be before project)\n"
                "include(pico_sdk_import.cmake)\n\n"
                )

    cmake_header3 = (
                "\n# Initialise the Pico SDK\n"
                "pico_sdk_init()\n\n"
                "# Add executable. Default name is the project name, version 0.1\n\n"
                )


    filename = Path(folder) / CMAKELIST_FILENAME

    file = open(filename, 'w')

    file.write(cmake_header1)

    # OK, for the path, CMake will accept forward slashes on Windows, and thats
    # seemingly a bit easier to handle than the backslashes

    p = str(params.sdkPath).replace('\\','/')
    p = '\"' + p + '\"'

    file.write('set(PICO_SDK_PATH ' + p + ')\n\n')
    file.write(cmake_header2)
    file.write('project(' + params.projectName + ' C CXX ASM)\n')

    if params.exceptions:
        file.write("\nset(PICO_CXX_ENABLE_EXCEPTIONS 1)\n")

    if params.rtti:
        file.write("\nset(PICO_CXX_ENABLE_RTTI 1)\n")

    file.write(cmake_header3)

    # add the preprocessor defines for overall configuration
    if params.configs:
        file.write('# Add any PICO_CONFIG entries specified in the Advanced settings\n')
        for c, v in params.configs.items():
            if v == "True":
                v = "1"
            elif v == "False":
                v = "0"
            file.write('add_compile_definitions(' + c + '=' + v + ')\n')
        file.write('\n')

    # No GUI/command line to set a different executable name at this stage
    executableName = params.projectName

    if params.wantCPP:
        file.write('add_executable(' + params.projectName + ' ' + params.projectName + '.cpp )\n\n')
    else:
        file.write('add_executable(' + params.projectName + ' ' + params.projectName + '.c )\n\n')

    file.write('pico_set_program_name(' + params.projectName + ' "' + executableName + '")\n')
    file.write('pico_set_program_version(' + params.projectName + ' "0.1")\n\n')

    if params.wantRunFromRAM:
        file.write('# no_flash means the target is to run from RAM\n')
        file.write('pico_set_binary_type(' + params.projectName + ' no_flash)\n\n')

    # Console output destinations
    if params.wantUART:
        file.write('pico_enable_stdio_uart(' + params.projectName + ' 1)\n')
    else:
        file.write('pico_enable_stdio_uart(' + params.projectName + ' 0)\n')

    if params.wantUSB:
        file.write('pico_enable_stdio_usb(' + params.projectName + ' 1)\n\n')
    else:
        file.write('pico_enable_stdio_usb(' + params.projectName + ' 0)\n\n')

    # Standard libraries
    file.write('# Add the standard library to the build\n')
    file.write('target_link_libraries(' + params.projectName + ' ' + STANDARD_LIBRARIES + ')\n\n')


    # Selected libraries/features
    if (params.features):
        file.write('# Add any user requested libraries\n')
        file.write('target_link_libraries(' + params.projectName + '\n')
        for feat in params.features:
            if (feat in features_list):
                file.write("        " + features_list[feat][LIB_NAME] + '\n')
        file.write('        )\n\n')

    file.write('pico_add_extra_outputs(' + params.projectName + ')\n\n')

    file.close()
Пример #48
0
 def relpath(self):
     """Return path relative to layout root"""
     root = object_session(self).query(LayoutInfo).first().root
     return str(Path(self.path).relative_to(root))
Пример #49
0
def test_root_dir():
    result = return_root_dir()
    assert Path(result).is_dir()
    print(result)
Пример #50
0
 def _path(self):
     return Path(self.path)
import json
import time
from dataclasses import dataclass
from typing import List
from dataclasses_json import dataclass_json
from pathlib import Path
from common import create_template, build_synonyms

version = int(round(time.time() * 1000))
Path("result/lor").mkdir(parents=True, exist_ok=True)


@dataclass_json
@dataclass
class CardData:
    name: str
    cardCode: str
    associatedCardRefs: List[str]
    collectible: bool
    supertype: str


def get_current_synonyms(card_name: str, synonyms: dict):
    if card_name in synonyms.keys():
        return synonyms[card_name]
    else:
        return []


glossary_template = create_template("data/lor/lor_card_glossary.html")
csv_row_template = create_template("data/csv_row.csv")
Пример #52
0
        print(m)
    sys.exit(-1)

if args.name == None and not args.gui and not args.list and not args.configs:
    print("No project name specfied\n")
    sys.exit(-1)

# load/parse any configuration dictionary we may have
LoadConfigurations()

p = CheckSDKPath(args.gui)

if p == None:
    sys.exit(-1)

sdkPath = Path(p)

if args.gui:
    RunGUI(sdkPath, args) # does not return, only exits

projectRoot = Path(os.getcwd())

if args.list or args.configs:
    if args.list:
        print("Available project features:\n")
        for feat in features_list:
            print(feat.ljust(6), '\t', features_list[feat][GUI_TEXT])
        print('\n')

    if args.configs:
        print("Available project configuration items:\n")
Пример #53
0
import numpy as np
import random
import copy
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).resolve().parent))
import Parameters.DE as DE

from Plot import plot

fbest = -1*sys.maxsize
Wbest = []

mi = DE.min
ma = DE.max
G = DE.G
P = DE.population
F = DE.F
CR = DE.CR

class DE:
	def __init__(self,func,N,**kwargs):
		self.game = func
		self.w_list = [[random.uniform(mi/10,ma/10) for i in range(N)] for j in range(P)]
		self.save = kwargs['save']
		
	class Population:
		def __init__(self,w,f):
			self.w = w
			self.f = f
Пример #54
0
    def __init__(self,
                 path,
                 img_size=640,
                 batch_size=16,
                 augment=False,
                 hyp=None,
                 rect=False,
                 image_weights=False,
                 cache_images=False,
                 single_cls=False,
                 stride=32,
                 pad=0.0):
        try:
            f = []  # image files
            for p in path if isinstance(path, list) else [path]:
                p = str(Path(p))  # os-agnostic
                parent = str(Path(p).parent) + os.sep
                if os.path.isfile(p):  # file
                    with open(p, 'r') as t:
                        t = t.read().splitlines()
                        f += [
                            x.replace('./', parent)
                            if x.startswith('./') else x for x in t
                        ]  # local to global path
                elif os.path.isdir(p):  # folder
                    f += glob.iglob(p + os.sep + '*.*')
                else:
                    raise Exception('%s does not exist' % p)
            self.img_files = sorted([
                x.replace('/', os.sep) for x in f
                if os.path.splitext(x)[-1].lower() in img_formats
            ])
        except Exception as e:
            raise Exception('Error loading data from %s: %s\nSee %s' %
                            (path, e, help_url))

        n = len(self.img_files)
        assert n > 0, 'No images found in %s. See %s' % (path, help_url)
        bi = np.floor(np.arange(n) / batch_size).astype(np.int)  # batch index
        nb = bi[-1] + 1  # number of batches

        self.n = n  # number of images
        self.batch = bi  # batch index of image
        self.img_size = img_size
        self.augment = augment
        self.hyp = hyp
        self.image_weights = image_weights
        self.rect = False if image_weights else rect
        self.mosaic = self.augment and not self.rect  # load 4 images at a time into a mosaic (only during training)
        self.mosaic_border = [-img_size // 2, -img_size // 2]
        self.stride = stride

        # Define labels
        self.label_files = [
            x.replace('images',
                      'labels').replace(os.path.splitext(x)[-1], '.txt')
            for x in self.img_files
        ]

        # Check cache
        cache_path = str(Path(
            self.label_files[0]).parent) + '.cache'  # cached labels
        if os.path.isfile(cache_path):
            cache = torch.load(cache_path)  # load
            if cache['hash'] != get_hash(self.label_files +
                                         self.img_files):  # dataset changed
                cache = self.cache_labels(cache_path)  # re-cache
        else:
            cache = self.cache_labels(cache_path)  # cache

        # Get labels
        labels, shapes = zip(*[cache[x] for x in self.img_files])
        self.shapes = np.array(shapes, dtype=np.float64)
        self.labels = list(labels)

        # Rectangular Training  https://github.com/ultralytics/yolov3/issues/232
        if self.rect:
            # Sort by aspect ratio
            s = self.shapes  # wh
            ar = s[:, 1] / s[:, 0]  # aspect ratio
            irect = ar.argsort()
            self.img_files = [self.img_files[i] for i in irect]
            self.label_files = [self.label_files[i] for i in irect]
            self.labels = [self.labels[i] for i in irect]
            self.shapes = s[irect]  # wh
            ar = ar[irect]

            # Set training image shapes
            shapes = [[1, 1]] * nb
            for i in range(nb):
                ari = ar[bi == i]
                mini, maxi = ari.min(), ari.max()
                if maxi < 1:
                    shapes[i] = [maxi, 1]
                elif mini > 1:
                    shapes[i] = [1, 1 / mini]

            self.batch_shapes = np.ceil(
                np.array(shapes) * img_size / stride + pad).astype(
                    np.int) * stride

        # Cache labels
        create_datasubset, extract_bounding_boxes, labels_loaded = False, False, False
        nm, nf, ne, ns, nd = 0, 0, 0, 0, 0  # number missing, found, empty, datasubset, duplicate
        pbar = tqdm(self.label_files)
        for i, file in enumerate(pbar):
            l = self.labels[i]  # label
            if l.shape[0]:
                assert l.shape[1] == 5, '> 5 label columns: %s' % file
                assert (l >= 0).all(), 'negative labels: %s' % file
                assert (l[:, 1:] <= 1).all(
                ), 'non-normalized or out of bounds coordinate labels: %s' % file
                if np.unique(l,
                             axis=0).shape[0] < l.shape[0]:  # duplicate rows
                    nd += 1  # print('WARNING: duplicate rows in %s' % self.label_files[i])  # duplicate rows
                if single_cls:
                    l[:, 0] = 0  # force dataset into single-class mode
                self.labels[i] = l
                nf += 1  # file found

                # Create subdataset (a smaller dataset)
                if create_datasubset and ns < 1E4:
                    if ns == 0:
                        create_folder(path='./datasubset')
                        os.makedirs('./datasubset/images')
                    exclude_classes = 43
                    if exclude_classes not in l[:, 0]:
                        ns += 1
                        # shutil.copy(src=self.img_files[i], dst='./datasubset/images/')  # copy image
                        with open('./datasubset/images.txt', 'a') as f:
                            f.write(self.img_files[i] + '\n')

                # Extract object detection boxes for a second stage classifier
                if extract_bounding_boxes:
                    p = Path(self.img_files[i])
                    img = cv2.imread(str(p))
                    h, w = img.shape[:2]
                    for j, x in enumerate(l):
                        f = '%s%sclassifier%s%g_%g_%s' % (
                            p.parent.parent, os.sep, os.sep, x[0], j, p.name)
                        if not os.path.exists(Path(f).parent):
                            os.makedirs(
                                Path(f).parent)  # make new output folder

                        b = x[1:] * [w, h, w, h]  # box
                        b[2:] = b[2:].max()  # rectangle to square
                        b[2:] = b[2:] * 1.3 + 30  # pad
                        b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int)

                        b[[0, 2]] = np.clip(b[[0, 2]], 0,
                                            w)  # clip boxes outside of image
                        b[[1, 3]] = np.clip(b[[1, 3]], 0, h)
                        assert cv2.imwrite(f, img[
                            b[1]:b[3],
                            b[0]:b[2]]), 'Failure extracting classifier boxes'
            else:
                ne += 1  # print('empty labels for image %s' % self.img_files[i])  # file empty
                # os.system("rm '%s' '%s'" % (self.img_files[i], self.label_files[i]))  # remove

            pbar.desc = 'Scanning labels %s (%g found, %g missing, %g empty, %g duplicate, for %g images)' % (
                cache_path, nf, nm, ne, nd, n)
        if nf == 0:
            s = 'WARNING: No labels found in %s. See %s' % (
                os.path.dirname(file) + os.sep, help_url)
            print(s)
            assert not augment, '%s. Can not train without labels.' % s

        # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM)
        self.imgs = [None] * n
        if cache_images:
            gb = 0  # Gigabytes of cached images
            pbar = tqdm(range(len(self.img_files)), desc='Caching images')
            self.img_hw0, self.img_hw = [None] * n, [None] * n
            for i in pbar:  # max 10k images
                self.imgs[i], self.img_hw0[i], self.img_hw[i] = load_image(
                    self, i)  # img, hw_original, hw_resized
                gb += self.imgs[i].nbytes
                pbar.desc = 'Caching images (%.1fGB)' % (gb / 1E9)
Пример #55
0
}


if os.getenv("SQL_DEBUG"):
    LOGGING["filters"]["require_debug_true"] = {
        "()": "django.utils.log.RequireDebugTrue"
    }
    LOGGING["handlers"]["console"] = {
        "level": "DEBUG",
        "filters": ["require_debug_true"],
        "class": "logging.StreamHandler",
    }
    LOGGING["loggers"]["django.db.backends"] = {
        "level": "DEBUG",
        "handlers": ["console"],
    }

INDY_HOLDER_ID = "TheOrgBook_Holder"
INDY_VERIFIER_ID = "TheOrgBook_Verifier"

custom_settings_file = Path(
    BASE_DIR,
    "custom_settings_" + str(os.getenv("TOB_THEME")).lower() + ".py",
)
if custom_settings_file.exists():
    with open(custom_settings_file) as source_file:
        print(
            "Loading custom settings file: {}".format(custom_settings_file.name)
        )
        exec(source_file.read())
def main(args):
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.multi_gpu is None else '0,1,2,3'
    '''CREATE DIR'''
    experiment_dir = Path('./experiment/')
    experiment_dir.mkdir(exist_ok=True)
    file_dir = Path(str(experiment_dir) +'/%sNuScenesSemSeg-'%args.model_name+ str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')))
    file_dir.mkdir(exist_ok=True)
    checkpoints_dir = file_dir.joinpath('checkpoints/')
    checkpoints_dir.mkdir(exist_ok=True)
    log_dir = file_dir.joinpath('logs/')
    log_dir.mkdir(exist_ok=True)

    '''LOG'''
    args = parse_args()
    logger = logging.getLogger(args.model_name)
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler = logging.FileHandler(str(log_dir) + '/train_%s_semseg.txt'%args.model_name)
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    logger.info('---------------------------------------------------TRANING---------------------------------------------------')
    logger.info('PARAMETER ...')
    logger.info(args)
    print('Load data...')

    nusc = NuScenes(version='v1.0-trainval', dataroot='/mnt/raid/chengnan/NuScenes/', verbose=True)
    dataset = NuscenesDatasetSemseg(nusc, npoints_lidar=16384, npoints_radar=1024, split='train_detect')
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=args.batchsize, collate_fn=dataset.collate_fn,
                                             shuffle=True, num_workers=int(args.workers))
    test_dataset = NuscenesDatasetSemseg(nusc, npoints_lidar=16384, npoints_radar=1024, split='val_small')
    testdataloader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batchsize, collate_fn=dataset.collate_fn,
                                                 shuffle=True, num_workers=int(args.workers))

    num_classes = 11
    model = PointNet2SemSeg5(num_classes)
    labelweight = torch.tensor([0.84423709, 3.562451, 8.4504371, 5.04570442, 1.69708204, 6.41300778, 6.44816675, 4.88638126, 5.20078234, 4.82712436, 3.74396562])
    loss_function = torch.nn.CrossEntropyLoss(labelweight.cuda())
    #loss_function = torch.nn.CrossEntropyLoss(reduction='none')

    if args.pretrain is not None:
        model.load_state_dict(torch.load(args.pretrain))
        print('load model %s'%args.pretrain)
        logger.info('load model %s'%args.pretrain)
    else:
        print('Training from scratch')
        logger.info('Training from scratch')
    pretrain = args.pretrain
    init_epoch = int(pretrain[-14:-11]) if args.pretrain is not None else 0

    if args.optimizer == 'SGD':
        optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
    elif args.optimizer == 'Adam':
        optimizer = torch.optim.Adam(
            model.parameters(),
            lr=args.learning_rate,
            betas=(0.9, 0.999),
            eps=1e-08,
            weight_decay=args.weight_decay
        )
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
    LEARNING_RATE_CLIP = 1e-5

    '''GPU selection and multi-GPU'''
    if args.multi_gpu is not None:
        device_ids = [int(x) for x in args.multi_gpu.split(',')]
        torch.backends.cudnn.benchmark = True
        model.cuda(device_ids[0])
        model = torch.nn.DataParallel(model, device_ids=device_ids)
    else:
        model.cuda()

    history = defaultdict(lambda: list())
    best_acc = 0
    best_acc_epoch = 0
    best_mIoU = 0
    best_mIoU_epoch = 0

    for epoch in range(init_epoch,args.epoch):
        scheduler.step()
        lr = max(optimizer.param_groups[0]['lr'],LEARNING_RATE_CLIP)
        print('Learning rate:%f' % lr)
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr
        train_loss_sum = 0.0
        train_acc_sum = 0.0
        for i, data in enumerate(dataloader):
            lidar_points, _, _, _, target, _, _, _   = data
            lidar_points, target = lidar_points.float(), target.long()
            lidar_points = lidar_points.transpose(2, 1)
            lidar_points, target = lidar_points.cuda(), target.cuda()
            optimizer.zero_grad()
            model = model.train()
            pred = model(lidar_points[:,:3,:],lidar_points[:,3:,:])
            #pred = model(lidar_points[:,:3,:],None)            
            loss = loss_function(pred, target)
            history['loss'].append(loss.item())
            train_loss_sum += loss.item()
            loss.backward()
            optimizer.step()
            # Train acc
            pred_val = torch.argmax(pred, 1)
            correct = torch.sum(((pred_val == target)&(target>0)).float())
            seen = torch.sum((target>0).float())+1e-08
            train_acc = correct/seen if seen!=0 else correct
            history['train_acc_per_iter'].append(train_acc.item())
            train_acc_sum += train_acc.item()
            if (i+1)%5 == 0:
                print('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc.item(), loss.item()))
                logger.info('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc.item(), loss.item()))
            if (i+1)%200 == 0:
                
                train_loss_avg = np.sum(history['loss'][-200:])/200.0
                train_acc_avg = np.sum(history['train_acc_per_iter'][-200:])/200.0
                history['train_it_acc'].append(train_acc_avg)
                print('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc_avg, train_loss_avg))
                logger.info('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc_avg, train_loss_avg))
                
                #Test acc
                test_losses = []
                total_correct = 0
                total_seen = 0
                total_correct_class = [0 for _ in range(num_classes)]
                total_seen_class = [0 for _ in range(num_classes)]
                total_intersection_class = [0 for _ in range(num_classes)]
                total_union_class = [0 for _ in range(num_classes)]
        
                for j, data in enumerate(testdataloader):
                    lidar_points, _, _, _, target, _, _, _   = data
                    lidar_points, target = lidar_points.float(), target.long()
                    lidar_points = lidar_points.transpose(2, 1)
                    lidar_points, target = lidar_points.cuda(), target.cuda()
                    model = model.eval()
                    with torch.no_grad():
                        pred = model(lidar_points[:,:3,:],lidar_points[:,3:,:])
                        #pred = model(lidar_points[:,:3,:],None)
                    loss = loss_function(pred, target)
                    test_losses.append(loss.item())
                    #first convert torch tensor to numpy array
                    pred_np = pred.cpu().numpy() #[B,C,N]
                    target_np = target.cpu().numpy() #[B,N]
                    # point wise acc
                    pred_val = np.argmax(pred_np, 1) #[B,N]
                    correct = np.sum((pred_val == target_np) & (target_np>0))
                    total_correct += correct
                    total_seen += np.sum(target_np>0)
            
            
                    # point wise acc and IoU per class
                    for l in range(num_classes):
                        total_seen_class[l] += np.sum((target_np==l))
                        total_correct_class[l] += np.sum((pred_val==l) & (target_np==l))
                        total_intersection_class[l] += np.sum((pred_val==l) & (target_np==l))
                        total_union_class[l] += np.sum(((pred_val==l) | (target_np==l)))
                
                test_loss = np.mean(test_losses)
                test_point_acc = total_correct/float(total_seen)
                history['test_it_point_acc'].append(test_point_acc)
                test_avg_class_point_acc = np.mean(np.array(total_correct_class[1:])/(np.array(total_seen_class[1:],dtype=np.float)+1e-6))
                history['test_it_avg_class_point_acc'].append(test_avg_class_point_acc)
                test_avg_class_point_IoU = np.mean(np.array(total_intersection_class[1:])/(np.array(total_union_class[1:],dtype=np.float)+1e-6))
                history['test_it_avg_class_point_IoU'].append(test_avg_class_point_IoU)

                print('[Epoch %d/%d] [Iteration %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), test_point_acc, test_loss))
                logger.info('[Epoch %d/%d] [Iteration %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), test_point_acc, test_loss))
                print('Whole scene point wise accuracy: %f' % (test_point_acc))
                logger.info('Whole scene point wise accuracy: %f' % (test_point_acc))
                print('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
                logger.info('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
                print('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
                logger.info('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
        
                per_class_point_str = 'point based --------\n'
                for l in range(0,num_classes):
                    per_class_point_str += 'class %d weight: %f, acc: %f, IoU: %f;\n' % (l,labelweight[l],total_correct_class[l]/float(total_seen_class[l]), total_intersection_class[l]/(float(total_union_class[l])+1e-6))
                logger.info(per_class_point_str)                
                if test_point_acc > best_acc:
                    best_acc = test_point_acc
                    best_acc_epoch = epoch+1
                    torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestacc.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_acc))
                    logger.info('Save best acc model..')
                    print('Save best acc model..')
                if test_avg_class_point_IoU > best_mIoU:
                    best_mIoU = test_avg_class_point_IoU
                    best_mIoU_epoch = epoch+1
                    torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestmIoU.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_mIoU))
                    logger.info('Save best mIoU model..')
                    print('Save best mIoU model..')                
                
                
        train_loss_avg = train_loss_sum/len(dataloader)
        train_acc_avg = train_acc_sum/len(dataloader)
        history['train_acc'].append(train_acc_avg)
        print('[Epoch %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, train_acc_avg, train_loss_avg))
        logger.info('[Epoch %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, train_acc_avg, train_loss_avg))
        
        #Test acc
        test_losses = []
        total_correct = 0
        total_seen = 0
        total_correct_class = [0 for _ in range(num_classes)]
        total_seen_class = [0 for _ in range(num_classes)]
        total_intersection_class = [0 for _ in range(num_classes)]
        total_union_class = [0 for _ in range(num_classes)]
        
        for j, data in enumerate(testdataloader):
            lidar_points, _, _, _, target, _, _, _   = data
            lidar_points, target = lidar_points.float(), target.long()
            lidar_points = lidar_points.transpose(2, 1)
            lidar_points, target = lidar_points.cuda(), target.cuda()
            model = model.eval()
            with torch.no_grad():
                pred = model(lidar_points[:,:3,:],lidar_points[:,3:,:])
                #pred = model(lidar_points[:,:3,:],None)
            loss = loss_function(pred, target)
            test_losses.append(loss.item())
            #first convert torch tensor to numpy array
            pred_np = pred.cpu().numpy() #[B,C,N]
            target_np = target.cpu().numpy() #[B,N]
            # point wise acc
            pred_val = np.argmax(pred_np, 1) #[B,N]
            correct = np.sum((pred_val == target_np) & (target_np>0))
            total_correct += correct
            total_seen += np.sum(target_np>0)
            
            
            # point wise acc and IoU per class
            for l in range(num_classes):
                total_seen_class[l] += np.sum((target_np==l))
                total_correct_class[l] += np.sum((pred_val==l) & (target_np==l))
                total_intersection_class[l] += np.sum((pred_val==l) & (target_np==l))
                total_union_class[l] += np.sum(((pred_val==l) | (target_np==l)))
                
        test_loss = np.mean(test_losses)
        test_point_acc = total_correct/float(total_seen)
        history['test_point_acc'].append(test_point_acc)
        test_avg_class_point_acc = np.mean(np.array(total_correct_class[1:])/(np.array(total_seen_class[1:],dtype=np.float)+1e-6))
        history['test_avg_class_point_acc'].append(test_avg_class_point_acc)
        test_avg_class_point_IoU = np.mean(np.array(total_intersection_class[1:])/(np.array(total_union_class[1:],dtype=np.float)+1e-6))
        history['test_avg_class_point_IoU'].append(test_avg_class_point_IoU)

        print('[Epoch %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, test_point_acc, test_loss))
        logger.info('[Epoch %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, test_point_acc, test_loss))
        print('Whole scene point wise accuracy: %f' % (test_point_acc))
        logger.info('Whole scene point wise accuracy: %f' % (test_point_acc))
        print('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
        logger.info('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
        print('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
        logger.info('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
        
        per_class_point_str = 'point based --------\n'
        for l in range(0,num_classes):
            per_class_point_str += 'class %d weight: %f, acc: %f, IoU: %f;\n' % (l,labelweight[l],total_correct_class[l]/float(total_seen_class[l]), total_intersection_class[l]/(float(total_union_class[l])+1e-6))
        logger.info(per_class_point_str)
            
        if (epoch+1) % 1 == 0:
            torch.save(model.state_dict(), '%s/%s_%.3d_%.4d.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1))
            logger.info('Save model..')
            print('Save model..')
        if test_point_acc > best_acc:
            best_acc = test_point_acc
            best_acc_epoch = epoch+1
            torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestacc.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_acc))
            logger.info('Save best acc model..')
            print('Save best acc model..')
        if test_avg_class_point_IoU > best_mIoU:
            best_mIoU = test_avg_class_point_IoU
            best_mIoU_epoch = epoch+1
            torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestmIoU.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_mIoU))
            logger.info('Save best mIoU model..')
            print('Save best mIoU model..')
    print('Best point wise accuracy is %f at epoch %d.' % (best_acc, best_acc_epoch))
    logger.info('Best point wise accuracy is %f at epoch %d.' % (best_acc, best_acc_epoch))
    print('Best class averaged point wise IoU is %f at epoch %d.' % (best_mIoU, best_mIoU_epoch))
    logger.info('Best class averaged point wise IoU is %f at epoch %d.' % (best_mIoU, best_mIoU_epoch))
    plot_loss_curve(history['loss'], str(log_dir))
    plot_acc_curve(history['train_it_acc'], history['test_it_point_acc'], str(log_dir))
    #plot_acc_curve(history['train_it_acc'], history['test_it_avg_class_point_IoU'], str(log_dir))
    print('FINISH.')
    logger.info('FINISH')
Пример #57
0
def main(args=None):
    set_random_seed(63)
    chainer.global_config.autotune = True
    chainer.cuda.set_max_workspace_size(512 * 1024 * 1024)
    parser = argparse.ArgumentParser()
    parser.add_argument('--batchsize',
                        '-b',
                        type=int,
                        default=64,
                        help='Number of images in each mini-batch')
    parser.add_argument('--learnrate',
                        '-l',
                        type=float,
                        default=0.01,
                        help='Learning rate for SGD')
    parser.add_argument('--epoch',
                        '-e',
                        type=int,
                        default=80,
                        help='Number of sweeps over the dataset to train')
    parser.add_argument('--gpu',
                        '-g',
                        type=int,
                        default=0,
                        help='GPU ID (negative value indicates CPU)')
    parser.add_argument('--out',
                        '-o',
                        default='result',
                        help='Directory to output the result')
    parser.add_argument('--resume',
                        '-r',
                        default='',
                        help='Resume the training from snapshot')
    parser.add_argument('--loss-function',
                        choices=['focal', 'sigmoid'],
                        default='focal')
    parser.add_argument('--optimizer',
                        choices=['sgd', 'adam', 'adabound'],
                        default='adam')
    parser.add_argument('--size', type=int, default=224)
    parser.add_argument('--limit', type=int, default=None)
    parser.add_argument('--data-dir', type=str, default='data')
    parser.add_argument('--lr-search', action='store_true')
    parser.add_argument('--pretrained', type=str, default='')
    parser.add_argument('--backbone',
                        choices=['resnet', 'seresnet', 'debug_model'],
                        default='resnet')
    parser.add_argument('--log-interval', type=int, default=100)
    parser.add_argument('--find-threshold', action='store_true')
    parser.add_argument('--finetune', action='store_true')
    parser.add_argument('--mixup', action='store_true')
    args = parser.parse_args() if args is None else parser.parse_args(args)

    print(args)

    if args.mixup and args.loss_function != 'focal':
        raise ValueError('mixupを使うときはfocal lossしか使えません(いまんところ)')

    train, test, cooccurrence = get_dataset(args.data_dir, args.size,
                                            args.limit, args.mixup)
    base_model = backbone_catalog[args.backbone](args.dropout)

    if args.pretrained:
        print('loading pretrained model: {}'.format(args.pretrained))
        chainer.serializers.load_npz(args.pretrained, base_model, strict=False)
    model = TrainChain(base_model,
                       1,
                       loss_fn=args.loss_function,
                       cooccurrence=cooccurrence,
                       co_coef=0)
    if args.gpu >= 0:
        chainer.backends.cuda.get_device_from_id(args.gpu).use()
        model.to_gpu()

    if args.optimizer in ['adam', 'adabound']:
        optimizer = Adam(alpha=args.learnrate,
                         adabound=args.optimizer == 'adabound',
                         weight_decay_rate=1e-5,
                         gamma=5e-7)
    elif args.optimizer == 'sgd':
        optimizer = chainer.optimizers.MomentumSGD(lr=args.learnrate)

    optimizer.setup(model)

    if not args.finetune:
        print('最初のエポックは特徴抽出層をfreezeします')
        model.freeze_extractor()

    train_iter = chainer.iterators.MultiprocessIterator(train,
                                                        args.batchsize,
                                                        n_processes=8,
                                                        n_prefetch=2)
    test_iter = chainer.iterators.MultithreadIterator(test,
                                                      args.batchsize,
                                                      n_threads=8,
                                                      repeat=False,
                                                      shuffle=False)

    if args.find_threshold:
        # train_iter, optimizerなど無駄なsetupもあるが。。
        print('thresholdを探索して終了します')
        chainer.serializers.load_npz(join(args.out, 'bestmodel_loss'),
                                     base_model)
        print('lossがもっとも小さかったモデルに対しての結果:')
        find_threshold(base_model, test_iter, args.gpu, args.out)

        chainer.serializers.load_npz(join(args.out, 'bestmodel_f2'),
                                     base_model)
        print('f2がもっとも大きかったモデルに対しての結果:')
        find_threshold(base_model, test_iter, args.gpu, args.out)
        return

    # Set up a trainer
    updater = training.updaters.StandardUpdater(
        train_iter,
        optimizer,
        device=args.gpu,
        converter=lambda batch, device: chainer.dataset.concat_examples(
            batch, device=device))
    trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)

    # Evaluate the model with the test dataset for each epoch
    trainer.extend(FScoreEvaluator(test_iter, model, device=args.gpu))

    if args.optimizer == 'sgd':
        # Adamにweight decayはあんまりよくないらしい
        optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(5e-4))
        trainer.extend(extensions.ExponentialShift('lr', 0.1),
                       trigger=(3, 'epoch'))
        if args.lr_search:
            print('最適な学習率を探します')
            trainer.extend(LRFinder(1e-7, 1, 5, optimizer),
                           trigger=(1, 'iteration'))
    elif args.optimizer in ['adam', 'adabound']:
        if args.lr_search:
            print('最適な学習率を探します')
            trainer.extend(LRFinder(1e-7, 1, 5, optimizer, lr_key='alpha'),
                           trigger=(1, 'iteration'))

        trainer.extend(extensions.ExponentialShift('alpha', 0.2),
                       trigger=triggers.EarlyStoppingTrigger(
                           monitor='validation/main/loss'))

    # Take a snapshot of Trainer at each epoch
    trainer.extend(
        extensions.snapshot(filename='snaphot_epoch_{.updater.epoch}'),
        trigger=(10, 'epoch'))

    # Take a snapshot of Model which has best val loss.
    # Because searching best threshold for each evaluation takes too much time.
    trainer.extend(extensions.snapshot_object(model.model, 'bestmodel_loss'),
                   trigger=triggers.MinValueTrigger('validation/main/loss'))
    trainer.extend(extensions.snapshot_object(model.model, 'bestmodel_f2'),
                   trigger=triggers.MaxValueTrigger('validation/main/f2'))
    trainer.extend(extensions.snapshot_object(model.model,
                                              'model_{.updater.epoch}'),
                   trigger=(5, 'epoch'))

    # Write a log of evaluation statistics for each epoch
    trainer.extend(
        extensions.LogReport(trigger=(args.log_interval, 'iteration')))

    trainer.extend(
        extensions.PrintReport([
            'epoch', 'lr', 'elapsed_time', 'main/loss', 'main/co_loss',
            'validation/main/loss', 'validation/main/co_loss',
            'validation/main/precision', 'validation/main/recall',
            'validation/main/f2', 'validation/main/threshold'
        ]))

    trainer.extend(extensions.ProgressBar(update_interval=args.log_interval))
    trainer.extend(extensions.observe_lr(),
                   trigger=(args.log_interval, 'iteration'))
    trainer.extend(CommandsExtension())
    save_args(args, args.out)

    trainer.extend(lambda trainer: model.unfreeze_extractor(),
                   trigger=(1, 'epoch'))

    if args.resume:
        # Resume from a snapshot
        chainer.serializers.load_npz(args.resume, trainer)

    # save args with pickle for prediction time
    pickle.dump(args, open(str(Path(args.out).joinpath('args.pkl')), 'wb'))

    # Run the training
    trainer.run()

    # find optimal threshold
    chainer.serializers.load_npz(join(args.out, 'bestmodel_loss'), base_model)
    print('lossがもっとも小さかったモデルに対しての結果:')
    find_threshold(base_model, test_iter, args.gpu, args.out)

    chainer.serializers.load_npz(join(args.out, 'bestmodel_f2'), base_model)
    print('f2がもっとも大きかったモデルに対しての結果:')
    find_threshold(base_model, test_iter, args.gpu, args.out)
Пример #58
0
import numpy as np
from pathlib import Path
import xarray
import ctypes as ct
import os

from .build import build, get_libpath

SDIR = Path(__file__).parent
BDIR = SDIR / "build"

# NOTE: must be str() for Windows, even with py37
dllfn = get_libpath(BDIR, "wmm15")
if dllfn is not None:
    libwmm = ct.cdll.LoadLibrary(str(dllfn))
else:
    build()
    dllfn = get_libpath(BDIR, "wmm20")
    if dllfn:
        libwmm = ct.cdll.LoadLibrary(str(dllfn))
    else:
        raise ModuleNotFoundError(f"could not find {dllfn}")


def wmm(glats: np.ndarray, glons: np.ndarray, alt_km: float,
        yeardec: float) -> xarray.Dataset:
    """
    wmm computes the value of the world magnetic model at grid points specified by glats and
    glons, for a single altitude value. glats and glons should be in degrees.

    glats and glons should be generated from something like np.meshgrid, so they should be
        d = {}
        listDoctors = get_all_doctor_ids()
        for i in listDoctors:
            doctorURI=getDoctorData(i)
            doctorDATA=get_uri(doctorURI)
            d[str(i)] = str(doctorDATA['Doctor Name :'])

        h = {}
        listHospitals=get_all_hospital_ids()
        for i in listHospitals:
            hospitalURI = getHospitalData(i)
            hospitalDATA = get_uri(hospitalURI)
            h[str(i)] = str(hospitalDATA['Hospital Name :'])

         
        output_path = Path('PatientReport.txt')

        text = [
            {"Input date" : patientDATA['Date & Time :'][0],
                "Input time" : patientDATA['Date & Time :'][1],
                "Patient's name" : patientDATA['Patient Name :'],
                "Patient's gender" : patientDATA['Patient Gender :'],
                "Doctor's name" : patientDATA['Doctor Name :'],
                "Hospital name" : patientDATA['Hospital Name :']
                },
                "DOCTOR VISITS",
                "________________________________________",
                L,
                "LIST OF DOCTORS",
                "________________________________________",
                d,
Пример #60
0
# builtin imports
import pickle
import re

# A lot of functions from os.path
# in python 2 moved to os. and changed
# their signature. Pathlib can be
# installed under python2.X with
# pip install pathlib2 and is in
# standard library in Python 3,
# hence we use it as a compatiblity
# library
try:
    from pathlib import Path
    Path().expanduser()
except (ImportError, AttributeError):
    from pathlib2 import Path


# other library imports
import Levenshtein
import ase.atoms
import ase.io
import numpy as np


# local imports
from .ase_tools import gas_phase_references

np.set_printoptions(threshold=500, linewidth=1800, edgeitems=80)