class DirectoryLocator(Locator):
def __init__(self, path, recurse=False):
self.path = Path(path)
self.recurse = recurse
self._filemap = None
@property
def filemap(self):
"""dist -> ver -> path mapping"""
if self._filemap is None:
self._filemap = defaultdict(lambda: defaultdict(list))
if self.recurse:
files = self.path.rglob("*")
else:
files = self.path.glob("*")
for f in files:
filetype, dist, ver = parse_filename(f.name)
self._filemap[dist][ver].append(str(f.resolve()))
return self._filemap
def distributions(self):
return sorted(self.filemap.keys())
def versions(self, distribution):
return sorted(self.filemap[distribution.lower().replace('-','_')].keys())
def get(self, distribution, version):
# Reformat the data...
return self.filemap[distribution.lower().replace('-','_')][version]
def getbasetemp(self):
""" return base temporary directory. """
if self._basetemp is None:
if self._given_basetemp is not None:
basetemp = Path(self._given_basetemp)
ensure_reset_dir(basetemp)
else:
from_env = os.environ.get("PYTEST_DEBUG_TEMPROOT")
temproot = Path(from_env or tempfile.gettempdir())
user = get_user() or "unknown"
# use a sub-directory in the temproot to speed-up
# make_numbered_dir() call
rootdir = temproot.joinpath("pyt-{}".format(user))
rootdir.mkdir(exist_ok=True)
basetemp = make_numbered_dir_with_cleanup(
prefix="",
root=rootdir,
keep=3,
lock_timeout=LOCK_TIMEOUT,
)
assert basetemp is not None
self._basetemp = t = basetemp
self._trace("new basetemp", t)
return t
else:
return self._basetemp
def collect_json(source_dir):
ret = []
source_path = Path(source_dir)
for file_path in source_path.glob('*.json'):
with file_path.open() as fp:
ret.append(json.load(fp))
return sorted(ret, key=lambda each: each['name'])
def main():
"""Print virtualenv and python version."""
workon_home = os.environ.get('WORKON_HOME')
workon_home = Path(workon_home)
for virtualenv in workon_home.iterdir():
if virtualenv.is_dir():
for python_bin in Path(f'{virtualenv}/bin/').iterdir():
if python_bin.name == 'python':
virtual_environment = str(virtualenv).rpartition('/')[-1]
command = [f'{python_bin}',
'-c',
"import sys;print(sys.version.split()[0]);"
]
stdout, _ = Popen(command, stdout=PIPE).communicate()
stdout = stdout.decode('utf-8')
python_version = stdout.strip()
if python_bin.name == 'pip':
command = [f'{python_bin}',
'freeze'
]
stdout, _ = Popen(command, stdout=PIPE).communicate()
stdout = stdout.decode('utf-8')
packages = [p.strip() for p in stdout.split()]
with open(f'virtualenvs-{os.uname()[1].split(".")[0]}.md', 'a') as f:
f.write(template.render(virtualenv=virtual_environment,
version=python_version,
packages=packages))
def execute(self, context):
w = context.world
if w:
# First, verify this is a valid Uru directory...
path = Path(self.filepath)
# Blendsucks likes to tack filenames onto our doggone directories...
if not path.is_dir():
path = path.parent
if not ((path / "UruExplorer.exe").is_file() or (path / "plClient.exe").is_file()):
self.report({"ERROR"}, "The selected directory is not a copy of URU.")
return {"CANCELLED"}
# New game?
games = w.plasma_games
new_game = self.game_index == -1
if new_game:
games.active_game_index = len(games.games)
game = games.games.add()
else:
game = games.games[self.game_index]
# Setup game...
game.path = str(path)
if (path / "cypython22.dll").is_file():
game.version = "pvPots"
else:
game.version = "pvMoul"
game.name = path.name
return {"FINISHED"}
else:
return {"CANCELLED"}
def iter_file(path, formatter=lambda x: x):
path = Path(path)
if not path.exists():
abort('Path does not exist: {}'.format(path))
with path.open() as f:
for l in f:
yield formatter(l)
def _setup_user_dir(self):
"""Returns user config dir, create it when it doesn't exist."""
user_dir = Path(os.path.expanduser('~/.thefuck'))
rules_dir = user_dir.joinpath('rules')
if not rules_dir.is_dir():
rules_dir.mkdir(parents=True)
self.user_dir = user_dir
def extract_notes_from_summary(notes, summary_file):
new_summary = Path(summary_file.name).read_text()
# Remove comments.
new_summary = new_summary.split(NEWLINE)
new_summary = [l for l in new_summary if not l.startswith(DATE_PREFIX)]
new_summary = NEWLINE.join(new_summary)
segments = new_summary.split(NOTE_HEADER)[1:]
if len(segments) > len(notes):
raise Exception("Notes were added in the process of editing the summary file.")
if len(segments) < len(notes):
raise Exception("Notes were removed in the process of editing the summary file.")
extracted_notes = type(notes)()
for (path, note), seg in zip(notes.items(), segments):
lines = seg.split(NEWLINE)
lines = lines[1:]
seg = NEWLINE.join(lines)
seg = seg.strip()
if cfg['show_tags']:
_note = Note.from_string(path, seg)
else:
_note = Note(path, seg, note.tags)
extracted_notes[path] = _note
return extracted_notes
def test_save(im_format="png", ask_input=False):
img_path = "test.{}".format(im_format)
fig = plt.figure()
data = [i for i in range(20)]
plt.plot(data)
fig.tight_layout()
try:
fig.savefig(img_path)
plt.close(fig)
resp = ""
if ask_input:
resp = input("Do you want to check the generated image? (yes/no):").strip().lower()
if resp != "yes":
p = Path(img_path)
p.unlink()
except Exception as exc:
print("Error: Saving {} does not work")
print(exc)
return -1
return 0
def main():
""" Main function for program """
global doc_counter
global reading
global vocab
global word_count
global doc_words
global doc_word_count
global doc_info_string
if len(sys.argv) != 3:
print("usage: .py <input_directory> <output_directory>\n")
sys.exit(1)
input_directory = Path(sys.argv[1])
output_directory = sys.argv[2]
for year_directory in input_directory.iterdir():
for day_file in year_directory.iterdir():
print('Processing: {}'.format(day_file))
with day_file.open() as f:
lines = f.readlines()
for line in lines:
process_line(line, output_file)
vocab_file = open('vocab.txt', 'w')
print(len(vocab))
for word in vocab:
vocab_file.write("%s\n" % word)
def main():
# get raw
report = Path(prefix + '_scrape/pdf').glob('*.pdf')
# locate storage
archive = scrape_util.ArchiveFolder(argv, prefix)
# write csv file for each historical report
for this_report in report:
# read raw
system(scrape_util.pdftotext.format(str(this_report)))
temp_txt = this_report.with_suffix('.txt')
with temp_txt.open('r') as io:
line = [this_line.strip() for this_line in io.readlines()]
temp_txt.unlink()
sale_date = get_sale_date(line)
io_name = Path(prefix + '_scrape/' + prefix + '_' + sale_date.strftime('%y-%m-%d') + '.csv')
sale_head = get_sale_head(line)
this_default_sale = default_sale.copy()
this_default_sale.update({
'sale_year': sale_date.year,
'sale_month': sale_date.month,
'sale_day': sale_date.day,
'sale_head': sale_head,
})
with io_name.open('w') as io:
writer = csv.DictWriter(io, scrape_util.header, lineterminator='\n')
writer.writeheader()
write_sale(line, this_default_sale, writer)
def main():
requirements_dir = Path(__file__).parent / '..' / 'requirements'
for requirement_file in requirements_dir.glob('*.txt'):
print(requirement_file.name)
with open(str(requirement_file), 'r') as f:
for req in f:
# Remove trailing and leading whitespace.
req = req.strip()
if not req:
# skip empty or white space only lines
continue
elif req.startswith('#'):
continue
# Get the name of the package
req = re.split('<|>|=|!|;', req)[0]
try:
# use pkg_resources to reliably get the version at install
# time by package name. pkg_resources needs the name of the
# package from pip, and not "import".
# e.g. req is 'scikit-learn', not 'sklearn'
version = pkg_resources.get_distribution(req).version
print(req.rjust(20), version)
except pkg_resources.DistributionNotFound:
print(req.rjust(20), 'is not installed')
def __init__(self, name, prefix, directory, *,
expect_handler=None, chunk_size=256*1024,
response_factory=StreamResponse):
assert prefix.startswith('/'), prefix
assert prefix.endswith('/'), prefix
super().__init__(
'GET', self.handle, name, expect_handler=expect_handler)
self._prefix = prefix
self._prefix_len = len(self._prefix)
try:
directory = Path(directory)
if str(directory).startswith('~'):
directory = Path(os.path.expanduser(str(directory)))
directory = directory.resolve()
if not directory.is_dir():
raise ValueError('Not a directory')
except (FileNotFoundError, ValueError) as error:
raise ValueError(
"No directory exists at '{}'".format(directory)) from error
self._directory = directory
self._chunk_size = chunk_size
self._response_factory = response_factory
if bool(os.environ.get("AIOHTTP_NOSENDFILE")):
self._sendfile = self._sendfile_fallback
def joinpath(*paths, resolve=False):
"""Helper function to determine paths in config file"""
base_dir = Path(__file__).parent.parent.resolve()
result = base_dir.joinpath(*paths)
if resolve:
result.resolve()
return str(result)
def save_to_dropbox_and_server(card_image, dropbox_path):
on_server = Path(dropbox_destination.lstrip("/"))
if not on_server.parent.exists():
on_server.parent.mkdir(parents=True)
card_image.save(on_server)
with on_server.open("rb") as image_file:
self.dropbox.put_file(dropbox_path, image_file)
def main():
############ prepare values
options, arguments = arguments_parse()
text = stdin_read()
contentId = options.force_id if options.force_id is not None else contentId_obtain(text, options.id_field)
diffPath = config.DIFF_PATH % contentId
txtPath = config.TXT_PATH % contentId
filepath = Path(txtPath)
############ do your job
### No revised file found ? Why are you even calling me ?
if not filepath.is_file():
y_error('\n -- No new revision file found\n')
print('\n -- No new revision file found\n')
exit(1)
### Ok, go on..
textNew = f_read(txtPath)
dmp = diff_match_patch()
rawPatch = dmp.patch_make(text, textNew)
txtPatch = dmp.patch_toText(rawPatch)
f_write(diffPath, txtPatch)
print("\nPatch for " + contentId + " Written !\n")
############ job's done !
exit(0)
def main():
global run, summarize_log
seed(37429)
args = argparser.parse_args()
outdir = Path(args.outdir, data_file_stem(args.data))
if not outdir.exists():
outdir.mkdir(parents=True)
summarize_log = \
partial(summarize_log, args.points, args.step_bin, args.node_bin, outdir)
run = partial(run, args.executable, args.data, args.keep_history)
summaries = [{} for _ in range(args.history + 1)]
naive_summary = None
if 'naive' in args.algorithm:
run(args.history, 'naive')
naive_summary = summarize_log('naive')
for h in range(1, args.history + 1):
sys.stderr.write('HISTORY {}\n'.format(h))
sys.stderr.flush()
summaries[h]['naive'] = naive_summary
for a in args.algorithm:
if a == 'naive':
continue
run(h, a)
summaries[h][a] = summarize_log('{}-{}'.format(a, h))
prof_start()
for a in args.algorithm:
keys = sorted(summaries[1][a].keys())
for k in keys:
with Path(outdir, '{}-{}.json'.format(a, k)).open('w') as out:
json.dump(
[(h, summaries[h][a][k]) for h in range(1, args.history + 1)], out)
prof_stop('save_across_history')
def _cli():
########################################################################
parser = argparse.ArgumentParser(
description="Check whether loci changed or not by signal metrics (raw, rpm, "
"rpkm) for interesting loci sets",
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('--signal', metavar="PATH",
# /mnt/stripe/bio/experiments/signal_consensus_peaks/
# /mnt/stripe/bio/experiments/signal_loci_of_interest/
default="/mnt/stripe/bio/experiments/signal",
help="Processed signal dir")
parser.add_argument('-o', '--out', required=True, metavar="PATH",
help="Output dir")
args = parser.parse_args()
signal_root = Path(args.signal)
# /mnt/stripe/bio/experiments/aging/signal@loci_stats
results_dir = Path(args.out)
results_dir.mkdir(parents=True, exist_ok=True)
########################################################################
# signal_dfs_by_datatype, signal_dfs_by_loci = build_signal_dfs(results_dir, signal_root)
# print(signal_dfs_by_datatype[("H3K4me1", "rpkm")].head())
stats_test(results_dir, signal_root)
def test_all():
cwd = Path(getcwd())
all_omts = [p.as_posix() for p in cwd.glob('**/*.omt')]
th = TallyHolder()
if environ.get('TRAVIS'):
if not environ.get('OMV_ENGINE'):
tallies = [parse_omt(t) for t in all_omts]
else:
engine = environ.get('OMV_ENGINE').lower()
tallies = [parse_omt(t)
for t in all_omts
if load_yaml(t)['engine'].lower() == engine]
else:
tallies = [parse_omt(t) for t in all_omts]
for t in tallies:
th.add(t)
results = [t.all_passed() for t in tallies]
inform('')
inform("%i test(s) run" % len(tallies),
overline='-', underline='-', center=True)
inform('')
if all(results):
inform("All tests passing!", underline='=', center=True)
else:
failed = [trim_path(t.omt) for t in tallies if not t.all_passed()]
inform("Some test(s) failed: ", failed, underline='=')
if is_verbose():
print('\n'+th.summary()+'\n')
assert all(results)
def extractLog (gameLog, sessionType, logType):
'''
Extracts logs from compressed game log file, if not already extracted.
Returns path to state log.
'''
if year != 2016:
gameIdRe = re.compile('game-(\d+)-{}-logs.tar.gz'.format(sessionType))
elif year == 2016:
gameIdRe = re.compile('game-(\d+)-{}.tar.gz'.format(sessionType))
path = Path(gameLog)
m = gameIdRe.search(str(path))
if m:
gameId = m.group(1)
if year != 2016:
logPath = Path(path.parent, 'log',
'powertac-{}-{}.{}'.format(sessionType, gameId, logType))
elif year == 2016:
logPath = Path(path.parent, 'log',
'powertac-{}-2016_finals_{}.{}'.format(sessionType, str(int(gameId)-654), logType))
if not logPath.exists():
pathdir = path.parent
path = path.as_posix()
pathdir = pathdir.as_posix()
tar = tarfile.open(path)
tar.extractall(pathdir)
tar.close()
return logPath
else:
gameId = 'xx'
print('Failed to find game ID in ' + str(path))
return False
def download_data_locally(self, start_year=2001, end_year=2011, local_dir="/BIG1/huziy/noaa_oisst_daily"):
"""
Download the data to a local server
:param start_year:
:param end_year:
:param local_dir:
"""
for y in range(start_year, end_year + 1):
# --
for month in range(1, 13):
for day in range(1, calendar.monthrange(y, month)[1] + 1):
file_url = self.get_url_for(year=y, month=month, day=day)
print("Fetching {} ...".format(file_url))
try:
with xr.open_dataset(file_url) as ds:
# assert isinstance(ds, xr.Dataset)
local_filepath = Path(local_dir) / Path(file_url).name
if local_filepath.exists():
print("{} already exists, skipping ...")
continue
ds.to_netcdf(path=str(local_filepath))
print("Saved {} to {}".format(file_url, local_filepath))
except OSError as err:
print(err)
print("Could not find {}".format(file_url))
def micro(ctx, publish, write, filetype):
"""Create a micro post"""
# Create a new file in the micro directory
now = datetime.datetime.today()
filename = "{}-{}.{}".format(str(now.day) + str(now.month) + str(now.year), str(now.second) + str(now.minute) + str(now.hour) ,filetype)
#blotpath = Path.home() / 'Rambling' / 'hill'
blotpath = os.getenv("BLOTDIR")
if blotpath:
micropath = Path(blotpath) / 'micro'
micropath = os.path.expanduser(micropath)
#click.echo(micropath)
# should check if the micropath exists
if not Path(micropath).exists():
click.echo("A micro directory does not exist. Creating one now...")
p = Path(micropath)
p.mkdir()
if write:
with open(str(micropath) + '/' + filename, mode='w') as f:
f.write("Tags: micropost\n\n" + write)
else:
#click.echo("Open {}!".format(os.getenv("EDITOR"))
subprocess.call(["nano", "{}/{}".format(micropath, filename)])
if publish:
ctx.invoke(publish)
else:
no_blotpath()
def score(truth_path: pathlib.Path, prediction_path: pathlib.Path) -> Dict[str, Dict[str, float]]:
for truth_file in truth_path.iterdir():
if re.match(r'^ISIC.*GroundTruth\.csv$', truth_file.name):
break
else:
raise ScoreException('Internal error, truth file could not be found.')
prediction_files = [
prediction_file
for prediction_file in prediction_path.iterdir()
if prediction_file.suffix.lower() == '.csv'
]
if len(prediction_files) > 1:
raise ScoreException(
'Multiple prediction files submitted. Exactly one CSV file should be submitted.'
)
elif len(prediction_files) < 1:
raise ScoreException(
'No prediction files submitted. Exactly one CSV file should be submitted.'
)
prediction_file = prediction_files[0]
with truth_file.open('rb') as truth_file_stream, prediction_file.open(
'rb'
) as prediction_file_stream:
return compute_metrics(truth_file_stream, prediction_file_stream)
def read_cargo_toml(key, manifest=None):
"""Read a value from Cargo.toml manifest.
:param key: Key to read from [package], or entire path to a key.
It may contain dots.
:param manifest: Optional path to the manifest,
or a file-like object with it opened
:return: The value
"""
if not isinstance(key, (list, tuple)):
key = (key,) if key.startswith('package.') else ('package', key)
key = list(chain.from_iterable(k.split('.') for k in key))
if not key:
raise ValueError("key must not be empty")
# Read the content of Cargo.toml.
manifest = manifest or Path.cwd() / 'Cargo.toml'
if hasattr(manifest, 'read'):
content = toml.load(manifest)
else:
manifest = Path(manifest)
with manifest.open() as f:
content = toml.load(f)
# Get the value.
value = content
for k in key:
value = value[k]
return value
def __init__(self, projdescpath, verify=True, project=None, ):
projdescpath = Path(str(projdescpath)).resolve()
if not projdescpath.exists():
raise Exception(projdescpath)
projdesc = Json.load_json_from(projdescpath)
self.projdesc = projdesc
for i in ["testinfo", "testfolder", "projectfolder"]:
key = ("experiment_config",i)
if not projdesc[key]:
print('Missing:\n\n File "{key}", line 1, in {projdescpath}\n\n'.format(projdescpath=projdescpath, key=("experiment_config",i)), file=sys.stdout)
raise ValueError("Missing key in config: `{}` from: `{}`".format(key, projdescpath))
self._testinfo = generatetestinfoclass(**projdesc["experiment_config"]["testinfo"])
names = self.projdesc.experiment_config.name.split('|' if '|' in self.projdesc.experiment_config.name else ';')
self.experiment_name = names[0]
self.test_name = names[1:]
self.project = projdescpath.parent if not project else project
files = DataTree(projdesc.experiment_config["projectfolder"]["filestructure"])
self._files = self.parsefolders(files, verify, parent=self.project)
for name, file in self._files.items():
self[name] = file
def isnew(needfixes):
#This function prevents repeat notifications for the same aircraft
#A list of aircraft needing repair is stored in a text file
dirs=AppDirs("nattgew-xpp","Nattgew")
filename=Path(dirs.user_data_dir).joinpath('aog.txt')
try:
filename.touch(exist_ok=True) #Create file if it doesn't exist
oldnews=[] #List of aircraft needing fixes already notified
with filename.open() as f:
for aog in f: #Loop over all aircraft in the file
aog=aog.strip()
#print("Checking for previously AOG: "+aog)
for current in needfixes: #Loop over all aircraft currently in need of repair
#print("Testing current AOG: "+current[0])
if current[0]==aog: #Aircraft was already listed in the file
#print("Matched, adding to list")
oldnews.append(current)
break
#print("Came up with oldnews:")
#print(oldnews)
with filename.open('w') as f: #Overwrite the file with the new list of aircraft
for current in needfixes:
#print("Writing "+current[0]+" to file")
f.write(current[0]+"\n")
#print("Will remove oldnews from needfixes list:")
#print(needfixes)
for oldie in oldnews: #Remove aircraft already notified from the list
#print("Removing "+oldie[0]+" from notify list")
needfixes.remove(oldie)
except IOError:
print("Could not open file: "+str(file))
return needfixes
def create_file(file_name):
name = Path(file_name)
create_dir(name.parent)
if name.exists():
name.unlink()
print("create file " + str(name))
return open(str(name), "w")
def create_dir(dir_name):
name = Path(dir_name)
if name.is_file():
raise (str(name) + " is file")
if not name.exists():
name.mkdir(parents=True)
print("create directory " + str(name))
def create(self, f):
_ensure(self.container, parents=True)
_ensure(self.tmp, parents=True)
hash = self.create_hash()
temp_file = tempfile.NamedTemporaryFile(
dir=str(self.tmp),
delete=False,
)
temp_file_path = Path(temp_file.name)
try:
with temp_file as tf:
for chunk in FileWrapper(f):
hash.update(chunk)
tf.write(chunk)
key = hash.hexdigest()
path = self.get_path(key)
_ensure(path.parent)
except:
temp_file_path.unlink()
else:
temp_file_path.rename(path)
return key
def run_pylint(module_path, pylint_options, ignore_prefixes=tuple()):
"""Runs Pylint. Returns a boolean indicating success"""
pylint_stats = Path('/run/user/{}/pylint_stats'.format(os.getuid()))
if not pylint_stats.parent.is_dir(): #pylint: disable=no-member
pylint_stats = Path('/run/shm/pylint_stats')
os.environ['PYLINTHOME'] = str(pylint_stats)
input_paths = list()
if not module_path.exists():
print('ERROR: Cannot find', module_path)
exit(1)
if module_path.is_dir():
for path in module_path.rglob('*.py'):
ignore_matched = False
for prefix in ignore_prefixes:
if path.parts[:len(prefix)] == prefix:
ignore_matched = True
break
if ignore_matched:
continue
input_paths.append(str(path))
else:
input_paths.append(str(module_path))
runner = lint.Run((*input_paths, *pylint_options), do_exit=False)
if pylint_stats.is_dir():
shutil.rmtree(str(pylint_stats))
if runner.linter.msg_status != 0:
print('WARNING: Non-zero exit status:', runner.linter.msg_status)
return False
return True
def load_feeds(article_generator):
settings = article_generator.settings
articles = article_generator.articles
# prepare blog articles
blog_feed_path = settings.get('FEED_ALL_ATOM_URL',
settings['FEED_ALL_ATOM'])
blog_feed = dict(
title='blog',
lang='en',
link=f"{settings['SITEURL']}/blog/",
href=f"{settings['FEED_DOMAIN']}/{blog_feed_path}",
)
blog_entries = [
dict(
title=article.title,
lang=article.lang,
link=f"{settings['SITEURL']}/{article.url}",
date=article.date.date(),
feed_title=blog_feed['title'],
feed_link=blog_feed['link'],
translations=[
dict(
title=translation.title,
lang=translation.lang,
link=f"{settings['SITEURL']}/{translation.url}",
date=translation.date.date(),
feed_title=blog_feed['title'],
feed_link=blog_feed['link'],
) for translation in article.translations
],
alternates=[],
) for article in articles
if getattr(article, 'home', '').lower() != 'false'
]
# create a lookup mapping of alternate entries
alternates_mapping = dict(
itertools.chain.from_iterable(((alternate['url'], i)
for alternate in article.alternates)
for i, article in enumerate(articles)))
# external feeds
path = Path('./content/data/feeds.yml')
feeds = yaml.load(path.read_text(), SCHEMA).data
entries = []
for entry in itertools.chain.from_iterable(map(parse_feed, feeds)):
try:
i = alternates_mapping[entry['link']]
except KeyError:
entries.append(entry)
else:
blog_entries[i]['alternates'].append(entry)
# merge & sort entries
feeds_entries = sorted(blog_entries + entries,
key=itemgetter('date'),
reverse=True)
# set context variables
article_generator.context['feeds_entries'] = feeds_entries
article_generator.context['feeds'] = [blog_feed] + feeds
bert_model_size_type = 'small'
bert_model_name = 'bert'
# bert_model_name = 'heBERT'
# bert_model_name = 'mBERT'
bert_version = f'{bert_model_name}-{bert_model_size_type}-{bert_tokenizer_type}-{bert_corpus_name}-{bert_vocab_size}-{epochs}'
tokenizer_version = f'{bert_model_name}-{bert_tokenizer_type}-{bert_corpus_name}-{bert_vocab_size}'
# tokenizer_version = f'{bert_tokenizer_type}-{bert_corpus_name}-{bert_vocab_size}'
# bert_version = f'{bert_model_name}'
# tokenizer_version = f'{bert_model_name}'
md_strategry = "morph-pipeline"
# md_strategry = "morph-sequence"
# md_strategry = "segment-only"
# Data
raw_root_path = Path(f'data/raw/{tb_data_src}')
if tb_name == 'HTB':
partition = tb.ud(raw_root_path, tb_name)
elif tb_name == 'hebtb':
if tb_schema == "UD":
partition = tb.spmrl_conllu(raw_root_path, tb_name)
else:
partition = tb.spmrl(raw_root_path, tb_name)
else:
partition = {'train': None, 'dev': None, 'test': None}
preprocessed_data_root_path = Path(
f'data/preprocessed/{tb_data_src}/{tb_name}/{tokenizer_version}')
def load_preprocessed_data_samples(data_root_path, partition,
label_names) -> dict:
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# Run experiments
gpu_percent = 0
if num_gpus > 0:
gpu_percent = configs.get("gpu_percentage", 0.5)
resources_per_trial = {"cpu": 1, "gpu": gpu_percent}
print("resources_per_trial =", resources_per_trial)
for exp in configs:
print("experiment =", exp)
config = configs[exp]
config["name"] = exp
# Stop criteria. Default to total number of iterations/epochs
stop_criteria = {"training_iteration": config.get("iterations")}
stop_criteria.update(config.get("stop", {}))
print("stop_criteria =", stop_criteria)
# Make sure path and data_dir are relative to the project location,
# handling both ~/nta and ../results style paths.
path = config.get("path", ".")
config["path"] = str(Path(path).expanduser().resolve())
data_dir = config.get("data_dir", "data")
config["data_dir"] = str(Path(data_dir).expanduser().resolve())
tune.run(
SpeechExperimentTune,
name=config["name"],
stop=stop_criteria,
config=config,
resources_per_trial=resources_per_trial,
num_samples=config.get("repetitions", 1),
local_dir=config.get("path", None),
upload_dir=config.get("upload_dir", None),
sync_function=config.get("sync_function", None),
checkpoint_freq=config.get("checkpoint_freq", 0),
checkpoint_at_end=config.get("checkpoint_at_end", False),
export_formats=config.get("", None),
search_alg=config.get("search_alg", None),
scheduler=config.get("scheduler", None),
verbose=config.get("verbose", 2),
resume=config.get("resume", False),
queue_trials=config.get("queue_trials", False),
reuse_actors=config.get("reuse_actors", False),
trial_executor=config.get("trial_executor", None),
raise_on_failed_trial=config.get("raise_on_failed_trial", True),
)
ray.shutdown()
""" Django settings for profiles_project project. Generated by 'django-admin startproject' using Django 3.2. For more information on this file, see https://docs.djangoproject.com/en/3.2/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.2/ref/settings/ """ from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 'django-insecure-9yt!9q9yr(08bw3fg04h!ise&7$aq+7@y3$nl671oi6lp+4-yt' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [
from pathlib import Path
import pandas as pd
import pickle
import click
import json
project_dir = Path(__file__).resolve().parents[1]
raw_data_dir = "{}/{}".format(project_dir, "data/raw")
output_dir = "{}/{}".format(project_dir, "data/output")
model_dir = "{}/{}".format(project_dir, "models")
params_dir = "{}/{}".format(project_dir, "params")
test_filepath = "{}/{}".format(raw_data_dir, "cs-test.csv")
default_model_json = "{}/{}".format(params_dir, "init_model.json")
TARGET_COLUMN = "SeriousDlqin2yrs"
@click.command()
@click.option('--model_json', default=default_model_json)
def predict(model_json):
"""
Predict class probabilities
Parameters:
model_json (str): Filepath to JSON containing model
parameters
"""
with open(model_json, "r") as f:
model_dict = json.loads(f.read())
@author: dberke
A simple script to scrape all the ADP files in
/Volumes/External Storage/HARPS/* and construct a CSV database of their
filenames, dates of observation, name of origin file, and object.
"""
import pandas as pd
from astropy.io import fits
from pathlib import Path
from tqdm import tqdm
databaseLines = []
baseDir = Path('/Volumes/External Storage/HARPS')
ADPfiles = baseDir.glob('*/ADP*.fits')
filelist = [file for file in ADPfiles]
print(len(filelist))
for file in tqdm(filelist):
with fits.open(file) as hdul:
header = hdul[0].header
date_obs = header['DATE-OBS']
arcfile = header['ARCFILE']
origfile = header['ORIGFILE']
try:
obj = header['HDNUM']
except KeyError:
obj = header['OBJECT']
def _write_paths(self):
with open(Path(self.output_dir / 'paths.txt'), 'w') as f:
f.write('model path: {}\n'.format(str(self.model_path)))
f.write('train path: {}\n'.format(str(self.train_file_path)))
f.write('val path: {}\n'.format(str(self.val_file_path)))
f.write('predict path: {}\n'.format(str(self.predict_file_path)))
def main():
main_dir = Path(
r'P:\Synchronize\IWS\Testings\fourtrans_practice\multisite_phs_spec_corr\5min\v7'
)
os.chdir(main_dir)
suff = '__RR1D_RTsum'
in_data_files = [
Path(r'../neckar_1min_ppt_data_20km_buff_Y2009__RRD_RTsum.pkl'),
Path(f'orig/ts_OK{suff}.csv'),
Path(f'ifted/ts_OK{suff}.csv'),
]
data_labels = ['REF', 'OK', 'FT']
lags = np.arange(1, 10 + 1, dtype=np.int64)
ranks_flag = False
sep = ';'
fig_size = (15, 7)
dpi = 200
beg_time = '2009-01-01 00:00:00'
end_time = '2009-03-31 23:59:00'
out_dir = Path(f'cmpr_figs__lag_corrs{suff}')
out_dir.mkdir(exist_ok=True)
rows = 2
cols = len(in_data_files)
axes = plt.subplots(rows,
cols,
squeeze=False,
figsize=fig_size,
sharex=True,
sharey=True)[1]
row = 0
for i in range(len(in_data_files)):
print('Going through:', in_data_files[i])
if in_data_files[i].suffix == '.csv':
data_df = pd.read_csv(in_data_files[i], sep=sep, index_col=0)
# data_df.index = pd.to_datetime(data_df.index, format=time_fmt)
elif in_data_files[i].suffix == '.pkl':
data_df = pd.read_pickle(in_data_files[i])
else:
raise NotImplementedError(
f'Unknown extension of in_data_file: {in_data_files[i].suffix}!'
)
data_df = data_df.loc[beg_time:end_time]
# if i == 0:
# data_df.dropna(axis=1, how='any', inplace=True)
#
# else:
# data_df.dropna(axis=0, how='any', inplace=True)
assert np.all(np.isfinite(data_df.values))
if ranks_flag:
data_df = data_df.rank(axis=0)
lags_corrs_all = []
for stn in data_df:
stn_vals = data_df[stn].values.copy()
lags_corrs = []
for lag in lags:
stn_no_lag_vals, stn_lag_vals = roll_real_2arrs(
stn_vals, stn_vals, lag)
lag_corr = np.corrcoef(stn_no_lag_vals, stn_lag_vals)[0, 1]
lags_corrs.append((lag, lag_corr))
lags_corrs_all.extend(lags_corrs)
lags_corrs = np.array(lags_corrs)
axes[row, i].plot(lags_corrs[:, 0],
lags_corrs[:, 1],
lw=1,
alpha=0.1,
color='k')
lags_corrs_all = np.array(lags_corrs_all)
lags_corrs_med = []
for lag in lags:
med_lag_corr = np.median(lags_corrs_all[lags_corrs_all[:,
0] == lag,
1])
lags_corrs_med.append((lag, med_lag_corr))
lags_corrs_med = np.array(lags_corrs_med)
axes[row, i].plot(lags_corrs_med[:, 0],
lags_corrs_med[:, 1],
lw=2,
alpha=0.75,
color='red',
label='median')
axes[row, i].set_title(data_labels[i])
axes[row, i].legend()
axes[row, i].grid()
axes[row, i].set_axisbelow(True)
if i == 0:
axes[row, i].set_ylabel('Correlation (-)')
axes[1, 1].plot(lags_corrs_med[:, 0],
lags_corrs_med[:, 1],
lw=2,
alpha=0.75,
label=f'{data_labels[i]}')
row = 1
for i in range(len(in_data_files)):
if i == 1:
axes[row, i].legend()
axes[row, i].grid()
axes[row, i].set_axisbelow(True)
axes[row, i].set_xlabel('Lag (steps)')
continue
axes[row, i].set_axis_off()
if ranks_flag:
rank_lab = '__ranks'
else:
rank_lab = ''
out_fig_name = f'lag_corrs{rank_lab}.png'
plt.savefig(str(out_dir / out_fig_name), bbox_inches='tight', dpi=dpi)
plt.close()
# plt.show()
return
def label_filename(self):
return Path(self.url.split("/")[-1])
# import pprint
# import pymongo
from pathlib import Path
from mongo_db.db_connection import DBConnection
from mongo_db.db_controller import DBController
with DBConnection('butterfly') as client:
# print(client._db.collection_names())
client.drop_collections('users', 'projects', 'topics')
controller = DBController(client)
db = client.db
with open(Path(__file__).parent / 'db.json') as f:
data = json.load(f)
users_json = data['users']
projects_json = data['projects']
topics_json = data['topics']
# Popola la collezione users da db.json
for user in users_json:
# NON usare questo metodo per inserire utenti! Usare
# controller.insert_user()
result = controller.collection('users').insert_one(user)
if result is not None:
print(result.inserted_id)
# Popola la collezione projects da db.json
class IndexDB:
fname = "pds_indices_db.toml"
fpath = Path.home() / f".{fname}"
def __init__(self):
"""Initialize index database.
Will copy the package's version to user's home folder at init,
so that user doesn't need to edit file in package to add new indices.
Adding new index URLs to the package's config file pds_indices_db.toml
is highly encouraged via pull request.
"""
if not self.fpath.exists():
with resource_path("planetarypy.pdstools.data", self.fname) as p:
self.config = self.read_from_file(p)
else:
self.config = self.read_from_file()
def read_from_file(self, path=None):
"""Read the config.
Writing this short method to decouple IndexDB from choice of config file format.
Parameters
----------
path : str, pathlib.Path
Path to the config file to open.
"""
if path is None:
path = self.fpath
return toml.load(path)
def write_to_file(self):
"Write the config to user's home copy."
with open(self.fpath, "w") as f:
toml.dump(self.config, f)
def get_by_path(self, nested_key):
"""Get sub-dictionary by nested key.
Parameters
----------
nested_key: str
A nested key in the toml format, separated by '.', e.g. cassini.uvis.ring_summary
"""
mapList = nested_key.split(".")
d = copy.deepcopy(self.config)
for k in mapList:
d = d[k]
return Index(nested_key, **d)
def get(self, nested_key):
"alias to get_by_path"
return self.get_by_path(nested_key)
def set_by_path(self, nested_key, value):
"""Set a nested dictionary key to new value.
Parameters
----------
nested_key : str
A nested key in the toml format, separated by '.', e.g. cassini.uvis.summary
value : str
New value for dictionary key
"""
dic = self.config
keys = nested_key.split(".")
for key in keys[:-1]:
dic = dic.setdefault(key, {})
dic[keys[-1]] = value
def list_indices(self):
"Print index database in pretty form, using toml.dumps"
print(toml.dumps(self.config))
print(
"Use indices.download('mission.instrument.index') to download in index file."
)
print("For example: indices.download('cassini.uvis.moon_summary'")
def update_timestamp(self, index):
self.set_by_path(f"{index.key}.timestamp",
index.new_timestamp.isoformat())
self.write_to_file()
def download(self,
key=None,
label_url=None,
local_dir="",
convert_to_hdf=True,
force=False):
"""Wrapping URLs for downloading PDS indices and their label files.
Parameters
----------
key : str, optional
Period-separated key into the available index files, e.g. cassini.uvis.moon_summary
label_url : str, optional
Alternative to using the index system, the user can provide the URL to a label
for an index. The table file has to be in the same folder, as usual.
local_dir: str, pathlib.Path, optional
Path for local storage. Default: current directory and filename from URL
convert_to_hdf : bool
Switch to convert the index automatically to a faster loading HDF file
"""
if label_url is None:
if key is not None:
index = self.get_by_path(key)
else:
raise SyntaxError("One of key or label_url needs to be given.")
# check timestamp
if not index.needs_download and not force:
print("Stored index is up-to-date.")
return index.local_hdf_path
if not local_dir:
local_dir = index.local_dir
label_url = index.url
logger.info("Downloading %s." % label_url)
local_label_path, _ = utils.download(label_url, local_dir)
data_url = replace_url_suffix(label_url)
logger.info("Downloading %s.", data_url)
local_data_path, _ = utils.download(data_url, local_dir)
self.update_timestamp(index)
if convert_to_hdf is True:
label = IndexLabel(local_label_path)
df = label.read_index_data()
savepath = local_data_path.with_suffix(".hdf")
df.to_hdf(savepath, "df")
print(f"Downloaded and converted to pandas HDF: {savepath}")
def __repr__(self):
return toml.dumps(self.config)
FILE = settings.DATA_INPUT / '2021-02-13_GuiaPetroleo.xlsx'
def save_dataframes_unique(file):
df = get_dataframe(file, worksheet=0, usecols=("A:O"))
branches = find_branches(df)
branches = find_duplicates(branches)
headquartes = find_headquarters(df)
headquartes = find_duplicates(headquartes)
worksheets = {'filiais': branches, 'matrizes': headquartes}
writer = create_excel_file(worksheets, filename='petroleo_empresas')
writer.save()
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
try:
Path.mkdir(Path(settings.DATA_INPUT))
Path.mkdir(Path(settings.DATA_OUTPUT))
Path.mkdir(Path(settings.REPORTS_PATH))
Path.mkdir(Path(settings.PROFILE_PATH))
except FileExistsError as e:
pass
else:
print("Pastas de saída criadas")
save_dataframes_unique(FILE)
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
from dateutil import parser
from tqdm import tqdm
from .. import utils
from .._config import config
from .scraper import CTXIndex
try:
# 3.6 compatibility
from importlib_resources import path as resource_path
except ModuleNotFoundError:
from importlib.resources import path as resource_path
logger = logging.getLogger(__name__)
indices_root = Path(config["data_archive"]["path"]) / "indices"
@dataclass
class Index:
"""Index manager class.
Parameters
----------
key : str
Nested key in form of mission.instrument.index_name
url : str
URL to index
timestamp : str
Timestamp in ISO time format yy-mm-ddTHH:MM:SS.
This is usually read by the IndexDB class from the config file and its
def __init__(self, labelpath):
self.path = Path(labelpath)
"search for table name pointer and store key and fpath."
tuple = [i for i in self.pvl_lbl if i[0].startswith("^")][0]
self.tablename = tuple[0][1:]
self.index_name = tuple[1]
def _dirname(self):
return Path(self.dirname)
def label_path(self):
return Path(urlsplit(self.url).path)
def GenerateCMake(folder, params):
cmake_header1 = ("# Generated Cmake Pico project file\n\n"
"cmake_minimum_required(VERSION 3.13)\n\n"
"set(CMAKE_C_STANDARD 11)\n"
"set(CMAKE_CXX_STANDARD 17)\n\n"
"# initalize pico_sdk from installed location\n"
"# (note this can come from environment, CMake cache etc)\n"
)
cmake_header2 = ("# Pull in Pico SDK (must be before project)\n"
"include(pico_sdk_import.cmake)\n\n"
)
cmake_header3 = (
"\n# Initialise the Pico SDK\n"
"pico_sdk_init()\n\n"
"# Add executable. Default name is the project name, version 0.1\n\n"
)
filename = Path(folder) / CMAKELIST_FILENAME
file = open(filename, 'w')
file.write(cmake_header1)
# OK, for the path, CMake will accept forward slashes on Windows, and thats
# seemingly a bit easier to handle than the backslashes
p = str(params.sdkPath).replace('\\','/')
p = '\"' + p + '\"'
file.write('set(PICO_SDK_PATH ' + p + ')\n\n')
file.write(cmake_header2)
file.write('project(' + params.projectName + ' C CXX ASM)\n')
if params.exceptions:
file.write("\nset(PICO_CXX_ENABLE_EXCEPTIONS 1)\n")
if params.rtti:
file.write("\nset(PICO_CXX_ENABLE_RTTI 1)\n")
file.write(cmake_header3)
# add the preprocessor defines for overall configuration
if params.configs:
file.write('# Add any PICO_CONFIG entries specified in the Advanced settings\n')
for c, v in params.configs.items():
if v == "True":
v = "1"
elif v == "False":
v = "0"
file.write('add_compile_definitions(' + c + '=' + v + ')\n')
file.write('\n')
# No GUI/command line to set a different executable name at this stage
executableName = params.projectName
if params.wantCPP:
file.write('add_executable(' + params.projectName + ' ' + params.projectName + '.cpp )\n\n')
else:
file.write('add_executable(' + params.projectName + ' ' + params.projectName + '.c )\n\n')
file.write('pico_set_program_name(' + params.projectName + ' "' + executableName + '")\n')
file.write('pico_set_program_version(' + params.projectName + ' "0.1")\n\n')
if params.wantRunFromRAM:
file.write('# no_flash means the target is to run from RAM\n')
file.write('pico_set_binary_type(' + params.projectName + ' no_flash)\n\n')
# Console output destinations
if params.wantUART:
file.write('pico_enable_stdio_uart(' + params.projectName + ' 1)\n')
else:
file.write('pico_enable_stdio_uart(' + params.projectName + ' 0)\n')
if params.wantUSB:
file.write('pico_enable_stdio_usb(' + params.projectName + ' 1)\n\n')
else:
file.write('pico_enable_stdio_usb(' + params.projectName + ' 0)\n\n')
# Standard libraries
file.write('# Add the standard library to the build\n')
file.write('target_link_libraries(' + params.projectName + ' ' + STANDARD_LIBRARIES + ')\n\n')
# Selected libraries/features
if (params.features):
file.write('# Add any user requested libraries\n')
file.write('target_link_libraries(' + params.projectName + '\n')
for feat in params.features:
if (feat in features_list):
file.write(" " + features_list[feat][LIB_NAME] + '\n')
file.write(' )\n\n')
file.write('pico_add_extra_outputs(' + params.projectName + ')\n\n')
file.close()
def relpath(self):
"""Return path relative to layout root"""
root = object_session(self).query(LayoutInfo).first().root
return str(Path(self.path).relative_to(root))
def test_root_dir():
result = return_root_dir()
assert Path(result).is_dir()
print(result)
def _path(self):
return Path(self.path)
import json
import time
from dataclasses import dataclass
from typing import List
from dataclasses_json import dataclass_json
from pathlib import Path
from common import create_template, build_synonyms
version = int(round(time.time() * 1000))
Path("result/lor").mkdir(parents=True, exist_ok=True)
@dataclass_json
@dataclass
class CardData:
name: str
cardCode: str
associatedCardRefs: List[str]
collectible: bool
supertype: str
def get_current_synonyms(card_name: str, synonyms: dict):
if card_name in synonyms.keys():
return synonyms[card_name]
else:
return []
glossary_template = create_template("data/lor/lor_card_glossary.html")
csv_row_template = create_template("data/csv_row.csv")
print(m)
sys.exit(-1)
if args.name == None and not args.gui and not args.list and not args.configs:
print("No project name specfied\n")
sys.exit(-1)
# load/parse any configuration dictionary we may have
LoadConfigurations()
p = CheckSDKPath(args.gui)
if p == None:
sys.exit(-1)
sdkPath = Path(p)
if args.gui:
RunGUI(sdkPath, args) # does not return, only exits
projectRoot = Path(os.getcwd())
if args.list or args.configs:
if args.list:
print("Available project features:\n")
for feat in features_list:
print(feat.ljust(6), '\t', features_list[feat][GUI_TEXT])
print('\n')
if args.configs:
print("Available project configuration items:\n")
import numpy as np import random import copy import sys from pathlib import Path sys.path.append(str(Path(__file__).resolve().parent)) import Parameters.DE as DE from Plot import plot fbest = -1*sys.maxsize Wbest = [] mi = DE.min ma = DE.max G = DE.G P = DE.population F = DE.F CR = DE.CR class DE: def __init__(self,func,N,**kwargs): self.game = func self.w_list = [[random.uniform(mi/10,ma/10) for i in range(N)] for j in range(P)] self.save = kwargs['save'] class Population: def __init__(self,w,f): self.w = w self.f = f
def __init__(self,
path,
img_size=640,
batch_size=16,
augment=False,
hyp=None,
rect=False,
image_weights=False,
cache_images=False,
single_cls=False,
stride=32,
pad=0.0):
try:
f = [] # image files
for p in path if isinstance(path, list) else [path]:
p = str(Path(p)) # os-agnostic
parent = str(Path(p).parent) + os.sep
if os.path.isfile(p): # file
with open(p, 'r') as t:
t = t.read().splitlines()
f += [
x.replace('./', parent)
if x.startswith('./') else x for x in t
] # local to global path
elif os.path.isdir(p): # folder
f += glob.iglob(p + os.sep + '*.*')
else:
raise Exception('%s does not exist' % p)
self.img_files = sorted([
x.replace('/', os.sep) for x in f
if os.path.splitext(x)[-1].lower() in img_formats
])
except Exception as e:
raise Exception('Error loading data from %s: %s\nSee %s' %
(path, e, help_url))
n = len(self.img_files)
assert n > 0, 'No images found in %s. See %s' % (path, help_url)
bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index
nb = bi[-1] + 1 # number of batches
self.n = n # number of images
self.batch = bi # batch index of image
self.img_size = img_size
self.augment = augment
self.hyp = hyp
self.image_weights = image_weights
self.rect = False if image_weights else rect
self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)
self.mosaic_border = [-img_size // 2, -img_size // 2]
self.stride = stride
# Define labels
self.label_files = [
x.replace('images',
'labels').replace(os.path.splitext(x)[-1], '.txt')
for x in self.img_files
]
# Check cache
cache_path = str(Path(
self.label_files[0]).parent) + '.cache' # cached labels
if os.path.isfile(cache_path):
cache = torch.load(cache_path) # load
if cache['hash'] != get_hash(self.label_files +
self.img_files): # dataset changed
cache = self.cache_labels(cache_path) # re-cache
else:
cache = self.cache_labels(cache_path) # cache
# Get labels
labels, shapes = zip(*[cache[x] for x in self.img_files])
self.shapes = np.array(shapes, dtype=np.float64)
self.labels = list(labels)
# Rectangular Training https://github.com/ultralytics/yolov3/issues/232
if self.rect:
# Sort by aspect ratio
s = self.shapes # wh
ar = s[:, 1] / s[:, 0] # aspect ratio
irect = ar.argsort()
self.img_files = [self.img_files[i] for i in irect]
self.label_files = [self.label_files[i] for i in irect]
self.labels = [self.labels[i] for i in irect]
self.shapes = s[irect] # wh
ar = ar[irect]
# Set training image shapes
shapes = [[1, 1]] * nb
for i in range(nb):
ari = ar[bi == i]
mini, maxi = ari.min(), ari.max()
if maxi < 1:
shapes[i] = [maxi, 1]
elif mini > 1:
shapes[i] = [1, 1 / mini]
self.batch_shapes = np.ceil(
np.array(shapes) * img_size / stride + pad).astype(
np.int) * stride
# Cache labels
create_datasubset, extract_bounding_boxes, labels_loaded = False, False, False
nm, nf, ne, ns, nd = 0, 0, 0, 0, 0 # number missing, found, empty, datasubset, duplicate
pbar = tqdm(self.label_files)
for i, file in enumerate(pbar):
l = self.labels[i] # label
if l.shape[0]:
assert l.shape[1] == 5, '> 5 label columns: %s' % file
assert (l >= 0).all(), 'negative labels: %s' % file
assert (l[:, 1:] <= 1).all(
), 'non-normalized or out of bounds coordinate labels: %s' % file
if np.unique(l,
axis=0).shape[0] < l.shape[0]: # duplicate rows
nd += 1 # print('WARNING: duplicate rows in %s' % self.label_files[i]) # duplicate rows
if single_cls:
l[:, 0] = 0 # force dataset into single-class mode
self.labels[i] = l
nf += 1 # file found
# Create subdataset (a smaller dataset)
if create_datasubset and ns < 1E4:
if ns == 0:
create_folder(path='./datasubset')
os.makedirs('./datasubset/images')
exclude_classes = 43
if exclude_classes not in l[:, 0]:
ns += 1
# shutil.copy(src=self.img_files[i], dst='./datasubset/images/') # copy image
with open('./datasubset/images.txt', 'a') as f:
f.write(self.img_files[i] + '\n')
# Extract object detection boxes for a second stage classifier
if extract_bounding_boxes:
p = Path(self.img_files[i])
img = cv2.imread(str(p))
h, w = img.shape[:2]
for j, x in enumerate(l):
f = '%s%sclassifier%s%g_%g_%s' % (
p.parent.parent, os.sep, os.sep, x[0], j, p.name)
if not os.path.exists(Path(f).parent):
os.makedirs(
Path(f).parent) # make new output folder
b = x[1:] * [w, h, w, h] # box
b[2:] = b[2:].max() # rectangle to square
b[2:] = b[2:] * 1.3 + 30 # pad
b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int)
b[[0, 2]] = np.clip(b[[0, 2]], 0,
w) # clip boxes outside of image
b[[1, 3]] = np.clip(b[[1, 3]], 0, h)
assert cv2.imwrite(f, img[
b[1]:b[3],
b[0]:b[2]]), 'Failure extracting classifier boxes'
else:
ne += 1 # print('empty labels for image %s' % self.img_files[i]) # file empty
# os.system("rm '%s' '%s'" % (self.img_files[i], self.label_files[i])) # remove
pbar.desc = 'Scanning labels %s (%g found, %g missing, %g empty, %g duplicate, for %g images)' % (
cache_path, nf, nm, ne, nd, n)
if nf == 0:
s = 'WARNING: No labels found in %s. See %s' % (
os.path.dirname(file) + os.sep, help_url)
print(s)
assert not augment, '%s. Can not train without labels.' % s
# Cache images into memory for faster training (WARNING: large datasets may exceed system RAM)
self.imgs = [None] * n
if cache_images:
gb = 0 # Gigabytes of cached images
pbar = tqdm(range(len(self.img_files)), desc='Caching images')
self.img_hw0, self.img_hw = [None] * n, [None] * n
for i in pbar: # max 10k images
self.imgs[i], self.img_hw0[i], self.img_hw[i] = load_image(
self, i) # img, hw_original, hw_resized
gb += self.imgs[i].nbytes
pbar.desc = 'Caching images (%.1fGB)' % (gb / 1E9)
}
if os.getenv("SQL_DEBUG"):
LOGGING["filters"]["require_debug_true"] = {
"()": "django.utils.log.RequireDebugTrue"
}
LOGGING["handlers"]["console"] = {
"level": "DEBUG",
"filters": ["require_debug_true"],
"class": "logging.StreamHandler",
}
LOGGING["loggers"]["django.db.backends"] = {
"level": "DEBUG",
"handlers": ["console"],
}
INDY_HOLDER_ID = "TheOrgBook_Holder"
INDY_VERIFIER_ID = "TheOrgBook_Verifier"
custom_settings_file = Path(
BASE_DIR,
"custom_settings_" + str(os.getenv("TOB_THEME")).lower() + ".py",
)
if custom_settings_file.exists():
with open(custom_settings_file) as source_file:
print(
"Loading custom settings file: {}".format(custom_settings_file.name)
)
exec(source_file.read())
def main(args):
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.multi_gpu is None else '0,1,2,3'
'''CREATE DIR'''
experiment_dir = Path('./experiment/')
experiment_dir.mkdir(exist_ok=True)
file_dir = Path(str(experiment_dir) +'/%sNuScenesSemSeg-'%args.model_name+ str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')))
file_dir.mkdir(exist_ok=True)
checkpoints_dir = file_dir.joinpath('checkpoints/')
checkpoints_dir.mkdir(exist_ok=True)
log_dir = file_dir.joinpath('logs/')
log_dir.mkdir(exist_ok=True)
'''LOG'''
args = parse_args()
logger = logging.getLogger(args.model_name)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler = logging.FileHandler(str(log_dir) + '/train_%s_semseg.txt'%args.model_name)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.info('---------------------------------------------------TRANING---------------------------------------------------')
logger.info('PARAMETER ...')
logger.info(args)
print('Load data...')
nusc = NuScenes(version='v1.0-trainval', dataroot='/mnt/raid/chengnan/NuScenes/', verbose=True)
dataset = NuscenesDatasetSemseg(nusc, npoints_lidar=16384, npoints_radar=1024, split='train_detect')
dataloader = torch.utils.data.DataLoader(dataset, batch_size=args.batchsize, collate_fn=dataset.collate_fn,
shuffle=True, num_workers=int(args.workers))
test_dataset = NuscenesDatasetSemseg(nusc, npoints_lidar=16384, npoints_radar=1024, split='val_small')
testdataloader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batchsize, collate_fn=dataset.collate_fn,
shuffle=True, num_workers=int(args.workers))
num_classes = 11
model = PointNet2SemSeg5(num_classes)
labelweight = torch.tensor([0.84423709, 3.562451, 8.4504371, 5.04570442, 1.69708204, 6.41300778, 6.44816675, 4.88638126, 5.20078234, 4.82712436, 3.74396562])
loss_function = torch.nn.CrossEntropyLoss(labelweight.cuda())
#loss_function = torch.nn.CrossEntropyLoss(reduction='none')
if args.pretrain is not None:
model.load_state_dict(torch.load(args.pretrain))
print('load model %s'%args.pretrain)
logger.info('load model %s'%args.pretrain)
else:
print('Training from scratch')
logger.info('Training from scratch')
pretrain = args.pretrain
init_epoch = int(pretrain[-14:-11]) if args.pretrain is not None else 0
if args.optimizer == 'SGD':
optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
elif args.optimizer == 'Adam':
optimizer = torch.optim.Adam(
model.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=args.weight_decay
)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
LEARNING_RATE_CLIP = 1e-5
'''GPU selection and multi-GPU'''
if args.multi_gpu is not None:
device_ids = [int(x) for x in args.multi_gpu.split(',')]
torch.backends.cudnn.benchmark = True
model.cuda(device_ids[0])
model = torch.nn.DataParallel(model, device_ids=device_ids)
else:
model.cuda()
history = defaultdict(lambda: list())
best_acc = 0
best_acc_epoch = 0
best_mIoU = 0
best_mIoU_epoch = 0
for epoch in range(init_epoch,args.epoch):
scheduler.step()
lr = max(optimizer.param_groups[0]['lr'],LEARNING_RATE_CLIP)
print('Learning rate:%f' % lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
train_loss_sum = 0.0
train_acc_sum = 0.0
for i, data in enumerate(dataloader):
lidar_points, _, _, _, target, _, _, _ = data
lidar_points, target = lidar_points.float(), target.long()
lidar_points = lidar_points.transpose(2, 1)
lidar_points, target = lidar_points.cuda(), target.cuda()
optimizer.zero_grad()
model = model.train()
pred = model(lidar_points[:,:3,:],lidar_points[:,3:,:])
#pred = model(lidar_points[:,:3,:],None)
loss = loss_function(pred, target)
history['loss'].append(loss.item())
train_loss_sum += loss.item()
loss.backward()
optimizer.step()
# Train acc
pred_val = torch.argmax(pred, 1)
correct = torch.sum(((pred_val == target)&(target>0)).float())
seen = torch.sum((target>0).float())+1e-08
train_acc = correct/seen if seen!=0 else correct
history['train_acc_per_iter'].append(train_acc.item())
train_acc_sum += train_acc.item()
if (i+1)%5 == 0:
print('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc.item(), loss.item()))
logger.info('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc.item(), loss.item()))
if (i+1)%200 == 0:
train_loss_avg = np.sum(history['loss'][-200:])/200.0
train_acc_avg = np.sum(history['train_acc_per_iter'][-200:])/200.0
history['train_it_acc'].append(train_acc_avg)
print('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc_avg, train_loss_avg))
logger.info('[Epoch %d/%d] [Iteration %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), train_acc_avg, train_loss_avg))
#Test acc
test_losses = []
total_correct = 0
total_seen = 0
total_correct_class = [0 for _ in range(num_classes)]
total_seen_class = [0 for _ in range(num_classes)]
total_intersection_class = [0 for _ in range(num_classes)]
total_union_class = [0 for _ in range(num_classes)]
for j, data in enumerate(testdataloader):
lidar_points, _, _, _, target, _, _, _ = data
lidar_points, target = lidar_points.float(), target.long()
lidar_points = lidar_points.transpose(2, 1)
lidar_points, target = lidar_points.cuda(), target.cuda()
model = model.eval()
with torch.no_grad():
pred = model(lidar_points[:,:3,:],lidar_points[:,3:,:])
#pred = model(lidar_points[:,:3,:],None)
loss = loss_function(pred, target)
test_losses.append(loss.item())
#first convert torch tensor to numpy array
pred_np = pred.cpu().numpy() #[B,C,N]
target_np = target.cpu().numpy() #[B,N]
# point wise acc
pred_val = np.argmax(pred_np, 1) #[B,N]
correct = np.sum((pred_val == target_np) & (target_np>0))
total_correct += correct
total_seen += np.sum(target_np>0)
# point wise acc and IoU per class
for l in range(num_classes):
total_seen_class[l] += np.sum((target_np==l))
total_correct_class[l] += np.sum((pred_val==l) & (target_np==l))
total_intersection_class[l] += np.sum((pred_val==l) & (target_np==l))
total_union_class[l] += np.sum(((pred_val==l) | (target_np==l)))
test_loss = np.mean(test_losses)
test_point_acc = total_correct/float(total_seen)
history['test_it_point_acc'].append(test_point_acc)
test_avg_class_point_acc = np.mean(np.array(total_correct_class[1:])/(np.array(total_seen_class[1:],dtype=np.float)+1e-6))
history['test_it_avg_class_point_acc'].append(test_avg_class_point_acc)
test_avg_class_point_IoU = np.mean(np.array(total_intersection_class[1:])/(np.array(total_union_class[1:],dtype=np.float)+1e-6))
history['test_it_avg_class_point_IoU'].append(test_avg_class_point_IoU)
print('[Epoch %d/%d] [Iteration %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), test_point_acc, test_loss))
logger.info('[Epoch %d/%d] [Iteration %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, i+1, len(dataloader), test_point_acc, test_loss))
print('Whole scene point wise accuracy: %f' % (test_point_acc))
logger.info('Whole scene point wise accuracy: %f' % (test_point_acc))
print('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
logger.info('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
print('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
logger.info('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
per_class_point_str = 'point based --------\n'
for l in range(0,num_classes):
per_class_point_str += 'class %d weight: %f, acc: %f, IoU: %f;\n' % (l,labelweight[l],total_correct_class[l]/float(total_seen_class[l]), total_intersection_class[l]/(float(total_union_class[l])+1e-6))
logger.info(per_class_point_str)
if test_point_acc > best_acc:
best_acc = test_point_acc
best_acc_epoch = epoch+1
torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestacc.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_acc))
logger.info('Save best acc model..')
print('Save best acc model..')
if test_avg_class_point_IoU > best_mIoU:
best_mIoU = test_avg_class_point_IoU
best_mIoU_epoch = epoch+1
torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestmIoU.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_mIoU))
logger.info('Save best mIoU model..')
print('Save best mIoU model..')
train_loss_avg = train_loss_sum/len(dataloader)
train_acc_avg = train_acc_sum/len(dataloader)
history['train_acc'].append(train_acc_avg)
print('[Epoch %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, train_acc_avg, train_loss_avg))
logger.info('[Epoch %d/%d] TRAIN acc/loss: %f/%f ' % (epoch+1, args.epoch, train_acc_avg, train_loss_avg))
#Test acc
test_losses = []
total_correct = 0
total_seen = 0
total_correct_class = [0 for _ in range(num_classes)]
total_seen_class = [0 for _ in range(num_classes)]
total_intersection_class = [0 for _ in range(num_classes)]
total_union_class = [0 for _ in range(num_classes)]
for j, data in enumerate(testdataloader):
lidar_points, _, _, _, target, _, _, _ = data
lidar_points, target = lidar_points.float(), target.long()
lidar_points = lidar_points.transpose(2, 1)
lidar_points, target = lidar_points.cuda(), target.cuda()
model = model.eval()
with torch.no_grad():
pred = model(lidar_points[:,:3,:],lidar_points[:,3:,:])
#pred = model(lidar_points[:,:3,:],None)
loss = loss_function(pred, target)
test_losses.append(loss.item())
#first convert torch tensor to numpy array
pred_np = pred.cpu().numpy() #[B,C,N]
target_np = target.cpu().numpy() #[B,N]
# point wise acc
pred_val = np.argmax(pred_np, 1) #[B,N]
correct = np.sum((pred_val == target_np) & (target_np>0))
total_correct += correct
total_seen += np.sum(target_np>0)
# point wise acc and IoU per class
for l in range(num_classes):
total_seen_class[l] += np.sum((target_np==l))
total_correct_class[l] += np.sum((pred_val==l) & (target_np==l))
total_intersection_class[l] += np.sum((pred_val==l) & (target_np==l))
total_union_class[l] += np.sum(((pred_val==l) | (target_np==l)))
test_loss = np.mean(test_losses)
test_point_acc = total_correct/float(total_seen)
history['test_point_acc'].append(test_point_acc)
test_avg_class_point_acc = np.mean(np.array(total_correct_class[1:])/(np.array(total_seen_class[1:],dtype=np.float)+1e-6))
history['test_avg_class_point_acc'].append(test_avg_class_point_acc)
test_avg_class_point_IoU = np.mean(np.array(total_intersection_class[1:])/(np.array(total_union_class[1:],dtype=np.float)+1e-6))
history['test_avg_class_point_IoU'].append(test_avg_class_point_IoU)
print('[Epoch %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, test_point_acc, test_loss))
logger.info('[Epoch %d/%d] TEST acc/loss: %f/%f ' % (epoch+1, args.epoch, test_point_acc, test_loss))
print('Whole scene point wise accuracy: %f' % (test_point_acc))
logger.info('Whole scene point wise accuracy: %f' % (test_point_acc))
print('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
logger.info('Whole scene class averaged point wise accuracy: %f' % (test_avg_class_point_acc))
print('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
logger.info('Whole scene class averaged point wise IoU: %f' % (test_avg_class_point_IoU))
per_class_point_str = 'point based --------\n'
for l in range(0,num_classes):
per_class_point_str += 'class %d weight: %f, acc: %f, IoU: %f;\n' % (l,labelweight[l],total_correct_class[l]/float(total_seen_class[l]), total_intersection_class[l]/(float(total_union_class[l])+1e-6))
logger.info(per_class_point_str)
if (epoch+1) % 1 == 0:
torch.save(model.state_dict(), '%s/%s_%.3d_%.4d.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1))
logger.info('Save model..')
print('Save model..')
if test_point_acc > best_acc:
best_acc = test_point_acc
best_acc_epoch = epoch+1
torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestacc.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_acc))
logger.info('Save best acc model..')
print('Save best acc model..')
if test_avg_class_point_IoU > best_mIoU:
best_mIoU = test_avg_class_point_IoU
best_mIoU_epoch = epoch+1
torch.save(model.state_dict(), '%s/%s_%.3d_%.4d_%.4f_bestmIoU.pth' % (checkpoints_dir,args.model_name, epoch+1, i+1, best_mIoU))
logger.info('Save best mIoU model..')
print('Save best mIoU model..')
print('Best point wise accuracy is %f at epoch %d.' % (best_acc, best_acc_epoch))
logger.info('Best point wise accuracy is %f at epoch %d.' % (best_acc, best_acc_epoch))
print('Best class averaged point wise IoU is %f at epoch %d.' % (best_mIoU, best_mIoU_epoch))
logger.info('Best class averaged point wise IoU is %f at epoch %d.' % (best_mIoU, best_mIoU_epoch))
plot_loss_curve(history['loss'], str(log_dir))
plot_acc_curve(history['train_it_acc'], history['test_it_point_acc'], str(log_dir))
#plot_acc_curve(history['train_it_acc'], history['test_it_avg_class_point_IoU'], str(log_dir))
print('FINISH.')
logger.info('FINISH')
def main(args=None):
set_random_seed(63)
chainer.global_config.autotune = True
chainer.cuda.set_max_workspace_size(512 * 1024 * 1024)
parser = argparse.ArgumentParser()
parser.add_argument('--batchsize',
'-b',
type=int,
default=64,
help='Number of images in each mini-batch')
parser.add_argument('--learnrate',
'-l',
type=float,
default=0.01,
help='Learning rate for SGD')
parser.add_argument('--epoch',
'-e',
type=int,
default=80,
help='Number of sweeps over the dataset to train')
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out',
'-o',
default='result',
help='Directory to output the result')
parser.add_argument('--resume',
'-r',
default='',
help='Resume the training from snapshot')
parser.add_argument('--loss-function',
choices=['focal', 'sigmoid'],
default='focal')
parser.add_argument('--optimizer',
choices=['sgd', 'adam', 'adabound'],
default='adam')
parser.add_argument('--size', type=int, default=224)
parser.add_argument('--limit', type=int, default=None)
parser.add_argument('--data-dir', type=str, default='data')
parser.add_argument('--lr-search', action='store_true')
parser.add_argument('--pretrained', type=str, default='')
parser.add_argument('--backbone',
choices=['resnet', 'seresnet', 'debug_model'],
default='resnet')
parser.add_argument('--log-interval', type=int, default=100)
parser.add_argument('--find-threshold', action='store_true')
parser.add_argument('--finetune', action='store_true')
parser.add_argument('--mixup', action='store_true')
args = parser.parse_args() if args is None else parser.parse_args(args)
print(args)
if args.mixup and args.loss_function != 'focal':
raise ValueError('mixupを使うときはfocal lossしか使えません（いまんところ）')
train, test, cooccurrence = get_dataset(args.data_dir, args.size,
args.limit, args.mixup)
base_model = backbone_catalog[args.backbone](args.dropout)
if args.pretrained:
print('loading pretrained model: {}'.format(args.pretrained))
chainer.serializers.load_npz(args.pretrained, base_model, strict=False)
model = TrainChain(base_model,
1,
loss_fn=args.loss_function,
cooccurrence=cooccurrence,
co_coef=0)
if args.gpu >= 0:
chainer.backends.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
if args.optimizer in ['adam', 'adabound']:
optimizer = Adam(alpha=args.learnrate,
adabound=args.optimizer == 'adabound',
weight_decay_rate=1e-5,
gamma=5e-7)
elif args.optimizer == 'sgd':
optimizer = chainer.optimizers.MomentumSGD(lr=args.learnrate)
optimizer.setup(model)
if not args.finetune:
print('最初のエポックは特徴抽出層をfreezeします')
model.freeze_extractor()
train_iter = chainer.iterators.MultiprocessIterator(train,
args.batchsize,
n_processes=8,
n_prefetch=2)
test_iter = chainer.iterators.MultithreadIterator(test,
args.batchsize,
n_threads=8,
repeat=False,
shuffle=False)
if args.find_threshold:
# train_iter, optimizerなど無駄なsetupもあるが。。
print('thresholdを探索して終了します')
chainer.serializers.load_npz(join(args.out, 'bestmodel_loss'),
base_model)
print('lossがもっとも小さかったモデルに対しての結果:')
find_threshold(base_model, test_iter, args.gpu, args.out)
chainer.serializers.load_npz(join(args.out, 'bestmodel_f2'),
base_model)
print('f2がもっとも大きかったモデルに対しての結果:')
find_threshold(base_model, test_iter, args.gpu, args.out)
return
# Set up a trainer
updater = training.updaters.StandardUpdater(
train_iter,
optimizer,
device=args.gpu,
converter=lambda batch, device: chainer.dataset.concat_examples(
batch, device=device))
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
# Evaluate the model with the test dataset for each epoch
trainer.extend(FScoreEvaluator(test_iter, model, device=args.gpu))
if args.optimizer == 'sgd':
# Adamにweight decayはあんまりよくないらしい
optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(5e-4))
trainer.extend(extensions.ExponentialShift('lr', 0.1),
trigger=(3, 'epoch'))
if args.lr_search:
print('最適な学習率を探します')
trainer.extend(LRFinder(1e-7, 1, 5, optimizer),
trigger=(1, 'iteration'))
elif args.optimizer in ['adam', 'adabound']:
if args.lr_search:
print('最適な学習率を探します')
trainer.extend(LRFinder(1e-7, 1, 5, optimizer, lr_key='alpha'),
trigger=(1, 'iteration'))
trainer.extend(extensions.ExponentialShift('alpha', 0.2),
trigger=triggers.EarlyStoppingTrigger(
monitor='validation/main/loss'))
# Take a snapshot of Trainer at each epoch
trainer.extend(
extensions.snapshot(filename='snaphot_epoch_{.updater.epoch}'),
trigger=(10, 'epoch'))
# Take a snapshot of Model which has best val loss.
# Because searching best threshold for each evaluation takes too much time.
trainer.extend(extensions.snapshot_object(model.model, 'bestmodel_loss'),
trigger=triggers.MinValueTrigger('validation/main/loss'))
trainer.extend(extensions.snapshot_object(model.model, 'bestmodel_f2'),
trigger=triggers.MaxValueTrigger('validation/main/f2'))
trainer.extend(extensions.snapshot_object(model.model,
'model_{.updater.epoch}'),
trigger=(5, 'epoch'))
# Write a log of evaluation statistics for each epoch
trainer.extend(
extensions.LogReport(trigger=(args.log_interval, 'iteration')))
trainer.extend(
extensions.PrintReport([
'epoch', 'lr', 'elapsed_time', 'main/loss', 'main/co_loss',
'validation/main/loss', 'validation/main/co_loss',
'validation/main/precision', 'validation/main/recall',
'validation/main/f2', 'validation/main/threshold'
]))
trainer.extend(extensions.ProgressBar(update_interval=args.log_interval))
trainer.extend(extensions.observe_lr(),
trigger=(args.log_interval, 'iteration'))
trainer.extend(CommandsExtension())
save_args(args, args.out)
trainer.extend(lambda trainer: model.unfreeze_extractor(),
trigger=(1, 'epoch'))
if args.resume:
# Resume from a snapshot
chainer.serializers.load_npz(args.resume, trainer)
# save args with pickle for prediction time
pickle.dump(args, open(str(Path(args.out).joinpath('args.pkl')), 'wb'))
# Run the training
trainer.run()
# find optimal threshold
chainer.serializers.load_npz(join(args.out, 'bestmodel_loss'), base_model)
print('lossがもっとも小さかったモデルに対しての結果:')
find_threshold(base_model, test_iter, args.gpu, args.out)
chainer.serializers.load_npz(join(args.out, 'bestmodel_f2'), base_model)
print('f2がもっとも大きかったモデルに対しての結果:')
find_threshold(base_model, test_iter, args.gpu, args.out)
import numpy as np
from pathlib import Path
import xarray
import ctypes as ct
import os
from .build import build, get_libpath
SDIR = Path(__file__).parent
BDIR = SDIR / "build"
# NOTE: must be str() for Windows, even with py37
dllfn = get_libpath(BDIR, "wmm15")
if dllfn is not None:
libwmm = ct.cdll.LoadLibrary(str(dllfn))
else:
build()
dllfn = get_libpath(BDIR, "wmm20")
if dllfn:
libwmm = ct.cdll.LoadLibrary(str(dllfn))
else:
raise ModuleNotFoundError(f"could not find {dllfn}")
def wmm(glats: np.ndarray, glons: np.ndarray, alt_km: float,
yeardec: float) -> xarray.Dataset:
"""
wmm computes the value of the world magnetic model at grid points specified by glats and
glons, for a single altitude value. glats and glons should be in degrees.
glats and glons should be generated from something like np.meshgrid, so they should be
d = {}
listDoctors = get_all_doctor_ids()
for i in listDoctors:
doctorURI=getDoctorData(i)
doctorDATA=get_uri(doctorURI)
d[str(i)] = str(doctorDATA['Doctor Name :'])
h = {}
listHospitals=get_all_hospital_ids()
for i in listHospitals:
hospitalURI = getHospitalData(i)
hospitalDATA = get_uri(hospitalURI)
h[str(i)] = str(hospitalDATA['Hospital Name :'])
output_path = Path('PatientReport.txt')
text = [
{"Input date" : patientDATA['Date & Time :'][0],
"Input time" : patientDATA['Date & Time :'][1],
"Patient's name" : patientDATA['Patient Name :'],
"Patient's gender" : patientDATA['Patient Gender :'],
"Doctor's name" : patientDATA['Doctor Name :'],
"Hospital name" : patientDATA['Hospital Name :']
},
"DOCTOR VISITS",
"________________________________________",
L,
"LIST OF DOCTORS",
"________________________________________",
d,
# builtin imports
import pickle
import re
# A lot of functions from os.path
# in python 2 moved to os. and changed
# their signature. Pathlib can be
# installed under python2.X with
# pip install pathlib2 and is in
# standard library in Python 3,
# hence we use it as a compatiblity
# library
try:
from pathlib import Path
Path().expanduser()
except (ImportError, AttributeError):
from pathlib2 import Path
# other library imports
import Levenshtein
import ase.atoms
import ase.io
import numpy as np
# local imports
from .ase_tools import gas_phase_references
np.set_printoptions(threshold=500, linewidth=1800, edgeitems=80)