def save(self, storage: FileStorage, path: PurePath, **kwargs) -> PurePath:
"""
Save the given file in this bucket and returns its relative path
:param storage: The file to be saved
:param path: The relative path where to save the file in this bucket. It will be modified in
case of conflicts if
:py:attr:`flask_googlestorage.LocalBucket.resolve_conflicts` is set to
``True``. Note that the path should be secured beforehand. You may use
:py:func:`flask_googlestorage.utils.secure_path` for this.
:returns: The relative path where the file was saved. You may use this value to get the URL
or delete the file later.
"""
(self.destination / path.parent).mkdir(parents=True, exist_ok=True)
filepath = self.destination / path
if self.resolve_conflicts and filepath.exists():
stem, suffix = path.stem, path.suffix
count = 0
while filepath.exists():
count += 1
path = path.with_name(f"{stem}_{count}{suffix}")
filepath = self.destination / path
storage.save(filepath)
return path
def inner(path: PurePath) -> Optional[PurePath]:
match = to_pattern(regex).fullmatch(path.name)
if match:
groups = [match.group(0)]
groups.extend(match.groups())
return path.with_name(target.format(*groups))
return None
def main():
argp = argparse.ArgumentParser()
argp.add_argument("--log-files",
nargs="+",
help="these are the collected log files",
required=True)
argp.add_argument("--output", help="the output file", default=None)
argp.add_argument("--svg",
help="the output file",
action="store_true",
default=False)
args = argp.parse_args()
#assert len(args.log_files) == 1
output = args.output
if output is None:
input_file = PurePath(args.log_files[0])
output = str(input_file.with_name(input_file.name + ".out.json"))
parser = ParserSvg() if args.svg else ParserJson()
files = []
for logfile_name in args.log_files:
with open(logfile_name) as logfile:
files.append(list(logfile))
parser.add_files(files)
parser.output(output)
if args.svg:
parser._write_svg("/tmp/deadlock.svg")
def load_file(session: Session, path: PurePath, content: str):
data = deserialize_file(path.suffix, content)
if "_type" in data:
cls = Model._decl_class_registry[data["_type"]] # type: ignore
else:
for model_class in Model._decl_class_registry.values():
if hasattr(model_class, "babu_paths"):
for path_glob in model_class.babu_paths:
if path.match(path_glob):
cls = model_class
break
inst = cls()
inst.id = path.with_name(path.stem).as_posix()
for k, v in data.items():
if k == "_type":
continue
if k == CONTENT:
inspector = inspect(cls)
for col in inspector.columns:
if isinstance(col.type, Content):
k = col.name
break
else:
raise ValueError(f"Class {cls} has no content column")
setattr(inst, k, v)
session.add(inst)
session.commit()
def organize_file(self, dir_path):
def mime_filter(my_file):
mime = mimetypes.guess_type(my_file)
return False if mime[0] is None or mime[0].lower().find(
args.mime.lower()) == -1 else True
def no_filter(my_file):
return True
file_filter = mime_filter if args.mime else no_filter
files = [
my_file for my_file in dir_path.glob('**/*')
if my_file.is_file() and file_filter(my_file)
]
for file_path in files:
old_file = PurePath(file_path)
current_name = old_file.name
if not self.is_current_file_name_valid_for_rename(old_file):
print('Skipping "{0}"'.format(current_name))
continue
dir_name = old_file.parent.name
new_name = self.extract_file_name(old_file,
dir_name) + old_file.suffix
print(
'Converting "{0}" to "{1}" using directory name "{2}"'.format(
current_name, new_name, dir_name))
new_file = old_file.with_name(new_name)
if not args.dry_run:
file_path.rename(new_file)
def undraft(pathlike):
"""
Remove the leading `_` from a path, if any.
"""
path = PurePath(pathlike)
if path.stem.startswith("_"):
return path.with_name(re.sub(r'^_', "", path.name))
else:
return path
async def sticker_downloader(_, message):
"""Copying @DownloadStickersBot"""
LOGGER.info(message.from_user)
sticker = message.sticker
pack_name = sticker.set_name
pack_link = f"http://t.me/addstickers/{pack_name}" if pack_name else ""
sticker_ext = PurePath(message.sticker.file_name).suffix
zipfile_name = f"{str(message.from_user.id)}_{pack_name}{sticker_ext}.zip"
Path(Config.WORK_DIR).mkdir(
exist_ok=True
) # Cause TemporaryDirectory failed to create parent directory
with TemporaryDirectory(dir=Config.WORK_DIR) as tempdir:
status = await message.reply_text(
text=String.DOWNLOAD_START, reply_to_message_id=message.message_id
)
try:
download_to = await message.download(
file_name=f"{tempdir}/", progress_args=String.DOWNLOAD_START
)
except ValueError as e:
await status.edit_text(text=str(e))
return False
await status.edit_text(
text=String.STICKER_INFO.format(
sticker.set_name, sticker.emoji, sticker.file_id
)
)
final_location = PurePath(download_to)
LOGGER.info(final_location) # /app/DOWNLOADS/<tempdir>/sticker.webp
zipfile_path = (
str(final_location.parent) + f"/{zipfile_name}"
) # /app/DOWNLOADS/<tempdir>/<zipfile_name>
with ZipFile(zipfile_path, "w") as stickerZip:
stickerZip.write(final_location, final_location.name)
if sticker_ext == ".webp":
png_location = Path(final_location.with_name(final_location.stem + ".png"))
LOGGER.info(png_location) # /app/DOWNLOADS/<tempdir>/sticker.png
# https://stackoverflow.com/a/21669827/4723940
Image.open(download_to).convert("RGB").save(png_location, "PNG")
await status.reply_photo(
photo=png_location, reply_to_message_id=status.message_id
)
await status.reply_document(
document=zipfile_path,
caption=pack_link,
reply_to_message_id=status.message_id,
)
def upload_excel_file(file: FileStorage, meta: Dict):
excel_sheets = pandas.read_excel(
file, sheet_name=None) # type: Dict[str, pandas.DataFrame]
excel_sheets = {
sheet: data
for (sheet, data) in excel_sheets.items() if not data.empty
}
basename = PurePath(cast(str, file.filename)).with_suffix('')
try:
db_files = []
def push_file(name: PurePath, data: pandas.DataFrame):
db_file = csv_file_schema.load({
'name': name.with_suffix('.csv').name,
'table': data.to_csv(index=False),
'meta': meta
})
db_files.append(db_file)
db.session.add(db_file)
if len(excel_sheets) == 1:
# single file case, simple name
data = next(iter(excel_sheets.values()))
push_file(basename, data)
else:
for sheet, data in excel_sheets.items():
push_file(basename.with_name('%s-%s' % (basename.name, sheet)),
data)
db.session.flush()
serialized = [_serialize_csv_file(f) for f in db_files]
db.session.commit()
return jsonify(serialized), 201
except Exception:
for f in db_files:
if f.uri.is_file():
f.uri.unlink()
db.session.rollback()
raise
def main():
parser = argparse.ArgumentParser(description='Basic image processing application with kernel '
'convolution.')
parser.add_argument('operation', choices=['sobel', 'sobel_gradient', 'gaussian', 'examples'],
help='The kernel operation to perform.')
parser.add_argument('src', help='Source image path.')
parser.add_argument('-o', help='Output image path.')
args = parser.parse_args()
in_path = PurePath(args.src)
if args.o is None:
out_path = in_path.with_name(f'{in_path.stem}_out{in_path.suffix}')
else:
out_path = PurePath(args.o)
img = cv2.imread(str(in_path), cv2.IMREAD_GRAYSCALE)
out = operations[args.operation](img)
cv2.imwrite(str(out_path), out)
def export(self):
#チップの幅と高さを得る
cw = self.to_int(self.entry10001_text.get())
if cw is None:
cw = 48
self.entry10001_text.set('{0:d}'.format(48))
ch = self.to_int(self.entry10002_text.get())
if ch is None:
ch = 48
self.entry10002_text.set('{0:d}'.format(48))
#元ファイルのパス
pp = PurePath(self.entry10000_text.get())
#出力
img = Image.open(self.entry10000_text.get())
for i in range(img.height // ch):
for j in range(img.width // cw):
pt = pp.with_name(pp.stem + '_y{:02d}'.format(i) +
'x{:02d}'.format(j) + pp.suffix)
box = (j * cw, i * ch, j * cw + cw, i * ch + ch)
reg = img.crop(box)
reg.save(str(pt), quality=75)
img.close()
async def _handle_link(
self,
element: IliasPageElement,
element_path: PurePath,
) -> Optional[Awaitable[None]]:
log.explain_topic(f"Decision: Crawl Link {fmt_path(element_path)}")
log.explain(f"Links type is {self._links}")
link_template_maybe = self._links.template()
link_extension = self._links.extension()
if not link_template_maybe or not link_extension:
log.explain("Answer: No")
return None
else:
log.explain("Answer: Yes")
element_path = element_path.with_name(element_path.name +
link_extension)
maybe_dl = await self.download(element_path, mtime=element.mtime)
if not maybe_dl:
return None
return self._download_link(element, link_template_maybe, maybe_dl)
def load_requirements(path: PurePath) -> List[str]:
""" Load dependencies from a requirements.txt style file, ignoring comments etc. """
res = []
with open(path) as fd:
for line in fd.readlines():
while line.endswith('\n') or line.endswith('\\'):
line = line[:-1]
line = line.strip()
if not line or line.startswith('-') or line.startswith('#'):
continue
res += [line]
return res
here = PurePath(__file__)
README = open(here.with_name('README.md')).read()
install_requires = load_requirements(here.with_name('requirements.txt'))
test_requires = load_requirements(here.with_name('test_requirements.txt'))
setup(
name='eduid-scimapi',
version=version,
description="External SCIM API for eduID",
classifiers=[
'Framework :: Falcon',
],
keywords='eduid',
author='Fredrik Thulin',
author_email='*****@*****.**',
url='https://www.eduid.se/',
#!/usr/bin/env python3
import os
from pathlib import PurePath
from asciidocapi import AsciiDocAPI
man_pages = [
'man/gnome-shell.1',
'subprojects/extensions-tool/man/gnome-extensions.1',
]
sourceroot = os.environ.get('MESON_SOURCE_ROOT')
distroot = os.environ.get('MESON_DIST_ROOT')
asciidoc = AsciiDocAPI()
for man_page in man_pages:
page_path = PurePath(man_page)
src = PurePath(sourceroot, page_path.with_suffix('.txt'))
dst = PurePath(distroot, page_path)
stylesheet = src.with_name('stylesheet.xsl')
asciidoc.options('--xsl-file', os.fspath(stylesheet))
asciidoc.execute(os.fspath(src), outfile=os.fspath(dst))
def evaluate_all(
experiments: Iterable[Tuple[Iterable[Estimator], int]],
dataset: UCIMultiClassDataset,
log_trainer: Trainer,
log_epsilon: float,
tgt_trainer: Trainer,
tgt_epsilon: float,
max_num_workers: int,
random_reward_prob: float = 0.0,
device=None,
):
action_space = ActionSpace(dataset.num_actions)
config_path = PurePath(dataset.config_file)
data_name = config_path.stem
log_model_name = data_name + "_" + log_trainer.__class__.__name__ + ".pickle"
log_model_file = str(config_path.with_name(log_model_name))
tgt_model_name = data_name + "_" + tgt_trainer.__class__.__name__ + ".pickle"
tgt_model_file = str(config_path.with_name(tgt_model_name))
log_trainer.load_model(log_model_file)
tgt_trainer.load_model(tgt_model_file)
if not log_trainer.is_trained or not tgt_trainer.is_trained:
(
train_x,
train_y,
train_r,
val_x,
val_y,
val_r,
test_x,
test_y,
test_r,
train_choices,
) = dataset.train_val_test_split((0.2, 0.8))
trainer_data = TrainingData(train_x, train_y, None, val_x, val_y, None)
if not log_trainer.is_trained:
log_trainer.train(trainer_data)
log_trainer.save_model(log_model_file)
if not tgt_trainer.is_trained:
tgt_trainer.train(trainer_data)
tgt_trainer.save_model(tgt_model_file)
log_results = log_trainer.predict(dataset.features)
assert log_results.probabilities is not None
log_policy = MultiClassPolicy(action_space, log_results.probabilities,
log_epsilon)
tgt_results = tgt_trainer.predict(dataset.features)
assert tgt_results.probabilities is not None
tgt_policy = MultiClassPolicy(action_space, tgt_results.probabilities,
tgt_epsilon)
tasks = []
# pyre-fixme[61]: `train_choices` may not be initialized here.
test_queries = list(set(range(len(dataset))) - set(train_choices))
for estimators, num_samples in experiments:
samples = []
for _ in range(num_samples):
qid = random.sample(test_queries, 1)
label = int(dataset.labels[qid].item())
log_action, log_action_probabilities = log_policy(qid)
log_reward = 1.0 if log_action.value == label else 0.0
tgt_action, tgt_action_probabilities = tgt_policy(qid)
ground_truth_reward = 1.0 if tgt_action.value == label else 0.0
item_feature = dataset.features[qid]
random_reward = random.random() < random_reward_prob
samples.append(
LogSample(
context=qid,
log_action=log_action,
log_reward=random.randint(0, 1)
if random_reward else log_reward,
log_action_probabilities=log_action_probabilities,
tgt_action_probabilities=tgt_action_probabilities,
tgt_action=tgt_action,
ground_truth_reward=ground_truth_reward,
item_feature=item_feature,
))
tasks.append(
(estimators, BanditsEstimatorInput(action_space, samples, False)))
evaluator = Evaluator(tasks, max_num_workers)
results = evaluator.evaluate()
Evaluator.report_results(results)
return results
with open(path) as fd:
for line in fd.readlines():
while line.endswith('\n') or line.endswith('\\'):
line = line[:-1]
line = line.strip()
if not line or line.startswith('-') or line.startswith('#'):
continue
res += [line]
return res
version = '0.0.4'
here = PurePath(__file__)
install_requires = load_requirements(here.with_name('requirements.txt'))
test_requires = load_requirements(here.with_name('test_requirements.txt'))
README = 'SUNET haproxy status'
try:
README = open(os.path.join(here, 'README.rst')).read()
except IOError:
pass
setup(
name='haproxy_status',
version=version,
description='SUNET haproxy status page',
long_description=README,
# TODO: add classifiers
classifiers=[
def inner(path: PurePath) -> Optional[PurePath]:
if path.name == source:
return path.with_name(target)
return None
def intermediate_path(job: dict,
render_path: pathlib.PurePath) -> pathlib.PurePath:
"""Determine the intermediate render output path."""
name = f'{render_path.name}__intermediate-{job["_created"]:%Y-%m-%d_%H%M%S}'
return render_path.with_name(name)
def replace_filename():
'''I can only replace the filename. Fails on PurePaths that never had a filename to begin with'''
relative = PurePath('austinchang/tutorials/python/language/python_37/popular_modules/pathlib_/purepath/methods.r2d2.py')
print(relative.with_name('davidchang')) # austinchang/tutorials/python/language/python_37/popular_modules/pathlib_/purepath/davidchang
abs_ = PurePath('/')
print(abs_.with_name('hi')) # ValueError: PurePosixPath('/') has an empty name
def load_requirements(path: PurePath) -> List[str]:
""" Load dependencies from a requirements.txt style file, ignoring comments etc. """
res = []
with open(path) as fd:
for line in fd.readlines():
while line.endswith('\n') or line.endswith('\\'):
line = line[:-1]
line = line.strip()
if not line or line.startswith('-') or line.startswith('#'):
continue
res += [line]
return res
here = PurePath(__file__)
README = open(here.with_name('README.rst')).read()
NEWS = open(here.with_name('NEWS.txt')).read()
install_requires = load_requirements(here.with_name('requirements.txt'))
tests_require = load_requirements(here.with_name('test_requirements.txt'))
python_implementation_str = python_implementation()
setup(
name='pyFF',
version=__version__,
description="Federation Feeder",
long_description=README + '\n\n' + NEWS,
classifiers=[
# Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
'Programming Language :: Python :: 3',
def evaluate_all(
experiments: Iterable[Tuple[Iterable[Estimator], int]],
dataset: UCIMultiClassDataset,
log_trainer: Trainer,
log_epsilon: float,
tgt_trainer: Trainer,
tgt_epsilon: float,
max_num_workers: int,
device=None,
):
action_space = ActionSpace(dataset.num_actions)
config_path = PurePath(dataset.config_file)
data_name = config_path.stem
log_model_name = data_name + "_" + log_trainer.__class__.__name__ + ".pickle"
log_model_file = str(config_path.with_name(log_model_name))
tgt_model_name = data_name + "_" + tgt_trainer.__class__.__name__ + ".pickle"
tgt_model_file = str(config_path.with_name(tgt_model_name))
log_trainer.load_model(log_model_file)
tgt_trainer.load_model(tgt_model_file)
if not log_trainer.is_trained or not tgt_trainer.is_trained:
(
train_x,
train_y,
train_r,
val_x,
val_y,
val_r,
test_x,
test_y,
test_r,
) = dataset.train_val_test_split((0.8, 0.8))
trainer_data = TrainingData(train_x, train_y, None, val_x, val_y, None)
if not log_trainer.is_trained:
log_trainer.train(trainer_data)
log_trainer.save_model(log_model_file)
if not tgt_trainer.is_trained:
tgt_trainer.train(trainer_data)
tgt_trainer.save_model(tgt_model_file)
log_results = log_trainer.predict(dataset.features)
log_policy = MultiClassPolicy(action_space, log_results.probabilities,
log_epsilon)
tgt_results = tgt_trainer.predict(dataset.features)
tgt_policy = MultiClassPolicy(action_space, tgt_results.probabilities,
tgt_epsilon)
inputs = []
tasks = []
total_queries = len(dataset)
for estimators, num_samples in experiments:
samples = []
for i in range(num_samples):
qid = random.randrange(total_queries)
label = int(dataset.labels[qid].item())
log_action, log_action_probabilities = log_policy(qid)
log_reward = 1.0 if log_action.value == label else 0.0
tgt_action, tgt_action_probabilities = tgt_policy(qid)
ground_truth_reward = 1.0 if tgt_action.value == label else 0.0
item_feature = dataset.features[qid]
samples.append(
LogSample(
context=qid,
log_action=log_action,
log_reward=log_reward,
log_action_probabilities=log_action_probabilities,
tgt_action_probabilities=tgt_action_probabilities,
tgt_action=tgt_action,
ground_truth_reward=ground_truth_reward,
item_feature=item_feature,
))
tasks.append(
(estimators, BanditsEstimatorInput(action_space, samples, False)))
logging.info("start evaluating...")
st = time.perf_counter()
evaluator = Evaluator(tasks, max_num_workers)
results = evaluator.evaluate()
Evaluator.report_results(results)
logging.info(f"evaluating done in {time.perf_counter() - st}s")
return results
def intermediate_path(job: dict, render_path: pathlib.PurePath) -> pathlib.PurePath:
"""Determine the intermediate render output path."""
name = f'{render_path.name}__intermediate-{job["_created"]:%Y-%m-%d_%H%M%S}'
return render_path.with_name(name)
