def post_to_imgur(imgur: ImgurClient, config: configparser, dir: str,
memes: Memes) -> str:
"""
Uploads images and creates a public albumb of the memes.
"""
# Upload images
image_ids: List[str] = []
for meme in memes:
response = imgur.upload_from_path(path=os.path.join(dir, meme),
config=None,
anon=False)
image_ids.append(response["id"])
# Create album
ymd = str(date.today())
title = config.get("daily-memes", "imgur_album_title_prefix")
desc = config.get("daily-memes", "imgur_album_desc")
album = imgur.create_album({
"title": f"{title} {ymd}",
"desc": desc,
"ids": ",".join(image_ids),
"privacy": "public"
})
return album["id"]
def _add_profile_config(option_file: configparser, profile: str,
region: str) -> None:
config_name = f'profile {profile}'
if not option_file.has_section(config_name):
option_file.add_section(config_name)
option_file.set(config_name, 'region', region)
option_file.set(config_name, 'output', 'json')
def __init__(self, config: configparser, wiki_extpath, model_dir_abspath):
self.wiki_glob_path = os.path.join(wiki_extpath, '**', "wiki_*")
self.prefix = os.path.join(model_dir_abspath,
config.get('train', 'model_name'))
self.vocab_size = config.get('train', 'vocab_size')
if not os.path.exists(model_dir_abspath):
os.mkdir(model_dir_abspath)
# Sentence Piece は、パスに スペースが入ってると動かない...。ので、一旦別の場所に移動する。
self.is_space_in_path = ' ' in model_dir_abspath
if self.is_space_in_path:
print('[確認](SP_Trainer) パスにスペースが入っていたため、回避処理を開始します。')
os.path.splitdrive(os.path.abspath(__file__))
self.root_dir = os.path.join(self.get_root_path(), 'tmp',
'SentencePiece', 'out')
# Wikipedia extractデータ をコピー
if os.path.exists(self.root_dir) and len(os.listdir(
self.root_dir)) == 0:
os.rmdir(self.root_dir)
if not os.path.exists(self.root_dir):
shutil.copytree(wiki_extpath, self.root_dir)
self.wiki_glob_path = os.path.join(self.root_dir, '**', "wiki_*")
# model ファイルは、通常の場所にもコピーする。という方向で。
self.prefix = os.path.join(self.root_dir,
config.get('train', 'model_name'))
self.prefix_tmp = os.path.join(model_dir_abspath,
config.get('train', 'model_name'))
print('[確認](SP_Trainer) パス スペース問題 回避のためのコピー完了。')
def ask_for_config(config_filename: str, config: configparser) -> None:
print('no config file, please fill in missing values')
for v, tip in config_values.items():
config.set(SECTION, v, input(f"{v}({tip}):"))
with open(config_filename, 'w') as configfile:
config.write(configfile)
print(f"configuration saved in {os.path.realpath(configfile.name)}")
def initialC(self):
data = ConfigParser()
data.read('validationData/initial_constants.ini')
name = self.name + '-'\
+ repr(self.index[0]) + '-' + repr(self.index[1])
C = []
for i in range(4):
C.append((float)(data.get(name, 'C' + repr(i + 1))))
return C
def initialC(self):
data = ConfigParser()
data.read('validationData/initial_constants.ini')
name = self.name + '-'\
+ repr(self.index[0]) + '-' + repr(self.index[1])
C = []
for i in range(4):
C.append((float)(data.get(name, 'C' + repr(i + 1))))
return C
def _remove_unused_configs(config_file: configparser,
profile_group: ProfileGroup):
used_profiles = profile_group.list_profile_names()
used_profiles.append('access-key')
for config_name in config_file.sections():
profile = config_name.replace('profile ', '')
if profile not in used_profiles:
config_file.remove_section(config_name)
return config_file
def _add_profile_credentials(credentials_file: configparser, profile: str,
secrets: dict) -> None:
if not credentials_file.has_section(profile):
credentials_file.add_section(profile)
credentials_file.set(profile, 'aws_access_key_id',
str(secrets['AccessKeyId']))
credentials_file.set(profile, 'aws_secret_access_key',
str(secrets['SecretAccessKey']))
credentials_file.set(profile, 'aws_session_token',
str(secrets['SessionToken']))
def setup_imgur(config: configparser) -> ImgurClient:
"""
Setups Imgur and logs in the user.
"""
imgur = client = ImgurClient(client_id=config.get("daily-memes",
"imgur_client_id"),
client_secret=config.get(
"daily-memes", "imgur_client_secret"))
imgur.set_user_auth(access_token=config.get("daily-memes",
"imgur_access_token"),
refresh_token=config.get("daily-memes",
"imgur_refresh_token"))
return imgur
def _getPlugboardSettings(self, config: configparser) -> dict:
try:
return dict(wiring[1].split(',')
for wiring in config.items('plugboard'))
except ValueError as error:
raise Exception('Unable to parse plugboard configuration')
def read_config(self):
""" Read config from config file
"""
conf = ConfigParser()
conf.read(self.configfile)
self.host = conf.get("jobs", "host")
self.queue = conf.get("jobs", "queue")
self.rhost = conf.get("results", "host")
self.rqueue = conf.get("results", "queue")
self.username = conf.get("credentials", "username")
self.password = conf.get("credentials", "password")
def read_config(self):
""" Read config from config file
"""
conf = ConfigParser()
conf.read(self.configfile)
self.host = conf.get("jobs", "host")
self.queue = conf.get("jobs", "queue")
self.rhost = conf.get("results", "host")
self.rqueue = conf.get("results", "queue")
self.username = conf.get("credentials", "username")
self.password = conf.get("credentials", "password")
def _write_file(path: str, config_parser: configparser) -> None:
with open(path, 'w') as file:
config_parser.write(file)
def reaload_dbconfig(self, parsed_config: configparser):
"""
reload Database config
:param parsed_config: config Object
:return:
"""
# mysql config
mysql_host = parsed_config.get('mysql', 'Host', fallback='localhost')
mysql_id = parsed_config.getint('mysql', 'ID')
mysql_port = parsed_config.getint('mysql', 'Port', fallback=3306)
mysql_user = parsed_config.get('mysql', 'User')
mysql_passwd = parsed_config.get('mysql', 'Passwd')
mysql_database = parsed_config.get('mysql', 'Database')
mysql_charset = parsed_config.get('mysql', 'Charset')
mysql_pool_size = parsed_config.getint('mysql', 'MaxPoolSize')
# redis config
redis_host = parsed_config.get('redis', 'Host', fallback='localhost')
redis_id = parsed_config.getint('redis', 'ID')
redis_port = parsed_config.getint('redis', 'Port')
redis_passwd = parsed_config.get('redis', 'Passwd')
redis_timeout = parsed_config.getint('redis', 'Timeout')
self.dbconfig.__setitem__(
'mysql',
config.to_formated_dbconfig(mysql_id,
mysql_host,
mysql_port,
mysql_passwd,
database=mysql_database,
user=mysql_user,
charset=mysql_charset,
pool_size=mysql_pool_size))
self.dbconfig.__setitem__(
'redis',
config.to_formated_dbconfig(redis_id,
redis_host,
redis_port,
redis_passwd,
timeout=redis_timeout))
def get_writer_config_dict(config_parser: configparser):
config_dict = dict(config_parser.items('file'))
config_dict['columns'] = dict(config_parser.items('columns'))
return config_dict
def _getReflector(self, config: configparser) -> str:
try:
return config.get('reflector', 'reflector')
except ValueError as error:
raise Exception('Unable to parse reflector configuration')
def _getRotorsSettings(self, config: configparser) -> dict:
try:
return dict(rotor_position
for rotor_position in config.items('rotor_positions'))
except ValueError as error:
raise Exception('Unable to parse rotor_positions configuration')
# help_functions.py
from help_functions import *
# extract_patches.py
from extract_patches import recompone
from extract_patches import recompone_overlap
from extract_patches import paint_border
from extract_patches import kill_border
from extract_patches import pred_only_FOV
from extract_patches import get_data_testing
from extract_patches import get_data_testing_overlap
# pre_processing.py
from pre_processing import my_PreProc
#config file to read from
config = ConfigParser()
config.read('configuration.txt')
#===========================================
#run the training on invariant or local
path_data = config.get('data paths', 'path_local')
#original test images (for FOV selection)
DRIVE_test_imgs_original = path_data + config.get('data paths', 'test_imgs_original')
test_imgs_orig = load_hdf5(DRIVE_test_imgs_original)
full_img_height = test_imgs_orig.shape[2]
full_img_width = test_imgs_orig.shape[3]
#the border masks provided by the DRIVE
DRIVE_test_border_masks = path_data + config.get('data paths', 'test_border_masks')
test_border_masks = load_hdf5(DRIVE_test_border_masks)
# dimension of the patches
patch_height = config.getint('data attributes', 'patch_height')
def _try_load_section(self, section: str, config_parser: configparser,
data: dict) -> bool:
if section in config_parser:
data.update(dict(config_parser.items(section)))
return True
return False
def _getRotorsIds(self, config: configparser) -> dict:
try:
return dict(rotor for rotor in config.items('rotors'))
except ValueError as error:
raise Exception('Unable to parse rotors configuration')
