def pull_metadata_from_server():
"""Automatically metadata from server"""
file_user_config = os.path.join(utils.get_base_path(),
utils.FILE_USER_CONFIG)
folder_base = utils.get_base_path()
username = None
if os.path.exists(file_user_config):
with open(file_user_config, 'r') as f_in:
configs = json.load(f_in)
username = configs.get('username', None)
# Ask for username if not available
if username is None:
username = input('Username to access Hanoi server: ')
utils.update_user_config({'username': username})
print('Loading metadata folder from server...')
commands = [
'rsync', '-vur', '-e', 'ssh{}'.format(utils.SERVER_PORT), '--delete',
'--progress', '{}@{}:{}'.format(username, utils.SERVER_IP,
utils.SERVER_FOLDER_METADATA),
'{}/'.format(folder_base)
]
subprocess.run(commands)
print('Finished!')
def persist_activity(self, activity_id, format='TCX'):
client = self.get_client()
if format == 'TCX':
data = client.download_activity(
activity_id, dl_fmt=client.ActivityDownloadFormat.TCX)
output_file = f'{str(activity_id)}.tcx'
elif format == 'GPX':
data = client.download_activity(
activity_id, dl_fmt=client.ActivityDownloadFormat.GPX)
output_file = f'{str(activity_id)}.gpx'
elif format == 'ZIP':
data = client.download_activity(
activity_id, dl_fmt=client.ActivityDownloadFormat.ORIGINAL)
output_file = f'{str(activity_id)}.zip'
else:
raise ActivityExtractorException(
'Invalid or not supported activity format given. Supported fomarts are: TCX, GPX and ZIP'
)
data_dir = os.path.join(get_base_path(), 'garmin', 'data')
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
with open(os.path.join(data_dir, output_file), 'wb') as fb:
fb.write(data)
def generate_dummy_activity(self):
file_path = os.path.join(get_base_path(), 'strava', 'example-activity.json')
for _ in range(MAX_RETRIES):
with open(file_path, 'r') as file:
activity = json.load(file)
fake_activity(activity)
for segment in activity['segment_efforts']:
fake_segment(activity, segment)
for lap in activity['laps']:
fake_lap(activity, lap)
for split in activity['splits_metric']:
fake_split_metric(activity, split)
try:
verify_data_integrity(activity)
except AssertionError:
logger.info('Generated activity data not consistent. Trying again...')
continue
return activity
else:
logger.warning(f'Generating consistent activity data failed {MAX_RETRIES} times. Aborting and using inconsistent values')
def _get_access_token():
token_path = os.path.join(get_base_path(), 'strava', 'strava-token.json')
try:
with open(token_path, 'r') as file:
creds = json.load(file)
return creds['access_token']
except FileNotFoundError:
return None
def pull_data_from_server():
"""Automatically update image data from server"""
file_user_config = os.path.join(utils.get_base_path(),
utils.FILE_USER_CONFIG)
folder_data = utils.get_base_path(utils.FOLDER_DATA)
folder_work = utils.get_base_path(utils.FOLDER_WORK)
folder_data_zipped = utils.get_base_path(utils.FOLDER_DATA_ZIPPED)
username = None
if os.path.exists(file_user_config):
with open(file_user_config, 'r') as f_in:
configs = json.load(f_in)
username = configs.get('username', None)
# Ask for username if not available
if username is None:
username = input('Username to access Hanoi server: ')
utils.update_user_config({'username': username})
print('Loading image folder from server...')
os.makedirs(folder_data, exist_ok=True)
commands = [
'rsync', '-vur', '-e', 'ssh{}'.format(utils.SERVER_PORT), '--delete',
'--progress', '{}@{}:{}'.format(username, utils.SERVER_IP,
utils.SERVER_FOLDER_DATA_ZIPPED),
'{}'.format(folder_work)
]
subprocess.run(commands)
zip_data = set([
os.path.splitext(os.path.basename(each_file))[0]
for each_file in glob.glob(os.path.join(folder_data_zipped, '*.zip'))
])
image_data = set([
os.path.basename(each_file[:-1])
for each_file in glob.glob(os.path.join(folder_data, '*/'))
])
update_data = list(zip_data.difference(image_data))
for each_file in update_data:
source_zip_file = os.path.join(folder_data_zipped,
'{}.zip'.format(each_file))
with zipfile.ZipFile(source_zip_file, 'r') as f_zipped:
print('Unzipping {}...'.format(source_zip_file))
f_zipped.extractall(folder_data)
def add_clean_images(input_folder, input_json, force=False):
"""Add the images into data folder
@NOTE: this method assumes that the images in `input_folder` and the
content in `input_json` is good to copy.
# Arguments
input_folder [str]: the path to input image folder
input_json [str]: the path to base json information
force [bool]: if True, then will force copy even if the output folder
and output json exist
# Returns
[str]: the path to output folder image
"""
# retrieve output folder name
input_folder = input_folder[:-1] if input_folder[
-1] == '/' else input_folder
data_folder = utils.get_base_path(utils.FOLDER_DATA)
folder_name = os.path.basename(input_folder)
output_folder = os.path.join(data_folder, folder_name)
# check if the output folder currently exists
if os.path.exists(output_folder) and not force:
raise AttributeError(
'the `output_folder` already exists: {}'.format(output_folder))
# retrieve output metadata name
metadata_folder = utils.get_base_path(utils.FOLDER_METADATA)
output_json = os.path.join(metadata_folder, os.path.basename(input_json))
# check if the output json file currently exists
if os.path.exists(output_json) and not force:
raise AttributeError(
'the `output_json` already exists: {}'.format(output_json))
subprocess.run(['cp', '-r', input_folder, output_folder])
subprocess.run(['cp', input_json, output_json])
print('The output folder is {}'.format(output_folder))
print('The output json is {}'.format(output_json))
return output_folder, output_json
def fresh_initialize():
"""Initialize the minisystem"""
# Working files and directories
folder_base = utils.get_base_path()
folder_data = utils.get_base_path(utils.FOLDER_DATA)
folder_metadata = utils.get_base_path(utils.FOLDER_METADATA)
folder_versions = utils.get_base_path(utils.FOLDER_VERSIONS)
folder_work = utils.get_base_path(utils.FOLDER_WORK)
folder_data_zipped = utils.get_base_path(utils.FOLDER_DATA_ZIPPED)
file_bad = os.path.join(folder_metadata, utils.FILE_BAD)
file_user_config = os.path.join(folder_base, utils.FILE_USER_CONFIG)
# Creating folders
os.makedirs(folder_data, exist_ok=True)
os.makedirs(folder_metadata, exist_ok=True)
os.makedirs(folder_versions, exist_ok=True)
os.makedirs(folder_work, exist_ok=True)
os.makedirs(folder_data_zipped, exist_ok=True)
# Creating files
with open(file_bad, 'w') as f_out:
f_out.write('')
with open(file_user_config, 'w') as f_out:
json.dump({}, f_out)
# Set up
username = input('Username to access Hanoi server (to sync image data): ')
utils.update_user_config({'username': username})
print('Initialized!')
def main():
signal(SIGINT, sig_handler)
args = parse_args()
default_branch = args.branch
username = utils.get_username(args)
if args.gerrit == "submit":
gerrit.submit(default_branch, username, args.owner)
sys.exit(0)
elif args.gerrit == "vote":
gerrit.vote(default_branch, username, args.owner)
sys.exit(0)
base_path = utils.get_base_path(default_branch)
config_dict = utils.get_config_dict(args.config, default_branch)
if args.path_to_crowdin == "crowdin" and not utils.check_dependencies():
sys.exit(1)
if args.upload_sources:
upload.upload_sources_crowdin(default_branch, config_dict,
args.path_to_crowdin)
elif args.upload_translations:
upload.upload_translations_crowdin(default_branch, config_dict,
args.path_to_crowdin)
elif args.download:
xml_files = utils.get_xml_files(base_path, default_branch)
download.download_crowdin(
base_path,
default_branch,
xml_files,
username,
config_dict,
args.path_to_crowdin,
)
elif args.unzip:
xml_files = utils.get_xml_files(base_path, default_branch)
from_zip.unzip(args.unzip, base_path, default_branch, xml_files,
username)
if download.has_created_commits() or upload.has_uploaded():
print("\nDone!")
sys.exit(0)
else:
print("\nNothing to commit")
sys.exit(2)
def get_token(code):
params = {
"client_id": os.getenv('STRAVA_CLIENT_ID'),
"client_secret": os.getenv('STRAVA_CLIENT_SECRET'),
"code": code,
"grant_type": "authorization_code"
}
response = requests.post("https://www.strava.com/oauth/token", params)
if response.status_code != 200:
logger.error(
f'Retrieving access token failed. {response.url} - {response.text}'
)
return
token_data = response.json()
token_path = os.path.join(get_base_path(), 'strava-token.json')
with open(token_path, 'w') as f:
json.dump(token_data, f, indent=4, sort_keys=True)
return token_data.get('access_token')
def main():
load_dotenv()
instagram_path = os.path.join(get_base_path(),
os.getenv('INSTAGRAM_DIR', 'instagram'))
images_path = get_images_path()
os.makedirs(instagram_path, exist_ok=True)
bot = Bot(base_path=instagram_path)
bot.login(username=os.getenv('INSTAGRAM_LOGIN'),
password=os.getenv('INSTAGRAM_PASSWORD'))
for filename in os.listdir(images_path):
image_path = os.path.join(images_path, filename)
if os.path.isfile(image_path):
bot.upload_photo(image_path, caption='')
# remove tmp files
for filename in filter(lambda x: x.endswith('.REMOVE_ME'),
os.listdir(images_path)):
os.unlink(os.path.join(images_path, filename))
model.reset_context_params()
# this will take the mean over the batches
logger.summarise_inner_loop(mode='valid')
# keep track of best models
logger.update_best_model(model, save_path)
if __name__ == '__main__':
args = arguments.parse_args()
# --- settings ---
if not os.path.exists(os.path.join(utils.get_base_path(), 'result_files')):
os.mkdir(os.path.join(utils.get_base_path(), 'result_files'))
if not os.path.exists(os.path.join(utils.get_base_path(), 'result_plots')):
os.mkdir(os.path.join(utils.get_base_path(), 'result_plots'))
path = os.path.join(utils.get_base_path(), 'result_files',
utils.get_path_from_args(args))
log_interval = 100
if (not os.path.exists(path + '.npy')) or args.rerun:
print('Starting experiment. Logging under filename {}'.format(path +
'.npy'))
run(args, num_workers=1, log_interval=log_interval, save_path=path)
else:
print(
'Found results in {}. If you want to re-run, use the argument --rerun'
def check_central_repo():
"""Check that there is no problem in central repo
This method can confirm that:
- the number of files in json is equal to the number of files in folder
- the filepath in json is legit
- the filename format is <Project>_<YYYYMMDD>_<index>.<ext>
- important files and folders exist
"""
# check important files and folders exist
print('\n=> Check important files and folders exist')
for each_folder in [
utils.FOLDER_DATA, utils.FOLDER_WORK, utils.FOLDER_DATA_ZIPPED,
utils.FOLDER_METADATA, utils.FOLDER_SCRIPT, utils.FOLDER_VERSIONS
]:
folder_path = utils.get_base_path(each_folder)
if not os.path.exists(folder_path):
print(':WARNING: folder {} does not exist'.format(folder_path))
for each_file in [
os.path.join(utils.FOLDER_METADATA, utils.FILE_BAD),
utils.FILE_USER_CONFIG
]:
file_path = utils.get_base_path(each_file)
if not os.path.exists(file_path):
print(':WARNING: file {} does not exist'.format(file_path))
features = utils.get_all_metadata()
folder_data = utils.get_base_path(utils.FOLDER_DATA) + '/'
images = glob.glob(os.path.join(folder_data, '**', '*.*'), recursive=True)
images = set([each_file.replace(folder_data, '') for each_file in images])
# check the number of files is equal to the number of files in folder
print('\n=> Check the number of file in json is equal to that in folder')
if len(features) == len(images):
print('Equal files ({})'.format(len(features)))
else:
print(':WARNING: not equal: json {}, files {}'.format(
len(features), len(images)))
# check filename format is <Project>_<YYYYMMDD>_<index>.<ext>
print('\n=> Check filename format is <Project>_<YYYYMMDD>_<index>.<ext>')
for each_file in list(images):
filename = os.path.basename(each_file)
filename, _ = os.path.splitext(filename)
bad = False
try:
_, date, idx = filename.split('_')
except ValueError:
bad = True
try:
date = arrow.get(date, 'YYYYMMDD')
except arrow.parser.ParserError:
bad = True
try:
idx = int(idx)
except ValueError:
bad = True
if bad:
print(':WARNING: {}'.format(each_file))
# check filepath in json is legit
print('\n=> Check file path in json is legit')
features_keys = set(features.keys())
only_features_keys = features_keys.difference(images)
only_files = images.difference(features_keys)
if len(only_features_keys) > 0:
print('there are {} files in json but does not in folder, they are {}'.
format(len(only_features_keys), only_features_keys))
if len(only_files) > 0:
print(
'there are {} files in folder but not in json, they are {}'.format(
len(only_files), only_files))
print('\nFinished!')
def generate_train_test(version_name, condition, version=None):
"""Generate the train and test
Under this method:
- POC samples have 80% probability in test set
- collection samples have 20% probability in test set
- synthetic samples have 0% probabilty in test set
# Arguments
field [str]: the field. Currently support name, address, all
version [str]: <major>.<minor>
# Returns
[str]: path to train version
[str]: path to test version
"""
folder_version = os.path.join(utils.get_base_path(utils.FOLDER_VERSIONS),
version_name)
folder_data = utils.get_base_path(utils.FOLDER_DATA)
folder_metadata = utils.get_base_path(utils.FOLDER_METADATA)
# sanity check
os.makedirs(folder_version, exist_ok=True)
# get the next minor version
if version is None:
olds = glob.glob(
os.path.join(folder_version,
'{}-train*.*.txt'.format(version_name)))
if len(olds) == 0:
version = '1.0'
else:
olds = [os.path.basename(each_file) for each_file in olds]
olds = [os.path.splitext(each_file)[0] for each_file in olds]
olds = [
each_file.replace('{}-train'.format(version_name), '')
for each_file in olds
]
olds = [each_file.split('.') for each_file in olds]
old_versions = []
for major, minor in olds:
try:
old_versions.append((int(major), int(minor)))
except ValueError:
continue
old_versions.sort(key=lambda obj: obj[0])
old_versions.sort(key=lambda obj: obj[1])
major, minor = old_versions[-1]
version = '{}.{}'.format(major, minor + 1)
# get valid images
all_images = set(
glob.glob(os.path.join(folder_data, '**', '*.*'), recursive=True))
# get bad images
bad_images = set(utils.get_bad_files())
out_images = list(all_images.difference(bad_images))
out_images.sort()
# get label
features = utils.get_all_metadata()
folder_data += '/'
finals_train, finals_test = [], []
for each_file in out_images:
filename = each_file.replace(folder_data, '')
proceed = True
for key, value in condition.items():
if value != features[filename].get(key, None):
proceed = False
if not proceed:
continue
label = features[filename][utils.KEY_LABEL]
origin = features[filename][utils.KEY_ORIGIN]
# train/test split based on origin is poc, collection or synthetic
if origin == utils.VALUE_ORIGIN_POC:
if random.random() > 0.8:
finals_train.append('{},{}'.format(filename, label))
else:
finals_test.append('{},{}'.format(filename, label))
elif origin == utils.VALUE_ORIGIN_COLLECTION:
if random.random() > 0.2:
finals_train.append('{},{}'.format(filename, label))
else:
finals_test.append('{},{}'.format(filename, label))
else:
finals_train.append('{},{}'.format(filename, label))
# dump the versions
target_train = os.path.join(folder_version,
'{}-train{}.txt'.format(version_name, version))
target_test = os.path.join(folder_version,
'{}-test{}.txt'.format(version_name, version))
with open(target_train, 'w') as f_out:
f_out.write('\n'.join(finals_train))
with open(target_test, 'w') as f_out:
f_out.write('\n'.join(finals_test))
print('The generated train {}'.format(target_train))
print('The generated test {}'.format(target_test))
return target_train, target_test
model.reset_context_params()
# this will take the mean over the batches
logger.summarise_inner_loop(mode='valid')
# keep track of best models
logger.update_best_model(model, save_path)
if __name__ == '__main__':
args = arguments.parse_args()
# --- settings ---
if not os.path.exists(os.path.join(utils.get_base_path(), 'result_files')):
os.mkdir(os.path.join(utils.get_base_path(), 'result_files'))
if not os.path.exists(os.path.join(utils.get_base_path(), 'result_plots')):
os.mkdir(os.path.join(utils.get_base_path(), 'result_plots'))
path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args))
log_interval = 100
if (not os.path.exists(path + '.npy')) or args.rerun:
print('Starting experiment. Logging under filename {}'.format(path + '.npy'))
run(args, num_workers=1, log_interval=log_interval, save_path=path)
else:
print('Found results in {}. If you want to re-run, use the argument --rerun'.format(path))
# -------------- plot -----------------
for elem in root.iter():
if elem.text.startswith('YOUR'):
elem.text = "MY_VALUE"
assert check_number_of_elements_containing_text(root, 'YOUR') == 0
xml_string_for_updated_tree = ElementTree.tostring(root).decode()
parsed_string = xmltodict.parse(xml_string_for_updated_tree)
target_json_file = open(path_to_destination_file, 'w')
target_json_file.write(json.dumps(parsed_string, indent=4))
target_json_file.close()
if not os.path.isfile(path_to_destination_file):
raise FileNotFoundError("Result file isn't created")
else:
raise FileNotFoundError(
"Source file cannot be found by specified path")
def check_number_of_elements_containing_text(root, value):
number = 0
for elem in root.iter():
if elem.text.startswith(value):
number += 1
return number
if __name__ == '__main__':
path_to_data_file = os.path.join(get_base_path(),
'file_samples/xml/test_data.xml')
path_to_target_file = os.path.join(
get_base_path(), 'file_samples/json/updated_test_data.json')
data_file_processing(path_to_data_file, path_to_target_file)
def main():
args = parse_command_line_arguments()
api = pyyoutube.Api(api_key=config.YT_API_KEY)
playlists = api.get_playlist_by_id(
playlist_id=args.playlist, parts=["snippet"]
).items
paths = map(lambda playlist: utils.get_base_path(args, playlist), playlists)
for playlist, path in zip(playlists, paths):
os.makedirs(path, exist_ok=args.force)
with open(os.path.join(path, config.INDEX), "w") as f:
frontmatter = config.FRONTMATTER["series"].format(
date=playlist.snippet.publishedAt,
description=args.description
or playlist.snippet.description.split("\n")[0],
title=args.title or playlist.snippet.title,
playlist_id=args.playlist,
)
f.write(frontmatter)
path_chapter = os.path.join(path, config.DIR_CHAPTER)
os.makedirs(path_chapter, exist_ok=args.force)
with open(os.path.join(path_chapter, config.INDEX), "w") as f:
f.write(config.FRONTMATTER["chapter"])
playlist_items = api.get_playlist_items(
playlist_id=args.playlist, parts=["snippet"], count=None
).items
snippets = [playlist.snippet for playlist in playlist_items]
snippets_grouped = groupby(
enumerate(snippets), lambda ix: ix[0] // config.YT_MAX_RESULTS
)
videos = [
api.get_video_by_id(
video_id=[s.resourceId.videoId for _, s in snippets]
).items
for _, snippets in snippets_grouped
]
videos = chain.from_iterable(videos)
snippet_frontmatter = []
for snippet, video in zip(snippets, videos):
index = snippet.position + 1
frontmatter = config.FRONTMATTER["video"].format(
date=snippet.publishedAt,
title=snippet.title,
description=snippet.description.split('\n', 1)[0],
# Add 1 so the weight starts at 1, not 0.
weight=index,
video_id=snippet.resourceId.videoId,
video_duration=video.contentDetails.get_video_seconds_duration(),
)
path = os.path.join(
path_chapter,
"{}_{}.md".format(index, utils.sanitize_title(snippet.title)),
)
with open(path, "w",) as f:
f.write(frontmatter)
def persist_activity(self, id, include_all_efforts=False):
activity = self.get_activity(id, include_all_efforts)
file_path = os.path.join(get_base_path(), 'strava', 'data',
f'activity-{id}.json')
with open(file_path, 'w') as f:
json.dump(activity, f, indent=4, sort_keys=True)
def __init__(self, dict_result):
self.list_dat_csv = self.__get_file_list()
self.base_path = utils.get_base_path()
self.dict_result = dict_result
# --- settings ---
args.k_shot = 1
args.lr_inner = 1.0
args.lr_meta = 'decay'
args.num_grad_steps_inner = 2
args.num_grad_steps_eval = 2
args.model = 'cnn'
args.num_context_params = 100
if args.k_shot == 1:
args.tasks_per_metaupdate = 4
else:
args.tasks_per_metaupdate = 2
path = os.path.join(utils.get_base_path(), 'result_files', datetime_folder, utils.get_path_from_args(args))
try:
training_stats, validation_stats = np.load(path + '.npy')
except FileNotFoundError:
print('You need to run the experiments first and make sure the results are saved at {}'.format(path))
raise FileNotFoundError
# initialise logger
logger = Logger(args)
logger.print_header()
for num_grad_steps in [2]:
print('\n --- ', num_grad_steps, '--- \n')
# initialise logger
def main():
parser = generate_parser()
args = parser.parse_args()
if args.path is not None and args.folders is None:
parser.error("When using --path, you should specify a folder structure"
" using --folders. Use -h for help")
elif args.folders is not None and len(args.playlists) != 1:
parser.error("When using --folders, multiple playlists aren't allowed."
" Use -h for help")
api = yt.Api(api_key=cfg.YT_API_KEY)
playlists = api.get_playlist_by_id(playlist_id=args.playlists,
parts=["snippet"]).items
paths = map(lambda playlist: u.get_base_path(args, playlist), playlists)
for playlist, path in zip(playlists, paths):
os.makedirs(path, exist_ok=args.force)
with open(os.path.join(path, cfg.INDEX), "w") as f:
frontmatter = cfg.FRONTMATTER["series"].format(
date=playlist.snippet.publishedAt,
description=args.description
or playlist.snippet.description.split("\n")[0],
title=args.title or playlist.snippet.title,
playlist_id=args.playlists,
)
f.write(frontmatter)
path_chapter = os.path.join(path, cfg.DIR_CHAPTER)
os.makedirs(path_chapter, exist_ok=args.force)
with open(os.path.join(path_chapter, cfg.INDEX), "w") as f:
f.write(cfg.FRONTMATTER["chapter"])
playlist_items = api.get_playlist_items(playlist_id=args.playlists,
parts=["snippet"],
count=None).items
snippets = [playlist.snippet for playlist in playlist_items]
snippets_grouped = groupby(enumerate(snippets),
lambda ix: ix[0] // cfg.YT_MAX_RESULTS)
videos = [
api.get_video_by_id(
video_id=[s.resourceId.videoId for _, s in snippets]).items
for _, snippets in snippets_grouped
]
videos = chain.from_iterable(videos)
snippet_frontmatter = [(
snippet,
cfg.FRONTMATTER["video"].format(
date=snippet.publishedAt,
title=snippet.title,
description=snippet.description,
weight=snippet.position,
video_id=snippet.resourceId.videoId,
video_duration=video.contentDetails.get_video_seconds_duration(
),
),
) for snippet, video in zip(snippets, videos)]
for snippet, frontmatter in snippet_frontmatter:
path = os.path.join(
path_chapter,
"{}_{}.md".format(snippet.position,
u.sanitize_title(snippet.title)),
)
with open(
path,
"w",
) as f:
f.write(frontmatter)
def persist_activities(self):
file_path = os.path.join(get_base_path(), 'fitfile', 'data',
f'activity-{self.activity_id}.json')
activities = self.get_activities()
with open(file_path, 'w') as f:
json.dump(activities, f, indent=4, sort_keys=True)
import numpy as np
import pandas as pd
from scipy.sparse import triu
import pickle
from collections import defaultdict
from tqdm.autonotebook import tqdm
import sys
from data_preparer import S_to_A, S_to_B, incidence_to_hyperedges, prepare_node_hyperneighbors_map
from utils import get_library_path, get_base_path
library_path = get_library_path()
sys.path.append(library_path)
sys.path.append(os.path.join(library_path, "hynetworkx"))
base_path = get_base_path()
def get_avg(_list):
return sum(_list) * 1.0 / len(_list) if len(_list) > 0 else 0
def get_sum(_list):
return sum(_list) * 1.0 if len(_list) > 0 else 0
def get_l11(_list):
return sum(np.abs(_list)) * 1.0 if len(_list) > 0 else 0
def get_l22(_list):
from xml.etree import ElementTree
from src.data_file_processing import data_file_processing
import pytest
import json
import filecmp
import os
from utils import get_base_path
path_to_data_file = os.path.join(get_base_path(),
'file_samples/xml/test_data.xml')
path_to_target_file = os.path.join(get_base_path(),
'file_samples/json/updated_test_data.json')
path_to_result_template_file = os.path.join(
get_base_path(), 'file_samples/json/result_file_template.json')
wrong_path = os.path.join(get_base_path())
path_to_xml_for_comparison = os.path.join(get_base_path(),
'file_samples/xml/updated_data.xml')
def test_valid_file_processing_and_content_of_result_file():
data_file_processing(path_to_data_file, path_to_target_file)
with open(path_to_target_file, 'r') as file1:
data1 = json.load(file1)
with open(path_to_result_template_file, 'r') as file2:
data2 = json.load(file2)
assert data1 == data2
def test_invalid_path_to_source_file():
with pytest.raises(FileNotFoundError):
def generate_new_version(version=None, name='version'):
"""Default choice for generating new text file version
The default behavior is: All images - Bad images
# Argumments
version [str]: type <major>.<minor> . If None, <minor> will be
increased by 1
name [str]: name of the text file
# Returns
[str]: the path to new version text file
"""
folder_version = utils.get_base_path(utils.FOLDER_VERSIONS)
folder_data = utils.get_base_path(utils.FOLDER_DATA)
folder_metadata = utils.get_base_path(utils.FOLDER_METADATA)
# get the next minor version
if version is None:
olds = glob.glob(os.path.join(folder_version,
'{}*.*.txt'.format(name)))
if len(olds) == 0:
version = '1.0'
else:
olds = [os.path.basename(each_file) for each_file in olds]
olds = [os.path.splitext(each_file)[0] for each_file in olds]
olds = [each_file.replace(name, '') for each_file in olds]
olds = [each_file.split('.') for each_file in olds]
old_versions = []
for major, minor in olds:
try:
old_versions.append((int(major), int(minor)))
except ValueError:
continue
old_versions.sort(key=lambda obj: obj[0])
old_versions.sort(key=lambda obj: obj[1])
major, minor = old_versions[-1]
version = '{}.{}'.format(major, minor + 1)
# get valid images
all_images = set(
glob.glob(os.path.join(folder_data, '**', '*.*'), recursive=True))
bad_images = set(utils.get_bad_files())
out_images = list(all_images.difference(bad_images))
out_images.sort()
# get label
features = utils.get_all_metadata()
folder_data += '/'
finals = []
for each_file in out_images:
filename = each_file.replace(folder_data, '')
label = (features[filename] if isinstance(features[filename], str) else
features[filename][utils.KEY_LABEL])
finals.append('{},{}'.format(filename, label))
# dump the version
target_txt = os.path.join(folder_version, '{}{}.txt'.format(name, version))
with open(target_txt, 'w') as f_out:
f_out.write('\n'.join(finals))
print('The generated version {}'.format(target_txt))
return target_txt
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--n_iter',
type=int,
default=60000,
help='number of meta-iterations')
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--lr', type=float, default=0.001)
parser.add_argument('--lr_decay', type=float, default=0.5)
parser.add_argument('--data_path',
type=str,
default='../data',
help='folder which contains image data')
parser.add_argument('--n_way', type=int, default=16, help='test on val')
parser.add_argument('--k_shot', type=int, default=1, help='test on val')
parser.add_argument('--k_query', type=int, default=15, help='test on val')
parser.add_argument('--num_filters',
type=int,
default=128,
help='number of filters per conv-layer')
parser.add_argument('--out_dim',
type=int,
default=64,
help='number of filters last conv-layer')
args = parser.parse_args()
if not os.path.exists(os.path.join(get_base_path(), 'pre_train_file')):
os.mkdir(os.path.join(get_base_path(), 'pre_train_file'))
pre_train(args)
# utils.set_seed(args.seed)
args.context_in_type = ['mix', 'mix', 'mix', 'mix']
args.k_shot = 5
args.lr_inner = 0.1
args.num_grad_steps_inner = 2
args.num_grad_steps_eval = 2
args.num_context_params = 100
if args.k_shot == 1:
args.tasks_per_metaupdate = 2
in_channel = 5
else:
args.tasks_per_metaupdate = 2
in_channel = 8
path = os.path.join(utils.get_base_path(), 'result_files',
utils.get_path_from_args(args))
try:
training_stats, validation_stats = np.load(path + '.npy',
allow_pickle=True)
print('load path:[{}]'.format(path))
except FileNotFoundError:
print(
'You need to run the experiments first and make sure the results are saved at {}'
.format(path))
raise FileNotFoundError
Logger.print_header()
for num_grad_steps in [2]:
for k, v in enumerate(inner_loss_model.parameters()):
v.grad = critic_meta_grad[k] / float(args.tasks_per_metaupdate)
v.grad.data.clamp_(-10, 10)
# the meta-optimiser only operates on the shared parameters, not the context parameters
meta_optimiser.step()
meta_critic_optimiser.step()
critic_scheduler.step()
scheduler.step()
model.reset_context_params()
if __name__ == '__main__':
args = parse_args()
if not os.path.exists(os.path.join(utils.get_base_path(), 'result_files')):
os.mkdir(os.path.join(utils.get_base_path(), 'result_files'))
if not os.path.exists(os.path.join(utils.get_base_path(), 'result_plots')):
os.mkdir(os.path.join(utils.get_base_path(), 'result_plots'))
if os.path.exists(os.path.join(utils.get_base_path(), 'pre_train_file', 'pre_train.pth')) and args.load_pre:
pre_train_path = os.path.join(utils.get_base_path(), 'pre_train_file', 'pre_train.pth')
else:
pre_train_path = None
path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args))
log_interval = 500
if (not os.path.exists(path + '.npy')) or args.rerun:
print('Starting experiment. Logging under filename {}'.format(path + '.npy'))
run(args, log_interval=log_interval, pre_train_path=pre_train_path, save_path=path)
else:
