def pull_metadata_from_server(): """Automatically metadata from server""" file_user_config = os.path.join(utils.get_base_path(), utils.FILE_USER_CONFIG) folder_base = utils.get_base_path() username = None if os.path.exists(file_user_config): with open(file_user_config, 'r') as f_in: configs = json.load(f_in) username = configs.get('username', None) # Ask for username if not available if username is None: username = input('Username to access Hanoi server: ') utils.update_user_config({'username': username}) print('Loading metadata folder from server...') commands = [ 'rsync', '-vur', '-e', 'ssh{}'.format(utils.SERVER_PORT), '--delete', '--progress', '{}@{}:{}'.format(username, utils.SERVER_IP, utils.SERVER_FOLDER_METADATA), '{}/'.format(folder_base) ] subprocess.run(commands) print('Finished!')
def persist_activity(self, activity_id, format='TCX'): client = self.get_client() if format == 'TCX': data = client.download_activity( activity_id, dl_fmt=client.ActivityDownloadFormat.TCX) output_file = f'{str(activity_id)}.tcx' elif format == 'GPX': data = client.download_activity( activity_id, dl_fmt=client.ActivityDownloadFormat.GPX) output_file = f'{str(activity_id)}.gpx' elif format == 'ZIP': data = client.download_activity( activity_id, dl_fmt=client.ActivityDownloadFormat.ORIGINAL) output_file = f'{str(activity_id)}.zip' else: raise ActivityExtractorException( 'Invalid or not supported activity format given. Supported fomarts are: TCX, GPX and ZIP' ) data_dir = os.path.join(get_base_path(), 'garmin', 'data') if not os.path.isdir(data_dir): os.mkdir(data_dir) with open(os.path.join(data_dir, output_file), 'wb') as fb: fb.write(data)
def generate_dummy_activity(self): file_path = os.path.join(get_base_path(), 'strava', 'example-activity.json') for _ in range(MAX_RETRIES): with open(file_path, 'r') as file: activity = json.load(file) fake_activity(activity) for segment in activity['segment_efforts']: fake_segment(activity, segment) for lap in activity['laps']: fake_lap(activity, lap) for split in activity['splits_metric']: fake_split_metric(activity, split) try: verify_data_integrity(activity) except AssertionError: logger.info('Generated activity data not consistent. Trying again...') continue return activity else: logger.warning(f'Generating consistent activity data failed {MAX_RETRIES} times. Aborting and using inconsistent values')
def _get_access_token(): token_path = os.path.join(get_base_path(), 'strava', 'strava-token.json') try: with open(token_path, 'r') as file: creds = json.load(file) return creds['access_token'] except FileNotFoundError: return None
def pull_data_from_server(): """Automatically update image data from server""" file_user_config = os.path.join(utils.get_base_path(), utils.FILE_USER_CONFIG) folder_data = utils.get_base_path(utils.FOLDER_DATA) folder_work = utils.get_base_path(utils.FOLDER_WORK) folder_data_zipped = utils.get_base_path(utils.FOLDER_DATA_ZIPPED) username = None if os.path.exists(file_user_config): with open(file_user_config, 'r') as f_in: configs = json.load(f_in) username = configs.get('username', None) # Ask for username if not available if username is None: username = input('Username to access Hanoi server: ') utils.update_user_config({'username': username}) print('Loading image folder from server...') os.makedirs(folder_data, exist_ok=True) commands = [ 'rsync', '-vur', '-e', 'ssh{}'.format(utils.SERVER_PORT), '--delete', '--progress', '{}@{}:{}'.format(username, utils.SERVER_IP, utils.SERVER_FOLDER_DATA_ZIPPED), '{}'.format(folder_work) ] subprocess.run(commands) zip_data = set([ os.path.splitext(os.path.basename(each_file))[0] for each_file in glob.glob(os.path.join(folder_data_zipped, '*.zip')) ]) image_data = set([ os.path.basename(each_file[:-1]) for each_file in glob.glob(os.path.join(folder_data, '*/')) ]) update_data = list(zip_data.difference(image_data)) for each_file in update_data: source_zip_file = os.path.join(folder_data_zipped, '{}.zip'.format(each_file)) with zipfile.ZipFile(source_zip_file, 'r') as f_zipped: print('Unzipping {}...'.format(source_zip_file)) f_zipped.extractall(folder_data)
def add_clean_images(input_folder, input_json, force=False): """Add the images into data folder @NOTE: this method assumes that the images in `input_folder` and the content in `input_json` is good to copy. # Arguments input_folder [str]: the path to input image folder input_json [str]: the path to base json information force [bool]: if True, then will force copy even if the output folder and output json exist # Returns [str]: the path to output folder image """ # retrieve output folder name input_folder = input_folder[:-1] if input_folder[ -1] == '/' else input_folder data_folder = utils.get_base_path(utils.FOLDER_DATA) folder_name = os.path.basename(input_folder) output_folder = os.path.join(data_folder, folder_name) # check if the output folder currently exists if os.path.exists(output_folder) and not force: raise AttributeError( 'the `output_folder` already exists: {}'.format(output_folder)) # retrieve output metadata name metadata_folder = utils.get_base_path(utils.FOLDER_METADATA) output_json = os.path.join(metadata_folder, os.path.basename(input_json)) # check if the output json file currently exists if os.path.exists(output_json) and not force: raise AttributeError( 'the `output_json` already exists: {}'.format(output_json)) subprocess.run(['cp', '-r', input_folder, output_folder]) subprocess.run(['cp', input_json, output_json]) print('The output folder is {}'.format(output_folder)) print('The output json is {}'.format(output_json)) return output_folder, output_json
def fresh_initialize(): """Initialize the minisystem""" # Working files and directories folder_base = utils.get_base_path() folder_data = utils.get_base_path(utils.FOLDER_DATA) folder_metadata = utils.get_base_path(utils.FOLDER_METADATA) folder_versions = utils.get_base_path(utils.FOLDER_VERSIONS) folder_work = utils.get_base_path(utils.FOLDER_WORK) folder_data_zipped = utils.get_base_path(utils.FOLDER_DATA_ZIPPED) file_bad = os.path.join(folder_metadata, utils.FILE_BAD) file_user_config = os.path.join(folder_base, utils.FILE_USER_CONFIG) # Creating folders os.makedirs(folder_data, exist_ok=True) os.makedirs(folder_metadata, exist_ok=True) os.makedirs(folder_versions, exist_ok=True) os.makedirs(folder_work, exist_ok=True) os.makedirs(folder_data_zipped, exist_ok=True) # Creating files with open(file_bad, 'w') as f_out: f_out.write('') with open(file_user_config, 'w') as f_out: json.dump({}, f_out) # Set up username = input('Username to access Hanoi server (to sync image data): ') utils.update_user_config({'username': username}) print('Initialized!')
def main(): signal(SIGINT, sig_handler) args = parse_args() default_branch = args.branch username = utils.get_username(args) if args.gerrit == "submit": gerrit.submit(default_branch, username, args.owner) sys.exit(0) elif args.gerrit == "vote": gerrit.vote(default_branch, username, args.owner) sys.exit(0) base_path = utils.get_base_path(default_branch) config_dict = utils.get_config_dict(args.config, default_branch) if args.path_to_crowdin == "crowdin" and not utils.check_dependencies(): sys.exit(1) if args.upload_sources: upload.upload_sources_crowdin(default_branch, config_dict, args.path_to_crowdin) elif args.upload_translations: upload.upload_translations_crowdin(default_branch, config_dict, args.path_to_crowdin) elif args.download: xml_files = utils.get_xml_files(base_path, default_branch) download.download_crowdin( base_path, default_branch, xml_files, username, config_dict, args.path_to_crowdin, ) elif args.unzip: xml_files = utils.get_xml_files(base_path, default_branch) from_zip.unzip(args.unzip, base_path, default_branch, xml_files, username) if download.has_created_commits() or upload.has_uploaded(): print("\nDone!") sys.exit(0) else: print("\nNothing to commit") sys.exit(2)
def get_token(code): params = { "client_id": os.getenv('STRAVA_CLIENT_ID'), "client_secret": os.getenv('STRAVA_CLIENT_SECRET'), "code": code, "grant_type": "authorization_code" } response = requests.post("https://www.strava.com/oauth/token", params) if response.status_code != 200: logger.error( f'Retrieving access token failed. {response.url} - {response.text}' ) return token_data = response.json() token_path = os.path.join(get_base_path(), 'strava-token.json') with open(token_path, 'w') as f: json.dump(token_data, f, indent=4, sort_keys=True) return token_data.get('access_token')
def main(): load_dotenv() instagram_path = os.path.join(get_base_path(), os.getenv('INSTAGRAM_DIR', 'instagram')) images_path = get_images_path() os.makedirs(instagram_path, exist_ok=True) bot = Bot(base_path=instagram_path) bot.login(username=os.getenv('INSTAGRAM_LOGIN'), password=os.getenv('INSTAGRAM_PASSWORD')) for filename in os.listdir(images_path): image_path = os.path.join(images_path, filename) if os.path.isfile(image_path): bot.upload_photo(image_path, caption='') # remove tmp files for filename in filter(lambda x: x.endswith('.REMOVE_ME'), os.listdir(images_path)): os.unlink(os.path.join(images_path, filename))
model.reset_context_params() # this will take the mean over the batches logger.summarise_inner_loop(mode='valid') # keep track of best models logger.update_best_model(model, save_path) if __name__ == '__main__': args = arguments.parse_args() # --- settings --- if not os.path.exists(os.path.join(utils.get_base_path(), 'result_files')): os.mkdir(os.path.join(utils.get_base_path(), 'result_files')) if not os.path.exists(os.path.join(utils.get_base_path(), 'result_plots')): os.mkdir(os.path.join(utils.get_base_path(), 'result_plots')) path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args)) log_interval = 100 if (not os.path.exists(path + '.npy')) or args.rerun: print('Starting experiment. Logging under filename {}'.format(path + '.npy')) run(args, num_workers=1, log_interval=log_interval, save_path=path) else: print( 'Found results in {}. If you want to re-run, use the argument --rerun'
def check_central_repo(): """Check that there is no problem in central repo This method can confirm that: - the number of files in json is equal to the number of files in folder - the filepath in json is legit - the filename format is <Project>_<YYYYMMDD>_<index>.<ext> - important files and folders exist """ # check important files and folders exist print('\n=> Check important files and folders exist') for each_folder in [ utils.FOLDER_DATA, utils.FOLDER_WORK, utils.FOLDER_DATA_ZIPPED, utils.FOLDER_METADATA, utils.FOLDER_SCRIPT, utils.FOLDER_VERSIONS ]: folder_path = utils.get_base_path(each_folder) if not os.path.exists(folder_path): print(':WARNING: folder {} does not exist'.format(folder_path)) for each_file in [ os.path.join(utils.FOLDER_METADATA, utils.FILE_BAD), utils.FILE_USER_CONFIG ]: file_path = utils.get_base_path(each_file) if not os.path.exists(file_path): print(':WARNING: file {} does not exist'.format(file_path)) features = utils.get_all_metadata() folder_data = utils.get_base_path(utils.FOLDER_DATA) + '/' images = glob.glob(os.path.join(folder_data, '**', '*.*'), recursive=True) images = set([each_file.replace(folder_data, '') for each_file in images]) # check the number of files is equal to the number of files in folder print('\n=> Check the number of file in json is equal to that in folder') if len(features) == len(images): print('Equal files ({})'.format(len(features))) else: print(':WARNING: not equal: json {}, files {}'.format( len(features), len(images))) # check filename format is <Project>_<YYYYMMDD>_<index>.<ext> print('\n=> Check filename format is <Project>_<YYYYMMDD>_<index>.<ext>') for each_file in list(images): filename = os.path.basename(each_file) filename, _ = os.path.splitext(filename) bad = False try: _, date, idx = filename.split('_') except ValueError: bad = True try: date = arrow.get(date, 'YYYYMMDD') except arrow.parser.ParserError: bad = True try: idx = int(idx) except ValueError: bad = True if bad: print(':WARNING: {}'.format(each_file)) # check filepath in json is legit print('\n=> Check file path in json is legit') features_keys = set(features.keys()) only_features_keys = features_keys.difference(images) only_files = images.difference(features_keys) if len(only_features_keys) > 0: print('there are {} files in json but does not in folder, they are {}'. format(len(only_features_keys), only_features_keys)) if len(only_files) > 0: print( 'there are {} files in folder but not in json, they are {}'.format( len(only_files), only_files)) print('\nFinished!')
def generate_train_test(version_name, condition, version=None): """Generate the train and test Under this method: - POC samples have 80% probability in test set - collection samples have 20% probability in test set - synthetic samples have 0% probabilty in test set # Arguments field [str]: the field. Currently support name, address, all version [str]: <major>.<minor> # Returns [str]: path to train version [str]: path to test version """ folder_version = os.path.join(utils.get_base_path(utils.FOLDER_VERSIONS), version_name) folder_data = utils.get_base_path(utils.FOLDER_DATA) folder_metadata = utils.get_base_path(utils.FOLDER_METADATA) # sanity check os.makedirs(folder_version, exist_ok=True) # get the next minor version if version is None: olds = glob.glob( os.path.join(folder_version, '{}-train*.*.txt'.format(version_name))) if len(olds) == 0: version = '1.0' else: olds = [os.path.basename(each_file) for each_file in olds] olds = [os.path.splitext(each_file)[0] for each_file in olds] olds = [ each_file.replace('{}-train'.format(version_name), '') for each_file in olds ] olds = [each_file.split('.') for each_file in olds] old_versions = [] for major, minor in olds: try: old_versions.append((int(major), int(minor))) except ValueError: continue old_versions.sort(key=lambda obj: obj[0]) old_versions.sort(key=lambda obj: obj[1]) major, minor = old_versions[-1] version = '{}.{}'.format(major, minor + 1) # get valid images all_images = set( glob.glob(os.path.join(folder_data, '**', '*.*'), recursive=True)) # get bad images bad_images = set(utils.get_bad_files()) out_images = list(all_images.difference(bad_images)) out_images.sort() # get label features = utils.get_all_metadata() folder_data += '/' finals_train, finals_test = [], [] for each_file in out_images: filename = each_file.replace(folder_data, '') proceed = True for key, value in condition.items(): if value != features[filename].get(key, None): proceed = False if not proceed: continue label = features[filename][utils.KEY_LABEL] origin = features[filename][utils.KEY_ORIGIN] # train/test split based on origin is poc, collection or synthetic if origin == utils.VALUE_ORIGIN_POC: if random.random() > 0.8: finals_train.append('{},{}'.format(filename, label)) else: finals_test.append('{},{}'.format(filename, label)) elif origin == utils.VALUE_ORIGIN_COLLECTION: if random.random() > 0.2: finals_train.append('{},{}'.format(filename, label)) else: finals_test.append('{},{}'.format(filename, label)) else: finals_train.append('{},{}'.format(filename, label)) # dump the versions target_train = os.path.join(folder_version, '{}-train{}.txt'.format(version_name, version)) target_test = os.path.join(folder_version, '{}-test{}.txt'.format(version_name, version)) with open(target_train, 'w') as f_out: f_out.write('\n'.join(finals_train)) with open(target_test, 'w') as f_out: f_out.write('\n'.join(finals_test)) print('The generated train {}'.format(target_train)) print('The generated test {}'.format(target_test)) return target_train, target_test
model.reset_context_params() # this will take the mean over the batches logger.summarise_inner_loop(mode='valid') # keep track of best models logger.update_best_model(model, save_path) if __name__ == '__main__': args = arguments.parse_args() # --- settings --- if not os.path.exists(os.path.join(utils.get_base_path(), 'result_files')): os.mkdir(os.path.join(utils.get_base_path(), 'result_files')) if not os.path.exists(os.path.join(utils.get_base_path(), 'result_plots')): os.mkdir(os.path.join(utils.get_base_path(), 'result_plots')) path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args)) log_interval = 100 if (not os.path.exists(path + '.npy')) or args.rerun: print('Starting experiment. Logging under filename {}'.format(path + '.npy')) run(args, num_workers=1, log_interval=log_interval, save_path=path) else: print('Found results in {}. If you want to re-run, use the argument --rerun'.format(path)) # -------------- plot -----------------
for elem in root.iter(): if elem.text.startswith('YOUR'): elem.text = "MY_VALUE" assert check_number_of_elements_containing_text(root, 'YOUR') == 0 xml_string_for_updated_tree = ElementTree.tostring(root).decode() parsed_string = xmltodict.parse(xml_string_for_updated_tree) target_json_file = open(path_to_destination_file, 'w') target_json_file.write(json.dumps(parsed_string, indent=4)) target_json_file.close() if not os.path.isfile(path_to_destination_file): raise FileNotFoundError("Result file isn't created") else: raise FileNotFoundError( "Source file cannot be found by specified path") def check_number_of_elements_containing_text(root, value): number = 0 for elem in root.iter(): if elem.text.startswith(value): number += 1 return number if __name__ == '__main__': path_to_data_file = os.path.join(get_base_path(), 'file_samples/xml/test_data.xml') path_to_target_file = os.path.join( get_base_path(), 'file_samples/json/updated_test_data.json') data_file_processing(path_to_data_file, path_to_target_file)
def main(): args = parse_command_line_arguments() api = pyyoutube.Api(api_key=config.YT_API_KEY) playlists = api.get_playlist_by_id( playlist_id=args.playlist, parts=["snippet"] ).items paths = map(lambda playlist: utils.get_base_path(args, playlist), playlists) for playlist, path in zip(playlists, paths): os.makedirs(path, exist_ok=args.force) with open(os.path.join(path, config.INDEX), "w") as f: frontmatter = config.FRONTMATTER["series"].format( date=playlist.snippet.publishedAt, description=args.description or playlist.snippet.description.split("\n")[0], title=args.title or playlist.snippet.title, playlist_id=args.playlist, ) f.write(frontmatter) path_chapter = os.path.join(path, config.DIR_CHAPTER) os.makedirs(path_chapter, exist_ok=args.force) with open(os.path.join(path_chapter, config.INDEX), "w") as f: f.write(config.FRONTMATTER["chapter"]) playlist_items = api.get_playlist_items( playlist_id=args.playlist, parts=["snippet"], count=None ).items snippets = [playlist.snippet for playlist in playlist_items] snippets_grouped = groupby( enumerate(snippets), lambda ix: ix[0] // config.YT_MAX_RESULTS ) videos = [ api.get_video_by_id( video_id=[s.resourceId.videoId for _, s in snippets] ).items for _, snippets in snippets_grouped ] videos = chain.from_iterable(videos) snippet_frontmatter = [] for snippet, video in zip(snippets, videos): index = snippet.position + 1 frontmatter = config.FRONTMATTER["video"].format( date=snippet.publishedAt, title=snippet.title, description=snippet.description.split('\n', 1)[0], # Add 1 so the weight starts at 1, not 0. weight=index, video_id=snippet.resourceId.videoId, video_duration=video.contentDetails.get_video_seconds_duration(), ) path = os.path.join( path_chapter, "{}_{}.md".format(index, utils.sanitize_title(snippet.title)), ) with open(path, "w",) as f: f.write(frontmatter)
def persist_activity(self, id, include_all_efforts=False): activity = self.get_activity(id, include_all_efforts) file_path = os.path.join(get_base_path(), 'strava', 'data', f'activity-{id}.json') with open(file_path, 'w') as f: json.dump(activity, f, indent=4, sort_keys=True)
def __init__(self, dict_result): self.list_dat_csv = self.__get_file_list() self.base_path = utils.get_base_path() self.dict_result = dict_result
# --- settings --- args.k_shot = 1 args.lr_inner = 1.0 args.lr_meta = 'decay' args.num_grad_steps_inner = 2 args.num_grad_steps_eval = 2 args.model = 'cnn' args.num_context_params = 100 if args.k_shot == 1: args.tasks_per_metaupdate = 4 else: args.tasks_per_metaupdate = 2 path = os.path.join(utils.get_base_path(), 'result_files', datetime_folder, utils.get_path_from_args(args)) try: training_stats, validation_stats = np.load(path + '.npy') except FileNotFoundError: print('You need to run the experiments first and make sure the results are saved at {}'.format(path)) raise FileNotFoundError # initialise logger logger = Logger(args) logger.print_header() for num_grad_steps in [2]: print('\n --- ', num_grad_steps, '--- \n') # initialise logger
def main(): parser = generate_parser() args = parser.parse_args() if args.path is not None and args.folders is None: parser.error("When using --path, you should specify a folder structure" " using --folders. Use -h for help") elif args.folders is not None and len(args.playlists) != 1: parser.error("When using --folders, multiple playlists aren't allowed." " Use -h for help") api = yt.Api(api_key=cfg.YT_API_KEY) playlists = api.get_playlist_by_id(playlist_id=args.playlists, parts=["snippet"]).items paths = map(lambda playlist: u.get_base_path(args, playlist), playlists) for playlist, path in zip(playlists, paths): os.makedirs(path, exist_ok=args.force) with open(os.path.join(path, cfg.INDEX), "w") as f: frontmatter = cfg.FRONTMATTER["series"].format( date=playlist.snippet.publishedAt, description=args.description or playlist.snippet.description.split("\n")[0], title=args.title or playlist.snippet.title, playlist_id=args.playlists, ) f.write(frontmatter) path_chapter = os.path.join(path, cfg.DIR_CHAPTER) os.makedirs(path_chapter, exist_ok=args.force) with open(os.path.join(path_chapter, cfg.INDEX), "w") as f: f.write(cfg.FRONTMATTER["chapter"]) playlist_items = api.get_playlist_items(playlist_id=args.playlists, parts=["snippet"], count=None).items snippets = [playlist.snippet for playlist in playlist_items] snippets_grouped = groupby(enumerate(snippets), lambda ix: ix[0] // cfg.YT_MAX_RESULTS) videos = [ api.get_video_by_id( video_id=[s.resourceId.videoId for _, s in snippets]).items for _, snippets in snippets_grouped ] videos = chain.from_iterable(videos) snippet_frontmatter = [( snippet, cfg.FRONTMATTER["video"].format( date=snippet.publishedAt, title=snippet.title, description=snippet.description, weight=snippet.position, video_id=snippet.resourceId.videoId, video_duration=video.contentDetails.get_video_seconds_duration( ), ), ) for snippet, video in zip(snippets, videos)] for snippet, frontmatter in snippet_frontmatter: path = os.path.join( path_chapter, "{}_{}.md".format(snippet.position, u.sanitize_title(snippet.title)), ) with open( path, "w", ) as f: f.write(frontmatter)
def persist_activities(self): file_path = os.path.join(get_base_path(), 'fitfile', 'data', f'activity-{self.activity_id}.json') activities = self.get_activities() with open(file_path, 'w') as f: json.dump(activities, f, indent=4, sort_keys=True)
import numpy as np import pandas as pd from scipy.sparse import triu import pickle from collections import defaultdict from tqdm.autonotebook import tqdm import sys from data_preparer import S_to_A, S_to_B, incidence_to_hyperedges, prepare_node_hyperneighbors_map from utils import get_library_path, get_base_path library_path = get_library_path() sys.path.append(library_path) sys.path.append(os.path.join(library_path, "hynetworkx")) base_path = get_base_path() def get_avg(_list): return sum(_list) * 1.0 / len(_list) if len(_list) > 0 else 0 def get_sum(_list): return sum(_list) * 1.0 if len(_list) > 0 else 0 def get_l11(_list): return sum(np.abs(_list)) * 1.0 if len(_list) > 0 else 0 def get_l22(_list):
from xml.etree import ElementTree from src.data_file_processing import data_file_processing import pytest import json import filecmp import os from utils import get_base_path path_to_data_file = os.path.join(get_base_path(), 'file_samples/xml/test_data.xml') path_to_target_file = os.path.join(get_base_path(), 'file_samples/json/updated_test_data.json') path_to_result_template_file = os.path.join( get_base_path(), 'file_samples/json/result_file_template.json') wrong_path = os.path.join(get_base_path()) path_to_xml_for_comparison = os.path.join(get_base_path(), 'file_samples/xml/updated_data.xml') def test_valid_file_processing_and_content_of_result_file(): data_file_processing(path_to_data_file, path_to_target_file) with open(path_to_target_file, 'r') as file1: data1 = json.load(file1) with open(path_to_result_template_file, 'r') as file2: data2 = json.load(file2) assert data1 == data2 def test_invalid_path_to_source_file(): with pytest.raises(FileNotFoundError):
def generate_new_version(version=None, name='version'): """Default choice for generating new text file version The default behavior is: All images - Bad images # Argumments version [str]: type <major>.<minor> . If None, <minor> will be increased by 1 name [str]: name of the text file # Returns [str]: the path to new version text file """ folder_version = utils.get_base_path(utils.FOLDER_VERSIONS) folder_data = utils.get_base_path(utils.FOLDER_DATA) folder_metadata = utils.get_base_path(utils.FOLDER_METADATA) # get the next minor version if version is None: olds = glob.glob(os.path.join(folder_version, '{}*.*.txt'.format(name))) if len(olds) == 0: version = '1.0' else: olds = [os.path.basename(each_file) for each_file in olds] olds = [os.path.splitext(each_file)[0] for each_file in olds] olds = [each_file.replace(name, '') for each_file in olds] olds = [each_file.split('.') for each_file in olds] old_versions = [] for major, minor in olds: try: old_versions.append((int(major), int(minor))) except ValueError: continue old_versions.sort(key=lambda obj: obj[0]) old_versions.sort(key=lambda obj: obj[1]) major, minor = old_versions[-1] version = '{}.{}'.format(major, minor + 1) # get valid images all_images = set( glob.glob(os.path.join(folder_data, '**', '*.*'), recursive=True)) bad_images = set(utils.get_bad_files()) out_images = list(all_images.difference(bad_images)) out_images.sort() # get label features = utils.get_all_metadata() folder_data += '/' finals = [] for each_file in out_images: filename = each_file.replace(folder_data, '') label = (features[filename] if isinstance(features[filename], str) else features[filename][utils.KEY_LABEL]) finals.append('{},{}'.format(filename, label)) # dump the version target_txt = os.path.join(folder_version, '{}{}.txt'.format(name, version)) with open(target_txt, 'w') as f_out: f_out.write('\n'.join(finals)) print('The generated version {}'.format(target_txt)) return target_txt
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--n_iter', type=int, default=60000, help='number of meta-iterations') parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--lr_decay', type=float, default=0.5) parser.add_argument('--data_path', type=str, default='../data', help='folder which contains image data') parser.add_argument('--n_way', type=int, default=16, help='test on val') parser.add_argument('--k_shot', type=int, default=1, help='test on val') parser.add_argument('--k_query', type=int, default=15, help='test on val') parser.add_argument('--num_filters', type=int, default=128, help='number of filters per conv-layer') parser.add_argument('--out_dim', type=int, default=64, help='number of filters last conv-layer') args = parser.parse_args() if not os.path.exists(os.path.join(get_base_path(), 'pre_train_file')): os.mkdir(os.path.join(get_base_path(), 'pre_train_file')) pre_train(args)
# utils.set_seed(args.seed) args.context_in_type = ['mix', 'mix', 'mix', 'mix'] args.k_shot = 5 args.lr_inner = 0.1 args.num_grad_steps_inner = 2 args.num_grad_steps_eval = 2 args.num_context_params = 100 if args.k_shot == 1: args.tasks_per_metaupdate = 2 in_channel = 5 else: args.tasks_per_metaupdate = 2 in_channel = 8 path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args)) try: training_stats, validation_stats = np.load(path + '.npy', allow_pickle=True) print('load path:[{}]'.format(path)) except FileNotFoundError: print( 'You need to run the experiments first and make sure the results are saved at {}' .format(path)) raise FileNotFoundError Logger.print_header() for num_grad_steps in [2]:
for k, v in enumerate(inner_loss_model.parameters()): v.grad = critic_meta_grad[k] / float(args.tasks_per_metaupdate) v.grad.data.clamp_(-10, 10) # the meta-optimiser only operates on the shared parameters, not the context parameters meta_optimiser.step() meta_critic_optimiser.step() critic_scheduler.step() scheduler.step() model.reset_context_params() if __name__ == '__main__': args = parse_args() if not os.path.exists(os.path.join(utils.get_base_path(), 'result_files')): os.mkdir(os.path.join(utils.get_base_path(), 'result_files')) if not os.path.exists(os.path.join(utils.get_base_path(), 'result_plots')): os.mkdir(os.path.join(utils.get_base_path(), 'result_plots')) if os.path.exists(os.path.join(utils.get_base_path(), 'pre_train_file', 'pre_train.pth')) and args.load_pre: pre_train_path = os.path.join(utils.get_base_path(), 'pre_train_file', 'pre_train.pth') else: pre_train_path = None path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args)) log_interval = 500 if (not os.path.exists(path + '.npy')) or args.rerun: print('Starting experiment. Logging under filename {}'.format(path + '.npy')) run(args, log_interval=log_interval, pre_train_path=pre_train_path, save_path=path) else: