def cache_match_schedule(): """Requests the match schedule from TBA and adds it to the cache.""" # HACK: Only pulls the match schedule once since the caching built # into tba_communicator.py is not complete. matches = tba_communicator.request_matches() for match_data in matches: # 'qm' stands for qualification match if match_data['comp_level'] == 'qm': red_teams = match_data['alliances']['red']['team_keys'] blue_teams = match_data['alliances']['blue']['team_keys'] match_number = match_data['match_number'] # Remove 'frc' from team number # (e.g. 'frc1678' -> '1678') red_teams = [team[3:] for team in red_teams] blue_teams = [team[3:] for team in blue_teams] final_match_data = { 'matchNumber': match_number, 'redTeams': red_teams, 'blueTeams': blue_teams, } with open( utils.create_file_path( f'data/cache/match_schedule/{match_number}.json'), 'w') as file: json.dump(final_match_data, file)
def __exit__(self, exc_type, exc_val, exc_tb): if not self._lines: return try: self._file = open(self._filename, 'w') except IOError: create_file_path(self._filename) self._file = open(self._filename, 'w') for line in self._lines: self._file.write(line + '\n') self._file.close() if self._executable: os.system('chmod a+x %s' % self._filename)
def save_image(self): self.__txt_path.setText(self.__txt_path.displayText()) save_path = self.__txt_path.text() created = create_dir_path(save_path) if created: self.label_log.append("create an directory:\n{}".format(save_path)) file_path = create_file_path(save_path) save_image = self.__canvas.get_current_image() save_image.save(file_path) file_path_ = os.path.abspath(file_path) self.label_log.append("image saved in path:\n{}".format(file_path_))
def temp_super_stream_handler(snapshot): """Runs when any new tempSuper datas are uploaded""" data = snapshot['data'] path = snapshot['path'] # This occurs when the entirety of tempSuper datas are updated # (stream initialization, all tempSuper data deleted, or first # tempSuper created) if path == '/': # This means that all tempSuper datas have been wiped and we # should wipe our local copy. if data is None: delete_cache_data_folder('temp_super') return elif path.count('/') == 1: # This is moving the path into the data so it is in the same # format as data at the path '/'. This allows us to use the # same code to save the data in our local cache later on. # The '[1:]' removes the slash at the beginning of the path data = {path[1:]: data} # If there is more than 1 slash in the path, the data is multiple # children deep. tempSupers are only one child deep and this will # only trigger if invalid data is sent to Firebase. else: print('Error: Invalid tempSuper data received') return # This saves each tempSuper data in a separate text file. for temp_super_name, temp_super_value in data.items(): # This means that this tempSuper has been deleted from Firebase # and we should delete it from our local copy. if temp_super_value is None: os.remove( utils.create_file_path( f'data/cache/temp_super/{temp_super_name}.txt')) else: with open( utils.create_file_path( f'data/cache/temp_super/{temp_super_name}.txt'), 'w') as file: file.write(temp_super_value)
def save_data(file_path, data): """Saves data in 'cache' and 'upload_queue' directories. file_path is the relative file path to a JSON file from inside the 'cache' or 'upload_queue' folder. (string) data is a dictionary that the JSON file is updated with.""" # Removes preceding slash if file_path[0] == '/': file_path = file_path[1:] for directory in ['cache', 'upload_queue']: absolute_path = utils.create_file_path(f'data/{directory}/{file_path}') update_json_file(absolute_path, data)
def logging_setup(path_to_log, log_level, log_filemode): if not log_level in TRUE_LOG_LEVELS: log_level = 'DEBUG' if not log_filemode in TRUE_FILE_MODES: log_filemode = 'a' path = create_file_path(path_to_log) logging.basicConfig(filename=path, level=log_level, filemode=log_filemode, format=MESSAGE_FORMAT_FOR_LOGGER, datefmt=DATE_FORMAT_FOR_LOGGER)
def get_image_paths(): """Gets dictionary of image paths""" csv_rows = dict() for team in TEAMS_LIST: # Makes the team key a list with the team number in it. csv_rows[team] = { 'full_robot': '', 'drivetrain': '', 'mechanism': [], } # Iterates through each device in the tablets folder for device in os.listdir(utils.create_file_path('data/tablets')): # If the device is a phone serial number if device not in ['9AQAY1EV7J', '9AMAY1E54G', '9AMAY1E53P']: continue device_dir = utils.create_file_path(f'data/tablets/{device}/') # Iterates through all of files in the phone's folder for file in os.listdir(device_dir): # Tries to match the file name with the regular expression result = re.fullmatch(PATH_PATTERN, file) # If the regular expression matched if result: # Team number is the result of the first capture type team_num = result.group(1) if team_num not in TEAMS_LIST: continue # Photo type is the result of the second capture group photo_type = result.group(2) # There can be multiple mechanism photos, so we need to handle differently if photo_type.startswith('mechanism'): csv_rows[team_num]['mechanism'].append(os.path.join(device_dir, file)) # Otherwise just add the photo path to its specified place in csv_rows else: csv_rows[team_num][photo_type] = os.path.join(device_dir, file) return csv_rows
def temp_timd_stream_handler(temp_timd_name, temp_timd_value_): """Runs when any new tempTIMDs are uploaded""" # HACK: Remove trailing '\n' (newlines) in compressed tempTIMD # data. This is a bug in the Scout app. temp_timd_value_ = temp_timd_value_.rstrip('\n') # This means that this tempTIMD has been deleted from Firebase # and we should delete our local copy. if temp_timd_value_ is None: os.remove( utils.create_file_path( f'data/cache/temp_timds/{temp_timd_name}.txt')) # Causes the corresponding TIMD to be recalculated register_modified_temp_timd(temp_timd_name) else: with open( utils.create_file_path( f'data/cache/temp_timds/{temp_timd_name}.txt'), 'w') as file: file.write(temp_timd_value_) timd_name = temp_timd_name.split('-')[0] # This means an already existing tempTIMD has been modified # and needs to be recalculated. if temp_timd_name in LATEST_CALCULATIONS_BY_TIMD.get(timd_name, []): register_modified_temp_timd(temp_timd_name)
def pull_device_data(): """Pulls tablet data from attached tablets.""" # Parses 'adb devices' to find num of devices so that don't try to pull from nothing devices = get_attached_devices() data = {'qr': [], 'obj_pit': [], 'subj_pit': []} if not devices: return data device_file_paths = [] device_file_path = utils.create_file_path('data/tablets') # Pull all files from the 'Download' folder on the tablet pull_device_files(device_file_path, '/storage/emulated/0/Download') # Iterates through the 'data' folder for device_dir in os.listdir(device_file_path): if device_dir in TABLET_SERIAL_NUMBERS.keys(): device_file_paths.append(device_dir) # If the folder name is a device serial, it must be a tablet folder for device in device_file_paths: # Iterate through the downloads folder in the device folder download_directory = os.path.join(device_file_path, device) for file in os.listdir(download_directory): for dataset, pattern in FILENAME_REGEXES.items(): if re.fullmatch(pattern, file): with open(os.path.join(download_directory, file)) as data_file: # QR data is just read if dataset == 'qr': file_contents = data_file.read().rstrip('\n') else: file_contents = json.load(data_file) data[dataset].append(file_contents) break # Filename will only match one regex # Add QRs to database and make sure that only QRs that should be decompressed are added to queue data['qr'] = qr_code_uploader.upload_qr_codes(data['qr']) for dataset in ['obj_pit', 'subj_pit']: current_data = local_database_communicator.read_dataset(dataset) modified_data = [] for datapoint in data[dataset]: if datapoint in current_data: continue # Specify query to ensure that each team only has one entry local_database_communicator.update_dataset( f'raw.{dataset}', datapoint, {'team_number': datapoint['team_number']}) modified_data.append({'team_number': datapoint['team_number']}) utils.log_info(f'{len(modified_data)} items uploaded to {dataset}') data[dataset] = modified_data return data
def full_data_export(): """Writes the current export to a timestamped directory. Returns the directory path written""" current_time = datetime.datetime.now() timestamp_str = current_time.strftime('%Y-%m-%d_%H:%M:%S') # Creates directory if it does not exist directory_path = utils.create_file_path(f'data/exports/export_{timestamp_str}') # Team data team_file_path = os.path.join(directory_path, f'team_export_{timestamp_str}.csv') export_team_data(team_file_path) # Team in match data timd_file_path = os.path.join(directory_path, f'timd_export_{timestamp_str}.csv') export_tim_data(timd_file_path) # TBA match data tba_file_path = os.path.join(directory_path, f'tba_export_{timestamp_str}.csv') write_tba_data(tba_file_path) return directory_path
def decompress_temp_timd_headers(compressed_headers): """Decompress headers for a single tempTIMD. compressed_headers are non-timed data fields.""" with open(utils.create_file_path('data/assignments/assignments.json'), 'r') as file: file_data = json.load(file) # Decompressed scout name to compressed scout name scout_name_compression_values = file_data['letters'] # Reverses key:value pairs to enable accessing decompressed scout # name from compressed scout name scout_name_compression_values = {letter: scout_name for \ scout_name, letter in scout_name_compression_values.items()} if compressed_headers[-1] == ',': # Removes trailing comma. compressed_headers = compressed_headers[:-1] compressed_headers = compressed_headers.split(',') decompressed_headers = {} for header in compressed_headers: compressed_key = header[0] compressed_value = header[1:] decompressed_key = TEMP_TIMD_COMPRESSION_KEYS[compressed_key] if decompressed_key == 'scoutName': # Uses 'scout_name_compression_values' dictionary to decompress scout name decompressed_value = scout_name_compression_values[ compressed_value] else: decompressed_value = decompress_temp_timd_value(compressed_value) decompressed_headers[decompressed_key] = decompressed_value return decompressed_headers
#!/usr/bin/env python3 # Copyright (c) 2019 FRC Team 1678: Citrus Circuits """Changes font size of tablets for app consistency.""" # External imports import time # Internal imports import adb_communicator import utils def adb_font_size_enforcer(): """Enforce tablet font size to 1.30, the largest supported size""" devices = adb_communicator.get_attached_devices() # Wait for USB connection to initialize time.sleep(.1) for device in devices: # The -s flag specifies the device by its serial number. utils.run_command( f'adb -s {device} shell settings put system font_scale 1.30', return_output=False) FILE_PATH = utils.create_file_path('data/tablets') utils.run_command(f'rm -R {FILE_PATH}', True) utils.run_command('mkdir data/tablets', True) adb_font_size_enforcer()
#!/usr/bin/python3.6 """Collects and exports all Super Scout pushing battles to a CSV file.""" # External imports import csv import os # Internal imports import decompressor import utils TEMP_SUPER_FILES = os.listdir(utils.create_file_path('data/cache/temp_super')) # Sorts by match number, since Elo needs to be in chronological order TEMP_SUPER_FILES.sort( key=lambda file_name: file_name.split('-')[0].split('Q')[1]) PUSHING_BATTLES = [] for temp_super_file in TEMP_SUPER_FILES: with open( utils.create_file_path(f'data/cache/temp_super/{temp_super_file}'), 'r') as file: compressed_temp_super = file.read() decompressed_pushing_battles = \ decompressor.decompress_temp_super_pushing_battles( compressed_temp_super) PUSHING_BATTLES += decompressed_pushing_battles # Orders pushing battle keys for CSV export CSV_HEADERS = ['matchNumber', 'winner', 'loser', 'winMarginIsLarge'] with open(utils.create_file_path('data/exports/pushing-battles.csv'), 'w') as file: CSV_WRITER = csv.DictWriter(file, fieldnames=CSV_HEADERS)
#!/usr/bin/python3.6 """Calculate SPRs (Scout Precision Rankings). Used in consolidation and to identify and address issues with scouting. These issues are often caused by misunderstanding or actions that have an ambiguous input. With SPRs, these questions can be cleared up during scout training and competition to decrease errors in the future.""" # External imports import csv import json import os # Internal imports import decompressor import utils TEMP_TIMDS = os.listdir(utils.create_file_path('data/cache/temp_timds')) TIMDS = os.listdir(utils.create_file_path('data/cache/timds')) # Scout name to SPR breakdown dictionary # Example format: 'Sam C': {'placement': {'correct': 3}, {'total': 10}} SPRS = {} def register_value(scout_name_, data_field_, is_correct): """Registers correct or incorrect value in 'SPRS'.""" if SPRS.get(scout_name_) is None: SPRS[scout_name_] = {} if SPRS[scout_name_].get(data_field_) is None: SPRS[scout_name_][data_field_] = {} previous_breakdown = SPRS[scout_name_][data_field_] previous_correct = previous_breakdown.get('correct', 0)
#!/usr/bin/python3.6 """Calculates points prevented by a defender in all TIMDs and Teams. TIMD stands for Team in Match Data""" # External imports import json import os import subprocess # Internal imports import utils # Extracts TIMDs from cache and organizes them by match. TIMDS_BY_MATCH = {} for timd in os.listdir(utils.create_file_path('data/cache/timds')): with open(utils.create_file_path(f'data/cache/timds/{timd}'), 'r') as file: timd_data = json.load(file) if timd_data.get('calculatedData') is not None: # .split() removes '.json' file ending timd_name = timd.split('.')[0] match_number = timd_name.split('Q')[1] # Creates a blank dictionary for a match if it doesn't exist yet. if TIMDS_BY_MATCH.get(match_number) is None: TIMDS_BY_MATCH[match_number] = {} TIMDS_BY_MATCH[match_number][timd_name] = timd_data # Teams that have played defense in 1 or more matches DEFENDER_TEAMS = set() for match_number, timds in TIMDS_BY_MATCH.items(): # Pulls match schedule (for a single match from cache with open(
"""Sends web requests to The Blue Alliance (TBA) APIv3 Caches data to prevent duplicate data retrieval from the TBA API. API documentation: https://www.thebluealliance.com/apidocs/v3""" # External imports import json import requests import time # Internal imports import utils EVENT_CODE = '2019carv' with open(utils.create_file_path('data/api_keys/tba_key.txt')) as file: API_KEY = file.read() # Removes trailing newline (if it exists) from file data. # Many file editors will automatically add a newline at the end of files. API_KEY = API_KEY.rstrip('\n') def make_request(api_url, show_output=True, acceptable_cache_age=0): """Sends a single web request to the TBA API v3 and caches result. api_url is the url of the API request (the path after '/api/v3') show_output shows print statements about the status of the request. Defaults to True. acceptable_cache_age is the maximum age (in seconds) of data that can be pulled from the cache. Pulling from the cache is disabled by default.""" base_url = 'https://www.thebluealliance.com/api/v3/' full_url = base_url + api_url
def main(): """ Main Function. Training/Test/Plot """ args = parse_arguments() device = torch.device('cuda') # remove randomness set_random_seed(args.seed) # Set Dataset if args.dataset == 'mnist': dataset = MNIST(path='data/MNIST', n_class=args.n_class, select=args.select, select_novel_classes=args.select_novel_classes) elif args.dataset == 'fmnist': dataset = FMNIST(path='data/FMNIST', n_class=args.n_class, select=args.select) elif args.dataset == 'thyroid': dataset = THYROID(path='data/UCI') elif args.dataset == 'kddcup': dataset = KDDCUP(path='data/UCI') else: raise ValueError('Unknown dataset') checkpoints_dir = create_checkpoints_dir(args.dataset, args.fixed, args.mulobj, args.num_blocks, args.hidden_size, args.code_length, args.estimator) # Set Model if (args.autoencoder is None): print('No Autoencoder') c, h, w = dataset.shape # build Density Estimator if args.estimator == 'SOS': model = TinvSOS(args.num_blocks, c * h * w, args.hidden_size).cuda() # 1-D estimator from LSA elif args.estimator == 'EN': self.model = Estimator1D(code_length=c * h * w, fm_list=[32, 32, 32, 32], cpd_channels=100).cuda() else: raise ValueError('Unknown Estimator') else: if args.autoencoder == "LSA": print(f'Autoencoder:{args.autoencoder}') print(f'Density Estimator:{args.estimator}') if args.dataset in ['mnist', 'fmnist']: model = LSA_MNIST(input_shape=dataset.shape, code_length=args.code_length, num_blocks=args.num_blocks, est_name=args.estimator, hidden_size=args.hidden_size).cuda() elif args.dataset in ['kddcup']: model = LSA_KDDCUP(num_blocks=args.num_blocks, hidden_size=args.hidden_size, code_length=args.code_length, est_name=args.estimator).cuda() elif args.dataset in ['thyroid']: model = LSA_THYROID(num_blocks=args.num_blocks, hidden_size=args.hidden_size, code_length=args.code_length, est_name=args.estimator).cuda() else: ValueError("Unknown Dataset") else: raise ValueError('Unknown Autoencoder') # Result saved path result_file_path = create_file_path(args.mulobj, args.fixed, args.pretrained, model.name, args.dataset, args.score_normed, args.num_blocks, args.hidden_size, args.code_length, args.lam, args.checkpoint) print(checkpoints_dir) helper = OneClassTestHelper(dataset=dataset, model=model, score_normed=args.score_normed, lam=args.lam, checkpoints_dir=checkpoints_dir, result_file_path=result_file_path, batch_size=args.batch_size, lr=args.lr, epochs=args.epochs, before_log_epochs=args.before_log_epochs, code_length=args.code_length, mulobj=args.mulobj, test_checkpoint=args.checkpoint, log_step=args.log_step, device=device, fixed=args.fixed, pretrained=args.pretrained, load_lsa=args.load_lsa) if args.trainflag: cl = args.select helper.train_one_class_classification(cl) elif args.testflag: helper.test_classification() elif args.compute_AUROC: helper.compute_AUROC(log_step=args.log_step, epoch_max=args.epochs) elif args.plot_training_loss_auroc: helper.plot_training_loss_auroc(log_step=args.log_step) elif args.using_train_set: cl = args.select helper.test_one_class_classification_with_trainset(cl)
blue_teams = match_data['alliances']['blue']['team_keys'] match_number = match_data['match_number'] # Remove 'frc' from team number and convert to integer # (e.g. 'frc1678' -> 1678) red_teams = [int(team[3:]) for team in red_teams] blue_teams = [int(team[3:]) for team in blue_teams] FINAL_MATCH_DATA[match_number] = { 'matchNumber': match_number, 'redTeams': red_teams, 'blueTeams': blue_teams, } FIREBASE_UPLOAD.update({'Matches': FINAL_MATCH_DATA}) if FULL_WIPE is True: # Loads scout names from assignment file with open(utils.create_file_path('data/assignments/assignments.json'), 'r') as file: SCOUT_NAMES = json.load(file)['letters'].keys() FIREBASE_UPLOAD.update({ 'tempTIMDs': None, 'TIMDs': None, 'tempSuper': None, 'scoutManagement': { 'currentMatchNumber': 1, 'cycleNumber': 0, 'availability': {scout: 0 for scout in SCOUT_NAMES}, }, }) # Removes 'cache' and 'upload_queue' folders to remove outdated data
#!/usr/bin/env python3 # Copyright (c) 2019 FRC Team 1678: Citrus Circuits """Sets up a machine for use or testing of the server. To be run for every new clone of server-2020. """ # External imports import os import subprocess import venv # Internal imports import utils # Creates data folder utils.create_file_path('data/') # Creates path by joining the base directory and the target directory TARGET_PATH = utils.create_file_path('.venv', False) # Only run if file is directly called if __name__ == '__main__': # Create virtual environment print('Creating Virtual Environment...') # Clear any existing environments with clear=True # Install pip directly into installation # Set prompt to 'venv' instead of '.venv' venv.create(TARGET_PATH, clear=True, with_pip=True, prompt='venv') print('Virtual Environment Created') # Install pip packages # Set create_directories to false to avoid trying to create directory for pip
"""Removes zeroes from a list, then returns the average of the list. lis is the list that is averaged""" lis = [item for item in lis if item != 0] if lis == []: return None else: return sum(lis) / len(lis) # Uses default Firebase URL # DB stands for database DB = firebase_communicator.configure_firebase() # List of files (tempSuper datas) in the 'temp_super' cache directory. TEMP_SUPER_FILES = os.listdir(utils.create_file_path('data/cache/temp_super')) # Match number (string) to list of tempSuper files for that match FILES_BY_MATCH = {} for file_name in TEMP_SUPER_FILES: # tempSuper naming format: # S!Q{match_number}-{alliance_color} # (e.g. S!Q3-B is the blue alliance in match 3) match_number = file_name.split('-')[0].split('Q')[1] if FILES_BY_MATCH.get(match_number) is None: FILES_BY_MATCH[match_number] = [] FILES_BY_MATCH[match_number].append(file_name) for match_number, files in FILES_BY_MATCH.items(): compressed_data = {} for temp_super_file in files:
win = 2 if calculated_data['redPredictedScore'] > \ calculated_data.get('bluePredictedScore', 0) else 0 total = win + calculated_data['redChanceClimbRP'] + \ calculated_data['redChanceRocketRP'] return total else: win = 2 if calculated_data['bluePredictedScore'] > \ calculated_data.get('redPredictedScore', 0) else 0 total = win + calculated_data['blueChanceClimbRP'] + \ calculated_data['blueChanceRocketRP'] return total # Gathers the calculated data from all the teams. TEAMS = {} for team in os.listdir(utils.create_file_path('data/cache/teams')): with open(utils.create_file_path(f'data/cache/teams/{team}')) as file: team_data = json.load(file) # Checks if the team has calculated data before considering them for # predictions. if team_data.get('calculatedData') is not None: # '.split()' removes '.txt' file ending TEAMS[team.split('.')[0]] = team_data # Gathers the matches in the competition. These matches are cached from # the tba match schedule when the server first runs. MATCH_SCHEDULE = {} for match in os.listdir(utils.create_file_path('data/cache/match_schedule')): with open(utils.create_file_path( f'data/cache/match_schedule/{match}')) as file: match_data = json.load(file)
FINAL_DATA = {} # Firebase key names to the equivilent local cache key names FIREBASE_TO_CACHE_KEY = { 'TIMDs': 'timds', 'Teams': 'teams', 'Matches': 'matches', } FILES_TO_REMOVE = [] for firebase_key, cache_key in FIREBASE_TO_CACHE_KEY.items(): for file in os.listdir( utils.create_file_path(f'data/upload_queue/{cache_key}')): file_path = utils.create_file_path( f'data/upload_queue/{cache_key}/{file}') # Collects and adds the data from a single file to 'FINAL_DATA' FINAL_DATA.update(collect_file_data(file_path, firebase_key)) FILES_TO_REMOVE.append(file_path) # Before sending the data, iterates through all of it and removes any # NaNs (Not a Number) in the data. (Relies on NaN != NaN) # HACK: NaNs should be handled during calculation. for path, value in FINAL_DATA.items(): if path.split('/')[-1] == 'timeline': for action in value: for key, value_ in action.items(): if isinstance(value_, float) and value_ != value_:
TEAM_DATA_DB_PATHS = [ 'raw.obj_pit', 'raw.subj_pit', 'processed.calc_obj_team', 'processed.calc_subj_team', 'processed.calc_tba_team' ] TIM_DATA_DB_PATHS = [ 'processed.calc_obj_tim', 'processed.calc_tba_tim' ] DB_PATH_TO_SCHEMA_FILE = { 'raw.obj_pit': 'schema/obj_pit_collection_schema.yml', 'raw.subj_pit': 'schema/subj_pit_collection_schema.yml', 'processed.calc_obj_team': 'schema/calc_obj_team_schema.yml', 'processed.calc_subj_team': 'schema/calc_subj_team_schema.yml', 'processed.calc_tba_team': 'schema/calc_tba_team_schema.yml', 'processed.calc_obj_tim': 'schema/calc_obj_tim_schema.yml', 'processed.calc_tba_tim': 'schema/calc_tba_tim_schema.yml' } if __name__ == '__main__': EXPORT_PATH = full_data_export() LATEST_PATH = utils.create_file_path('data/exports/latest_export', False) # Remove latest export directory if it exists if os.path.exists(LATEST_PATH): os.remove(LATEST_PATH) # Symlink the latest_export directory to the export that was just made os.symlink(EXPORT_PATH, LATEST_PATH)
def make_request(api_url, show_output=True, acceptable_cache_age=0): """Sends a single web request to the TBA API v3 and caches result. api_url is the url of the API request (the path after '/api/v3') show_output shows print statements about the status of the request. Defaults to True. acceptable_cache_age is the maximum age (in seconds) of data that can be pulled from the cache. Pulling from the cache is disabled by default.""" base_url = 'https://www.thebluealliance.com/api/v3/' full_url = base_url + api_url request_headers = {'X-TBA-Auth-Key': API_KEY} # This cache is used with TBA's 'Last-Modified' and # 'If-Modified-Since' headers to prevent duplicate data downloads. # If the data has not changed since the last request, it will be # pulled from the cache. # Documentation of the 'Last-Modified' and 'If-Modified-Since' headers: # https://www.thebluealliance.com/apidocs#apiv3 try: with open(utils.create_file_path('data/cache/tba/tba.json'), 'r') as file_: cached_requests = json.load(file_) except FileNotFoundError: cached_requests = {} # Returns the cached request if it was pulled within the # 'acceptable_cache_age' limit. last_request_time = cached_requests.get(api_url, {}).get('last_requested', 0) if (time.time() - last_request_time) < acceptable_cache_age: return cached_requests[api_url]['data'] # 'cache_last_modified' is the time that the data in the cache was # published to TBA's API. cache_last_modified = cached_requests.get(api_url, {}).get('last_modified') if cache_last_modified is not None: request_headers['If-Modified-Since'] = cache_last_modified if show_output is True: print(f'Retrieving data from TBA...\nURL: {api_url}') while True: try: request_time = time.time() request = requests.get(full_url, headers=request_headers) except requests.exceptions.ConnectionError: # Errors will always be shown, even if 'show_output' is False. print('Error: No internet connection. Trying again in 3 seconds...') else: if show_output is True: print('TBA data successfully retrieved.') break time.sleep(3) # A 304 status code means the data was not modified since our last # request, and we can pull it from the cache. if request.status_code == 304: return cached_requests[api_url]['data'] # A 200 status code means the request was successful elif request.status_code == 200: # Updates local cache cached_requests[api_url] = { 'last_requested': request_time, 'last_modified': request.headers['Last-Modified'], 'data': request.json(), } with open(utils.create_file_path('data/cache/tba/tba.json'), 'w') as file_: json.dump(cached_requests, file_) return request.json() else: # Errors will always be shown, even if 'show_output' is False. print(f'Request failed with status code {request.status_code}') return {}
if { 'match_number': match, 'scout_id': id_ } not in items_to_ignore: utils.log_warning( f'Duplicate Scout ID {id_} for Match {match}') else: unique_scout_ids.append(id_) # Scout IDs are from 1-18 inclusive for id_ in range(1, 19): if id_ not in unique_scout_ids: if { 'match_number': match, 'scout_id': id_ } not in items_to_ignore: utils.log_warning( f'Scout ID {id_} missing from Match {match}') # Load latest match collection compression QR code schema SCHEMA = utils.read_schema('schema/match_collection_qr_schema.yml') MISSING_TIM_IGNORE_FILE_PATH = utils.create_file_path( 'data/missing_tim_ignore.yml') _GENERIC_DATA_FIELDS = _get_data_fields('generic_data') OBJECTIVE_QR_FIELDS = _GENERIC_DATA_FIELDS.union( _get_data_fields('objective_tim')) SUBJECTIVE_QR_FIELDS = _GENERIC_DATA_FIELDS.union( _get_data_fields('subjective_aim')) _TIMELINE_FIELDS = get_timeline_info()
def train(self): """ Training model. """ # switch to train self.net.train() self.log.writeline('# Start training.', False) plot_point = 0 # for tensorboard # loop epoch for ep in range(self.max_epoch): total_loss = 0 # total loss total_acc = 0 # total accuracy self.log.writeline(f'----- Epoch: {ep + 1} -----') subdivision = self.tms.subdivision # batch in one epoch (outer tqdm) outer_pbar = tqdm(self.train_loader, total=len(self.train_loader), ncols=100, bar_format='{l_bar}{bar:30}{r_bar}') outer_pbar.set_description('TRAIN') # batch process for batch_idx, items in enumerate(outer_pbar): imgs: torch.Tensor labels: torch.Tensor paths: torch.Tensor imgs, labels, paths = items self.optimizer.zero_grad() # init gradient batch_size = len(imgs) # batch size batch_result = torch.tensor([]) # all result of one batch batch_loss = 0 # total loss of one batch # generate arithmetic progression of mini batch sep = np.linspace(0, batch_size, subdivision + 1, dtype=np.int) # mini batch process for sd in range(subdivision): n, m = sep[sd], sep[sd + 1] # cutout data (N ~ M) mb_imgs = imgs[n:m].to(self.device) mb_labels = labels[n:m].to(self.device) mb_result = self.net(mb_imgs) # data into model loss = self.criterion(mb_result, mb_labels) # calculate loss loss.backward() # calculate gradient (back propagation) # concatenate result batch_result = torch.cat((batch_result, mb_result.cpu()), dim=0) batch_loss += float(loss.item()) # add loss value # end of this mini batch self.optimizer.step() # update parameters loss_val = batch_loss / subdivision # calc avg loss value # tensorboard log self.writer.add_scalar('data/loss', loss_val, plot_point) plot_point += 1 # label predicted = torch.max(batch_result.data, 1)[1].cpu() # predict labels = labels.cpu() self.all_pred = torch.cat((self.all_pred, predicted), dim=0) self.all_label = torch.cat((self.all_label, labels), dim=0) predicted = predicted.numpy() # predict label_ans = labels.numpy() # correct answer # cls_bool = [label_ans[i] for i, x in enumerate(pred_bool) if not x] pred_bool = (label_ans == predicted) # matching # index of mistake prediction false_step = [idx for idx, x in enumerate(pred_bool) if not x] # save image of mistake prediction for idx in false_step: fp = self.false_paths[ep] name = Path(str(paths[idx])).name img_path = Path(fp, f'batch_{batch_idx}-{name}') img_path.parent.mkdir(parents=True, exist_ok=True) save_image(imgs[idx], str(img_path)) # save # count of matched label acc_cnt = pred_bool.sum() # calc total total_acc += acc_cnt total_loss += loss_val * batch_size acc = acc_cnt / batch_size # accuracy # for tqdm message outer_pbar.set_postfix( ordered_dict=OrderedDict(loss=f'{loss_val:<.6f}', acc=f'{acc:<.6f}')) # for log ss = f'loss: {loss_val:<.6f} / acc: {acc:<.6f}' ss += f'\n -> ans : {label_ans}' ss += f'\n -> result: {predicted}' self.log.writeline(ss, debug_ok=False) # break # end of this batch # add confusion matrix to tensorboard cm = calc_confusion_matrix(self.all_label, self.all_pred, len(self.classes)) fig = plot_confusion_matrix(cm, list(self.classes.values())) add_to_tensorboard(self.writer, fig, 'confusion matrix', ep) # calclate total loss / accuracy size = len(self.train_loader.dataset) total_loss = total_loss / size total_acc = total_acc / size # for log self.log.writeline('\n---------------', debug_ok=False) self.log.writeline(f'Total loss: {total_loss}', debug_ok=False) self.log.writeline(f'Total acc: {total_acc}', debug_ok=False) self.log.writeline('---------------\n', debug_ok=False) # for tqdm print(f' Total loss: {total_loss}') print(f' Total acc: {total_acc}\n') # for tensorboard self.writer.add_scalar('data/total_acc', total_acc, ep) self.writer.add_scalar('data/total_loss', total_loss, ep) # exec test cycle if self.test_schedule[ep]: self.test_model.test(epoch=ep) # save pth cycle if self.pth_save_schedule[ep]: save_path = ul.create_file_path( self.pth_save_path, '', head=f'epoch{ep + 1}', ext='pth') progress = ul.ProgressLog(f'Saving model to \'{save_path}\'') self.save_model(save_path) # save progress.complete() # log self.log.writeline(f'# Saved model to \'{save_path}\'', debug_ok=False) # break # end of this epoch # export as json # self.writer.export_scalars_to_json(f'{self.tms.config_path}/all_scalars.json') self.writer.close()
'015d172c98041412': 'Scout 16', '015d188421480008': 'Scout 17', '015d2568753c0200': 'Scout 18', # Fire tablets without cases (Backups 1-5) 'G000H40563460VSC': 'Backup 1', 'G000H4056383066L': 'Backup 2', 'G000H40563460T65': 'Backup 3', 'G000H404610600EK': 'Backup 4', 'G0K0KH02623400GT': 'Backup 5', # Super scout tablets 'redacted': 'Red Super', 'redacted': 'Blue Super', 'redacted': 'Purple Super', } ASSIGNMENT_FILE_PATH = utils.create_file_path( 'data/assignments/assignments.txt') # List of devices to which 'assignments.txt' has already been sent DEVICES_WITH_FILE = [] def validate_file(device_id): """Validates that the assignment file was successfully transfered. Compares the assignments.txt on the tablet to the locally stored assignments.txt file. device_id is the serial number of the device""" # Reads the server version of assignments.txt with open(ASSIGNMENT_FILE_PATH, 'r') as file: computer_data = file.read()
import utils # Uses default firebase URL # DB stands for database DB = firebase_communicator.configure_firebase() if len(sys.argv) == 2: CYCLE_NUMBER = sys.argv[1] else: print('Error: Cycle number not being passed as an argument. Exiting...') sys.exit(0) # Each scout name is associated with a letter (for compression). # This opens the JSON file that stores the letters and loads the dict # that is used to swap names with letters. with open(utils.create_file_path('data/assignments/assignments.json'), 'r') as file: LETTERS = json.load(file)['letters'] AVAILABILITY = DB.child('scoutManagement/availability').get().val() AVAILABLE_SCOUTS = [ scout for scout, availability in AVAILABILITY.items() if availability == 1 ] # The base assignment string ASSIGNMENT_STRING = f'{CYCLE_NUMBER}_{firebase_communicator.URL}|' with open(utils.create_file_path('data/sprs/sprs.json'), 'r') as file: SPRS = json.load(file) # Sorts scouts from best SPR to worst SPR
continue # Join list returned by update_array with existing write operations write_operations.extend(update_array(path, changed_documents)) # Write changes to database # Ordered must be true because we pull outdated data before pushing new data # Throws error on lost connection try: if write_operations: CLOUD_DB.competitions.bulk_write(write_operations, ordered=True) except pymongo.errors.AutoReconnect: utils.log_warning('Cloud Database Write Timeout.') return None return 0 def add_competition_cloud(tba_event_key): """Adds competition document to cloud database.""" local_database_communicator.add_competition(CLOUD_DB, tba_event_key) # Connect to cloud database with open(utils.create_file_path('data/api_keys/cloud_password.txt')) as file: CLOUD_PASSWORD = file.read().rstrip('\n') DB_ADDRESS = f'mongodb+srv://server:{CLOUD_PASSWORD}@scouting-system-3das1.gcp.mongodb.net/test?retryWrites=true&w=majority' CLOUD_DB = pymongo.MongoClient(DB_ADDRESS).scouting_system_cloud # Creates cloud database indexes (if they don't already exist) CLOUD_DB.competitions.create_indexes([ pymongo.IndexModel('tba_event_key', unique=True), pymongo.IndexModel('year', unique=False) ])
for team, data in TEAMS.items() } mean = numpy.mean(list(averages.values())) sd = numpy.std(list(averages.values())) for team, average in averages.items(): if sd == 0.0: TEAMS[team]['calculatedData'][team_zscore_field] = 0.0 else: TEAMS[team]['calculatedData'][team_zscore_field] = (average - mean) / sd # Gathers the calculated data from all the teams. TEAMS = {} for team in os.listdir(utils.create_file_path('data/cache/teams')): with open(utils.create_file_path(f'data/cache/teams/{team}')) as file: team_data = json.load(file) if team_data.get('calculatedData') is not None: # '.split()' removes '.json' file ending TEAMS[team.split('.')[0]] = team_data # Each Z-Score data field to the average data field it is calculated from. SUPER_ZSCORE_DATA_FIELDS = { 'agilityZScore': 'avgAgility', 'speedZScore': 'avgSpeed', } # Calculates zscores for teams based on data fields in 'SUPER_ZSCORE_DATA_FIELDS' for zscore_name, average_name in SUPER_ZSCORE_DATA_FIELDS.items(): calculate_zscores(average_name, zscore_name)
final_match_data = { 'matchNumber': match_number, 'redTeams': red_teams, 'blueTeams': blue_teams, } with open( utils.create_file_path( f'data/cache/match_schedule/{match_number}.json'), 'w') as file: json.dump(final_match_data, file) # Deletes the entire 'cache' directory to remove any old data. # Checks if the directory exists before trying to delete it to avoid # causing an error. if os.path.isdir(utils.create_file_path('data/cache', False)): shutil.rmtree(utils.create_file_path('data/cache', False)) # Detects when CTRL+C is pressed, then runs handle_ctrl_c signal.signal(signal.SIGINT, handle_ctrl_c) # Creates all the database streams and stores them in global dict. STREAMS = create_streams() # In order to make match calculations, the match schedule must be taken # from TBA and put into the cache. cache_match_schedule() # Wipes 'temp_timds' cache folder delete_cache_data_folder('temp_timds') # Stores the keys of cached 'tempTIMDs'