def upload_qr_codes(qr_codes):
"""Uploads QR codes into the current competition document.
Prevents duplicate QR codes from being uploaded to the database.
qr_codes is a list of QR code strings to upload.
"""
# Gets the starting character for each QR code type, used to identify QR code type
schema = utils.read_schema('schema/match_collection_qr_schema.yml')
# Acquires current qr data using local_database_communicator.py
qr_data = local_database_communicator.read_dataset('raw.qr')
# Creates a set to store QR codes
# This is a set in order to prevent addition of duplicate qr codes
qr = set()
for qr_code in qr_codes:
if qr_code in qr_data:
pass
# Checks to make sure the qr is valid by checking its starting character. If the starting
# character doesn't match either of the options, the QR is printed out.
elif not (qr_code.startswith(
schema['subjective_aim']['_start_character'])
or qr_code.startswith(
schema['objective_tim']['_start_character'])):
utils.log_warning(f'Invalid QR code not uploaded: "{qr_code}"')
else:
qr.add(qr_code)
# Adds the QR codes to the local database if the set isn't empty
if qr != set():
local_database_communicator.append_to_dataset('raw.qr', list(qr))
return qr
def tba_request(api_url):
"""Sends a single web request to the TBA API v3 api_url is the suffix of the API request URL
(the part after '/api/v3').
"""
utils.log_info(f'tba request from {api_url} started')
full_url = f'https://www.thebluealliance.com/api/v3/{api_url}'
request_headers = {'X-TBA-Auth-Key': API_KEY}
cached = local_database_communicator.select_tba_cache(api_url)
# Check if cache exists
if cached != {}:
cached = cached[api_url]
request_headers['If-Modified-Since'] = cached['timestamp']
print(f'Retrieving data from {full_url}')
utils.log_info(f'tba request from {api_url} finished')
try:
request = requests.get(full_url, headers=request_headers)
except requests.exceptions.ConnectionError:
utils.log_warning('Error: No internet connection.')
return None
# A 200 status code means the request was successful
# 304 means that data was not modified since the last timestamp
# specified in request_headers['If-Modified-Since']
if request.status_code == 304:
return cached['data']
if request.status_code == 200:
formatted_data = {
'timestamp': request.headers['Last-Modified'],
'data': request.json()
}
local_database_communicator.overwrite_tba_data(formatted_data, api_url)
return request.json()
raise Warning(f'Request failed with status code {request.status_code}')
def update_array(path, change_list):
"""Updates an array of embedded documents. Return 0 on success, 1 if connection was lost."""
write_operations = []
# Return blank list if there are no changes at this path
if not change_list:
return write_operations
# Remove documents to be updated
write_operations.append(
pymongo.UpdateOne({'tba_event_key': utils.TBA_EVENT_KEY},
{'$pull': {
path: {
'$or': change_list
}
}}))
# Select documents to add
filter_change_list = []
for change in change_list:
equals = []
for key, value in change.items():
equals.append({'$eq': [f'$$item.{key}', value]})
filter_change_list.append({'$and': equals})
to_add = local_database_communicator.DB.competitions.aggregate([{
'$match': {
'tba_event_key': utils.TBA_EVENT_KEY
}
}, {
'$project': {
path: {
'$filter': {
'input': f'${path}',
'as': 'item',
'cond': {
'$or': filter_change_list
}
}
}
}
}])
# Aggregate returns a cursor object, so it must be converted to a list. `tba_event_key` is
# guaranteed to be unique, so there will always one and only one result.
to_add = list(to_add)[0]
# Remove `_id` so so the only item is the array nested in the directory structure
to_add.pop('_id')
# Remove nesting, making `to_add` only a list of changed documents
while isinstance(to_add, dict):
to_add = to_add[[*to_add.keys()][0]]
# No data matched or dataset does not exist, so warn & return blank list
if to_add is None:
utils.log_warning(f'No the dataset at {path} does not exist.')
return []
write_operations.append(
pymongo.UpdateOne({'tba_event_key': utils.TBA_EVENT_KEY},
{'$push': {
path: {
'$each': to_add
}
}}))
return write_operations
def write_tba_data(path):
"""Writes TBA Data to csv export. Path is a str representing the output absolute file path."""
data = format_tba_data()
if not data:
utils.log_warning('No TBA Data to export')
field_names = data[0].keys()
with open(path, 'w') as file:
writer = csv.DictWriter(file, field_names)
writer.writeheader()
for row in data:
writer.writerow(row)
utils.log_info('Exported TBA Data')
def _pull_file_from_device(self, device_path, host_path):
if self.adb.run(['pull', device_path, host_path]):
return True
# In non-root device, we can't pull /data/app/XXX/base.odex directly.
# Instead, we can first copy the file to /data/local/tmp, then pull it.
filename = device_path[device_path.rfind('/')+1:]
if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and
self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])):
self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename])
return True
log_warning('failed to pull %s from device' % device_path)
return False
def stop_recording(args):
adb = AdbHelper()
result = adb.run(['shell', 'pidof', 'simpleperf'])
if not result:
log_warning('No simpleperf process on device. The recording has ended.')
else:
adb.run(['shell', 'pkill', '-l', '2', 'simpleperf'])
print('Waiting for simpleperf process to finish...')
while adb.run(['shell', 'pidof', 'simpleperf']):
time.sleep(1)
adb.run(['shell', 'cat', '/data/local/tmp/simpleperf_output'])
adb.check_run(['pull', '/data/local/tmp/perf.data', args.perf_data_path])
print('The recording data has been collected in %s.' % args.perf_data_path)
def stop_recording(args):
adb = AdbHelper()
result = adb.run(['shell', 'pidof', 'simpleperf'])
if not result:
log_warning(
'No simpleperf process on device. The recording has ended.')
else:
adb.run(['shell', 'pkill', '-l', '2', 'simpleperf'])
print('Waiting for simpleperf process to finish...')
while adb.run(['shell', 'pidof', 'simpleperf']):
time.sleep(1)
adb.run(['shell', 'cat', '/data/local/tmp/simpleperf_output'])
adb.check_run(['pull', '/data/local/tmp/perf.data', args.perf_data_path])
print('The recording data has been collected in %s.' % args.perf_data_path)
def check_status_consistency(filename, row, i, log):
"""Check that the status is consistent with the requirements."""
# Checks if Status is one of Accepted, Opened, Rejected
# and checks for required information if so
if row["Status"] in ["Accepted", "Opened", "Rejected"]:
# The project apache/incubator-dubbo was renamed to apache/dubbo,
# so the Project URL name (old) doesn't match the PR Link name
# (new), despite them being the same project. This if statement is
# a workaround for that issue.
if (row["Project URL"] == "https://github.com/apache/incubator-dubbo"
and re.sub(r"\/pull\/\d+", "", row["PR Link"]).casefold()
== "https://github.com/apache/dubbo"):
pass
else:
check_pr_link(filename, row, i, log)
if row["Status"] in ["InspiredAFix", "Skipped", "MovedOrRenamed"]:
# Should contain a note
if row["Notes"] == "":
log_warning(
filename,
log,
i,
"Status " + row["Status"] + " should contain a note",
)
# If it contains a note, it should be a valid link
else:
check_notes(filename, row, i, log)
# Should contain a PR Link
if row["Status"] == "InspiredAFix":
if row["PR Link"] == "":
log_warning(
filename,
log,
i,
"Status " + row["Status"] + " should have a PR Link",
)
# If it contains a PR link, it should be a valid one
else:
check_pr_link(filename, row, i, log)
if row["Status"] == "" and row["PR Link"] != "":
check_pr_link(filename, row, i, log)
log_std_error(
filename, log, i, row,
"Status should not be empty when a PR link is provided.")
def check_scout_ids():
"""Checks unconsolidated TIMs in `tim_queue` to see which scouts have not sent data.
This operation is done by `scout_id` -- if a match is missing data, then the scout_id will not
have sent data for the match.
returns None -- warnings are issued directly through `utils.log_warning`.
"""
# Load matches or matches and ids to ignore from ignore file
if os.path.exists(MISSING_TIM_IGNORE_FILE_PATH):
with open(MISSING_TIM_IGNORE_FILE_PATH) as ignore_file:
items_to_ignore = yaml.load(ignore_file, Loader=yaml.Loader)
else:
items_to_ignore = []
matches_to_ignore = [
item['match_number'] for item in items_to_ignore if len(item) == 1
]
tims = local_database_communicator.read_dataset(
'processed.unconsolidated_obj_tim')
matches = {}
for tim in tims:
match_number = tim['match_number']
matches[match_number] = matches.get(match_number,
[]) + [tim['scout_id']]
for match, scout_ids in matches.items():
if match in matches_to_ignore:
continue
unique_scout_ids = []
for id_ in scout_ids:
if id_ in unique_scout_ids:
if {
'match_number': match,
'scout_id': id_
} not in items_to_ignore:
utils.log_warning(
f'Duplicate Scout ID {id_} for Match {match}')
else:
unique_scout_ids.append(id_)
# Scout IDs are from 1-18 inclusive
for id_ in range(1, 19):
if id_ not in unique_scout_ids:
if {
'match_number': match,
'scout_id': id_
} not in items_to_ignore:
utils.log_warning(
f'Scout ID {id_} missing from Match {match}')
def update_dataset(path, new_data, query, competition=utils.TBA_EVENT_KEY):
"""Updates a single dictionary within a dataset, if the query matches a dictionary within a
dataset, replace the data given, if it does not exist create a new dictionary and add the query
and data given by function parameter
'path' is the path to the dataset, (e.g. 'raw.qr', 'processed.calc_obj_tim'), use dot notation.
'data' is the data to either add, or to replace if the query matches in the dataset, must be a
dictionary.
'query' is a query to search through the given datset for the first occurence within a
dictionary.
'competition' is the tba event key.
"""
dataset = read_dataset(path, competition, **query)
new_document = {}
# If all queries matched the document
if dataset != []:
# New document gets the value of the first dictionary in dataset that matched queries
new_document = dataset[0]
for new_datum in new_data:
# Add each new datum to the new dictionary
new_document[new_datum] = new_data[new_datum]
# Delete the dictionary from the dataset
DB.competitions.update_one({'tba_event_key': competition},
{'$pull': {
path: query
}})
# If 'new_document' is still empty, the query did not match
else:
# Iterate through the keys in query
for key in query:
# Add them to the new document
new_document[key] = query[key]
# Iterate through new_data
for datum in new_data:
# Add the keys to the new document
if datum in new_document and new_document[datum] != new_data[datum]:
utils.log_warning('Query and new data are mismatched')
return None
new_document[datum] = new_data[datum]
# Update the dictionary in the dataset
DB.competitions.update_one({'tba_event_key': competition},
{'$push': {
path: new_document
}})
def _annotate_files(self):
"""Annotate Source files: add acc_period/period for each source file.
1. Annotate java source files, which have $JAVA_SRC_ROOT prefix.
2. Annotate c++ source files.
"""
dest_dir = self.config['annotate_dest_dir']
for key in self.file_periods:
from_path = key
if not os.path.isfile(from_path):
log_warning("can't find source file for path %s" % from_path)
continue
if from_path.startswith('/'):
to_path = os.path.join(dest_dir, from_path[1:])
elif is_windows() and ':\\' in from_path:
to_path = os.path.join(dest_dir, from_path.replace(':\\', os.sep))
else:
to_path = os.path.join(dest_dir, from_path)
is_java = from_path.endswith('.java')
self._annotate_file(from_path, to_path, self.file_periods[key], is_java)
def push_changes_to_db(local_change_list, server_restart):
"""Pushes changes to cloud database given the local changes.
Returns 0 on success, None on failure.
"""
# List of paths that should be directly added (do not point to a document to be updated)
direct_push = ['raw.qr']
# Stores PyMongo UpdateOne objects to be written in a bulk write
write_operations = []
for section_name, datafield in local_change_list.items():
for datafield_name, changed_documents in datafield.items():
path = '.'.join([section_name, datafield_name])
# Cloud data should be replaced on server restart, so all existing data should be
# removed so no outdated data remains
if server_restart:
write_operations.append(
pymongo.UpdateOne({'tba_event_key': utils.TBA_EVENT_KEY},
{'$set': {
path: []
}}))
if path in direct_push and changed_documents != []:
write_operations.append(
pymongo.UpdateOne(
{'tba_event_key': utils.TBA_EVENT_KEY},
{'$push': {
path: {
'$each': changed_documents
}
}}))
continue
# Join list returned by update_array with existing write operations
write_operations.extend(update_array(path, changed_documents))
# Write changes to database
# Ordered must be true because we pull outdated data before pushing new data
# Throws error on lost connection
try:
if write_operations:
CLOUD_DB.competitions.bulk_write(write_operations, ordered=True)
except pymongo.errors.AutoReconnect:
utils.log_warning('Cloud Database Write Timeout.')
return None
return 0
def _annotate_files(self):
"""Annotate Source files: add acc_period/period for each source file.
1. Annotate java source files, which have $JAVA_SRC_ROOT prefix.
2. Annotate c++ source files.
"""
dest_dir = self.config['annotate_dest_dir']
for key in self.file_periods:
from_path = key
if not os.path.isfile(from_path):
log_warning("can't find source file for path %s" % from_path)
continue
if from_path.startswith('/'):
to_path = os.path.join(dest_dir, from_path[1:])
elif is_windows() and ':\\' in from_path:
to_path = os.path.join(dest_dir,
from_path.replace(':\\', os.sep))
else:
to_path = os.path.join(dest_dir, from_path)
is_java = from_path.endswith('.java')
self._annotate_file(from_path, to_path, self.file_periods[key],
is_java)
def install_apk(device_serial):
"""Installs chosen APK to either phone or tablet depending on user input.
Convert serial number to human-readable format.
"""
device_name = adb_communicator.DEVICE_SERIAL_NUMBERS[device_serial]
print(f'Loading {LOCAL_FILE_PATH} onto {device_name}')
# Calls 'adb push' command, which uses the Android Debug Bridge (ADB) to send the APK file
# The -s flag specifies the device_serial by its serial number
# return_output=True returns the output of adb
utils.log_info(f'APK install started on {device_serial}')
validate = utils.run_command(
f'adb -s {device_serial} install -r {LOCAL_FILE_PATH}',
return_output=True)
# If .apk is loaded successfully, ADB will output a string containing 'Success'
if 'Success' in validate:
DEVICES_WITH_APK.append(device_serial)
print(f'Loaded {LOCAL_FILE_PATH} onto {device_name}')
utils.log_info(f'APK successefully installed on {device_serial}')
else:
utils.log_warning(
f'Failed Loading {LOCAL_FILE_PATH} onto {device_name}.')
import pymongo
# Internal imports
import adb_communicator
import calculate_obj_team
import calculate_obj_tims
import calculate_tba_tims
import decompressor
import local_database_communicator
import qr_code_uploader
import tba_communicator
import utils
try:
import cloud_database_communicator
except pymongo.errors.ConfigurationError:
utils.log_warning(f'Cloud database import failed. No internet.')
def get_empty_modified_data():
"""Returns empty modified data field."""
modified_data = {
'raw': {
'qr': [],
'obj_pit': [],
'subj_pit': []
},
'processed': {
'unconsolidated_obj_tim': [],
'calc_obj_tim': [],
'calc_tba_tim': [],
'subj_aim': [],
if device not in DEVICES_WITH_SCHEDULE and SEND_MATCH_SCHEDULE:
print(
f'\nAttempting to load {MATCH_SCHEDULE_LOCAL_PATH} onto {device_name}'
)
if adb_communicator.push_file(device,
MATCH_SCHEDULE_LOCAL_PATH,
MATCH_SCHEDULE_TABLET_PATH,
validate_file):
DEVICES_WITH_SCHEDULE.add(device)
print(
f'Loaded {MATCH_SCHEDULE_LOCAL_PATH} onto {device_name}'
)
else:
# Give both serial number and device name in warning
utils.log_warning(
f'FAILED sending {MATCH_SCHEDULE_LOCAL_PATH} to {device_name} ({device})'
)
if device not in DEVICES_WITH_LIST:
print(
f'\nAttempting to load {TEAM_LIST_LOCAL_PATH} onto {device_name}'
)
if adb_communicator.push_file(device, TEAM_LIST_LOCAL_PATH,
TEAM_LIST_TABLET_PATH,
validate_file):
DEVICES_WITH_LIST.add(device)
print(f'Loaded {TEAM_LIST_LOCAL_PATH} to {device_name}')
else:
# Give both serial number and device name in warning
utils.log_warning(
f'FAILED sending {TEAM_LIST_LOCAL_PATH} to {device_name} ({device})'
)
def update_calc_tba_tims(tims):
"""Returns a list of the TIM calcs that require TBA data to run.
Reads from the `calc_tba_tim` schema file to get data points that are pulled from TBA.
"""
# Pull TBA data
tba_api_url = f'event/{utils.TBA_EVENT_KEY}/matches'
tba_data = local_database_communicator.select_tba_cache(
tba_api_url)[tba_api_url]['data']
# Filter out matches such that we only have quals matches
# Create dictionary of match_number: match data to allow easier access
quals_matches = {
data['match_number']: data
for data in tba_data if data['comp_level'] == 'qm'
}
full_tim_refs = []
# Get team number and match number for TIMs referenced by `tims`
# This is needed because TIMs can be passed just using match number
for tim in tims:
if 'team_number' in tim:
# Ref is as specific as possible, refers to one team in one match
if 'match_number' in tim:
full_tim_refs.append(tim)
else: # Match refers to all a team's matches
for match_num, match_data in quals_matches.items():
if tim['team_number'] in get_team_list_from_match(
match_data):
full_tim_refs.append({
'team_number': tim['team_number'],
'match_number': match_num
})
# Ref refers to all TIMs from a match
elif 'match_number' in tim:
if tim['match_number'] in quals_matches:
for team in get_team_list_from_match(
quals_matches[tim['match_number']]):
full_tim_refs.append({
'team_number': team,
'match_number': tim['match_number']
})
else:
utils.log_warning(
f'Cannot find TBA data from q{tim["match_number"]} in cache'
)
else:
utils.log_warning(f'Invalid TBA TIM ref {tim}')
output_data = []
for ref in full_tim_refs:
out = copy.deepcopy(ref)
if quals_matches[out['match_number']]['score_breakdown'] is None:
utils.log_warning(
f'TBA TIM Calculation on {out["match_number"]} missing match data'
)
continue
# Get robot number (e.g. the `i` in initLineRobot1) and alliance color for TIM
number_result = utils.catch_function_errors(
get_robot_number_and_alliance, out['team_number'],
quals_matches[out['match_number']])
# `utils.catch_function_errors` returns `None` for errors, which must be handled before
# assigning variables to function results.
if number_result is None:
continue
robot_number, alliance = number_result
for key, values in TBA_SCHEMA['tba'].items():
filters = copy.deepcopy(values)
type_ = filters.pop('type')
if type_ != 'bool':
utils.log_warning(f'Type {type_} not recognized, skipping...')
break
for name, correct_value in filters.items():
# Detect entries like initLineRobot, which need a robot number after them
if name.endswith('Robot'):
del filters[name]
filters[f'{name}{robot_number}'] = correct_value
result = utils.catch_function_errors(
calc_tba_bool, quals_matches[out['match_number']], alliance,
filters)
if result is not None:
out[key] = result
else:
break
else:
output_data.append(out)
return output_data