def __init__(self, jobs_db=None, instances_db=None):
self.gce_ops_in_progress = BoxList()
self.instances_db: DB = instances_db or get_worker_instances_db()
self.jobs_db: DB = jobs_db or get_jobs_db()
self.gce = googleapiclient.discovery.build('compute', 'v1')
self.project: str = GCP_PROJECT
self.zone: str = GCP_ZONE
def save_problem_ci_results(ci_error, db, error, eval_data, gist, problem_ci,
results, should_merge):
if not should_merge:
# If problem_ci fails, don't save to aggregate bot scores collection
if ci_error:
log.error('Problem CI failed, not saving to bots '
'official scores as this is likely an issue '
'with the new version of the problem.')
problem_ci.status = PROBLEM_CI_STATUS_FAILED
problem_ci.error = ci_error
update_pr_status_problem_ci(ci_error, problem_ci, eval_data)
else:
log.info('Problem CI not yet finished')
else:
# Aggregate data from bot evals now that they're done
gists = BoxList()
for bot_eval_key in problem_ci.bot_eval_keys:
bot_eval = db.get(get_eval_db_key(bot_eval_key))
save_to_bot_scores(
bot_eval, bot_eval.eval_key,
Box(score=bot_eval.results.score, eval_key=bot_eval.eval_key))
gists.append(bot_eval.gist)
problem_ci.gists = gists
update_pr_status_problem_ci(error, problem_ci, eval_data)
problem_ci.status = PROBLEM_CI_STATUS_PASSED
db.set(problem_ci.id, problem_ci)
def test_to_csv(self):
data = BoxList([
{
"Number": 1,
"Name": "Chris",
"Country": "US"
},
{
"Number": 2,
"Name": "Sam",
"Country": "US"
},
{
"Number": 3,
"Name": "Jess",
"Country": "US"
},
{
"Number": 4,
"Name": "Frank",
"Country": "UK"
},
{
"Number": 5,
"Name": "Demo",
"Country": "CA"
},
])
file = Path(tmp_dir, "csv_file.csv")
data.to_csv(filename=file)
assert file.read_text().startswith("Number,Name,Country\n1,Chris,US")
assert data.to_csv().endswith(
"2,Sam,US\r\n3,Jess,US\r\n4,Frank,UK\r\n5,Demo,CA\r\n")
def assign_jobs(self) -> Tuple[BoxList, List]:
new_jobs = BoxList()
exceptions = []
for job in self.jobs_db.where('status', '==', JOB_STATUS_CREATED):
try:
if self.should_start_job(job):
log.info(f'Assigning job ' f'{box2json(job)}...')
self.assign_job(job)
new_jobs.append(job)
except Exception as e:
# Could have been a network failure, so just try again.
# More granular exceptions should be handled before this
# which can set the job to not run again
# if that's what's called for.
log.exception(f'Exception triggering eval for job {job}, '
f'will try again shortly.')
exceptions.append(e)
# TODO: Check for failed / crashed instance once per minute
# TODO: Stop instances if they have been idle for longer than timeout
# TODO: Cap total instances
# TODO: Cap instances per bot owner, using first part of docker tag
# TODO: Delete instances over threshold of stopped+started
return new_jobs, exceptions
def test_from_csv(self):
bl = BoxList.from_csv(filename=Path(test_root, "data", "csv_file.csv"))
assert bl[1].Name == "Sam"
b2 = BoxList.from_csv(
"Number,Name,Country\r\n1,Chris,US\r\n2,Sam"
",US\r\n3,Jess,US\r\n4,Frank,UK\r\n5,Demo,CA\r\n")
assert b2[2].Name == "Jess"
def check_gce_ops_in_progress(self):
ops_still_in_progress = BoxList()
for op in self.gce_ops_in_progress:
try:
op_result = Box(
self.gce.zoneOperations().get(project=self.project,
zone=self.zone,
operation=op.name).execute())
except:
log.exception('Could not get op_result')
break
if op_result.status == 'DONE':
if 'error' in op_result:
log.error(f'GCE operation resulted in an error: '
f'{op_result.error}\nOperation was:'
f'\n{box2json(op)}')
if op.operationType == 'insert':
# Retry the creation?
pass
# elif op.operationType == 'start':
#
else:
ops_still_in_progress.append(op)
self.gce_ops_in_progress = ops_still_in_progress
def test_box_config_propagate(self):
structure = Box(a=[Box(default_box=False)],
default_box=True,
box_inherent_settings=True)
assert structure._box_config["default_box"] is True
assert structure.a[0]._box_config["default_box"] is True
base = BoxList([BoxList([Box(default_box=False)])], default_box=True)
assert base[0].box_options["default_box"] is True
base2 = BoxList((BoxList([Box()], default_box=False), ),
default_box=True)
assert base2[0][0]._box_config["default_box"] is True
base3 = Box(a=[Box(default_box=False)],
default_box=True,
box_inherent_settings=True,
box_intact_types=[Box, BoxList])
base3.a.append(Box(default_box=False))
base3.a.append(BoxList(default_box=False))
for item in base3.a:
if isinstance(item, Box):
assert item._box_config["default_box"] is True
elif isinstance(item, BoxList):
assert item.box_options["default_box"] is True
def test_bad_csv(self):
data = BoxList([
{'test': 1},
{'bad': 2, 'data': 3}
])
file = Path(tmp_dir, 'csv_file.csv')
with pytest.raises(BoxError):
data.to_csv(file)
def __init__(self, sourceDir):
if not os.path.isdir(sourceDir):
logging.info('target dir %s not exist!', sourceDir)
raise 'IOError'
self.sourceDir = sourceDir
self.patientBaseDict = None
self.patientImageSetList = BoxList()
self.patientPlanList = BoxList()
def test_box_list_from_yaml(self):
alist = [{"item": 1}, {"CamelBad": 2}]
yaml_list = yaml.dump(alist)
bl = BoxList.from_yaml(yaml_list, camel_killer_box=True)
assert bl[0].item == 1
assert bl[1].camel_bad == 2
with pytest.raises(BoxError):
BoxList.from_yaml(yaml.dump({"a": 2}))
def test_box_list_from_json(self):
alist = [{"item": 1}, {"CamelBad": 2}]
json_list = json.dumps(alist)
bl = BoxList.from_json(json_list, camel_killer_box=True)
assert bl[0].item == 1
assert bl[1].camel_bad == 2
with pytest.raises(BoxError):
BoxList.from_json(json.dumps({"a": 2}))
def test_to_multiline(self):
a = BoxList([Box(a=1), Box(b=2), Box(three=5)])
a.to_json(tmp_json_file, multiline=True)
count = 0
with open(tmp_json_file) as f:
for line in f:
assert isinstance(json.loads(line), dict)
count += 1
assert count == 3
def list_instances(self, label) -> BoxList:
if label:
query_filter = f'labels.{label}:*'
else:
query_filter = None
ret = self.gce.instances().list(project=self.project,
zone=self.zone,
filter=query_filter).execute()
ret = BoxList(ret.get('items', []))
return ret
def test_box_list_from_yaml(self):
alist = [{"item": 1}, {"CamelBad": 2}]
yaml = YAML()
with StringIO() as sio:
yaml.dump(alist, stream=sio)
bl = BoxList.from_yaml(sio.getvalue(), camel_killer_box=True)
assert bl[0].item == 1
assert bl[1].camel_bad == 2
with pytest.raises(BoxError):
BoxList.from_yaml("a: 2")
def test_to_csv(self):
data = BoxList([
{'Number': 1, 'Name': 'Chris', 'Country': 'US'},
{'Number': 2, 'Name': 'Sam', 'Country': 'US'},
{'Number': 3, 'Name': 'Jess', 'Country': 'US'},
{'Number': 4, 'Name': 'Frank', 'Country': 'UK'},
{'Number': 5, 'Name': 'Demo', 'Country': 'CA'},
])
file = Path(tmp_dir, 'csv_file.csv')
data.to_csv(file)
assert file.read_text().startswith("Number,Name,Country\n1,Chris,US")
def test_box_list_from_tml(self):
alist = [{'item': 1}, {'CamelBad': 2}]
toml_list = toml.dumps({'key': alist})
bl = BoxList.from_toml(toml_string=toml_list, key_name='key', camel_killer_box=True)
assert bl[0].item == 1
assert bl[1].camel_bad == 2
with pytest.raises(BoxError):
BoxList.from_toml(toml.dumps({'a': 2}), 'a')
with pytest.raises(BoxError):
BoxList.from_toml(toml_list, 'bad_key')
def test_box_list_from_tml(self):
alist = [{"item": 1}, {"CamelBad": 2}]
toml_list = toml.dumps({"key": alist})
bl = BoxList.from_toml(toml_string=toml_list, key_name="key", camel_killer_box=True)
assert bl[0].item == 1
assert bl[1].camel_bad == 2
with pytest.raises(BoxError):
BoxList.from_toml(toml.dumps({"a": 2}), "a")
with pytest.raises(BoxError):
BoxList.from_toml(toml_list, "bad_key")
def reset(self):
if not DEVMODE:
return
for i in range(self.count() - 1, -1, -1):
self.removeTab(i)
self.addTab(self.get_textbox(Box(self.app.fastflix.config.dict())), "Config")
self.addTab(self.get_textbox(Box(self.get_ffmpeg_details())), "FFmpeg Details")
self.addTab(self.get_textbox(BoxList(self.app.fastflix.conversion_list)), "Queue")
self.addTab(self.get_textbox(Box(self.app.fastflix.encoders)), "Encoders")
self.addTab(self.get_textbox(BoxList(self.app.fastflix.audio_encoders)), "Audio Encoders")
if self.app.fastflix.current_video:
self.addTab(self.get_textbox(BoxList(self.app.fastflix.current_video)), "Current Video")
def test_hashing(self):
bx1 = Box(t=3, g=4, frozen_box=True)
bx2 = Box(g=4, t=3, frozen_box=True)
assert hash(bx1) == hash(bx2)
bl1 = BoxList([1, 2, 3, 4], frozen_box=True)
bl2 = BoxList([1, 2, 3, 4], frozen_box=True)
bl3 = BoxList([2, 1, 3, 4], frozen_box=True)
assert hash(bl2) == hash(bl1)
assert hash(bl3) != hash(bl2)
with pytest.raises(TypeError):
hash(BoxList([1, 2, 3]))
def city_search(self, city_name: str) -> BoxList:
"""Use the downloaded city data to find a city's ID, lat and lon"""
if not self.city_info:
try:
city_info = self._download_city_list()
except Exception as err:
print(
f'Could not download or transform city information: {err}')
return BoxList()
else:
if not city_info:
raise OpenWeatherError('Cannot load city data!')
self.city_info = city_info
return BoxList(city for city in self.city_info
if city_name.casefold() in city.name.casefold())
def test_from_multiline(self):
content = '{"a": 2}\n{"b": 3}\r\n \n'
with open(tmp_json_file, "w") as f:
f.write(content)
a = BoxList.from_json(filename=tmp_json_file, multiline=True)
assert a[1].b == 3
def get_sheet_names(self, export_sequence=True) -> list:
"""
Provides all the sheets. Ordering is dependent on export_sequence flag.
:param export_sequence: if True then sheets are provided in order that can be easily exported - uses DFS.
if False, provides sheet names as they appear in mapper yml configuration file.
:return: list of sheet names.
"""
if not export_sequence:
return list(self.parsed_sheets.keys())
visited = set()
graph = self.parsed_sheets.keys()
dfs_nodes = BoxList()
def dfs(node):
nonlocal visited, graph
if node not in visited:
visited.add(node)
dependents = self.parsed_sheets[node].dependent_sheets
for neighbour in dependents:
dfs(neighbour)
dfs_nodes.append(node)
while 1:
starting_node = set(graph) - set(visited)
if not starting_node:
break
dfs(starting_node.pop())
logging.info(dfs_nodes)
return dfs_nodes
def from_data(self, data):
self.raw_data = data
if isinstance(data, list):
self._boxed_data = BoxList(data, camel_killer_box=True)
else:
self._boxed_data = Box(data, camel_killer_box=True)
return self
def print_yaml(x: typing.Any) -> str:
if isinstance(x, dict):
return Box(x).to_yaml()
elif isinstance(x, list):
return BoxList(x).to_yaml()
else:
return str(x)
def _download_city_list(self):
if not self.city_file_location.exists():
self.city_file_location.parent.mkdir(parents=True, exist_ok=True)
reusables.download(
'http://bulk.openweathermap.org/sample/city.list.json.gz',
filename=str(self.city_file_location))
return BoxList.from_json(gzip.open(self.city_file_location).read())
def test_set_default(self):
test_dict = {
'key1': 'value1',
"Key 2": {
"Key 3": "Value 3",
"Key4": {
"Key5": "Value5"
}
}
}
a = Box(test_dict)
new = a.setdefault("key3", {'item': 2})
new_list = a.setdefault("lister", [{'gah': 7}])
assert a.setdefault("key1", False) == 'value1'
assert new == Box(item=2)
assert new_list == BoxList([{'gah': 7}])
assert a.key3.item == 2
assert a.lister[0].gah == 7
b = LightBox(test_dict)
new = b.setdefault("key3", {'item': 2})
new_list = b.setdefault("lister", [{'gah': 7}])
assert b.setdefault("key1", False) == 'value1'
assert new == Box(item=2)
assert new_list == [{'gah': 7}]
assert b.key3.item == 2
assert b.lister[0]["gah"] == 7
assert not isinstance(b.lister, BoxList)
def test_ror_boxes(self):
b = dict(c=1, d={"sub": 1}, e=1)
c = Box(d={"val": 2}, e=4)
assert c.__ror__(b) == Box(c=1, d={"sub": 1}, e=1)
assert c | b == Box(c=1, d={"sub": 1}, e=1)
with pytest.raises(BoxError):
BoxList() | Box()
def test_wait_for_problem_cis():
ret = wait_for_problem_cis(
BoxList([
Box(pr_number=96,
commit='4a6a67702931920ee9a6813247d09d88b433d7b0')
]))
return ret
def test_set_default_box_dots(self):
a = Box(box_dots=True)
a["x"] = {"y": 10}
a.setdefault("x.y", 20)
assert a["x.y"] == 10
a["lists"] = [[[{"test": "here"}], {1, 2}], (4, 5)]
assert list(_get_dot_paths(a)) == [
"x",
"x.y",
"lists",
"lists[0]",
"lists[0][0]",
"lists[0][0][0]",
"lists[0][0][0].test",
"lists[0][1]",
"lists[1]",
]
t = Box({"a": 1},
default_box=True,
box_dots=True,
default_box_none_transform=False)
assert t.setdefault("b", [1, 2]) == [1, 2]
assert t == Box(a=1, b=[1, 2])
assert t.setdefault("c", [{"d": 2}]) == BoxList([{"d": 2}])
def test_ior_boxes(self):
b = Box(c=1, d={"sub": 1}, e=1)
c = dict(d={"val": 2}, e=4)
b |= c
assert b == Box(c=1, d={"val": 2}, e=4)
with pytest.raises(BoxError):
a = Box()
a |= BoxList()