def produce(self,
inputs: Inputs,
timeout=None,
iterations=None) -> Outputs:
"""
Arguments:
- inputs: List( # Data
List( # Segments
CurveFitting
)
)
Returns:
- List(d3m_ndarray)
"""
with stopit.ThreadingTimeout(timeout) as timer:
outputs = List()
for cinput in inputs:
deg, num_feats = cinput[0].beta.shape
betas = np.array([segment.beta for segment in cinput]).reshape(
(-1, num_feats))
outputs.append(self.model.predict(betas))
if timer.state == timer.EXECUTED:
return d3m_ndarray(outputs)
else:
raise TimeoutError('ClusterCurveFitting exceeded time limit')
def solve_mpc_adam(self, graph, history, target):
"""
Args:
graph: DGL graph
history: tuple (tc history, control history)
- tc history [#.tc sensors x history length]
- control history [#. control sensors x history length]
target:
Returns:
"""
crit = torch.nn.MSELoss(reduction='none')
gamma_mask = get_discount_factor(target.shape[1],
1.0).view(1, -1).to(self.device)
with stopit.ThreadingTimeout(self.timeout) as to_ctx_mgr:
us = Actions(target[0, :], history[1].shape[0], 1, self.u_min,
self.u_max).to(self.device)
opt = torch.optim.Adam(us.parameters(), lr=1e-3)
with torch.no_grad():
h0 = self.model.filter_history(graph, history[0], history[1])
for i in range(self.max_iter):
opt.zero_grad()
prediction = self.predict_future(graph, h0, us())
loss = crit(prediction, target)
loss = (loss * gamma_mask).sum()
loss.backward()
opt.step()
us.us.data = us.us.data.clamp(min=0.0, max=1.0)
if to_ctx_mgr.state == to_ctx_mgr.TIMED_OUT:
us.us.data = us.us.data.clamp(min=0.0, max=1.0)
return us
def _run_match(self, match_id, home_team, away_team, home_agent,
away_agent):
game = Game(match_id,
home_team=deepcopy(home_team),
away_team=deepcopy(away_team),
home_agent=home_agent,
away_agent=away_agent,
config=self.config,
record=self.record)
game.config.fast_mode = True
game.config.competition_mode = True
print("Starting new match")
try:
with stopit.ThreadingTimeout(int(
self.config.time_limits.game)) as to_ctx_mgr:
game.init()
if to_ctx_mgr.state != to_ctx_mgr.EXECUTED:
print("Game timed out!")
game.end_time = time.time()
game.game_over = True
game.disqualified_agent = game.actor
return GameResult(game)
except Exception as e:
print(
f"Game crashed by {game.actor.name if game.actor is not None else 'the framework'}: ",
e)
game.disqualified_agent = game.actor
return GameResult(game, crashed=True)
return GameResult(game)
def fit(self, *, timeout: float = None, iterations: int = None) -> CallResult[None]:
"""
Arguments
- inputs: List( # Data
List( # Segments
[ deg, num_feats ], ...
)
),
"""
if self._fitted:
return CallResult(None)
if self._training_inputs is None:
raise Exception('Missing training data')
with stopit.ThreadingTimeout(timeout) as timer:
inputs_curve_fitting = self._training_inputs
num_data = sum([ len(x) for x in inputs_curve_fitting ]) # number of segments, each segment if formed by multiple data samples
deg, num_feats = inputs_curve_fitting[0][0].shape
betas = np.vstack([
np.array([
segment.flatten() for segment in cinput
]) for cinput in inputs_curve_fitting if len(cinput) > 0
])
self._model.fit(betas)
self._fitted = True
if timer.state == timer.EXECUTED:
return CallResult(None)
else:
raise TimeoutError('ClusterCurveFittingKMeans exceeded time limit')
def produce(self,
*,
inputs: Sequence[Input],
timeout: float = None,
iterations: int = None) -> Sequence[Output]:
"""
precond: run fit() before
to complete the data, based on the learned parameters, support:
-> greedy search
also support the untrainable methods:
-> iteratively regression
-> other
Parameters:
----------
data: pandas dataframe
label: pandas series, used for the evaluation of imputation
TODO:
----------
1. add evaluation part for __simpleImpute()
"""
if (not self.is_fitted):
# todo: specify a NotFittedError, like in sklearn
raise ValueError("Calling produce before fitting.")
if (timeout is None):
timeout = math.inf
if (iterations is None):
self._iterations_done = True
iterations = 30 # only works for iteratively_regre method
data = inputs.copy()
# record keys:
keys = data.keys()
# setup the timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mrg:
assert to_ctx_mrg.state == to_ctx_mrg.EXECUTING
# start completing data...
if (self.verbose > 0):
print("=========> iteratively regress method:")
data_clean = self.__regressImpute(data, self.best_imputation,
iterations)
if to_ctx_mrg.state == to_ctx_mrg.EXECUTED:
self.is_fitted = True
self._has_finished = True
return pd.DataFrame(data=data_clean, columns=keys)
elif to_ctx_mrg.state == to_ctx_mrg.TIMED_OUT:
print("Timed Out...")
self.is_fitted = False
self._has_finished = False
self._iterations_done = False
return
def timeout(func, *args, seconds=None, **kwargs):
if seconds is None:
return func(*args, **kwargs)
try:
start = time.time()
with stopit.ThreadingTimeout(seconds) as to_ctx_mgr:
result = func(*args, **kwargs)
duration = time.time() - start
# OK, let's check what happened
if to_ctx_mgr.state == to_ctx_mgr.EXECUTED:
# All's fine, everything was executed
return result
elif to_ctx_mgr.state == to_ctx_mgr.TIMED_OUT:
# Timeout occurred while executing the block
raise TimeoutError("Timed out after {0} seconds.".format(duration))
else:
traceback.print_tb()
raise RuntimeError("Something weird happened.")
except stopit.TimeoutException as e:
raise TimeoutError("Timed out after {0} seconds.".format(e.seconds))
duration = time.time() - seconds
def RunTestCaseOnWebsite((website, test_case, config)):
""" Runs a |test_case| on a |website|. In case when |test_case| has
failed it tries to rerun it. If run takes too long, then it is stopped.
"""
profile_path = tempfile.mkdtemp()
# The tests can be flaky. This is why we try to rerun up to 3 times.
attempts = 3
result = ("", "", False, "")
logger = logging.getLogger("run_tests")
for _ in xrange(attempts):
shutil.rmtree(path=profile_path, ignore_errors=True)
logger.log(SCRIPT_DEBUG, "Run of test case %s of website %s started",
test_case, website)
try:
with stopit.ThreadingTimeout(100) as timeout:
logger.log(SCRIPT_DEBUG,
"Run test with parameters: %s %s %s %s %s %s",
config.chrome_path, config.chromedriver_path,
profile_path, config.passwords_path, website,
test_case)
result = tests.RunTest(config.chrome_path,
config.chromedriver_path, profile_path,
config.passwords_path, website,
test_case)[0]
if timeout != timeout.EXECUTED:
result = (website, test_case, False, "Timeout")
_, _, success, _ = result
if success:
return result
except Exception as e:
result = (website, test_case, False, e)
return result
def _do_shot(self, player, board_to_shoot):
error = None
# Call get_shot and enforce the timeout
with stopit.ThreadingTimeout(MAX_SHOT_TIME, swallow_exc=False):
player_shot = player.get_shot()
# If this succeeded, ensure the player is returning a point
try:
assert isinstance(player_shot, Point)
# If not - fail
except AssertionError:
is_hit = None
is_sunk = False
ship_sunk = None
error = NotAPointError
return player_shot, is_hit, is_sunk, ship_sunk, error
# finally, if everything is ok, try and perform the shot
try:
is_hit, is_sunk, ship_sunk = board_to_shoot.shoot(player_shot)
except InvalidShotException:
is_hit = None
is_sunk = False
ship_sunk = None
error = InvalidShotException
return player_shot, is_hit, is_sunk, ship_sunk, error
def main(args, ws):
service = ws.webservices[args.service_name]
print(f'Testing {service}')
print('\n---------- LOGS ----------\n')
print(service.get_logs())
# NOQA: E501
x = [[
0, 1, 8, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0.5, 0.3,
0.610327781, 7, 1, -1, 0, -1, 1, 1, 1, 2, 1, 65, 1, 0.316227766,
0.669556409, 0.352136337, 3.464101615, 0.1, 0.8, 0.6, 1, 1, 6, 3, 6, 2,
9, 1, 1, 1, 12, 0, 1, 1, 0, 0, 1
],
[
4, 2, 5, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0.9, 0.5,
0.771362431, 4, 1, -1, 0, 0, 11, 1, 1, 0, 1, 103, 1, 0.316227766,
0.60632002, 0.358329457, 2.828427125, 0.4, 0.5, 0.4, 3, 3, 8, 4,
10, 2, 7, 2, 0, 3, 10, 0, 0, 1, 1, 0, 1
]]
print(f'x: {x}')
input_json = json.dumps({"data": x})
with stopit.ThreadingTimeout(10) as ctx:
assert ctx.state == ctx.EXECUTING
predictions = service.run(input_data=input_json)
print('\n---------- PREDICTION ----------\n')
print(predictions)
def timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, seconds=0, minutes=0, hours=0, sleep_between_cleanup_attempts=10
):
limit = seconds + 60 * minutes + 3600 * hours
with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
no_errors = False
try:
yield [t]
no_errors = True
finally:
attempts = 3
while attempts > 0:
attempts -= 1
try:
transformer.delete_model()
LOGGER.info("deleted SageMaker model {}".format(transformer.model_name))
_show_logs(transformer.model_name, "Models", sagemaker_session)
if no_errors:
_cleanup_logs(transformer.model_name, "Models", sagemaker_session)
break
except ClientError as ce:
if ce.response["Error"]["Code"] == "ValidationException":
pass
sleep(sleep_between_cleanup_attempts)
def produce(self,
*,
inputs: Inputs,
timeout: float = None,
iterations: int = None) -> CallResult[Outputs]:
if (timeout is None):
big_table = self._core(inputs)
self._has_finished = True
self._iterations_done = True
return CallResult(big_table, self._has_finished,
self._iterations_done)
else:
# setup the timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mrg:
assert to_ctx_mrg.state == to_ctx_mrg.EXECUTING
# core computations
big_table = self._core(inputs)
if to_ctx_mrg.state == to_ctx_mrg.EXECUTED:
self._has_finished = True
self._iterations_done = True
return CallResult(big_table, self._has_finished,
self._iterations_done)
elif to_ctx_mrg.state == to_ctx_mrg.TIMED_OUT:
self._has_finished = False
self._iterations_done = False
return CallResult(None, self._has_finished,
self._iterations_done)
def fit(self, *, timeout: float = None, iterations: int = None) -> None:
"""
train imputation parameters. Now support:
-> greedySearch
for the method that not trainable, do nothing:
-> interatively regression
-> other
Parameters:
----------
data: pandas dataframe
label: pandas series, used for the trainable methods
"""
# if already fitted on current dataset, do nothing
if self.is_fitted:
return True
if (timeout is None):
timeout = math.inf
if (iterations is None):
self._iterations_done = True
iterations = 30 # only works for iteratively_regre method
# setup the timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mrg:
assert to_ctx_mrg.state == to_ctx_mrg.EXECUTING
# start fitting...
data = self.train_x.copy()
if (self.train_y is not None):
label_col_name = "target_label" # name for label, assume no duplicate exists in data
data[label_col_name] = self.train_y
# start fitting
if (self.strategy == "greedy"):
if (self.train_y is None):
raise ValueError("label is nessary for greedy search")
print("=========> Greedy searched imputation:")
self.best_imputation = self.__imputationGreedySearch(
data, label_col_name)
elif (self.strategy == "iteratively_regre"):
print("=========> iteratively regress method:")
data_clean, self.best_imputation = self.__iterativeRegress(
data, iterations, label_col_name)
elif (self.strategy == "other"):
print("=========> other method:")
# no operation here because this method not needs to be trained
else:
raise ValueError("no such strategy: {}".format(self.strategy))
if to_ctx_mrg.state == to_ctx_mrg.EXECUTED:
self.is_fitted = True
self._has_finished = True
elif to_ctx_mrg.state == to_ctx_mrg.TIMED_OUT:
return
def timeout_and_delete_endpoint_by_name(
endpoint_name,
sagemaker_session,
seconds=0,
minutes=45,
hours=0,
sleep_between_cleanup_attempts=10,
):
limit = seconds + 60 * minutes + 3600 * hours
with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
no_errors = False
try:
yield [t]
no_errors = True
finally:
attempts = 3
while attempts > 0:
attempts -= 1
try:
sagemaker_session.delete_endpoint(endpoint_name)
LOGGER.info("deleted endpoint {}".format(endpoint_name))
_show_logs(endpoint_name, "Endpoints", sagemaker_session)
if no_errors:
_cleanup_logs(endpoint_name, "Endpoints",
sagemaker_session)
break
except ClientError as ce:
if ce.response["Error"]["Code"] == "ValidationException":
# avoids the inner exception to be overwritten
pass
# trying to delete the resource again in 10 seconds
sleep(sleep_between_cleanup_attempts)
def produce(self, inputs: Inputs, timeout=None, iterations=None) -> Outputs:
"""
Arguments:
- inputs: [ num_frames, num_feats ]
Returns:
- List([ num_frames, num_feats ], [ num_frames, num_feats], ...)
"""
with stopit.ThreadingTimeout(timeout) as timer:
outputs = Outputs()
for cinput in inputs:
if cinput.size == 0:
outputs.append(np.array([]))
continue
# dither
cinput2 = cinput + np.random.randn(*cinput.shape)*1e-9
norm = np.sqrt(np.power(cinput2, 2).sum(axis=1))
short_dists = np.sum(cinput2[:-self.short_dist]*cinput2[self.short_dist:],
axis=1)/(norm[:-self.short_dist]*norm[self.short_dist:])
long_dists = np.sum(cinput2[:-self.long_dist]*cinput2[self.long_dist:],
axis=1)/(norm[:-self.long_dist]*norm[self.long_dist:])
if timer.state == timer.EXECUTED:
return short_dists, long_dists
else:
raise TimeoutError('DiscontinuitySegmentation exceeded time limit')
def produce(self,
*,
inputs: Input,
timeout: float = None,
iterations: int = None) -> CallResult[Output]:
"""
precond: run fit() before
to complete the data, based on the learned parameters, support:
-> greedy search
also support the untrainable methods:
-> iteratively regression
-> other
Parameters:
----------
data: pandas dataframe
label: pandas series, used for the evaluation of imputation
TODO:
----------
1. add evaluation part for __simpleImpute()
"""
if (not self._is_fitted):
# todo: specify a NotFittedError, like in sklearn
raise ValueError("Calling produce before fitting.")
if (timeout is None):
timeout = math.inf
data = inputs.copy()
# record keys:
keys = data.keys()
index = data.index
# setup the timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mrg:
assert to_ctx_mrg.state == to_ctx_mrg.EXECUTING
# start completing data...
if self._verbose:
print("=========> impute using result from greedy search:")
data_clean = self.__simpleImpute(data, self._best_imputation)
value = None
if to_ctx_mrg.state == to_ctx_mrg.EXECUTED:
self._is_fitted = True
self._has_finished = True
self._iterations_done = True
value = pd.DataFrame(data_clean, index, keys)
elif to_ctx_mrg.state == to_ctx_mrg.TIMED_OUT:
print("Timed Out...")
self._is_fitted = False
self._has_finished = False
self._iterations_done = False
return CallResult(value, self._has_finished, self._iterations_done)
def produce(self,
*,
inputs: Inputs,
timeout: float = None,
iterations: int = None) -> CallResult[Outputs]:
with stopit.ThreadingTimeout(timeout) as timer:
sk_inputs = inputs
if self.hyperparams['use_semantic_types']:
sk_inputs = inputs.iloc[:, self._training_indices]
sk_output = self._model.predict(sk_inputs)
if sparse.issparse(sk_output):
sk_output = sk_output.toarray()
output = d3m_dataframe(
sk_output,
columns=self._target_names if self._target_names else None,
generate_metadata=False)
output.metadata = inputs.metadata.clear(for_value=output,
generate_metadata=True)
output.metadata = self._add_target_semantic_types(
metadata=output.metadata,
target_names=self._target_names,
source=self)
outputs = common_utils.combine_columns(
return_result=self.hyperparams['return_result'],
add_index_columns=self.hyperparams['add_index_columns'],
inputs=inputs,
column_indices=[],
columns_list=[output])
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('BBNMLPClassifier exceeded time limit')
def produce(self,
*,
inputs: Inputs,
timeout: float = None,
iterations: int = None) -> CallResult[Outputs]:
"""
Arguments:
- inputs: List[d3m_ndarray]
Returns:
- List[d3m_ndarray]
"""
with stopit.ThreadingTimeout(timeout) as timer:
outputs = Outputs()
metadata = inputs.metadata
for cinput in inputs:
if self.hyperparams['reseed']:
np.random.seed(self.random_seed)
coutput = cinput + self.hyperparams['level'] * (
np.random.rand(*cinput.shape) * 2 - 1)
outputs.append(d3m_ndarray(coutput, generate_metadata=False))
# Set metadata attribute.
outputs.metadata = metadata
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('SignalDither exceeded time limit')
def produce(self,
*,
inputs: Inputs,
timeout: float = None,
iterations: int = None) -> CallResult[Outputs]:
"""
Arguments:
- inputs: [ num_samples, num_channels ]
Returns:
- [ num_samples ]
"""
with stopit.ThreadingTimeout(timeout) as timer:
metadata_lookup = self.__class__._parse_metadata(
metadata=inputs.metadata)
if not metadata_lookup:
return None
outputs = Outputs()
metadata = self.__class__._can_accept(self=self,
method_name='produce',
arguments={
'inputs':
inputs.metadata,
},
hyperparams=self.hyperparams,
outputs=outputs)
csv_location_base_uris = inputs.metadata.query(
metadata_lookup['location_base_uris']
['selector'])['location_base_uris'][0]
for idx, row in inputs[metadata_lookup['primary_resource_id']
['selector'][0]].iterrows():
#for idx in range(len(inputs[metadata_lookup['primary_resource_id']['selector'][0]])):
#row = inputs[metadata_lookup['primary_resource_id']['selector'][0]][idx]
#d3mIndex = row[metadata_lookup['primary_key']['selector'][-1]]
d3mIndex = row['d3mIndex']
csv_fn = row[metadata_lookup['csv_fn']['selector'][-1]]
filename = os.path.join(csv_location_base_uris, csv_fn)
filename = re.sub('^file://', '', filename)
#csv_file= csv.load(filename)
csv_file = pd.read_csv(filename, index_col=0)
start = 0
end = len(csv_file)
outputs.append(csv_file)
metadata = metadata.update((idx, ), {'sampling_rate': 1})
metadata = metadata.update((),
{'dimension': {
'length': len(outputs)
}})
# Set metadata attribute.
outputs.metadata = metadata
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('Reader exceeded time limit')
def inner(*args, **kwargs):
result = None
with stopit.ThreadingTimeout(seconds) as to_ctx_mgr:
assert to_ctx_mgr.state == to_ctx_mgr.EXECUTING
result = sleep_30seconds()
if to_ctx_mgr.state == to_ctx_mgr.TIMED_OUT:
print("timed out")
return result
def process(self, **kwargs) -> None:
for x in self.iter_dependencies():
with (
stopit.ThreadingTimeout(self.timeout)
if self.timeout
else utils.NoOpContext()
):
yield from self.apply(x, **kwargs)
def produce(self, *, inputs: Inputs, timeout: float = None, iterations: int = None) -> CallResult[Outputs]:
"""
Arguments:
- inputs: [ num_frames, num_feats ]
Returns:
- List([ num_frames, num_feats ], [ num_frames, num_feats], ...)
"""
with stopit.ThreadingTimeout(timeout) as timer:
seg_len = self.hyperparams['seg_len']
seg_shift = self.hyperparams['seg_shift'] if self.hyperparams['seg_shift'] is not None \
else seg_len
outputs = Outputs()
metadata = inputs.metadata.clear({
'schema': metadata_module.CONTAINER_SCHEMA_VERSION,
'structural_type': Outputs,
'dimension': {
'length': len(outputs)
}
}, for_value=outputs).update((metadata_module.ALL_ELEMENTS,), {
'structural_type': List,
})
for input_id in range(len(inputs)):
cinput = inputs[input_id]
if cinput.size == 0:
outputs.append(d3m_ndarray(np.array([]), generate_metadata=False))
continue
sampling_rate = inputs.metadata.query((input_id,))['sampling_rate'] if 'sampling_rate' in inputs.metadata.query((input_id,)) else 1
frame_length = self._seg_length(seg_len, sampling_rate)
frame_shift = self._seg_shift(seg_shift, sampling_rate)
if cinput.shape[0] <= frame_length:
if len(cinput.shape) <= 2:
cinput = np.concatenate((cinput,
#np.matlib.repmat(cinput[-1], frame_length-cinput.shape[0], 1)
np.zeros((frame_length-cinput.shape[0],)+cinput.shape[1:], dtype=cinput.dtype)
))
shape = ((cinput.shape[0] - frame_length) // frame_shift + 1,
frame_length) + cinput.shape[1:]
strides = (cinput.strides[0]*frame_shift,cinput.strides[0]) + cinput.strides[1:]
coutput = np.lib.stride_tricks.as_strided(cinput, shape=shape, strides=strides)
segments = List()
for i in range(coutput.shape[0]):
segments.append(d3m_ndarray(coutput[i], generate_metadata=False))
outputs.append(segments)
# Set metadata attribute.
outputs.metadata = metadata
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('UniformSegmentation exceeded time limit')
def produce(self, *, inputs: Inputs, timeout: float = None, iterations: int = None) -> CallResult[Outputs]:
"""
Arguments:
- inputs: [ num_samples ]
Returns:
- [ num_windows, window_len ]
"""
with stopit.ThreadingTimeout(timeout) as timer:
outputs = Outputs()
metadata = inputs.metadata.clear({
'schema': metadata_module.CONTAINER_SCHEMA_VERSION,
'structural_type': Outputs,
'dimension': {
'length': len(outputs)
}
}, for_value=outputs).update((metadata_module.ALL_ELEMENTS,), {
'structural_type': d3m_ndarray,
})
for input_id in range(len(inputs)):
cinput = inputs[input_id]
# TODO: review the following because it's hacky
# It was done in the way to enable handling both audio (high sampling_rate) and frames
sampling_rate = inputs.metadata.query((input_id,))['sampling_rate'] if 'sampling_rate' in inputs.metadata.query((input_id,)) else 1
frame_length = self._frame_length(sampling_rate)
frame_shift = self._frame_shift(sampling_rate)
if cinput.size == 0:
outputs.append(d3m_ndarray(np.array([]), generate_metadata=False))
continue
if cinput.shape[0] <= frame_length:
if len(cinput.shape) <= 2:
cinput = np.concatenate((cinput,
#np.matlib.repmat(cinput[-1], frame_length-cinput.shape[0], 1)
np.zeros((frame_length-cinput.shape[0],)+cinput.shape[1:], dtype=cinput.dtype)
))
shape = ((cinput.shape[0] - frame_length) // frame_shift + 1,
frame_length) + cinput.shape[1:]
strides = (cinput.strides[0]*frame_shift,cinput.strides[0]) + cinput.strides[1:]
coutput = np.lib.stride_tricks.as_strided(cinput, shape=shape, strides=strides)
outputs.append(d3m_ndarray(
coutput.flatten() if self.hyperparams['flatten_output'] else coutput,
generate_metadata=False))
if 'sampling_rate' in inputs.metadata.query((input_id,)):
metadata = metadata.update((input_id,), { 'sampling_rate': inputs.metadata.query((input_id,))['sampling_rate'] })
#metadata = metadata.update((), { 'dimension': { 'length': len(outputs) } })
# Set metadata attribute.
outputs.metadata = metadata
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('SignalFramer exceeded time limit')
def produce(self,
*,
inputs: Inputs,
timeout: float = None,
iterations: int = None) -> CallResult[Outputs]:
with stopit.ThreadingTimeout(timeout) as timer:
x = self._tfidf.transform(inputs).toarray()
outputs = d3m_dataframe(x, generate_metadata=False)
metadata = inputs.metadata.clear(
{
'schema':
metadata_module.CONTAINER_SCHEMA_VERSION,
'structural_type':
type(outputs),
'semantic_types':
['https://metadata.datadrivendiscovery.org/types/Table'],
'dimension': {
'length':
outputs.shape[0],
'name':
'rows',
'semantic_types': [
'https://metadata.datadrivendiscovery.org/types/TabularRow'
]
}
},
for_value=outputs
).update(
((metadata_base.ALL_ELEMENTS, )), {
'dimension': {
'length':
outputs.shape[1],
'name':
'columns',
'semantic_types': [
'https://metadata.datadrivendiscovery.org/types/TabularColumn'
]
}
}
).update(
((metadata_base.ALL_ELEMENTS, metadata_base.ALL_ELEMENTS)),
{
#'structural_type': self._v.dtype,
'semantic_types': [
'https://metadata.datadrivendiscovery.org/types/Attribute'
],
})
# Set metadata attribute.
outputs.metadata = metadata
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('BBNTfidfTransformer exceeded time limit')
def fit(self, timeout=None, iterations=None):
# type: (float, int) -> None
"""
Fit the random forest distance to a set of labeled data by sampling and fitting
to pairwise constraints.
"""
# state/input checking
if self.fitted:
return
if not hasattr(self, 'X') or not hasattr(self, 'y'):
raise ValueError("Missing training data.")
if (not (isinstance(self.X, np.ndarray)
and isinstance(self.y, np.ndarray))):
raise TypeError(
'Training inputs and outputs must be numpy arrays.')
# store state in case of timeout
if hasattr(self, 'd'):
dtemp = self.d
else:
dtemp = None
rftemp = copy.deepcopy(self.rf)
# do fitting with timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mgr:
n = self.X.shape[0]
self.d = self.X.shape[1]
assert n > 0
assert self.d > 0
assert n == self.y.shape[0]
constraints = get_random_constraints(self.y,
int(self.class_cons / 3),
2 * int(self.class_cons / 3))
c1 = self.X[constraints[:, 0], :]
c2 = self.X[constraints[:, 1], :]
rfdfeat = np.empty(dtype=np.float32,
shape=(constraints.shape[0], self.d * 2))
rfdfeat[:, :self.d] = np.abs(c1 - c2)
rfdfeat[:, self.d:] = (c1 + c2) / 2
self.rf.fit(rfdfeat, constraints[:, 2])
self.fitted = True
# if we completed on time, return. Otherwise reset state and raise error.
if to_ctx_mgr.state == to_ctx_mgr.EXECUTED:
return
else:
self.d = dtemp
self.rf = rftemp
self.fitted = False
raise TimeoutError("RFD fitting timed out.")
def produce(self,
*,
inputs: Inputs,
timeout: float = None,
iterations: int = None) -> CallResult[Outputs]:
"""
Arguments:
- inputs: [ num_samples, num_channels ]
Returns:
- [ num_samples ]
"""
with stopit.ThreadingTimeout(timeout) as timer:
outputs = Outputs()
metadata = inputs.metadata.clear(
{
'schema': metadata_module.CONTAINER_SCHEMA_VERSION,
'structural_type': Outputs,
'dimension': {
'length': len(outputs)
}
},
for_value=outputs).update((metadata_module.ALL_ELEMENTS, ), {
'structural_type': d3m_ndarray,
})
for idx in range(len(inputs)):
cinput = inputs[idx]
if cinput.ndim > 2:
raise ValueError('Incompatible shape ' +
str(cinput.shape) + ' of cinput.')
elif cinput.ndim == 1:
coutput = cinput.copy()
else:
coutput = cinput.mean(axis=1)
outputs.append(d3m_ndarray(coutput))
if 'sampling_rate' in inputs.metadata.query((idx, )):
metadata = metadata.update(
(idx, ), {
'sampling_rate':
inputs.metadata.query((idx, ))['sampling_rate']
})
metadata = metadata.update((),
{'dimension': {
'length': len(outputs)
}})
# Set metadata attribute.
outputs.metadata = metadata
if timer.state == timer.EXECUTED:
return CallResult(outputs)
else:
raise TimeoutError('ChannelAverager exceeded time limit')
def produce(self,
*,
inputs: Input,
timeout: float = None,
iterations: int = None) -> CallResult[Output]:
"""
precond: run fit() before
to complete the data, based on the learned parameters, support:
-> greedy search
also support the untrainable methods:
-> iteratively regression
-> other
Parameters:
----------
data: pandas dataframe
label: pandas series, used for the evaluation of imputation
TODO:
----------
1. add evaluation part for __simpleImpute()
"""
if (timeout is None):
timeout = math.inf
if (iterations is None):
iterations = 100 # default value for mice
if isinstance(inputs, pd.DataFrame):
data = inputs.copy()
else:
data = inputs[0].copy()
# record keys:
keys = data.keys()
index = data.index
# setup the timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mrg:
assert to_ctx_mrg.state == to_ctx_mrg.EXECUTING
# start completing data...
if self._verbose: print("=========> impute by fancyimpute-mice:")
data_clean = self.__mice(data, iterations)
value = None
if to_ctx_mrg.state == to_ctx_mrg.EXECUTED:
self._has_finished = True
self._iterations_done = True
value = pd.DataFrame(data_clean, index, keys)
elif to_ctx_mrg.state == to_ctx_mrg.TIMED_OUT:
self._has_finished = False
self._iterations_done = False
return CallResult(value, self._has_finished, self._iterations_done)
def __enter__(self):
if self.n is None:
# timeout disabled
return None
if sys.platform.startswith("win"):
# Windows does not support signal-based timeout
self._obj = stopit.ThreadingTimeout(self.n)
else:
self._obj = stopit.SignalTimeout(self.n)
return self._obj.__enter__()
def run_with_timeout_in_thread(seconds, fun, *args, **kwargs):
if seconds > 0:
try:
with stopit.ThreadingTimeout(seconds, swallow_exc=False):
result = fun(*args, **kwargs)
return result
except stopit.TimeoutException:
# to match behavior of old run with timeout
raise TimeoutError
else:
return fun(*args, **kwargs)
def fit(self,
*,
timeout: float = None,
iterations: int = None) -> CallResult[None]:
"""
train imputation parameters. Now support:
-> greedySearch
for the method that not trainable, do nothing:
-> interatively regression
-> other
Parameters:
----------
data: pandas dataframe
label: pandas series, used for the trainable methods
"""
# if already fitted on current dataset, do nothing
if self._is_fitted:
return CallResult(None, self._has_finished, self._iterations_done)
if (timeout is None):
timeout = 2**31 - 1
# setup the timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mrg:
assert to_ctx_mrg.state == to_ctx_mrg.EXECUTING
if isinstance(self._train_x, pd.DataFrame):
data = self._train_x.copy()
label = self._train_y.copy()
else:
data = self._train_x[0].copy()
label = self._train_y[0].copy()
# start fitting...
# 1. to figure out what kind of problem it is and assign model and scorer
# now only support "classification" or "regresion" problem
self._set_model_scorer()
# 2. using the model and scorer to do greedy search
if self._verbose:
print("=========> Greedy searched imputation:")
self._best_imputation = self.__imputationGreedySearch(data, label)
if to_ctx_mrg.state == to_ctx_mrg.EXECUTED:
self._is_fitted = True
self._has_finished = True
self._iterations_done = True
elif to_ctx_mrg.state == to_ctx_mrg.TIMED_OUT:
print("Timed Out...")
self._is_fitted = False
self._has_finished = False
self._iterations_done = False
return CallResult(None, self._has_finished, self._iterations_done)
def produce(self, inputs, timeout=None, iterations=None):
# type: (Inputs, float, int) -> Outputs
with stopit.ThreadingTimeout(timeout) as to_ctx_mgr:
C = self.compute_sparse_coefficient_matrix(inputs)
W = self.build_adjacency_matrix(C)
labels = self.spectral_clustering(W, n_clusters = self._k)
labels = np.array(labels)
if to_ctx_mgr.state == to_ctx_mgr.EXECUTED:
return labels
else:
raise TimeoutError("SSC CVX fitting has timed out.")
