def test_bisect():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert temp.bisect_left('a') == 0
assert temp.bisect_right('f') == 6
assert temp.bisect('b') == 1
def init_schema_fixture(request):
return SortedDict(request.param)
def __init__(self,
base_path,
datatype: str = 'labeled',
inzip=True,
phase="training",
trainval_split=1,
trainval_random=False,
trainval_byseq=False,
nframes=0):
super().__init__(base_path,
inzip=inzip,
phase=phase,
nframes=nframes,
trainval_split=trainval_split,
trainval_random=trainval_random,
trainval_byseq=trainval_byseq)
self.datatype = datatype
if phase == "testing":
raise ValueError("There's no testing split for CADC dataset!")
if datatype != "labeled":
raise NotImplementedError(
"Currently only labeled data are supported!")
# count total number of frames
frame_count = dict()
_dates = ["2018_03_06", "2018_03_07", "2019_02_27"]
if self.inzip:
globs = [
self.base_path.glob(f"{date}/00*/{datatype}.zip")
for date in _dates
]
for archive in chain(*globs):
with ZipFile(archive) as data:
velo_files = (name for name in data.namelist()
if name.endswith(".bin"))
seq = archive.stem
frame_count[seq] = sum(1 for _ in velo_files)
else:
for date in _dates:
if not (self.base_path / date).exists():
continue
for archive in (self.base_path / date).iterdir():
if not archive.is_dir(): # skip calibration files
continue
seq = archive.name
frame_count[seq] = sum(
1 for _ in (archive / "velodyne_points" /
"data").iterdir())
if not len(frame_count):
raise ValueError(
"Cannot parse dataset or empty dataset, please check path, inzip option and file structure"
)
self.frame_dict = SortedDict(frame_count)
self.frames = split_trainval_seq(phase, self.frame_dict,
trainval_split, trainval_random,
trainval_byseq)
self._label_cache = {} # used to store parsed label data
self._calib_cache = {} # used to store parsed calibration data
self._timestamp_cache = {} # used to store parsed timestamp
self._pose_cache = {} # used to store parsed pose data
self._tracklet_cache = {} # used to store parsed tracklet data
self._tracklet_mapping = {} # inverse mapping for tracklets
def rc_chain_groups(self):
r = len(self.I)
# L = FastRBTree()
L_dict = SortedDict()
V = [0] * len(self.groups)
back_track = [-1] * len(self.groups)
self.I.sort()
# print self.I
## print self.groups
for i in range(r):
# I_list[i] is a start point, noticed we go through the chain from botton to top
if self.I[i][2] == 0:
k = self.I[i][1]
l_k = self.groups[k][0][1] - self.groups[k][0][2]
start_y = l_k
start_x = self.I[i][0] - self.groups[k][0][2]
end_x = self.groups[k][-1][0]
end_y = self.groups[k][-1][1]
diagonal = start_y + start_x
# find largest h_j strictly smaller than l_k and also not off diagonal
j_index, j_key, j_value = find_not_smaller_key(L_dict, l_k + 1)
# j_key = L.floor_key(l_k - 1)
if j_index != None:
v_j = sum([x[2] for x in self.groups[k]])
j = j_value[1]
prev_score = V[j]
else:
v_j = sum([x[2] for x in self.groups[k]])
j = -1
prev_score = 0
V[k] = prev_score + v_j
back_track[k] = j
# is a end point
else:
# print L
k = self.I[i][1]
h_k = self.groups[k][-1][1]
j_index, j_key, j_value = find_not_bigger_key(L_dict, h_k)
# j_key = L.ceiling_key(h_k)
if j_index != None:
j = j_value[1]
V_j = j_value[0]
if V[k] > V_j:
L_dict[h_k] = (V[k], k)
# L.insert(h_k, (V[k], k))
# L.insert(h_k, (V[k], k))
else:
# print len(L)
if len(L_dict) == 0 or L_dict.peekitem(0)[1][0] < V[k]:
L_dict[h_k] = (V[k], k)
"""
if len(L) == 0 or L.max_item()[1][0] < V[k]:
L[h_k] = (V[k], k)
"""
# L.insert(h_k, (V[k], k))
j1_index, j1_key, j1_value = find_not_bigger_key(L_dict, h_k)
# maybe mofify here'
# print "Vk", V[k]
# print L
if j1_index != None:
j1_index -= 1
while j1_index >= 0:
prev_j1_key = j1_key
prev_j1_value = j1_value
try:
# print j1_index
j1_key = L_dict.iloc[j1_index]
j1_value = L_dict[j1_key]
if V[k] > j1_value[0]:
del L_dict[j1_key]
j1_index -= 1
except KeyError:
# print prev_j1_item
if V[k] > prev_j1_value[0]:
del L_dict[prev_j1_key]
break
# print "DP finished"
try:
# print L
max_item = L_dict.peekitem(index=0)
max_value = max_item[1]
score = max_value[0]
current_j = max_value[1]
# backtrack
chain_index = []
chain_index.append(current_j)
while True:
prev_j = back_track[current_j]
if prev_j == -1:
break
else:
current_j = prev_j
chain_index.append(current_j)
optimal_chain = []
length = 0
for i in chain_index[::-1]:
optimal_chain.extend(self.groups[i])
length += sum([x[2] for x in self.groups[i]])
except ValueError:
optimal_chain = None
length = 0
return optimal_chain, length
#!/usr/bin/env python
from sortedcontainers import SortedDict
d = SortedDict()
d["b"] = "c"
d["c"] = "c"
d["y"] = "c"
d["a"] = "c"
for k in d.keys():
print(k)
def Predicted_ranking(self, session, after=False):
'''
Returns a tuple of players or teams that represents the preducted ranking given their skills
before (or after) the nominated session and a probability of that ranking in a 2-tuple.
Each list item can be:
A player (instance of the Leaderboard app's Player model)
A team (instance of the Leaderboard app's Team model)
A list of either players or teams - if a tie is predicted at that rank
:param session: an instance of the leaderboards model Rank
:param after: if true, gets and uses skills "after" rather than "before" the present ranks (recorded play session).
'''
# For predicted rankings we assume a partial play weighting of 1 (full participation)
# The recorded partial play waighting has no impact on what ranking we would predict for
# these performers, before or after the sessions skill updates.
def P(player, after):
p = session.performance(player)
return Performance(
p.trueskill_mu_after if after else p.trueskill_mu_before,
p.trueskill_sigma_after**2
if after else p.trueskill_sigma_before**2, 1)
# One dict to hold the performers and another to hold the performances
performers = SortedDict(
) # Keyed and sorted on trueskill_mu of the performer
performances = SortedDict(
) # Keyed and sorted on trueskill_mu of the performer
if session.team_play:
for team in session.teams:
p = self.team_performance(
[P(player, after) for player in team.players.all()])
k = -p.mu
if not k in performers:
performers[k] = []
performers[k].append(team)
if not p.mu in performances:
performances[k] = []
performances[k].append(p)
else:
for player in session.players:
p = P(player, after)
k = -p.mu
if not k in performers:
performers[k] = []
performers[k].append(player)
if not k in performances:
performances[k] = []
performances[k].append(p)
# Freeze and flatten
for k, tied_performers in performers.items():
if len(tied_performers) == 1:
performers[k] = tied_performers[0] # Remove the list
else:
performers[k] = tuple(tied_performers) # Freeze the list
for k, tied_performances in performances.items():
if len(tied_performances) == 1:
performances[k] = tied_performances[0] # Remove the list
else:
performances[k] = tuple(tied_performances) # Freeze the list
if settings.DEBUG:
log.debug(f"\nPredicted_ranking:")
for k in performers:
log.debug(f"\t{performers[k]}: {performances[k]}")
# Get the probability of this ranking
prob = self.P_ranking_performers(tuple(performances.values()))
# Return the ordered tuple
return (tuple(performers.values()), prob)
def init(self):
self.sd_bids = SortedDict()
self.sd_asks = SortedDict()
def test_getitem():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert all(
(temp[val] == pos) for pos, val in enumerate(string.ascii_lowercase))
def test_iter():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert all(lhs == rhs for lhs, rhs in zip(temp, string.ascii_lowercase))
def test_contains():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert all((val in temp) for val in string.ascii_lowercase)
def test_delitem():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
del temp['a']
temp._check()
def test_init_kwargs():
temp = SortedDict(a=1, b=2)
assert len(temp) == 2
assert temp['a'] == 1
assert temp['b'] == 2
temp._check()
def test_init_args():
temp = SortedDict([('a', 1), ('b', 2)])
assert len(temp) == 2
assert temp['a'] == 1
assert temp['b'] == 2
temp._check()
def test_init():
temp = SortedDict()
temp._check()
def count_event(self, event_start, event_end):
"""
Increment the counter for the given interval of time.
:event_start: A python `datetime`, pandas `Timestamp`, or Unix timestamp marking the beginning of the event interval.
:event_end: A python `datetime`, pandas `Timestamp`, or Unix timestamp for the end of the event interval, or `None` for
and event with an open interval.
Performs a right-bisection on the counter intervals, assigning counts to increasingly
finer slices based on the the event's timespan's intersection with the existing counter intervals.
"""
event_start = self.__ts2int(event_start)
event_end = None if (event_end is None or event_end is pandas.NaT
) else self.__ts2int(event_end)
to_remove = SortedSet()
to_add = SortedDict()
__counter = self.counter
__splits = self.splits
# get the next insertion index
index = self.__insertidx(start=event_start, end=event_end)
# move the index to the right, splitting the existing intervals and incrementing counts along the way
while index < len(self.counts) and (
event_end is None or self.__interval(index).start < event_end):
interval = self.__interval(index)
count = self.counts[interval]
start, end = interval.start, interval.end
# the event has a closed timespan: [event_start, event_end]
if event_end is not None:
# event fully spans and contains the interval
if event_start <= start and event_end >= end:
# increment the interval
to_add[CounterInterval(start, end)] = count + 1
self.counter += 1
# event starts before the interval and overlaps from the left
elif event_start <= start and event_end > start and event_end < end:
# subdivide and increment the affected sub-interval
# [start, end] -> [start, event_end]+, [event_end, end]
to_remove.add(interval)
to_add[CounterInterval(start, event_end)] = count + 1
to_add[CounterInterval(event_end, end)] = count
self.splits += 1
self.counter += 1
# event starts in the interval and overlaps on the right
elif event_start > start and event_start < end and event_end >= end:
# subdivide and increment the affected interval
# [start, end] -> [start, event_start], [event_start, end]+
to_remove.add(interval)
to_add[CounterInterval(start, event_start)] = count
to_add[CounterInterval(event_start, end)] = count + 1
self.splits += 1
self.counter += 1
# event is fully within and contained by the interval
elif event_start > start and event_end < end:
# subdivide and increment the affected interval
# [start, end] -> [start, event_start], [event_start, event_end]+, [event_end, end]
to_remove.add(interval)
to_add[CounterInterval(start, event_start)] = count
to_add[CounterInterval(event_start, event_end)] = count + 1
to_add[CounterInterval(event_end, end)] = count
self.splits += 2
self.counter += 1
# the event has an open timespan: [event_start, )
else:
# event starts before the interval
if event_start <= start:
# incrememnt the interval
to_add[CounterInterval(start, end)] = count + 1
self.counter += 1
# event starts inside the interval
elif event_start > start and event_start <= end:
# subdivide and increment the affected interval
# [start, end] -> [start, event_start], [event_start, end]+
to_remove.add(interval)
to_add[CounterInterval(start, event_start)] = count
to_add[CounterInterval(event_start, end)] = count + 1
self.splits += 1
self.counter += 1
index += 1
for r in to_remove:
self.counts.pop(r)
for k in to_add.keys():
self.counts[k] = to_add[k]
if self.debug:
debug = {
"start": event_start,
"end": event_end,
"index": index,
"remove": len(to_remove),
"split": int(self.splits - __splits),
"add": len(to_add),
"counter": int(self.counter - __counter)
}
print(", ".join([f"{k}: {v}" for k, v in debug.items()]))
self.events += 1
return self
def __init__(self):
self.timestampToPrice = SortedDict()
self.pricesCount = SortedDict()
from nitpick.files.pre_commit import PreCommitFile
from nitpick.files.pyproject_toml import PyProjectTomlFile
from nitpick.files.setup_cfg import SetupCfgFile
from nitpick.generic import get_subclasses
style_mapping = SortedDict({
"black.toml": "black_",
"flake8.toml": "flake8_",
"isort.toml": "isort_",
"mypy.toml": "mypy_",
"absent-files.toml": "Absent files",
"ipython.toml": "IPython_",
"package-json.toml": "package.json_",
"poetry.toml": "Poetry_",
"pre-commit/bash.toml": "Bash_",
"pre-commit/commitlint.toml": "commitlint_",
"pre-commit/general.toml": "pre-commit_ (hooks)",
"pre-commit/main.toml": "pre-commit_ (main)",
"pre-commit/python.toml": "pre-commit_ (Python hooks)",
"pylint.toml": "Pylint_",
"pytest.toml": "pytest_",
"python35-36-37.toml": "Python 3.5, 3.6 or 3.7",
"python35-36-37-38.toml": "Python 3.5, 3.6, 3.7 to 3.8",
"python36-37.toml": "Python 3.6 or 3.7",
"python36.toml": "Python 3.6",
"python37.toml": "Python 3.7",
})
file_classes = [PyProjectTomlFile, SetupCfgFile, PreCommitFile, JSONFile]
divider = ".. auto-generated-from-here"
docs_dir = Path(__file__).parent.absolute() # type: Path
styles_dir = docs_dir.parent / "styles" # type: Path
def transform_teragon_csv(teragon_csv, transpose=False, indexed=False):
"""transform Teragon's CSV response into a python dictionary,
which mirrors the JSON response we want to provide to API clients
Arguments:
teragon_csv {reference} -- reference to a CSV table on disk
or in memory
transpose {boolean} -- transpose Teragon table
indexed {boolean} -- return dictionary in indexed format or as records
Returns:
{dict} -- a dictionary representing the Terragon table, transformed
for ease of use in spatial/temporal data vizualation
"""
petl_table = etl.fromcsv(teragon_csv)
# print(petl_table)
# get iterable of column pairs (minus 'Timestamp')
# this is used to group the double columns representing a single
# data point in Teragon's CSV
h = list(etl.header(petl_table))
xy_cols = zip(*[iter(h[1:])] * 2)
# make a new header row
new_header = ['Timestamp']
fields_to_cut = []
for each in xy_cols:
# print(each)
# correct id, assembled from columns
id_col, note_col = each[0], each[1]
# assemble new id column, to replace of PX column (which has data)
# id_col = "{0}{1}".format(px[:3], px[4:])
# assemble new notes column, to replace of PY column (which has notes)
notes_col = "{0}-n".format(id_col)
# add those to our new header (array)
new_header.extend([id_col, notes_col])
# track fields that we might want to remove
fields_to_cut.append(notes_col)
# transform the table
table = etl \
.setheader(petl_table, new_header) \
.cutout(*tuple(fields_to_cut)) \
.select('Timestamp', lambda v: v.upper() != 'TOTAL') \
.convert('Timestamp', lambda t: parse(t).isoformat()) \
.replaceall('N/D', None)
# transpose the table, so that rows are cells/gauges and columns are times
# (note that this operation can take a while)
if transpose:
table = etl.transpose(table)
# if indexed: format data where cells/gauges or times are keys, and
# rainfall amounts are values
# otherwise, format as nested records (arrays of dicts)
if indexed:
data = SortedDict()
for row in etl.dicts(table):
inside = SortedDict()
for d in row.items():
if d[0] != 'Timestamp':
if d[1]:
v = float(d[1])
else:
v = d[1]
inside[d[0]] = v
data[row['Timestamp']] = inside
return data
else:
rows = []
# create a nested dictionary from the table
for row in etl.dicts(table):
data = []
for d in row.items():
if d[0] != 'Timestamp':
if d[1]:
v = float(d[1])
else:
v = d[1]
data.append({'id': d[0], 'v': v})
rows.append({"id": row['Timestamp'], "d": data})
# print(rows)
# print(json.dumps(rows, indent=2))
return rows
def _loads(self, contents: List[Dict[str, Any]]) -> None:
self._data = SortedDict()
for sensor_info in contents:
self.add(Sensor.loads(sensor_info))
def get_bin(max_bins):
"""
Returns a bin object with given max_bins
"""
return SortedDict({'maxbins': max_bins})
def validate(self,
protocol: str,
subset: str = 'development',
every: int = 1,
start: Union[int, Literal['last']] = 1,
end: Union[int, Literal['last']] = 100,
chronological: bool = False,
device: Optional[torch.device] = None,
batch_size: int = 32,
n_jobs: int = 1,
**kwargs):
# use last available epoch as starting point
if start == 'last':
start = self.get_number_of_epochs() - 1
# use last available epoch as end point
if end == 'last':
end = self.get_number_of_epochs() - 1
criterion = self.validation_criterion(protocol, **kwargs)
validate_dir = Path(
self.VALIDATE_DIR.format(
train_dir=self.train_dir_,
_criterion=f'_{criterion}' if criterion is not None else '',
protocol=protocol,
subset=subset))
params_yml = validate_dir / 'params.yml'
validate_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(log_dir=str(validate_dir), purge_step=start)
self.validate_dir_ = validate_dir
validation_data = self.validate_init(protocol, subset=subset)
if n_jobs > 1:
self.pool_ = multiprocessing.Pool(n_jobs)
progress_bar = tqdm(unit='iteration')
for i, epoch in enumerate(
self.validate_iter(start=start,
end=end,
step=every,
chronological=chronological)):
# {'metric': 'detection_error_rate',
# 'minimize': True,
# 'value': 0.9,
# 'pipeline': ...}
details = self.validate_epoch(epoch,
validation_data,
protocol=protocol,
subset=subset,
device=device,
batch_size=batch_size,
n_jobs=n_jobs,
**kwargs)
# initialize
if i == 0:
# what is the name of the metric?
metric = details['metric']
# should the metric be minimized?
minimize = details['minimize']
# epoch -> value dictionary
values = SortedDict()
# load best epoch and value from past executions
if params_yml.exists():
with open(params_yml, 'r') as fp:
params = yaml.load(fp, Loader=yaml.SafeLoader)
best_epoch = params['epoch']
best_value = params[metric]
values[best_epoch] = best_value
# metric value for current epoch
values[epoch] = details['value']
# send value to tensorboard
writer.add_scalar(f'validate/{protocol}.{subset}/{metric}',
values[epoch],
global_step=epoch)
# keep track of best value so far
if minimize:
best_epoch = values.iloc[np.argmin(values.values())]
best_value = values[best_epoch]
else:
best_epoch = values.iloc[np.argmax(values.values())]
best_value = values[best_epoch]
# if current epoch leads to the best metric so far
# store both epoch number and best pipeline parameter to disk
if best_epoch == epoch:
best = {
metric: best_value,
'epoch': epoch,
}
if 'pipeline' in details:
pipeline = details['pipeline']
best['params'] = pipeline.parameters(instantiated=True)
with open(params_yml, mode='w') as fp:
fp.write(yaml.dump(best, default_flow_style=False))
# create/update zip file for later upload to torch.hub
hub_zip = create_zip(validate_dir)
# progress bar
desc = (f'{metric} | '
f'Epoch #{best_epoch} = {100 * best_value:g}% (best) | '
f'Epoch #{epoch} = {100 * details["value"]:g}%')
progress_bar.set_description(desc=desc)
progress_bar.update(1)
def printSets(thisType, values, tabs, firstTab=""):
if not values:
return
if firstTab:
print(firstTab, thisType)
else:
print(thisType)
for x in values:
print(tabs, x)
if __name__ == "__main__":
folders = SortedDict()
documents = SortedSet()
config = SortedSet()
for x in os.listdir():
if x[0].isalnum() and os.path.isdir(os.getcwd() + f"/{x}"):
folders[x] = {
"Config Files": SortedSet(),
"Documents": SortedSet(),
"Folders": SortedSet()
}
newPath = os.getcwd() + f"/{x}"
for y in os.listdir(newPath):
def __init__(self):
self._epoch = datetime.now()
self._timers_by_expire_time = SortedDict()
def __init__(self):
self.calendar = SortedDict()
def __init__(self, capacity: int):
self.capacity = capacity
self.stkno = 0
self.empty, self.full, self.half = SortedDict(), SortedDict(), SortedDict()
def crop(self, support, mode='intersection'):
"""Crop annotation to new support
Parameters
----------
support : Segment or Timeline
If `support` is a `Timeline`, its support is used.
mode : {'strict', 'loose', 'intersection'}, optional
Controls how segments that are not fully included in `support` are
handled. 'strict' mode only keeps fully included segments. 'loose'
mode keeps any intersecting segment. 'intersection' mode keeps any
intersecting segment but replace them by their actual intersection.
Returns
-------
cropped : Annotation
Cropped annotation
Note
----
In 'intersection' mode, the best is done to keep the track names
unchanged. However, in some cases where two original segments are
cropped into the same resulting segments, conflicting track names are
modified to make sure no track is lost.
"""
# TODO speed things up by working directly with annotation internals
if isinstance(support, Segment):
support = Timeline(segments=[support], uri=self.uri)
return self.crop(support, mode=mode)
elif isinstance(support, Timeline):
cropped = self.__class__(uri=self.uri, modality=self.modality)
if mode == 'loose':
_tracks = {}
_labels = set([])
for segment, _ in \
self.get_timeline(copy=False).co_iter(support):
tracks = dict(self._tracks[segment])
_tracks[segment] = tracks
_labels.update(tracks.values())
cropped._tracks = SortedDict(_tracks)
cropped._labelNeedsUpdate = {label: True for label in _labels}
cropped._labels = {label: None for label in _labels}
cropped._timelineNeedsUpdate = True
cropped._timeline = None
return cropped
elif mode == 'strict':
_tracks = {}
_labels = set([])
for segment, other_segment in \
self.get_timeline(copy=False).co_iter(support):
if segment not in other_segment:
continue
tracks = dict(self._tracks[segment])
_tracks[segment] = tracks
_labels.update(tracks.values())
cropped._tracks = SortedDict(_tracks)
cropped._labelNeedsUpdate = {label: True for label in _labels}
cropped._labels = {label: None for label in _labels}
cropped._timelineNeedsUpdate = True
cropped._timeline = None
return cropped
elif mode == 'intersection':
for segment, other_segment in \
self.get_timeline(copy=False).co_iter(support):
intersection = segment & other_segment
for track, label in six.iteritems(self._tracks[segment]):
track = cropped.new_track(intersection,
candidate=track)
cropped[intersection, track] = label
return cropped
else:
raise NotImplementedError("unsupported mode: '%s'" % mode)
def start_generation(sconf):
""" This function controls the reading of rules and the actual
generation of traffic.
"""
global TOTAL_GENERATED_STREAMS
global TOTAL_GENERATED_PACKETS
global FINAL
myrulelist = RuleList()
if sconf.getRuleFile() and sconf.getRuleDir():
print("You must specify either a single rule file, "
"or a directory containing multiple rule files, not both.")
sconf.usage()
elif sconf.getRuleFile():
myrulelist.readRuleFile(sconf.getRuleFile())
elif sconf.getRuleDir():
myrulelist.readRuleFiles(sconf.getRuleDir())
else:
print("Random Content and Random headers")
sconf.setRandom(True)
if sconf.getIPV4Home() is not None:
set_ipv4_home(sconf.getIPV4Home())
if sconf.getIPV6Home() is not None:
set_ipv6_home(sconf.getIPV6Home())
allrules = myrulelist.getParsedRules()
# Retrieve Background Traffic percentage
back_traffic_percent = sconf.getBackgroundTraffic()
back_dist_list = None
back_absent_proto = None
# Get Background Traffic Rule if given
if myrulelist.getBackgroundTraffic():
bt_rule = myrulelist.getBackgroundTraffic()
back_traffic_percent = bt_rule.getBackgroundPercent()
back_dist_list = bt_rule.getProbabilityDist()
back_absent_proto = bt_rule.getAbsentProtocol()
current = 0
end = 0
current_sec = sconf.getFirstTimestamp()
current_usec = 0
total_generated_streams = 0
total_generated_packets = 0
flow_start_offset = 0
mix_count = sconf.getMixCount()
traffic_queue = SortedDict()
if sconf.getWriteRegEx():
return printRegEx(allrules)
# If we define a scan attack from the command line, add it to the traff
# here.
if sconf.getScan():
base_offset = 0
for t in sconf.getScanTargets():
if sconf.getRandomizeOffset():
base_offset += int(
random.normalvariate(sconf.getScanOffset(),
sconf.getScanOffset() / 4))
else:
base_offset += int(sconf.getScanOffset())
rule = Rule("Scan Attack")
r_ts = ScanAttackRule(sconf.getScanType(), t,
sconf.getTargetPorts(), None,
sconf.getScanDuration(),
sconf.getIntensity(), base_offset,
sconf.getScanReplyChance())
rule.addTS(r_ts)
conversation = Conversation(rule, sconf, current_sec)
sec, usec = conversation.getNextTimeStamp()
timekey = sec + (usec / 1000000)
if timekey in traffic_queue:
traffic_queue[timekey].append(conversation)
else:
traffic_queue[timekey] = [conversation]
total_generated_streams += conversation.getNumberOfStreams()
traffic_writer = TrafficWriter(sconf.getOutputFile(),
sconf.getFirstTimestamp())
if sconf.getEval() or sconf.getFullEval():
return build_eval_pcap(allrules, traffic_writer, sconf)
if sconf.getTrafficDuration() > 0:
end = sconf.getTrafficDuration() + sconf.getFirstTimestamp()
else:
end = sconf.getTotalStreams()
fd_result = open(sconf.getResultFile(),
'w') # for recording how packets are generated
while current < end:
myrule = None
if sconf.getMixMode() and mix_count >= 0:
if mix_count > 0 and allrules:
myrule = copy.deepcopy(random.choice(allrules))
mix_count = mix_count - 1
elif allrules:
myrule = copy.deepcopy(random.choice(allrules))
if sconf.getVerbosity():
print(myrule)
flow_start_offset = random.randint(1,
sconf.getConcurrentFlows() + 100000)
# Create background traffic conversation based on
# Background traffic rule
if back_traffic_percent > 0:
pick = random.randint(0, 99)
if pick < back_traffic_percent:
btrule = Rule("Background Traffic")
# Update the content with saved information
bt_rule = BackgroundTrafficRule()
bt_rule.updateContent(None, back_dist_list, back_absent_proto)
btrule.addTS(bt_rule)
conversation = Conversation(btrule, sconf, current_sec,
current_usec + flow_start_offset)
else:
conversation = Conversation(myrule, sconf, current_sec,
current_usec + flow_start_offset)
else:
conversation = Conversation(myrule, sconf, current_sec,
current_usec + flow_start_offset)
sec, usec = conversation.getNextTimeStamp()
timekey = timekey = sec + (usec / 1000000)
if timekey in traffic_queue:
traffic_queue[timekey].append(conversation)
else:
traffic_queue[timekey] = [conversation]
total_generated_streams += conversation.getNumberOfStreams()
# Need to track global value in case of interupt
TOTAL_GENERATED_STREAMS = total_generated_streams
if len(traffic_queue) >= sconf.getConcurrentFlows():
pkts, current_sec, current_usec = write_packets(
traffic_queue, traffic_writer, sconf, fd_result)
total_generated_packets += pkts
# Need to track global values in case of interupt
TOTAL_GENERATED_PACKETS = total_generated_packets
FINAL = current_sec
if sconf.getTrafficDuration() > 0:
current = current_sec
elif sconf.getTrafficDuration() <= 0:
current = total_generated_streams
while traffic_queue and len(traffic_queue) > 0:
pkts, current_sec, current_usec = write_packets(
traffic_queue, traffic_writer, sconf, fd_result)
total_generated_packets += pkts
# Track global values
TOTAL_GENERATED_PACKETS = total_generated_packets
FINAL = current_sec
traffic_writer.close_save_file()
fd_result.close()
return [total_generated_streams, total_generated_packets, current_sec]
def _init_sorting(self, tops):
sorted_tops = SortedDict()
for idx, entry in enumerate(tops):
self._update_sorting(sorted_tops, entry, idx)
return sorted_tops
def clear(self):
class_fields = fields(self.dataclass)
self.dicts = {field.name: SortedDict() for field in class_fields}
def test_index():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert temp.index('a') == 0
assert temp.index('f', 3, -3) == 5
