def test_process_events(process_event_task_mock):
events = [{'wait': 0.01, 'type': 'edit'} for x in range(5)]
p = Processor()
assert p.async_results == deque()
p._process_events(events)
assert len(p.async_results) == 5
process_event_task_mock.assert_called_with(events[0])
def __init__(self, args):
# Ignore warnings about SSL connections
warnings.simplefilter("ignore", ResourceWarning)
self.processor = Processor()
self.user_id = args["user"]
self.resultsLimit = args["top"]
self.table = None
def test_list_instances(client_mock):
client_mock().describe_instances.return_value = {
'Reservations': [{
'Instances': []
}]
}
processor = Processor()
actual = processor.list_instances()
assert actual == {'instances': []}
def main():
# Loading Parameters
parser = init_parameters()
args, _ = parser.parse_known_args()
# Updating Parameters (cmd > yaml > default)
args = update_parameters(parser, args)
# Setting save_dir
save_dir = get_save_dir(args)
U.set_logging(save_dir)
with open('{}/config.yaml'.format(save_dir), 'w') as f:
yaml.dump(vars(args), f)
# Processing
if args.generate_data or args.generate_label:
g = Generator(args)
g.start()
elif args.extract or args.visualization:
if args.extract:
p = Processor(args, save_dir)
p.extract()
if args.visualization:
v = Visualizer(args)
v.start()
else:
p = Processor(args, save_dir)
p.start()
def predict():
"""Predict url is for an API calls to predict the price of
a listing and returns json format predictions list
:return: json format price prediction
"""
try:
processor = Processor()
input_params = processor.process_input(request.data)
predictions = processor.predict(input_params)
json_input = json.loads(request.data)
insert_inference(json.dumps(json_input), json.dumps(predictions.tolist()))
return json.dumps({"Predicted Price": predictions.tolist()})
except (KeyError, json.JSONDecodeError, AssertionError, ValueError):
return json.dumps({"error": "CHECK INPUT"}), 400
except Exception as err:
return json.dumps({"error": "PREDICTION FAILED", "message": {err}}), 500
def process_file(cpp_path, build):
if not file_is_cpp(cpp_path):
return
print(cpp_path, "... ", end="")
fname = os.path.basename(cpp_path)
gml_path, is_script = reconstruct_gml_path(build, fname)
if not gml_path:
cprint("cannot find GML source, skipping!", "yellow")
return
try:
Processor.inject_types(cpp_path, gml_path)
if is_script:
Processor.handle_threading(cpp_path, gml_path)
cprint("processed", "green")
except FileNotFoundError:
pass
def home():
"""Homepage that has an input form where you can enter data
and it returns the prediction of a listing.
:return: html template based on GET or POST request
"""
if request.method == "POST":
try:
processor = Processor()
processed_form = processor.process_form(request.form.to_dict())
json_input, input_params = processed_form
prediction = processor.predict(input_params)
insert_inference(json_input, json.dumps(prediction.tolist()))
return render_template("prediction.html", prediction=prediction)
except:
return "Something went wrong, please try again later", 400
else:
return render_template("home.html")
def generate_processors(self, processors_amount):
self.processors = []
for i in range(0, processors_amount):
self.processors.append(
Processor(id=i,
tasks_list=self.tasks,
strategy=self.strategy,
arbitrationRule=self.arbitration,
expropriation=self.expropriation))
def test_partition_key_generation():
sample_request_id = '156be180-becf-11e9-890a-dfff977ea657'
event = {'requestContext': {'requestId': sample_request_id}}
partition_key = Processor.generate_partition_key(event)
assert type(partition_key) is str
assert len(partition_key) == 64
assert partition_key.isalnum()
def test_inactivity_value_no_file():
# this file does not exist
inactivity_file_name = TEST_BASE + 'input/inactivity_period_bad_file.txt'
output_file = ''
try:
Processor(inactivity_file_name, output_file)
except IOError:
assert True
def test_inactivity_value_bad():
# this file has a non-numeric value
inactivity_file_name = TEST_BASE + 'input/inactivity_period_bad_value.txt'
output_file = ''
try:
Processor(inactivity_file_name, output_file)
except ValueError:
assert True
def __init__(self, port):
print('initializing')
self.init_server(('localhost', port))
self.processor = Processor(self.sample_rate)
self.s2t = S2T(self.sample_rate)
self.chat = ChatMan('ckpt/39900_checkpoint.tar')
self.t2s = T2S(self.sample_rate)
# TODO: independence speaker
self.speaker = Audio('output', self.sample_rate, 1.0)
self.listen()
def test_inactivity_value():
inactivity_file_name = TEST_BASE + 'input/inactivity_period.txt'
output_file = ''
try:
proc = Processor(inactivity_file_name, output_file)
except ValueError:
assert False
else:
assert proc.inactivity_time == 2
def execute_and_print(processor: Processor):
c = processor.clock_update()
clear()
print(f"END OF CLOCK CYCLE {processor.clock_time}", end=' ')
print("RESULTS")
processor.first_stage.print_info_to_console()
processor.second_stage.print_to_console()
processor.third_stage.print_to_console()
processor.fourth_stage.print_to_console()
processor.fifth_stage.print_to_console()
return c
def test_output():
output_file_name = TEST_BASE + 'output/sessionization.txt'
raw_data = get_raw_log_data(TEST_BASE + 'input/log.csv')
inactivity_file_name = TEST_BASE + 'input/inactivity_period.txt'
proc = Processor(inactivity_file_name, output_file_name)
proc.process_logs(raw_data)
with open(output_file_name) as f:
results = f.readlines()
with open(TEST_BASE + 'output/ref.txt') as f:
expected = f.readlines()
assert len(results) == len(expected), 'Incorrect number of result items'
for i, r in enumerate(results):
expected_split = expected[i].split(',')
r_split = r.split(',')
for j, piece in enumerate(r_split):
assert piece == expected_split[j], \
'Incorrect value for piece {}'.format(j)
def main():
parser = Init_parameters()
# Update parameters by yaml
args = parser.parse_args()
if os.path.exists('/home/aayadi/projet/RA-GCNv22/configs/' + args.config +
'.yaml'):
with open(
'/home/aayadi/projet/RA-GCNv22/configs/' + args.config +
'.yaml', 'r') as f:
yaml_arg = yaml.load(f, Loader=yaml.FullLoader)
default_arg = vars(args)
for k in yaml_arg.keys():
if k not in default_arg.keys():
raise ValueError('Do NOT exist the parameter {}'.format(k))
parser.set_defaults(**yaml_arg)
else:
raise ValueError('Do NOT exist this config: {}'.format(args.config))
# Update parameters by cmd
args = parser.parse_args()
# Show parameters
print('\n************************************************')
print('The running config is presented as follows:')
v = vars(args)
for i in v.keys():
print('{}: {}'.format(i, v[i]))
print('************************************************\n')
# Processing
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(list(map(str, args.gpus)))
if args.visualization:
if args.extract:
p = Processor(args)
p.extract()
print('Starting visualizing ...')
v = Visualizer(args)
v.show_wrong_sample()
v.show_important_joints()
v.show_heatmap()
v.show_skeleton()
print('Finish visualizing!')
else:
p = Processor(args)
p.start()
def test_next_result_ready():
a = Mock()
a.ready.return_value = False
p = Processor()
p.async_results = deque([a])
res = p._next_result_ready()
assert res is False
a.ready.return_value = True
p.async_results = deque([a])
res = p._next_result_ready()
assert res is True
def list_instances(event, context):
processor = Processor()
instances = processor.list_instances()
return make_response(instances, 200)
class Metrics:
def __init__(self, args):
# Ignore warnings about SSL connections
warnings.simplefilter("ignore", ResourceWarning)
self.processor = Processor()
self.user_id = args["user"]
self.resultsLimit = args["top"]
self.table = None
def _load_table(self, event):
table = agate.Table.from_object(list(self.processor.messagesQueue))
event.set()
self.table = table
return
def _analyze_senders(self, event):
data = (self.table.pivot("fields/from").where(
lambda row: row["fields/from"] is not None).order_by(
"Count", reverse=True).limit(self.resultsLimit))
_values = data.columns.values()
data_keys = list(_values[0].values())
data_count = [[i] for i in list(map(int, list(_values[1].values())))]
event.set()
print(f"\n\n{helpers.h1_icn} Senders (top {self.resultsLimit})\n")
args = {
"stacked": False,
"width": 55,
"no_labels": False,
"format": "{:<,d}",
"suffix": "",
"vertical": False,
"different_scale": False,
}
chart(colors=[94], data=data_count, args=args, labels=data_keys)
def _analyze_count(self, event):
# Average emails per day
total = self.table.aggregate([("total", agate.Count())])["total"]
total_senders = (
self.table.distinct("fields/from").select("fields/from").aggregate(
[("total", agate.Count())])["total"])
if total == 0:
first_email_date = ""
last_email_date = None
else:
date_data = self.table.where(
lambda row: row["fields/date"] is not None).compute([(
"reduce_to_datetime",
agate.Formula(
agate.DateTime(datetime_format="%Y-%m-%d %H:%M:%S"),
lambda row: helpers.reduce_to_datetime(row[
"fields/date"]),
),
)])
first_email_date = (date_data.order_by("reduce_to_datetime").limit(
1).columns["fields/date"].values()[0])
last_email_date = (date_data.order_by(
"reduce_to_datetime",
reverse=True).limit(1).columns["fields/date"].values()[0])
event.set()
metrics = [
["Total emails", total],
["Senders", total_senders],
["First Email Date", first_email_date],
]
if last_email_date:
date_delta = helpers.convert_date(
last_email_date) - helpers.convert_date(first_email_date)
avg_email_per_day = total / date_delta.days
metrics.append(["Avg. Emails/Day", f"{avg_email_per_day:.2f}"])
print(f"\n\n{helpers.h1_icn} Stats\n")
print(termtables.to_string(metrics))
def _analyze_date(self, event):
table = self.table.where(
lambda row: row["fields/date"] is not None).compute([
(
"reduce_to_date",
agate.Formula(
agate.Text(),
lambda row: helpers.reduce_to_date(row["fields/date"]),
),
),
(
"reduce_to_year",
agate.Formula(
agate.Number(),
lambda row: helpers.reduce_to_year(row["fields/date"]),
),
),
(
"reduce_to_time",
agate.Formula(
agate.Number(),
lambda row: helpers.reduce_to_time(row["fields/date"]),
),
),
])
years = table.distinct(
"reduce_to_year").columns["reduce_to_year"].values()
_data = {}
for year in years:
_data[year] = (table.where(lambda row: row[
"reduce_to_year"] == year).select("reduce_to_date").pivot(
"reduce_to_date").order_by("reduce_to_date"))
event.set()
print(f"\n\n{helpers.h1_icn} Date\n")
for year in years:
data_keys = list(_data[year].columns["reduce_to_date"].values())
_counts = list(
map(int, list(_data[year].columns["Count"].values())))
_sum = sum(_counts)
data_count = [[i] for i in _counts]
args = {
"color": False,
"custom_tick": False,
"start_dt": f"{year}-01-01"
}
print(f"\n{helpers.h2_icn} Year {year} ({_sum:,} emails)\n")
calendar_heatmap(data=data_count, args=args, labels=data_keys)
def analyse(self):
"""
read from the messages queue, and generate:
1. Counter for From field
2. Counter for Time field (by hour)
"""
# {'id': '16f39fe119ee8427', 'labels': ['UNREAD', 'CATEGORY_UPDATES', 'INBOX'], 'fields': {'from': 'Coursera <*****@*****.**>', 'date': 'Tue, 24 Dec 2019 22:13:09 +0000'}}
with concurrent.futures.ThreadPoolExecutor() as executor:
progress = Spinner(f"{helpers.loader_icn} Loading messages ")
event = Event()
future = executor.submit(self._load_table, event)
while not event.isSet() and future.running():
progress.next()
time.sleep(0.1)
progress.finish()
progress = Spinner(f"{helpers.loader_icn} Analysing count ")
event = Event()
future = executor.submit(self._analyze_count, event)
while not event.isSet() and future.running():
progress.next()
time.sleep(0.1)
progress.finish()
progress = Spinner(f"{helpers.loader_icn} Analysing senders ")
event = Event()
future = executor.submit(self._analyze_senders, event)
while not event.isSet() and future.running():
progress.next()
time.sleep(0.1)
progress.finish()
progress = Spinner(f"{helpers.loader_icn} Analysing dates ")
event = Event()
future = executor.submit(self._analyze_date, event)
while not event.isSet() and future.running():
progress.next()
time.sleep(0.1)
progress.finish()
def start(self):
messages = self.processor.get_messages()
self.processor.get_metadata(messages)
self.analyse()
def test_processor_init():
p = Processor()
assert p.async_results is not None
assert isinstance(p.async_results, deque)
def test_processor_process(process_events_mock, write_outfile_mock):
events = [{'wait': 0.01, 'type': 'edit'} for x in range(5)]
p = Processor()
p.process(events=events)
process_events_mock.assert_called_once()
write_outfile_mock.assert_called_once()
from src.utils import _hash, log
url = 'tcp://10.9.70.170:4004'
family_name = 'poc-blockchain'
version = ['1.0']
def func(transation, context):
print('----')
print(transation)
try:
payload = structure.Payload()
payload.ParseFromString(transation.payload)
print(payload)
log.info(payload)
info = Actions[payload.action](transation, context, payload,
family_name)
log.info(info)
except Exception as e:
print(e)
log.exception(e)
if __name__ == '__main__':
handler = Handler(family_name, version, func)
processor = Processor(handler, url)
processor.start()
# entity = structure.Entity()
# entity1 = structure.Entity()
# entity.childs.append(entity1)
print("RESULTS")
processor.first_stage.print_info_to_console()
processor.second_stage.print_to_console()
processor.third_stage.print_to_console()
processor.fourth_stage.print_to_console()
processor.fifth_stage.print_to_console()
return c
if __name__ == '__main__':
print(
"*****************WELCOME TO THE MIPS PROCESSOR*****************************"
)
var = input("ENTER X to EXIT, OR PRESS ENTER TO CONTINUE: ")
while var != 'x' and var != 'X':
main_processor = Processor()
clear()
file_found = False
var3 = 'w'
while not file_found and var3 != 'x' and var3 != 'X':
file_found = True
print(
"TIP: ENTER PATHS RELATIVE TO src, OTHERWISE ENTER FULL PATH")
file_path = input("Please Enter asm file: ")
try:
main_processor.load_file_into_im(file_path)
except FileNotFoundError:
file_found = False
clear()
var3 = input(
"FILE NOT FOUND:\nPRESS X TO EXIT OR PRESS ENTER TO RETRY ENTERING FILE PATH: "
def run(self):
# Necessarities
single_core = Processor()
# Create User Output
HELPER[0].schedule_start_message(self, self.schedule_mode)
# Scheduling - loop
while len(PROCESS_LIST) > 0:
# Determine which Scheduler an get the decision accordingly
if self.schedule_mode == FCFS:
process_found, active_index, finish = first_come_first_serve(
single_core)
elif self.schedule_mode == SJF:
process_found, active_index, finish = smallest_job_first(
single_core)
elif self.schedule_mode == SRTF:
process_found, active_index, finish = shortest_remaining_time_first(
single_core)
elif self.schedule_mode == RR:
process_found, active_index, finish = round_robin(single_core)
elif self.schedule_mode == PBS:
process_found, active_index, finish = priority_based(
single_core)
elif self.schedule_mode == EDF:
process_found, active_index, finish = earliest_deadline_first(
single_core)
elif self.schedule_mode == HRRN:
process_found, active_index, finish = highest_response_ratio_next(
single_core)
else:
self.window.display_text("Error Occured! Please try again!")
return
# CPU got to wait, no Process arrived yet
if not process_found:
self.window.display_text(
f"No Process in Queue. Running Empty...")
self.window.display_text(
f"System-Clock: {single_core.get_clock_time_step()}")
single_core.work()
else:
# Put the CPU to work according to the decisions made
single_core.work_process(self, active_index,
self.schedule_mode, single_core,
finish)
# Finish Message
self.window.display_end_line()
self.window.display_text(f"Finished all Processes!")
self.window.display_text(
f"Average Waiting Time: {single_core.get_average_waiting()}")
return
def main():
parser = Init_parameters()
# Update parameters by yaml
args = parser.parse_args()
if os.path.exists('./configs/' + args.config + '.yaml'):
with open('./configs/' + args.config + '.yaml', 'r') as f:
yaml_arg = yaml.load(f, Loader=yaml.FullLoader)
default_arg = vars(args)
for k in yaml_arg.keys():
if k not in default_arg.keys():
raise ValueError('Do NOT exist the parameter {}'.format(k))
parser.set_defaults(**yaml_arg)
else:
raise ValueError('Do NOT exist this config: {}'.format(args.config))
# Update parameters by cmd
args = parser.parse_args()
# Show parameters
print('\n************************************************')
#if type(args.gpus) == int:
# n = args.gpus
# if n == 4:
# args.gpus = [0, 1, 2, 3]
# else:
# args.gpus = [0]
print('The running config is presented as follows:')
print_default_keys = ['config', 'batch_size', 'pretrained', 'model_stream']
print_eval_keys = [
'occlusion_part', 'occlusion_time', 'occlusion_block',
'occlusion_rand', 'jittering_joint', 'jittering_frame', 'sigma'
]
v = vars(args)
if '-g' in sys.argv or '--gpus' in sys.argv:
aa = args.gpus
args.gpus = [int(x) for x in aa.split(',')]
else:
if node == 'obama':
args.gpus = [0, 1, 2, 3]
elif node == 'puma':
args.gpus = [0]
else:
args.gpus = [0]
for i in v.keys():
if i in print_default_keys:
print('{}: {}'.format(i, v[i]))
if args.evaluate:
for i in v.keys():
if i in print_eval_keys:
if v[i]:
print('{}: {}'.format(i, v[i]))
print('************************************************\n')
# Processing
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(list(map(str, args.gpus)))
if args.visualization:
if args.extract:
p = Processor(args)
p.extract()
print('Starting visualizing ...')
v = Visualizer(args)
v.show_wrong_sample()
v.show_important_joints()
v.show_heatmap()
v.show_skeleton()
print('Finish visualizing!')
else:
if args.baseline:
p = Processor_BS(args)
else:
p = Processor(args)
p.start()
