def scrape():
vprint('working from %s' % getcwd())
handler = DataHandler(OUTPUT)
handler.read_data()
handler.remove_old_entries(DAYS_TO_KEEP)
added, per_url = 0, 0
for url in get_file_items(URLS_PATH):
vprint('requesting feed from %s...' % url)
feed = parse(url)
if not feed or 'status' not in feed:
tsprint('error: unable to reach target url. aborting.')
exit(-1)
status = feed['status']
if status != 200:
tsprint(
'error: request from %s responded with error code %s. skipping...'
% (url, status))
continue
vprint('status returned normal. scanning entries...')
for entry in feed['entries']:
try:
id_ = entry['id']
published = None
if 'published_parsed' in entry:
published = entry['published_parsed']
elif 'updated_parsed' in entry:
published = entry['updated_parsed']
else:
print('entries may not have dates. skipping...')
break
if not is_new(published) or handler.entry_exists(id_):
continue
tags = []
if 'www.reddit.com' in url or 'tags' not in entry or not entry[
'tags']:
tags = clean_unique_tags([entry['title']])
if per_url == 0:
vprint(
'no tags for entries, using title instead:\n %s' %
tags)
else:
tags = clean_unique_tags(
[tag['term'] for tag in entry['tags']])
if not tags: continue
extras = {}
if 'www.reddit.com' in url and 'summary' in entry:
m = search(r'href="(\S+)">\[link\]', entry['summary'])
if m: extras = {"dlink": m.group(1)}
handler.add_entry(url, id_, clean_entry(entry, tags, **extras))
added += 1
per_url += 1
except KeyError as e:
print('%s\nskipping...' % e)
per_url = 0
break
vprint('got %s entries from %s' % (per_url, url))
per_url = 0
handler.write_data()
tsprint('added %s new entries' % added)
tsprint('%s entries total' % len(handler.get_all_entries()))
def get_by_distance():
distance = request.args.get("dist")
latitudeN = request.args.get("latN")
longitudeE = request.args.get("lonE")
dh = DataHandler()
return dh.select_schools_by_distance(float(latitudeN), float(longitudeE),
float(distance))
def find_distance(i_):
path_to_covers = '/home/anosov/data/hard_base/covers/case_%i.dump' % (i_, )
base, ext = os.path.splitext(path_to_covers)
path_to_data = base + '__computing' + ext
h = DataHandler(path_to_covers, path_to_data)
tmp = h.distance_to_dead_ends
h.dump()
def runServer(self):
Logger.writeInfo("Opening socket...")
sock = socket(AF_INET, SOCK_STREAM)
sock.bind((self.host, self.port))
sock.listen(10)
handler = DataHandler()
try:
while True:
conn, addr = sock.accept()
Logger.writeInfo("Connected by {}".format(addr))
try:
data = Server.receiveAll(conn)
if data != None:
response = handler.process(data)
conn.sendall(response)
conn.close()
except Exception as e:
Logger.writeError(str(e))
Logger.writeInfo("Disconnected {}".format(addr))
finally:
Logger.writeInfo("Socket closed")
sock.close()
handler.close()
def test(path, run_id, runs):
# load parser and data handler
parser = CYKParser.load(path)
data_handler = DataHandler(config.test_set, run_id, runs)
# parse sentences in parallel
executor = ProcessPoolExecutor(config.processes)
futures = [
executor.submit(parse_tree, parser, sent, run_id)
for sent in data_handler.generator()
]
# following code is to track progress
kwargs = {
'total': len(futures),
'unit': 'nap',
'unit_scale': True,
'leave': True
}
for _ in tqdm(as_completed(futures), **kwargs):
pass
for future in futures:
if future.exception() is not None:
print(future.exception())
# stitch files if number of runs is 1
if runs == 1:
stitch_files()
print("Done parsing")
def __init__(self):
self.epoch_duration = 60
self.vk_client = vk_autorization(LOGIN, PASSWORD)
self.data = DataHandler()
self.view = View(self, self.data.get_players(),
self.data.get_settings())
self.view.show()
def perform_exp(self):
list_of_scores = []
data_handler = DataHandler(self.data, self.var_dict)
raw_X = data_handler.get_dummy_coded_data('dummy_only')
n_cat_dummy_var = raw_X.shape[1] - len(self.var_dict['numerical_vars'])
raw_clf_scores = self._get_classification_score(raw_X)
list_of_scores.append(
('raw', raw_clf_scores, raw_X.shape[1] - n_cat_dummy_var))
for n_init_bins in self.n_init_bins_list:
sb_X = self.semantic_binning.fit_transform(self.data, n_init_bins)
sb_clf_scores = self._get_classification_score(sb_X)
list_of_scores.append(('sb_{}'.format(n_init_bins), sb_clf_scores,
sb_X.shape[1] - n_cat_dummy_var))
for n_bins in self.n_bins_range:
ew_X = data_handler.get_dummy_coded_data('equal_width', n_bins)
ew_clf_scores = self._get_classification_score(ew_X)
list_of_scores.append(('ew_{}'.format(n_bins), ew_clf_scores,
ew_X.shape[1] - n_cat_dummy_var))
ef_X = data_handler.get_dummy_coded_data('equal_freq', n_bins)
ef_clf_scores = self._get_classification_score(ef_X)
list_of_scores.append(('ef_{}'.format(n_bins), ef_clf_scores,
ef_X.shape[1] - n_cat_dummy_var))
self.list_of_scores = list_of_scores
print('Experiment Finished !. Result Saved in Exp Instance..')
def __init__(self,master):
super().__init__(master)
self.data_handler = DataHandler("data.json")
self.category_lists = self.data_handler.get_categories_list()
self.category_data_labels = []
self.element = 'Sports'
#delete old labels..
for self.element in self.category_data_labels:
self.element.configure(text = "")
del self.element
#add new labels for name..
self.list_data_for_category = self.data_handler.get_news_for_category(self.element)
for i in range(len(self.list_data_for_category)):
self.element = self.list_data_for_category[i]
self.labelfont = ('times',15,'bold')
self.labelcontent_font = ('times',10,'bold','italic')
self.label_heading = Label(self, text = self.element['headline'])
self.label_heading.config(font = self.labelfont)
self.label_heading.grid(column = 0, row = 3*i)
self.label_content = Label(self, text = self.element['content'])
self.label_content.config(font = self.labelcontent_font)
self.label_content.grid(column = 0, row = 3*i+1)
self.label_content_separator = Label(self, text = '---------------------------')
self.label_content_separator.grid(column = 0, row=3*i+2)
self.category_data_labels.append(self.label_heading)
self.category_data_labels.append(self.label_content)
self.category_data_labels.append(self.label_content_separator)
self.pack()
def __init__(self, number_of_epochs=10):
self.verbose = True
self.number_of_channels = 2
self.data_handler = \
DataHandler(number_of_channels=self.number_of_channels,
number_of_negative_sets=50,
number_of_positive_sets=50,
number_of_test_sets=50,
verbose=self.verbose)
self.data_handler.load_training_data()
self.data_handler.load_test_data()
self.data_handler.preprocess_data()
self.mini_batch_size = 1
self.model = CNN(number_of_channels=self.number_of_channels,
number_of_filters=12,
regularization_coefficient=1e0,
learning_rate=0.001,
filter_length=12,
pool_size=512,
fully_connected_layer_neurons=8,
momentum=0.9,
perform_normalization="no",
update_type="adam",
pool_mode="average_exc_pad")
self.number_of_epochs = number_of_epochs
self.training_errors = []
self.test_errors = []
self.classifier = SVC(C=11., kernel="rbf", gamma=1. / (2 * 2.85))
def main():
opts = configs.model_config
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
gpu_config = tf.ConfigProto(device_count={'GPU': 1},
allow_soft_placement=False,
log_device_placement=False)
gpu_config.gpu_options.allow_growth = True
sess = tf.Session(config=gpu_config)
print('starting processing data ...')
data = DataHandler(opts)
print('starting initialising model ...')
opts['r_range_upper'] = data.train_r_max
opts['r_range_lower'] = data.train_r_min
model = Model_Decon(sess, opts)
opts['batch_size'] = 1
opts['va_sample_num'] = 6
opts['model_bn_is_training'] = False
print('starting testing policy using AC_Decon ...')
ac = AC_Decon(sess, opts, model)
ac.policy_test(data)
def test_replace_null_median(self):
'''This function adds np.nan values to copies of self.df_5 and self.df_30
and then sets the values of df_5 and df_30 in test_handler to the new
copies of df_5 and df_30. Then the replace_null(method='median')
function is called on test_handler whcih should replace the np.nan values
with median values. Finally assertEqual is called to check that the
replace_null function is working correctly.'''
df_5_replace_null_check = self.df_5.copy()
df_5_replace_null_check.loc[self.df_5.first_valid_index(),
'C'] = np.nan
df_30_replace_null_check = self.df_30.copy()
df_30_replace_null_check.loc[self.df_30.last_valid_index(),
'D'] = np.nan
test_handler = DataHandler()
test_handler.df_5 = df_5_replace_null_check
test_handler.df_30 = df_30_replace_null_check
test_handler.replace_null(method='median')
self.assertEqual(
test_handler.df_5.loc[test_handler.df_5.first_valid_index(), 'C'],
30.0)
self.assertEqual(
test_handler.df_30.loc[test_handler.df_30.last_valid_index(), 'D'],
4.0)
def __init__(self, ):
# Initialize data loader
self.data = DataHandler()
# Initialize model
self.ada_network = ADDANet()
def run_model(model_name, data):
dh = DataHandler()
dh.import_data(data)
dh.create_targets(-1)
model = build_model(model_name, dh)
m_eval = ModelEvaluator(model)
acc, std = m_eval.n_time_k_cross_fold(10, 5)
print('Accuracy: {}\nStandard Deviation: {}\n'.format(acc, std))
def dataLoader(self):
data_handler = DataHandler()
npz = data_handler.npzLoader(self.target_file)
data, label = npz[0], npz[1]
data /= 255.0
return data, label
def __init__(self, server_socket, buffer, socket_ip):
self.names = []
self.record = {}
self.buffer = buffer
self.connected_list = []
self.socket_ip = socket_ip
self.data_handler = DataHandler(socket_ip)
self.server_socket = server_socket
self.add_connection(server_socket)
def run():
global model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Create dataset
template_dataset = TemplateDataset(config)
training_loader, validation_loader, test_loader = template_dataset.get_loaders(
)
# Create the neural network
model = NN(net, optimizer, loss_function, lr_scheduler, metric, device,
config).to(device)
# Create the data handler
data_handler = DataHandler(training_loader, validation_loader, test_loader)
for epoch in range(config['epochs']):
# Training
model.train()
for i, data in enumerate(training_loader, 0):
x, y = data
x, y = x.to(device), y.to(device)
y_hat = model(x)
loss = model.backpropagate(y_hat, y)
result = model.evaluate(y_hat, y)
data_handler.train_loss.append(loss)
data_handler.train_metric.append(result)
with torch.no_grad():
model.eval()
# Validating
if validation_loader is not None:
for i, data in enumerate(validation_loader, 0):
x, y = data
x, y = x.to(device), y.to(device)
y_hat = model(x)
_, loss = model.calculate_loss(y_hat, y)
result = model.evaluate(y_hat, y)
data_handler.valid_loss.append(loss)
data_handler.valid_metric.append(result)
# Testing
if test_loader is not None:
for i, data in enumerate(test_loader, 0):
x, y = data
x, y = x.to(device), y.to(device)
y_hat = model(x)
_, loss = model.calculate_loss(y_hat, y)
result = model.evaluate(y_hat, y)
data_handler.test_loss.append(loss)
data_handler.test_metric.append(result)
model.lr_scheduler_step()
data_handler.epoch_end(epoch, model.get_lr())
data_handler.plot(loss=config['plot']['loss'],
metric=config['plot']['metric'])
def create_wordcloud(self):
movies = DataHandler().get_data()
movies['title'] = movies['title'].fillna("").astype('str')
title_corpus = ''.join(movies['title'])
wordcloud = WordCloud(stopwords=STOPWORDS, background_color='black', height=2000,
width=4000).generate(title_corpus)
plt.figure(figsize=(16, 8))
plt.imshow(wordcloud)
plt.show()
def clustering(self, data):
kmeans = KMeans(n_clusters=4,
init='k-means++',
max_iter=1000,
n_init=10)
prediction = kmeans.fit_predict(data)
df = DataHandler().read_data()
df['Clusters'] = prediction
return df, kmeans
def load_data(db_name,
label="training_0000",
n_imgs=None,
thresh=1e5,
step_size=1,
db_dir=None):
"""
loads rgb images and targets from an hdf5 database and returns them as a np array
Expects data to be saved in the following group stucture:
Training Data
training_0000/data/0000 using %04d to increment data name
Validation Data
validation_0000/data/0000 using %04d to increment data name
Both return an array with the rgb image saved under the 'rgb' key
and the target saved under the 'target' key
Parameters
----------
db_name: string
name of database to load from
label: string, Optional (Default: 'training_0000')
location in database to load from
n_imgs: int
how many images to load
"""
# TODO: specify the data format expected in the comment above
dat = DataHandler(db_dir=db_dir, db_name=db_name)
# load training images
images = []
targets = []
skip_list = ["datestamp", "timestamp"]
keys = np.array([
int(val) for val in dat.get_keys("%s" % label) if val not in skip_list
])
n_imgs = max(keys) if n_imgs is None else n_imgs
print("Total number of images in dataset: ", max(keys))
for nn in range(0, n_imgs, step_size):
data = dat.load(parameters=["rgb", "target"],
save_location="%s/%04d" % (label, nn))
if np.linalg.norm(data["target"]) < thresh:
images.append(data["rgb"])
targets.append(data["target"])
images = np.asarray(images)
targets = np.asarray(targets)
print("Total number of images within threshold: ", images.shape[0])
return images, targets
def __init__(self, file_sys_broadcast_addr, task_ping_addr,
current_worker_addr, job_url, job_id, function_url,
block_urls, task, answer_addr, load_byte, status_db_url,
block_id):
self.status_handler = StatusHandler(
status_db_url, self._verify_if_errors_in_fs,
self._reset_method_if_no_answer_from_fs)
self.data_handler = DataHandler(
job_url, job_id, self._verify_if_errors_in_fs,
self._reset_method_if_no_answer_from_fs)
self.task_ping_addr = task_ping_addr
self.file_sys_broadcast_addr = file_sys_broadcast_addr
self.file_sys_addrs = []
self.answer_addr = answer_addr
self._update_filesystem_nodes()
self.file_sys_addr = self._get_new_filesystem_node()
self.job_url = job_url
self.job_id = job_id
self.function_url = function_url
self.block_urls = block_urls
# print("Task_Exc: Este es el block_urls: ",self.block_urls)
self.task = task
self.load_byte = load_byte
self.current_worker_addr = current_worker_addr
self.status_db_url = status_db_url
self.block_id = block_id
self.map_fun, self.red_fun, self.comb = self.get_func()
self.record_readers = {
True: self.record_reader_byte,
False: self.record_reader_str
}
self.execute_methods = {
'map': self.execute_map_task,
'reduce': self.execute_reduce_task
}
self.start_listen_pings()
# nos aseguramos que si se cayo el master en el momento que me mando el mensaje de task, yo mismo pongo en
# submitted el bloque
to_update = [('state', mt.slices_states[1]),
('worker_ip', self.task_ping_addr[0]),
('worker_port', self.task_ping_addr[1])]
print(
"Task_Exc: ",
"Salvamos el estado del bloque {} a SUBMITTED en el filesys: {}".
format(block_id, self.file_sys_addr))
self.status_handler.update_status_row(self.file_sys_addr, 'block',
('block_id', block_id),
to_update)
self.execute_task = self.execute_methods[self.task]
self.record_reader = self.record_readers[load_byte]
def test_parser(self):
"""Tests the correct behaviour of the parser method: an args string without a required input is
passed to the method and the test checks if an exception is raised"""
data_handler = DataHandler('')
method_handler = MethodHandler(data_handler, '')
cli_app = CLI(method_handler)
args_exp = {'Ticker': {'Type': 'String', 'Required': 1}, 'StartDate': {'Type': 'Date', 'Required': 1},
'EndDate': {'Type': 'Date', 'Required': 1}, 'Currency': {'Type': 'String', 'Required': 1}}
args_obtained = 'AAPL 2021/01/01 2021/02/14'
with self.assertRaises(Exception):
res = cli_app.parse_args(args_obtained, args_exp)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=4)
parser.add_argument('--emb_dim', type=int, default=200)
parser.add_argument('--enc_hid_dim', type=int, default=128)
parser.add_argument('--dec_hid_dim', type=int, default=256)
parser.add_argument('--attn_size', type=int, default=200)
parser.add_argument('--epochs', type=int, default=12)
parser.add_argument('--learning_rate', type=float, default=2.5e-4)
parser.add_argument('--dataset_path', type=str, default='../data/Maluuba/')
parser.add_argument('--glove_path', type=str, default='../data/')
parser.add_argument('--checkpoint', type=str, default="./trainDir/")
config = parser.parse_args()
DEVICE = "/gpu:0"
logging.info("Loading Data")
handler = DataHandler(
emb_dim = config.emb_dim,
batch_size = config.batch_size,
train_path = config.dataset_path + "train.json",
val_path = config.dataset_path + "val.json",
test_path = config.dataset_path + "test.json",
vocab_path = "./vocab.json",
entities_path = config.dataset_path + "entities.json",
glove_path = config.glove_path)
logging.info("Loading Architecture")
model = DialogueModel(
device = DEVICE,
batch_size = config.batch_size,
inp_vocab_size = handler.input_vocab_size,
out_vocab_size = handler.output_vocab_size,
generate_size = handler.generate_vocab_size,
emb_init = handler.emb_init,
result_keys_vector = handler.result_keys_vector,
emb_dim = config.emb_dim,
enc_hid_dim = config.enc_hid_dim,
dec_hid_dim = config.dec_hid_dim,
attn_size = config.attn_size)
logging.info("Loading Trainer")
trainer = Trainer(
model=model,
handler=handler,
ckpt_path="./trainDir/",
num_epochs=config.epochs,
learning_rate = config.learning_rate)
trainer.trainData()
def main():
scaleModes = {
"50": "Normal mode",
"46": "Real-time mode"
}
board_handler = BoardHandler()
data_handler = DataHandler(scaleModes)
output_handler = OutputHandler()
gui = GuiHandler(board_handler, data_handler, output_handler)
gui.initWindow()
def main():
SERVER_IP = '127.0.0.1'
CLIENT_IP = '127.0.0.1'
SERVER_PORT = '8443'
CLIENT_PORT = '53432'
data_handler = DataHandler()
client_hello = get_packet('test_data/data/client_hello')
server_hello = get_packet('test_data/data/server_hello')
key_exchange = get_packet('test_data/data/client_key_exchange')
client_encrypted_data1 = get_packet('test_data/data/client_encrypted_data1')
server_encrypted_data1 = get_packet('test_data/data/server_encrypted_data1')
client_encrypted_data2 = get_packet('test_data/data/client_encrypted_data2')
server_encrypted_data2 = get_packet('test_data/data/server_encrypted_data2')
print("Starting Parser: ")
Parser(client_hello, data_handler,
CLIENT_IP, SERVER_IP, CLIENT_PORT, SERVER_PORT).parse()
Parser(server_hello, data_handler,
SERVER_IP, CLIENT_IP, SERVER_PORT, CLIENT_PORT).parse()
Parser(key_exchange, data_handler,
CLIENT_IP, SERVER_IP, CLIENT_PORT, SERVER_PORT).parse()
Parser(client_encrypted_data1, data_handler,
CLIENT_IP, SERVER_IP, CLIENT_PORT, SERVER_PORT).parse()
Parser(server_encrypted_data1, data_handler,
SERVER_IP, CLIENT_IP, SERVER_PORT, CLIENT_PORT).parse()
Parser(client_encrypted_data2, data_handler,
CLIENT_IP, SERVER_IP, CLIENT_PORT, SERVER_PORT).parse()
Parser(server_encrypted_data2, data_handler,
SERVER_IP, CLIENT_IP, SERVER_PORT, CLIENT_PORT).parse()
print("\nStarting Key Finder:")
KeyFinder(data_handler, '../keys/privkey.pem',
CLIENT_IP, SERVER_IP, CLIENT_PORT, SERVER_PORT,
MAC_KEY_LENGTH, SYMMETRIC_KEY_LENGTH, IV_LENGTH).find()
print("Handshake Table:\n")
print(data_handler.handshake_table)
print("Application Data:\n")
print(data_handler.app_data_table)
print("\nStarting Decryptor\n")
decryptor = Decryptor(data_handler,
CLIENT_IP, SERVER_IP, CLIENT_PORT, SERVER_PORT)
output = decryptor.start('results/')
for packet in output:
print(f'packet {packet}\n')
print(len(output))
def test_get_hist_prices(self):
"""Given a pre-defined set of inputs the test checks if the method correctly computes the results"""
data_handler = DataHandler('')
method_handler = MethodHandler(data_handler, '')
data = {dt.datetime(2021, 1, 1): 120, dt.datetime(2021, 1, 2): 122, dt.datetime(2021, 1, 3): 123,
dt.datetime(2021, 1, 4): 122, dt.datetime(2021, 1, 5): 119, dt.datetime(2021, 1, 8): 118,
dt.datetime(2021, 1, 9): 120, dt.datetime(2021, 1, 10): 122}
data_handler.load_db_manually('MSFT', data)
res_test = {dt.datetime(2021, 1, 3): 123, dt.datetime(2021, 1, 4): 122, dt.datetime(2021, 1, 5): 119,
dt.datetime(2021, 1, 8): 118, dt.datetime(2021, 1, 9): 120}
res = method_handler.get_hist_prices('MSFT', dt.datetime(2021, 1, 3), dt.datetime(2021, 1, 9), '')
self.assertEqual(res, res_test)
def store_analyzed_reviews(self, hotel_name, aspect_details, platforms):
for key, value in self.aspect_details.items():
value.review_list = {}
to_json = json.dumps(aspect_details, cls=MyEncoder)
data_handler = DataHandler()
data_handler.set_analyzed_reviews(hotel_name, to_json, platforms)
# ra = ReviewAnalyzer()
# ra.get_analyzed_reviews('Kingsbury', ['ALL'])
def get_analyzed_reviews(self, hotel_name, platforms):
data_handler = DataHandler()
reviews = data_handler.get_analyzed_reviews(hotel_name, platforms)
print(hotel_name)
print(reviews)
if reviews is None:
self.analyze_reviews(hotel_name, platforms)
reviews = data_handler.get_analyzed_reviews(hotel_name, platforms)
if reviews is None:
return None
return reviews[2]
def main():
path_to_covers = '/home/anosov/data/hard_base/covers/case_0.dump'
base, ext = os.path.splitext(path_to_covers)
path_to_data = base + '__computing' + ext
h = DataHandler(path_to_covers, path_to_data)
gui = GUI(h)
# gui.append(trivia.show_stat_rm)
# gui.append(trivia.show_equal_stat)
gui.append(trivia.case_0)
gui.run()
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.ui.graphicsView.setBackground(pg.mkColor(0.3))
self.plot_box = self.ui.graphicsView.addViewBox(row=1,
col=1,
lockAspect=True,
enableMouse=True,
invertY=True)
self.image_item = pg.ImageItem()
self.image_item.setOpts(axisOrder='row-major')
self.plot_box.addItem(self.image_item)
self.roi = None
self.ui.selectDataButton.toggled.connect(self.show_roi)
self.ui.resetSelectDataButton.clicked.connect(self.reset_roi)
self.settings_layout = QHBoxLayout()
self.settings_widget = QWidget()
self.settings_layout.addWidget(self.settings_widget)
self.ui.camSettingsWidget.setLayout(self.settings_layout)
self.data_handler = DataHandler()
for plugin in self.data_handler.plugins:
self.add_plugin(plugin.get_widget(), plugin.name)
self.data_handler.ndarray_available.connect(self.show_ndarray)
self.data_handler.camera_controls_changed.connect(
self.set_camera_controls)
self.ui.actionSave_image.triggered.connect(self.data_handler.save_file)
self.data_handler.enable_saturation_widget.connect(
self.enable_saturation_bar)
self.data_handler.saturation_changed.connect(
self.ui.progressBar.setValue)
self.data_handler.message.connect(self.show_message)
self.camera_dialog = CameraDialog()
self.ui.actionChoose_camera.triggered.connect(
self.camera_dialog.choose_camera)
self.camera_dialog.camera_changed.connect(
self.data_handler.change_camera)
self.camera_dialog.choose_first_camera()
self.ui.actionTune_camera_parameters.triggered.connect(self.tune_pid)
self.ui.actionShow_Settings.toggled.connect(self.show_settings)
self.ui.actionDraw_lines.toggled.connect(self.draw_lines)
self.hline = None
self.vline = None
def __init__(self, worker_broadcast_addr, filesystem_broadcast_addr,
tracker_addr_ping, tracker_ip, current_worker_addr, job_url,
job_id, data_type, client_addr, functions_url, map_data_url,
status_db_url):
self.worker_broadcast_addr = worker_broadcast_addr
self.status_handler = StatusHandler(
status_db_url, self._verify_if_errors_in_fs,
self._reset_method_if_no_answer_from_fs)
self.data_handler = DataHandler(
job_url, job_id, self._verify_if_errors_in_fs,
self._reset_method_if_no_answer_from_fs)
self.filesystem_broadcast_addr = filesystem_broadcast_addr
self.job_url = job_url
self.file_sys_addrs = []
self._update_filesystem_nodes()
self.file_sys_addr = self._get_new_filesystem_node()
self.client_addr = client_addr
self.job_id = job_id
self.current_worker_addr = current_worker_addr
self.data_type = data_type
self.states = ["map", "reduce"]
self.job_phase = self.states[0]
self.veto_workers = []
self.tracker_ip = tracker_ip
self.tracker_addr_ping = tracker_addr_ping
self.tracker_addr = (tracker_ip, '8080')
self.delimiters = [' ', '\n']
self.map_results = None
self.result_data_url = '{}/result_data'.format(self.job_url)
self.map_data_url = map_data_url
self.functions_url = functions_url
self.status_db_url = status_db_url
self.phases = [
'GETWORKERS', 'SLICES', 'SENDTASK', 'WAITANSWERS', 'GETRESULTS',
'DONE'
]
self.load_job_methods = {
'GETWORKERS': self.getting_workers,
'SLICES': self.getting_workers,
'SENDTASK': self._load_send_task_phase,
'WAITANSWERS': self._load_wait_results,
'GETRESULTS': self.getting_results,
}
self.status_phase = mt.task_phases[0]
self.pinging_process = None
self.get_data = self.data_handler.get_line_by_line_str
