async def start(resume): '''Connection to the API''' async with websockets.connect('wss://gateway.discord.gg?encoding=json&v=6', loop=asyncio.get_event_loop()) as websocket: start_time = time.time() if not resume: hello = parse_data(await websocket.recv()) payload = { "op": 2, "d": { "token": TOKEN, 'properties': { '$properties': os.name, '$browser': 'dapi-bot', '$device': 'dapi-bot' }, 'compress': True, 'large_threshold': 250, "presence": { "game": { "name": " with Discord API", "type": 0 }, "status": "dnd", "since": int(time.time()), "afk": False } } } await websocket.send(json.dumps(payload)) ready = parse_data(await websocket.recv()) SESSION_DATA[0] = ready['d']['session_id'] for i in ready['d']['guilds']: x = parse_data(await websocket.recv()) print('Bot connected') else: payload = { "op": 6, "d": { "token": TOKEN, "session_id": SESSION_DATA[0], "seq": SESSION_DATA[1] } } await websocket.send(json.dumps(payload)) hello = parse_data(await websocket.recv()) await websocket.recv() while True: if int(time.time() - start_time) + 1 % hello['d']['heartbeat_interval'] == 0: payload = {"op": 1, "d": SESSION_DATA[1]} await websocket.send(json.dumps(payload)) await websocket.recv() try: event = parse_data(await websocket.recv()) SESSION_DATA[1] = event['s'] print(event['t']) #Logs every event the WS sendss await commands.parse_event(event) except websockets.exceptions.ConnectionClosed: break
def get_data(query, variables): has_next = True cursor = None entities = [] spinner = Spinner('Fetching Github Data') while has_next: spinner.next() variables['cursor'] = cursor rate_limit = get_rate_limit(client) handle_rate_limit(rate_limit) results = json.loads(client.execute(query, variables)) if results['data'] and results['data']['search']['edges']: nodes = [ edge['node'] for edge in results['data']['search']['edges']] for node in nodes: entities.append(parse_data(node)) has_next = results['data']['search']['pageInfo']['hasNextPage'] cursor = results['data']['search']['pageInfo']['endCursor'] else: logger.warn(f'No data found: {results}') has_next = False spinner.finish() print('\n') return entities
def __init__(self, **kwargs): tau = kwargs.get("tau", None) filename = kwargs.get("filename", "infile") seqs, num_seqs, len_seqs = parse_data(filename) leaf_nodes = self.generate_leaves(seqs, tau) self.root = self.generate_tree(leaf_nodes, tau) self.curr = self.root
def main(): ut = UTILS() parameters=ut.read_parameter_file(file = "../para.dat") parameters['L']=2 nrange = np.arange(4,200,4,dtype=int) nrange = [nrange[18]] dt = 0.02 parameters['dt'] = dt mean_fid = [] std_fid = [] for n in nrange: #model = ut.quick_setup(argv=['T=%.3f'%T,'n_step=%i'%n_step],file='../para.dat') parameters['T'] = n*dt parameters['n_step']= n file_name = ut.make_file_name(parameters,root="../data/") res = parse_data(file_name,v=2) # results stored here ... mean_fid.append(np.mean(res['F'])) std_fid.append(np.std(res['F'])) prot = res['protocol'] n_step = len(res['protocol'][0]) plotting.protocol(np.arange(0,n_step)*dt,np.mean(res['protocol'],axis=0),title='$T=%.3f$'%(dt*n_step)) #plt.scatter(nrange*dt,std_fid) #plt.plot(nrange*dt,std_fid) plt.show()
def uploaded_data(content=None): if content is not None: df = utils.parse_data(content) freq_plot, time_plot = utils.generate_plot(df) return freq_plot, time_plot else: raise dash.exceptions.PreventUpdate
def test_parse_data_invalid(invalid_data_frame): invalid_df, cols = invalid_data_frame test_row = list(zip(invalid_df[cols].to_numpy()))[0][0] try: result = parse_data(test_row) except Exception as e: assert type(e) == KeyError
def get_data(self): with tf.name_scope('data'): #1.construct the dataset for train\validate\test. train_data, eval_data, test_data = utils.get_mnist_dataset( self.batch_size) # without val_data iterator = tf.data.Iterator.from_structure( train_data.output_types, train_data.output_shapes) #print(train_data.output_shapes) #((None, 28, 28), (None, 10)) self.train_init = iterator.make_initializer( train_data) # initializer for train_data self.eval_init = iterator.make_initializer(eval_data) self.test_init = iterator.make_initializer(test_data) #2.construct the dataset only for dumping the inter-layers test = utils.parse_data('data/mnist', 't10k', False) #refer to parse_data function test_set = tf.data.Dataset.from_tensor_slices( test) #construct the dataset iterator1 = test_set.make_one_shot_iterator() if self.dump == False: img, self.label = iterator.get_next( ) #fetch batch-size samples, but not always equals to batch_size. #print(img.shape, self.label.shape) #(?, 28, 28) (?, 10) else: img, self.label = iterator1.get_next() #print(img.shape, self.label.shape) #(28, 28) (10, ) length = self.label.shape[0] #equals to label.size self.label = tf.reshape( self.label, shape=[1, length]) #define the shaps as (1, 10) forcedly self.img = tf.reshape( img, shape=[-1, 28, 28, 1]) # -1 represents not-specific, here should be 1
def category(data, data_file): res = [] cat = parse_data(data_file) for k in cat: tmp = [k for e in cat[k] if e in data] res.extend(tmp) return res
def __init__(self, model_type, context_size): samples = [] targets = [] self.vocab = Vocab() self.vocab.add_context(raw_text) pairs = parse_data(raw_text, model_type, context_size) if model_type == 'NEG': self.unigram_dist = build_unigram(raw_text) self.positive_pairs = set( tuple(self.vocab.get_tensor(sample)) for sample, _ in pairs) for sample, target in pairs: if model_type == 'NEG': samples.append(self.vocab.get_tensor(sample)) targets.append(target) elif model_type == 'SKIPGRAM': samples.append(self.vocab.get_tensor([sample])) targets.append(self.vocab.get_tensor([target])) else: samples.append(self.vocab.get_tensor(sample)) targets.append(self.vocab.get_tensor([target])) self.samples = torch.tensor(samples).long() self.targets = torch.tensor(targets).long()
def make_histogram(parameters, version=2): utils = UTILS() file_name = utils.make_file_name(parameters, root="../data/") data = parse_data(file_name, v=version) fidelities = data['F'] plt.hist(fidelities, bins=200) plt.savefig('histogram/hist.pdf')
def on_any_event(event): if event.is_directory: return None elif event.event_type == 'created': # Take any action here when a file is first created. print("Received created event - %s." % event.src_path) if event.src_path.endswith('.json'): try: df = pd.read_json(event.src_path) if len(df) == 0: print("Data point is empty. Removing data point...") os.remove(event.src_path) except: print("Corrupted json file. Removing data point...") os.remove(event.src_path) else: print("Updating data...") Param.json_data = parse_data(Param.WATCH, Param.OUTPUT, Param.json_data, event.src_path, init=False) print("Reloading Supervisor...") os.system('sudo service supervisor reload') else: print("Ignoring")
def menu1(): """ Menu that will import the ship data """ # Declare we are using the global to add the JSON data to it global ship_dict (ship_dict) = utils.parse_data() choice = input("Enter 0 to exit or main for main menu: ") exec_menu(choice) return
def create_normalized_index(): db = connect_db() cursor = db.cursor() data = get_all_data_from_db(cursor) redis = connect_word_to_norm_word() words = parse_data(data) redis_normal_index(redis, words, normalize_bag_of_words(words))
def initialize(): print("Initializing...") print('reading all JSON files from: ' + Param.WATCH) Param.json_data = parse_data(Param.WATCH, Param.OUTPUT, Param.json_data, [], init=True) print("Done.")
def load_ships(list_widget): global ship_dict ship_dict = utils.parse_data() list_widget.clear() # Add all ships to the list for item in ship_dict: list_widget.addItem(item['Name']) alert = QMessageBox() alert.setText('Ships have been loaded from data.json') alert.exec_()
def test_parse_data_legacy(self, relation_get): _rel_data = {'testkey': repr({'test': 1})} relation_get.side_effect = lambda key, relid, unit: _rel_data.get(key) self.assertEqual( utils.parse_data('hacluster:1', 'neutron-api/0', 'testkey'), {'test': 1}) relation_get.assert_has_calls([ mock.call('json_testkey', 'neutron-api/0', 'hacluster:1'), mock.call('testkey', 'neutron-api/0', 'hacluster:1'), ])
def test(): ux, uy = parse_data() usps_data = load_usps(ux, uy, validation_size=5000, test_size=0) mnist_data = load_mnist(one_hot=True, validation_size=5000) x_original = mnist_data.dataset.validation._images wdgrl = WDGRL(input_dim=784, gp_param=10, training_steps=2000, D_train_steps=20) wdgrl.fit(data_src=usps_data, data_tar=mnist_data, draw_plot=True) x_new = wdgrl.transform(x_original)
def migrate_maas_dns(): """ Migrates the MAAS DNS HA configuration to write local IP address information to files. """ if not needs_maas_dns_migration(): log("MAAS DNS migration is not necessary.", INFO) return for relid in relation_ids('ha'): for unit in related_units(relid): resources = parse_data(relid, unit, 'resources') resource_params = parse_data(relid, unit, 'resource_params') if True in [ra.startswith('ocf:maas') for ra in resources.values()]: for resource in resource_params.keys(): if resource.endswith("_hostname"): res_ipaddr = get_ip_addr_from_resource_params( resource_params[resource]) log("Migrating MAAS DNS resource %s" % resource, INFO) write_maas_dns_address(resource, res_ipaddr)
def loc(ip, api_conf): if ip not in mem: with lock: # This check is necesary to check posible memoization of the thread that held the lock if ip not in mem: params = parse_data(api_conf) loc = requests.get("http://api.ipstack.com/{}".format(ip), params=params).json() mem[ip] = { "country": loc["country_name"], "is_eu": loc["location"]["is_eu"] } return mem[ip]
def test_parse_data_legacy(self, relation_get): _rel_data = { 'testkey': repr({'test': 1}) } relation_get.side_effect = lambda key, relid, unit: _rel_data.get(key) self.assertEqual(utils.parse_data('hacluster:1', 'neutron-api/0', 'testkey'), {'test': 1}) relation_get.assert_has_calls([ mock.call('json_testkey', 'neutron-api/0', 'hacluster:1'), mock.call('testkey', 'neutron-api/0', 'hacluster:1'), ])
def test_parse_data_json(self, relation_get): _rel_data = { 'json_testkey': json.dumps({'test': 1}), 'testkey': repr({'test': 1}) } relation_get.side_effect = lambda key, relid, unit: _rel_data.get(key) self.assertEqual( utils.parse_data('hacluster:1', 'neutron-api/0', 'testkey'), {'test': 1}) # NOTE(jamespage): as json is the preferred format, the call for # testkey should not occur. relation_get.assert_has_calls([ mock.call('json_testkey', 'neutron-api/0', 'hacluster:1'), ])
async def callback(callback): try: await dp.throttle('callback', rate=0.8) except Throttled: await bot.answer_callback_query(callback.id, text='Флуд') else: await bot.answer_callback_query(callback.id) data = parse_data(callback.data) if data['action'] == 'open': getattr(view, data['action'])(callback.message)
def ahorros_para_lograr_meta(): data = parse_data(request.get_json()) ini_dep = data.get('ini_dep') fin_bal = data.get('fin_bal') freq = data.get('freq') num_of_years = data.get('num_of_years') rate = data.get('rate') dep_when = data.get('dep_when') time_scale, rows_per_page = data.get('time_scale') periods, periods_m, periods_a = get_periods(freq, num_of_years) fv = fut_val(rate / (100 * freq), freq * num_of_years, ini_dep) reg_dep = -1 * payment(rate / (100 * freq), freq * num_of_years, 0, fin_bal + fv, dep_when) deposits, reg_deps, extra_deps = get_deposits(ini_dep, reg_dep, 0, 0, 0, periods) interests, agg_interests, agg_deposits, balances = get_balances( periods, deposits, ini_dep, rate, freq, dep_when) return jsonify({ 'reg_dep': reg_dep, 'time_scale': time_scale, 'total_dep': sum(deposits), 'total_int': sum(interests), 'fin_bal': balances[-1], 'periods': periods, 'agg_deposits': agg_deposits, 'agg_interests': agg_interests, 'balances': balances, 'table': get_table(periods, deposits, interests, balances), 'table_m': get_table_m(periods_m, deposits, interests, balances, freq), 'table_a': get_table_a(periods_a, deposits, interests, balances, freq) }), 200, HEADERS
def test_parse_data_json(self, relation_get): _rel_data = { 'json_testkey': json.dumps({'test': 1}), 'testkey': repr({'test': 1}) } relation_get.side_effect = lambda key, relid, unit: _rel_data.get(key) self.assertEqual(utils.parse_data('hacluster:1', 'neutron-api/0', 'testkey'), {'test': 1}) # NOTE(jamespage): as json is the preferred format, the call for # testkey should not occur. relation_get.assert_has_calls([ mock.call('json_testkey', 'neutron-api/0', 'hacluster:1'), ])
def main(): ut = UTILS() parameters = ut.read_parameter_file(file="../para.dat") parameters['L'] = 1 n_step = 400 nrange = np.arange(10, 800, 10, dtype=int) Trange = np.arange(0.05, 4.01, 0.05) dt = 0.005 parameters['dt'] = dt mean_fid = [] std_fid = [] n_fid = [] ed1 = [] ed2 = [] for n in nrange: #for n in nrange: #model = ut.quick_setup(argv=['T=%.3f'%T,'n_step=%i'%n_step],file='../para.dat') parameters['T'] = n * dt #parameters['T'] = T parameters['n_step'] = n #parameters['n_step']= n_step #parameters['dt'] = parameters['T']/parameters['n_step'] file_name = ut.make_file_name(parameters, root="../data/") res = parse_data(file_name) # results stored here ... print(n, '\t', len(res['F'])) mean_fid.append(np.max(res['F'])) n_fid.append(np.mean(res['n_fid'])) std_fid.append(np.std(res['F'])) tmp = 8 * (res['protocol'] - 0.5) ed1.append(Ed_Ad_OP(tmp)) #ed2.append(Ed_Ad_OP_2(res['protocol'],min_h=0, max_h=1)) plt.plot(nrange * dt, n_fid, label='ed1') #plt.plot(nrange*dt, ed2,label='ed2') plt.legend(loc='best') plt.show() exit() n_step = len(res['protocol'][0]) plotting.protocol(Trange, np.mean(res['protocol'], axis=0), title='$T=%.3f$' % (dt * n_step)) #plotting.protocol(np.arange(0,n_step)*dt,np.mean(res['protocol'],axis=0),title='$T=%.3f$'%(dt*n_step)) #plt.scatter(nrange*dt,std_fid) #plt.plot(nrange*dt,std_fid) plt.show()
def calculadora_de_ahorros(): data = parse_data(request.get_json()) ini_dep = data.get('ini_dep') reg_dep = data.get('reg_dep') freq = data.get('freq') num_of_years = data.get('num_of_years') rate = data.get('rate') extra_dep = data.get('extra_dep') extra_dep_start = data.get('extra_dep_start') extra_dep_f = data.get('extra_dep_f') dep_when = data.get('dep_when') time_scale, rows_per_page = data.get('time_scale') periods, periods_m, periods_a = get_periods(freq, num_of_years) deposits, reg_deps, extra_deps = get_deposits(ini_dep, reg_dep, extra_dep, extra_dep_start, extra_dep_f, periods) interests, agg_interests, agg_deposits, balances = get_balances( periods, deposits, ini_dep, rate, freq, dep_when) return jsonify({ 'time_scale': time_scale, 'total_dep': sum(deposits), 'total_int': sum(interests), 'fin_bal': balances[-1], 'periods': periods, 'agg_deposits': agg_deposits, 'agg_interests': agg_interests, 'balances': balances, 'table': get_table(periods, deposits, interests, balances), 'table_m': get_table_m(periods_m, deposits, interests, balances, freq), 'table_a': get_table_a(periods_a, deposits, interests, balances, freq) }), 200, HEADERS
async def main(): reader, writer = await asyncio.open_connection(reverse_ip, reverse_port) writer.write(b"con") await writer.drain() print("Connection to {} {} is established, ready to reverse proxy.".format( reverse_ip, reverse_port)) future_list = [] data = b'' while True: dat = await reader.read(2048) data += dat command_list, data = parse_data(data) for command in command_list: coro = make_pipe(command, reverse_ip, reverse_port) asyncio.ensure_future(coro) print("New future created")
def upload_courses(file_path: str, engine: MySQLEngine): """ Uploads all the courses to the MySQL database. :param self: :param file_path: the directory where to search the CSV files :param engine: a MySQLEngine where the data needs to be uploaded """ df = parse_data(file_path) df = df[['courseId', 'creditHours']] # default value of Interest- will need to be hardcoded. df['Interest'] = ['["None"]'] * len(df) gpa_df = extract_gpa_data(file_path) df = merge_gpa_data(df, gpa_df) engine.insert_df(df, 'courses')
def main(): utils = UTILS() model = utils.quick_setup(argv=['T=3.3'], file="../para.dat") parameters = utils.read_parameter_file(file="../para.dat") Trange = np.arange(0.1, 4.01, 0.1) # maybe trick is to bin stuff up --> ?! interval = [0.85, 1.0] #for t in Trange: parameters['T'] = 3.2 parameters['n_step'] = 100 parameters['n_quench'] = 1000 parameters['dt'] = parameters['T'] / parameters['n_step'] file_name = utils.make_file_name(parameters, root="../data/") data = parse_data(file_name, v=3) fid_series = data['fid_series'] protocols = data['protocol'] fid = data['F'] protocols = protocols[(fid < interval[1]) & (fid > interval[0])] # protocols within an interval
def __init__(self, training, args): self.training = training self.classification = True self.augment = False self.data_dir = 'D:/all/' self.df = pd.read_csv(os.path.join(self.data_dir,'stage_1_train_labels.csv')) self.truth = parse_data(self.df) if self.training: self.data_index, _, _,_ = train_test_split(np.arange(len(self.truth)), np.arange(len(self.truth)), test_size=0.25, random_state=42) else: _, self.data_index, _,_ = train_test_split(np.arange(len(self.truth)), np.arange(len(self.truth)), test_size=0.25, random_state=42) self.length = len(self.data_index) self.length = len(self.data_index) with open('val.txt','w') as f: for i in range(self.length): f.write(self.df['patientId'][self.data_index[i]] +'\n')
def calcAll(): """ Calculate the new macd-coefficients for all MACD-objects :return: List of serialized MACD-objects """ global macd_objects global data for macd in macd_objects: try: if macd.pair not in data: data[macd.pair] = fetch(macd.pair) # get data data[macd.pair] = parse_data( data[macd.pair]) # in each pair is stored sdf-data itself except Exception as err: pass sdf = macd.calculate_coefficient(data[macd.pair][macd.time_period]) data = dict() # empty data
def train(epoch, max_epoch, model, dataloader, optimizer, class_num, print_freq=20): acc_mean = 0.0 loss_mean = 0.0 cnt = 0 for batch_idx, data in enumerate(dataloader): imgs, labels = parse_data(data, class_num) batch_size = imgs.shape[0] # print(imgs.shape) # print(imgs[0]) outputs = model(imgs) # if batch_idx == 0: # print(outputs[0]) loss_deriv = L2Loss_deriv(outputs, labels) optimizer.backward(loss_deriv) optimizer.update(imgs) loss = L2Loss(outputs, labels) loss_mean = (loss * batch_size + cnt * loss_mean) / (cnt + batch_size) acc = accuracy(outputs, labels) acc_mean = (acc * batch_size + cnt * acc_mean) / (cnt + batch_size) cnt += batch_size if (batch_idx+1) % print_freq == 0: print('Epoch: [{0}/{1}][{2}/{3}] ' 'Loss {loss:.4f} ({loss_mean:.4f}) ' 'Acc {acc:.2f} ({acc_mean:.2f}) ' 'Lr {lr:.6f} '.format( epoch + 1, max_epoch, batch_idx + 1, len(dataloader), loss=loss, loss_mean=loss_mean, acc=acc, acc_mean=acc_mean, lr=optimizer.lr ) ) return loss_mean, acc_mean
def getInitiateId(yb, type): """ :param yb: 实例化yb对象 :param type: 返回数据类型0:最新表单 1:前一天表单 """ # 通过访问任务列表获取TaskId taskList = yb.taskList() TaskId = taskList.get('data')['list'][type]["LinkTo"].split('=')[1] CompletedList = yb.getTaskDetail(TaskId) # 我的任务 # 对< InitiateId >进行异常捕获,寻找到最新的值 try: InitiateId = CompletedList.get('data')['InitiateId'] except KeyError: if TaskId == 'view&id' or InitiateId is not None: for i in range(10): TaskId = taskList.get('data')['list'][i]["LinkTo"].split( '=')[1] if TaskId != 'view&id': CompletedList = yb.getTaskDetail(TaskId) InitiateId = CompletedList.get('data')['InitiateId'] # WeiSanJin.getInitiateId: print("WeiSanJin.异常处理 :<- InitiateId -> " + str(InitiateId)) break shareUrl = 'https://app.uyiban.com/workflow/client/#/share?initiateId=' + InitiateId # 防止链接失效,再次获取分享链接 ShareUrl = yb.getShareUrl(InitiateId) # 将最新的分享链接写入 config.txt with open('config.txt', 'w') as f: # print('\n' + "分享链接:" + shareUrl) f.write(shareUrl) # 返回最新的表单数据 return parse_data(shareUrl, yb)
def main(): reply = eg.boolbox(msg='Hello, do you have a pre-computed classifier?', choices=('Yes', 'No')) if reply == 1: filename = eg.fileopenbox(msg='Please specificy cached classifier file ending in .clfr') model = utils.load_classifiers(filename) reply = eg.boolbox(msg='Now that we have specified our classifier, we must now specify data to be classified. Are you ready to proceed?', choices=('Yes', 'No')) if reply == 0: sys.exit() filename = eg.fileopenbox(msg='Please specify data to be classified.') D = utils.parse_data(filename) outfilename = ''.join(filename.split('.')[:-1]) + '.lbls' print 'Classifying data...' with open(outfilename, 'w') as f: for d in D: f.write(str(model.classify_vector(d))) f.write('\n') print 'Done!' else: filename = eg.fileopenbox(msg='Please specify training data for your classifier') D = utils.parse_data(filename) reply = eg.boolbox(msg='Would you like to engage in manual or automatic construction of your classifier?', choices=('Manual', 'Automatic')) #manual selection if reply == 1: algs = eg.multchoicebox(msg='Please choose at least one algorithm:', choices=('k-Nearest Neighbors', 'Support Vector Machine', 'Naive Bayes')) alg_params = {alg : 'auto' for alg in algs} #storage for set of classifiers C = [] for alg in algs: reply = eg.boolbox(msg='Would you like to engage in manual or automatic parameter tuning for your ' + alg + ' algorithm?', choices=('Manual', 'Automatic')) if reply == 1: if alg[0] == 'k': params = eg.multenterbox(msg='Please select the following parameters for your ' + alg + ' algorithm:', fields=('k'), values=['1']) print 'Building ' + alg + ' classifier...' C.append(cl.kNN(D, k=int(params[0]))) print 'Done!\n' if alg[0] == 'S': reply = eg.boolbox(msg='What type of kernel would you like to use for your Support Vector Machine?', choices=('Radial Basis Function', 'Linear')) if reply == 1: params = eg.multenterbox(msg='Please select the following parameters for your RBF Support Vector Machine:', fields=('margin', 'gamma'), values=['1.0', '1.0']) print 'Building ' + alg + ' classifier...' C.append(cl.SVM(D, kernel_type='RBF', margin=float(params[0]), gamma=float(params[1]))) print 'Done!\n' else: params = eg.enterbox(msg='Please select the margin parameter for your Linear Support Vector Machine:', default='1.0') print 'Building ' + alg + ' classifier...' C.append(cl.SVM(D, kernel_type='Linear', margin=float(params[0]))) print 'Done!\n' if alg[0] == 'N': params = eg.enterbox(msg='Please select the binning threshold parameter for your Naive Bayes algorithm:', default='.01') print 'Building ' + alg + ' classifier...' C.append(cl.NB(D)) print 'Done!\n' else: if alg[0] == 'k': print 'Building ' + alg + ' classifier...' C.append(cl.kNN(D)) print 'Done!\n' if alg[0] == 'S': print 'Building ' + alg + ' classifier...' C.append(cl.SVM(D)) print 'Done!\n' if alg[0] == 'N': print 'Building ' + alg + ' classifier...' C.append(cl.NB(D)) print 'Done!\n' model = mcl.AdaBoost(C) #automatic selection else: print 'Constructing classifiers...' model = mcl.AdaBoost([cl.kNN(D), cl.SVM(D), cl.NB(D)]) print 'Done!\n' reply = eg.boolbox(msg='Now that we have specified our classifier, we must now specify data to be classified. Are you ready to proceed?', choices=('Yes', 'No')) if reply == 0: sys.exit() filename = eg.fileopenbox(msg='Please specify data to be classified.') D = utils.parse_data(filename) outfilename = ''.join(filename.split('.')[:-1]) print 'Classifying data...' with open(outfilename + '.lbls', 'w') as f: for d in D: f.write(str(model.classify_vector(d))) f.write('\n') print 'Done!' #cache our classifier utils.store_classifiers(outfilename + '.clfr', model) #give user some information on classifiers used open(outfilename + '.info', 'w').write(model.get_info())
""" Run data-level preprocessing so we don't have to do it every epoch """ import utils from collections import Counter import csv import codecs if __name__ == '__main__': for filename in [utils.TRAIN_FILENAME, utils.TRAIN_PLUS_DEV_FILENAME, utils.DEV_FILENAME, utils.TEST_FILENAME]: utils.parse_data(filename)
def ha_relation_changed(): # Check that we are related to a principle and that # it has already provided the required corosync configuration if not get_corosync_conf(): log('Unable to configure corosync right now, deferring configuration', level=INFO) return if relation_ids('hanode'): log('Ready to form cluster - informing peers', level=DEBUG) relation_set(relation_id=relation_ids('hanode')[0], ready=True) else: log('Ready to form cluster, but not related to peers just yet', level=INFO) return # Check that there's enough nodes in order to perform the # configuration of the HA cluster if len(get_cluster_nodes()) < int(config('cluster_count')): log('Not enough nodes in cluster, deferring configuration', level=INFO) return relids = relation_ids('ha') if len(relids) == 1: # Should only ever be one of these # Obtain relation information relid = relids[0] units = related_units(relid) if len(units) < 1: log('No principle unit found, deferring configuration', level=INFO) return unit = units[0] log('Parsing cluster configuration using rid: %s, unit: %s' % (relid, unit), level=DEBUG) resources = parse_data(relid, unit, 'resources') delete_resources = parse_data(relid, unit, 'delete_resources') resource_params = parse_data(relid, unit, 'resource_params') groups = parse_data(relid, unit, 'groups') ms = parse_data(relid, unit, 'ms') orders = parse_data(relid, unit, 'orders') colocations = parse_data(relid, unit, 'colocations') clones = parse_data(relid, unit, 'clones') locations = parse_data(relid, unit, 'locations') init_services = parse_data(relid, unit, 'init_services') else: log('Related to %s ha services' % (len(relids)), level=DEBUG) return if True in [ra.startswith('ocf:openstack') for ra in resources.values()]: apt_install('openstack-resource-agents') if True in [ra.startswith('ocf:ceph') for ra in resources.values()]: apt_install('ceph-resource-agents') if True in [ra.startswith('ocf:maas') for ra in resources.values()]: try: validate_dns_ha() except MAASConfigIncomplete as ex: log(ex.args[0], level=ERROR) status_set('blocked', ex.args[0]) # if an exception is raised the hook will end up in error state # which will obfuscate the workload status and message. return log('Setting up access to MAAS API', level=INFO) setup_maas_api() # Update resource_parms for DNS resources to include MAAS URL and # credentials for resource in resource_params.keys(): if resource.endswith("_hostname"): res_ipaddr = get_ip_addr_from_resource_params( resource_params[resource]) resource_params[resource] += ( ' maas_url="{}" maas_credentials="{}"' ''.format(config('maas_url'), config('maas_credentials'))) write_maas_dns_address(resource, res_ipaddr) # NOTE: this should be removed in 15.04 cycle as corosync # configuration should be set directly on subordinate configure_corosync() try_pcmk_wait() configure_cluster_global() configure_monitor_host() configure_stonith() # Only configure the cluster resources # from the oldest peer unit. if is_leader(): log('Deleting Resources' % (delete_resources), level=DEBUG) for res_name in delete_resources: if pcmk.crm_opt_exists(res_name): if ocf_file_exists(res_name, resources): log('Stopping and deleting resource %s' % res_name, level=DEBUG) if pcmk.crm_res_running(res_name): pcmk.commit('crm -w -F resource stop %s' % res_name) else: log('Cleanuping and deleting resource %s' % res_name, level=DEBUG) pcmk.commit('crm resource cleanup %s' % res_name) # Daemon process may still be running after the upgrade. kill_legacy_ocf_daemon_process(res_name) pcmk.commit('crm -w -F configure delete %s' % res_name) log('Configuring Resources: %s' % (resources), level=DEBUG) for res_name, res_type in resources.items(): # disable the service we are going to put in HA if res_type.split(':')[0] == "lsb": disable_lsb_services(res_type.split(':')[1]) if service_running(res_type.split(':')[1]): service_stop(res_type.split(':')[1]) elif (len(init_services) != 0 and res_name in init_services and init_services[res_name]): disable_upstart_services(init_services[res_name]) if service_running(init_services[res_name]): service_stop(init_services[res_name]) # Put the services in HA, if not already done so # if not pcmk.is_resource_present(res_name): if not pcmk.crm_opt_exists(res_name): if res_name not in resource_params: cmd = 'crm -w -F configure primitive %s %s' % (res_name, res_type) else: cmd = ('crm -w -F configure primitive %s %s %s' % (res_name, res_type, resource_params[res_name])) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) if config('monitor_host'): cmd = ('crm -F configure location Ping-%s %s rule ' '-inf: pingd lte 0' % (res_name, res_name)) pcmk.commit(cmd) else: # the resource already exists so it will be updated. code = pcmk.crm_update_resource(res_name, res_type, resource_params.get(res_name)) if code != 0: msg = "Cannot update pcmkr resource: {}".format(res_name) status_set('blocked', msg) raise Exception(msg) log('Configuring Groups: %s' % (groups), level=DEBUG) for grp_name, grp_params in groups.items(): if not pcmk.crm_opt_exists(grp_name): cmd = ('crm -w -F configure group %s %s' % (grp_name, grp_params)) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) log('Configuring Master/Slave (ms): %s' % (ms), level=DEBUG) for ms_name, ms_params in ms.items(): if not pcmk.crm_opt_exists(ms_name): cmd = 'crm -w -F configure ms %s %s' % (ms_name, ms_params) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) log('Configuring Orders: %s' % (orders), level=DEBUG) for ord_name, ord_params in orders.items(): if not pcmk.crm_opt_exists(ord_name): cmd = 'crm -w -F configure order %s %s' % (ord_name, ord_params) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) log('Configuring Clones: %s' % clones, level=DEBUG) for cln_name, cln_params in clones.items(): if not pcmk.crm_opt_exists(cln_name): cmd = 'crm -w -F configure clone %s %s' % (cln_name, cln_params) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) # Ordering is important here, colocation and location constraints # reference resources. All resources referenced by the constraints # need to exist otherwise constraint creation will fail. log('Configuring Colocations: %s' % colocations, level=DEBUG) for col_name, col_params in colocations.items(): if not pcmk.crm_opt_exists(col_name): cmd = 'crm -w -F configure colocation %s %s' % (col_name, col_params) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) log('Configuring Locations: %s' % locations, level=DEBUG) for loc_name, loc_params in locations.items(): if not pcmk.crm_opt_exists(loc_name): cmd = 'crm -w -F configure location %s %s' % (loc_name, loc_params) pcmk.commit(cmd) log('%s' % cmd, level=DEBUG) for res_name, res_type in resources.items(): if len(init_services) != 0 and res_name in init_services: # Checks that the resources are running and started. # Ensure that clones are excluded as the resource is # not directly controllable (dealt with below) # Ensure that groups are cleaned up as a whole rather # than as individual resources. if (res_name not in clones.values() and res_name not in groups.values() and not pcmk.crm_res_running(res_name)): # Just in case, cleanup the resources to ensure they get # started in case they failed for some unrelated reason. cmd = 'crm resource cleanup %s' % res_name pcmk.commit(cmd) for cl_name in clones: # Always cleanup clones cmd = 'crm resource cleanup %s' % cl_name pcmk.commit(cmd) for grp_name in groups: # Always cleanup groups cmd = 'crm resource cleanup %s' % grp_name pcmk.commit(cmd) for rel_id in relation_ids('ha'): relation_set(relation_id=rel_id, clustered="yes")
# # Push data live to API using Python on a RaspberryPi # import serial from utils import parse_data, send_data ser = serial.Serial('/dev/ttyACM0') while True: message = ser.readline() data = parse_data(message) if data: send_data(data)
else: print "no trimming" print "Algorithm type:", if algorithm_type == ClusterAlg.KMEANS: print "kmeans" elif algorithm_type == ClusterAlg.HIERARCHICAL: print "hierarchical" elif algorithm_type == ClusterAlg.SELFORGMAPS: print "selforgmaps" print "Number of points in SOM grid (xgrid * ygrid):", xgrid * ygrid # Parse input data. try: data = utils.parse_data(input_file_path) except IOError, err: sys.exit(err) # Import requested events. # TODO(karol): this may have to be moved from here to the plotting # file. try: events = { } if import_catastrophic_events: catastrophic_events = eventutils.import_events( "../data/wydarzenia-katastrofy-polska.txt") events[catastrophic_events[0]] = catastrophic_events[1] if import_economical_events: economical_events = eventutils.import_events(
event_paths['ekonomiczne'] = '../data/wydarzenia-ekonomiczne-polska.txt' event_paths['inne'] = '../data/wydarzenia-inne-polska.txt' event_file_path = event_paths[arg[7:]] if arg.startswith('xrange:'): if arg[7:]: set_xrange = 'set xrange [%s]' % arg[7:] # If error message is not empty print it and exit. if err_msg != "": print err_msg sys.exit(1) # Parse stock data. data = utils.parse_data("../data/notowania.txt") found_cluster = False plot_tmpfiles = [] if all_clusters: clusters_count = detect_clusters_count(clusters_file_path) for i in range(clusters_count): _, plot_data = plot_single_cluster(clusters_file_path, i, average_plot) plot_tmpfiles.append(plot_data) else: found_cluster, plot_data_tmpfile_path = plot_single_cluster(clusters_file_path, cluster_number, average_plot) if found_cluster: # Generate file with commands for gnuplot.
def setUp(self): self.data1 = utils.parse_data(self.test_file_path1) self.data2 = utils.parse_data(self.test_file_path2)