def aggregate(self, video, aggregated): cls = BatchIVA batch = Batch() ts_set = [] coder_dat_buf = dict() coder_dat_len = dict() videoL = 999999999999L for c in cls.coder_set: name = "+".join([c, video]) filename = os.path.join(cls.feedback_folder, "feedback/"+name+".txt") if not os.path.exists(filename): print filename, "not exists" continue feedback = [] videoLen = batch._processFeedbackFile( filename, feedback ) coder_dat_buf[name] = feedback coder_dat_len[name] = videoLen if videoLen < videoL: videoL = videoLen try: for k in coder_dat_buf.iterkeys(): v = coder_dat_buf[k] if len(v) == 0: pass elif coder_dat_len[k] > videoL + 1500: print coder_dat_len[k], ";", videoL, " [aggregate] ", video, k, " is outlier" else: ts_set.append(v) except Exception, exception: raise exception
def _matchWithFeature(self, c, v, coder_feature, window=1000): cls = BatchIVA batch = Batch() name = "+".join( [c, v] ) filename = os.path.join( cls.feedback_folder, "feedback/"+name+".txt" ) if not os.path.exists(filename): print filename, "not exists" return feedback = [] videoLen = batch._processFeedbackFile(filename, feedback) if c not in cls.coder_backchannel_num: cls.coder_backchannel_num[c] = len(feedback) else: cls.coder_backchannel_num[c] = cls.coder_backchannel_num[c] + len(feedback) try: for feature_name in cls.video_info_buffer[v].iterkeys(): feature_values = cls.video_info_buffer[v][feature_name] for feature_value in feature_values: for f in feedback: if feature_value[0] <= f and f <= feature_value[1]+window: if feature_name in coder_feature: coder_feature[feature_name] = coder_feature[feature_name]+1 else: coder_feature[feature_name] = 1 except Exception, exception: print exception
def load_keyboard_consensus(self, speaker_id): result = [] data_root = os.path.join(os.path.dirname(__file__), "../analysis/data/") batch = Batch() batch.load(data_root) batch._getDataOfVideo(speaker_id, result) return result
def driver_listener(transaction_queue): start = time() driver = Driver() i = 0 while True: batch_file = transaction_queue.get() batch = Batch() batch.load(batch_file) for transaction in batch.items: try: added = driver.run(transaction) duration = time() - start total = len(driver.hset) + len(driver.lset) print('Driver rate: {} of {} ({}|{})\r'.format(round(total / duration, 3), total, len(driver.hset), len(driver.lset)), flush=True, end='') if added: i += 1 except KeyboardInterrupt: raise KeyboardInterrupt except neobolt.exceptions.CypherSyntaxError: pass except Exception as e: print(e, flush=True) print(transaction.in_label, flush=True) print(transaction.out_label, flush=True) print(transaction.uuid, flush=True) print(transaction.from_uuid, flush=True) print(transaction.data, flush=True)
def __init__(self, dbname, rounds): isp = dbname[3:7] self.isp = isp self.ISP_edges_file = os.getcwd () + '/ISP_topo/'+str (isp) +'_edges.txt' self.ISP_nodes_file = os.getcwd () + '/ISP_topo/'+str (isp)+'_nodes.txt' self.rib_prefixes_file = os.getcwd() + '/rib_feeds/' + "rib20011204_prefixes.txt" self.rib_peerIPs_file = os.getcwd() + '/rib_feeds/' + "rib20011204_nodes.txt" print "for database " + dbname + "--------------------" create_db (dbname, Batch.username) # add comments if database_exists == 0: add_pgrouting_plpy_plsh_extension (dbname, Batch.username) load_schema (dbname, Batch.username, "/home/mininet/ravel/sql_scripts/primitive.sql") Batch_isp.init_ISP_topo (self, dbname) Batch.__init__(self,dbname, rounds) remove_profile_schema (self.cur) rib_feeds_all = os.getcwd() + '/rib_feeds/rib20011204_edges.txt' feeds = dbname[8:] self.rib_edges_file = os.getcwd() + '/rib_feeds/rib20011204_edges_' + str (feeds) + '.txt' os.system ("head -n " + str(feeds) + " " + rib_feeds_all + " > " + self.rib_edges_file) Batch_isp.init_rib (self)
def fetch(self): Batch.fetch(self) self.cur.execute("SELECT * FROM tm;") cs = self.cur.fetchall() self.tm = [[h['fid'], h['src'], h['dst'], h['vol'], h['fw'], h['lb']] for h in cs]
def _matchWithFeature(self, coderId, videoId, coder_feature, video_feature, window=1000): try: cls = DiveFeature feedback = [] name = "+".join( [coderId, videoId] ) filename = os.path.join( cls.data_root, "feedback/" + name + ".txt" ) batch = Batch() videoLen = batch._processFeedbackFile(filename, feedback) try: for feature_name in cls.video_info_buffer[videoId].iterkeys(): feature_values = cls.video_info_buffer[videoId][feature_name] for feature_value in feature_values: for f in feedback: if feature_value[0] <= f and f <= feature_value[1]+window: if feature_name in video_feature: video_feature[feature_name] = video_feature[feature_name] + 1 else: video_feature[feature_name] = 1 if feature_name in coder_feature: coder_feature[feature_name] = coder_feature[feature_name] + 1 else: coder_feature[feature_name] = 1 except Exception, exception: print exception except Exception, exception: print "_matchWithFeature => ", exception
def __init__(self, dbname, rounds): isp = dbname[3:7] self.isp = isp self.ISP_edges_file = os.getcwd() + '/ISP_topo/' + str( isp) + '_edges.txt' self.ISP_nodes_file = os.getcwd() + '/ISP_topo/' + str( isp) + '_nodes.txt' self.rib_prefixes_file = os.getcwd( ) + '/rib_feeds/' + "rib20011204_prefixes.txt" self.rib_peerIPs_file = os.getcwd( ) + '/rib_feeds/' + "rib20011204_nodes.txt" print "for database " + dbname + "--------------------" create_db(dbname, Batch.username) # add comments if database_exists == 0: add_pgrouting_plpy_plsh_extension(dbname, Batch.username) load_schema(dbname, Batch.username, "/home/mininet/ravel/sql_scripts/primitive.sql") Batch_isp.init_ISP_topo(self, dbname) Batch.__init__(self, dbname, rounds) remove_profile_schema(self.cur) rib_feeds_all = os.getcwd() + '/rib_feeds/rib20011204_edges.txt' feeds = dbname[8:] self.rib_edges_file = os.getcwd( ) + '/rib_feeds/rib20011204_edges_' + str(feeds) + '.txt' os.system("head -n " + str(feeds) + " " + rib_feeds_all + " > " + self.rib_edges_file) Batch_isp.init_rib(self)
def post(self): """ Execute GraphQL queries and mutations Use this endpoint to send http request to the GraphQL API. """ payload = request.json # Execute request on GraphQL API status, data = utils.execute_graphql_request(payload['query']) # Execute batch of indicators if status == 200 and 'executeBatch' in payload['query']: if 'id' in data['data']['executeBatch']['batch']: batch_id = str(data['data']['executeBatch']['batch']['id']) batch = Batch() batch.execute(batch_id) else: message = "Batch Id attribute is mandatory in the payload to be able to trigger the batch execution. Example: {'query': 'mutation{executeBatch(input:{indicatorGroupId:1}){batch{id}}}'" abort(400, message) # Test connectivity to a data source if status == 200 and 'testDataSource' in payload['query']: if 'id' in data['data']['testDataSource']['dataSource']: data_source_id = str( data['data']['testDataSource']['dataSource']['id']) data_source = DataSource() data = data_source.test(data_source_id) else: message = "Data Source Id attribute is mandatory in the payload to be able to test the connectivity. Example: {'query': 'mutation{testDataSource(input:{dataSourceId:1}){dataSource{id}}}'" abort(400, message) if status == 200: return jsonify(data) else: abort(500, data)
def clusterize(path, algo): attacks = list(read_csv(path)) cluster_builder = ClusterBuilder(nitems=len(attacks)) batch = Batch() addr_to_index = {} for index, attack in enumerate(attacks): batch.feed(attack) addr = attack.source_addr try: prev_index = addr_to_index[addr] except KeyError: pass else: cluster_builder.merge(prev_index, index) addr_to_index[addr] = index db = algo.fit([batch.features(attack) for attack in attacks]) specimen = [-1 for _ in range(len(set(db.labels_)))] for index, attack in enumerate(attacks): cluster = db.labels_[index] if cluster == -1: cluster_builder.mark_as_noise(index) continue if specimen[cluster] != -1: cluster_builder.merge(specimen[cluster], index) specimen[cluster] = index return Clusters(attacks=attacks, classes=cluster_builder.finalize())
def tenant_fullmesh(self, hosts): f = self.f cur = self.cur Batch.update_max_fid(self) fid = self.max_fid + 1 cur.execute("select max (counts) from clock;") ct = cur.fetchall()[0]['max'] + 1 for i in range(len(hosts)): for j in range(i + 1, len(hosts)): print "tenant_fullmesh: [" + str(hosts[i]) + "," + str( hosts[j]) + "]" t1 = time.time() cur.execute("INSERT INTO tenant_policy values (%s,%s,%s);", ([str(fid), int(hosts[i]), int(hosts[j])])) cur.execute("INSERT INTO p_spv values (%s,'on');", ([ct])) t2 = time.time() f.write('----rt*tenant: route ins----' + str((t2 - t1) * 1000) + '\n') f.flush() ct += 1 fid += 1
def run(self): with open(self.train_data, 'rb') as f: data = pickle.load(f) with open(self.train_labels, 'rb') as f: labels = pickle.load(f) labels = one_hot(sorted(list(set(labels))), labels) with open(self.test_data, 'rb') as f: data_t = pickle.load(f) with open(self.test_labels, 'rb') as f: labels_t = pickle.load(f) labels_t = one_hot(sorted(list(set(labels_t))), labels_t) b = Batch(data, labels, Params.batch_size) var = tf.trainable_variables() conv = [v for v in var if v.name.startswith("conv")] fool = [v for v in var if v.name.startswith("fooling")] fc = [v for v in var if v.name.startswith("fc")] smax = [v for v in var if v.name.startswith("soft_max")] cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=self.output)) optimiser = tf.train.AdamOptimizer(Params.learning_rate).minimize(cross_entropy, var_list=conv + fc + smax) # collect prediction in the batch correct_prediction = tf.equal(tf.argmax(self.pred, 1), tf.argmax(self.output, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) total_batch = int(len(data) / Params.batch_size) learning = [] if self.trainable: saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(Params.epoch): b.shuffle() avg_cost = 0 print ("{} epoch".format(epoch)) for i in range(total_batch): batch_x, batch_y = b.next_batch() _, cost = sess.run([optimiser, cross_entropy], feed_dict={model.input: batch_x, model.output: batch_y}) avg_cost += cost/total_batch acc = sess.run(accuracy, feed_dict={model.input: data_t, model.output: labels_t}) learning.append(acc) # saving the model if epoch % 10 == 0: pass # checkpoint_path = os.path.join(Params.checkpoint_path, 'model.ckpt') # save_path = saver.save(sess, checkpoint_path) # print("model saved to {}".format(checkpoint_path)) # print (filters) # print(avg_cost, acc) plt.plot(learning) plt.title('Epoch vs Test accuracy') plt.xlabel('Epoch') plt.ylabel('Test accuracy') plt.show()
def close (self): os.system ("cp "+ Batch.logfile + ' ' + self.logdest) if self.profile == True: os.system ("sudo mv "+ self.logdest + ' ' + ' /media/sf_share/ravel_plot/profile/log/') else: os.system ("sudo mv "+ self.logdest + ' ' + ' /media/sf_share/ravel_plot/fattree/log/') Batch.close (self)
class DistributionTestCase(unittest.TestCase): def setUp(self): self.job = Job() self.batch = Batch() self.resource = Resource() rootDir = os.environ['BOLT_DIR'] self.batch.readConfig(rootDir + configDir + "/" + batchConfig) self.resource.readConfig(rootDir + configDir + "/" + resourceConfig) def testParallelTaskDitributionPureMPI(self): """Pure MPI task distribution (fully populated).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(self.resource.numCoresPerNode()) self.job.setThreads(1) self.job.setParallelDistribution(self.resource, self.batch) correct = "aprun -n 1024 -N 32 -S 8 -d 1" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct)) def testParallelTaskDitributionHalfPopulate(self): """Pure MPI task distribution (half populated).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(16) self.job.setThreads(1) self.job.setParallelDistribution(self.resource, self.batch) correct = "aprun -n 1024 -N 16 -S 4 -d 2" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct)) def testParallelTaskDitributionTwoThreads(self): """Hybrid MPI/OpenMP task distribution (2 OpenMP threads).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(16) self.job.setThreads(2) self.job.setParallelDistribution(self.resource, self.batch) correct = "export OMP_NUM_THREADS=2\naprun -n 1024 -N 16 -S 4 -d 2" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct)) def testParallelTaskDitributionThreeThreads(self): """Hybrid MPI/OpenMP task distribution (3 OpenMP threads).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(10) self.job.setThreads(3) self.job.setParallelDistribution(self.resource, self.batch) correct = "export OMP_NUM_THREADS=3\naprun -n 1024 -N 10 -d 3" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct))
def execute_batch(self, max_nodes, ami, instance_type): batch_id = 'batch-%s' % uuid.uuid4() batch = Batch('received') batch.ami = ami batch.instance_type = instance_type batch.max_nodes = max_nodes self.client.set(batch_id, pickle.dumps(batch)) self.client.publish('batches', batch_id) return batch_id
def init_rib (self): cursor = self.cur ISP_edges_file = self.ISP_edges_file ISP_nodes_file = self.ISP_nodes_file rib_prefixes_file = self.rib_prefixes_file rib_peerIPs_file = self.rib_peerIPs_file rib_edges_file = self.rib_edges_file def peerIP_ISP_map (peerIP_nodes_file, ISP_nodes_file): pf = open (peerIP_nodes_file, "r").readlines () ispf = open (ISP_nodes_file, "r").readlines () node_map = {} for pn in pf: ISP_node = random.choice (ispf) ispf.remove (ISP_node) node_map[pn[:-1]] = int (ISP_node[:-1]) return node_map # map (randomly picked) ISP nodes (switch nodes in tp table) # to peer IPs in rib feeds nm = peerIP_ISP_map (rib_peerIPs_file, ISP_nodes_file) ISP_borders = nm.values () cursor.execute (""" DROP TABLE IF EXISTS borders CASCADE; CREATE UNLOGGED TABLE borders ( sid integer, peerip text ); """) # set up borders table, randomly pick 21 switches, and assign # each switch a unique peer IP for key in nm.keys(): cursor.execute ("""INSERT INTO borders (sid, peerip) VALUES (%s, %s)""", (nm[key], key)) cursor.execute (""" SELECT * FROM uhosts, borders WHERE hid = 100000 + sid; """) cs = self.cur.fetchall () sid2u_hid = {h['sid']: int (h['u_hid']) for h in cs} # print len (sid2u_hid) ribs = open (rib_edges_file, "r").readlines () Batch.update_max_fid (self) fid = self.max_fid + 1 for r in ribs: switch_id = int (nm [r.split ()[0]]) random_border = int(random.choice (ISP_borders)) if random_border != switch_id: cursor.execute ("INSERT INTO rtm VALUES (%s,%s,%s);", (fid, sid2u_hid[switch_id], sid2u_hid[random_border])) fid += 1
def __init__(self, args): self._bidsDir = args.bids_dir self._dicomDir = args.dicom_dir self._session = Session(args.session, args.participant, self._bidsDir) self._parser = getattr(studyparser, args.algorithm)(self._dicomDir, self._session) self._yes = args.yes self._codeDir = os.path.join(self._bidsDir, 'code') utils.make_directory_tree(self._codeDir) self._batch = Batch(self._codeDir, self._session)
def execute_batch(self, max_nodes, ami, instance_type, email=''): batch_id = 'batch-%s' % str(uuid.uuid4())[31:36] batch = Batch('received') batch.ami = ami batch.instance_type = instance_type batch.max_nodes = max_nodes batch.email = email self.client.set(batch_id, pickle.dumps(batch)) self.client.publish('batches', batch_id) return batch_id
def regress(simu, policy, policy_type, nb_trajs, render=False) -> None: batch = Batch() simu.env.set_reward_flag(False) simu.env.set_duration_flag(False) if policy_type == "bernoulli" or policy_type == "discrete": batch = perform_expert_episodes_bangbang(simu, batch, nb_trajs, render) else: batch = perform_expert_episodes_continuous(simu, batch, nb_trajs, render) # print("size: ", batch.size()) batch.train_policy_through_regress(policy)
def close(self): os.system("cp " + Batch.logfile + ' ' + self.logdest) if self.profile == True: os.system("sudo mv " + self.logdest + ' ' + ' /media/sf_share/ravel_plot/profile/log/') else: os.system("sudo mv " + self.logdest + ' ' + ' /media/sf_share/ravel_plot/fattree/log/') Batch.close(self)
def get_decode_data(hps, vocab, data_path, randomize=False): tf.logging.info('Fetching data..') filelist = glob.glob(data_path) inputs = [] total_examples = 0 total_batches = 0 for f in filelist: reader = open(f, 'rb') while True: len_bytes = reader.read(8) if not len_bytes: break str_len = struct.unpack('q', len_bytes)[0] example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0] e = example_pb2.Example.FromString(example_str) try: article_text = e.features.feature['article'].bytes_list.value[ 0].decode() if len(article_text) == 0: #tf.logging.warning('Found an example with empty article text. Skipping it.') pass else: abstract_text = e.features.feature[ 'abstract'].bytes_list.value[0].decode() abstract_sentences = [ sent.strip() for sent in data.abstract2sents(abstract_text) ] example = Example(article_text, abstract_sentences, vocab, hps) inputs.append(example) total_examples = total_examples + 1 except ValueError: #tf.logging.error('Failed to get article or abstract from example') continue batches = [] tf.logging.info('Creating batches..') if randomize: random.shuffle(inputs) example = inputs[0] b = [example for _ in range(hps.beam_size)] batches.append(Batch(b, hps, vocab)) total_batches = 1 total_examples = 1 else: for i in range(0, len(inputs)): b = [inputs[i] for _ in range(hps.beam_size)] batches.append(Batch(b, hps, vocab)) total_batches = total_batches + 1 tf.logging.info('[TOTAL Batches] : %i', total_batches) tf.logging.info('[TOTAL Examples] : %i', total_examples) tf.logging.info('Creating batches..COMPLETE') return batches
def main(svc_input, configs): logger = Logger("查询日志", verbose=True) log_file_name = "log%s_%s.txt" % (svc_input.replace("?", "#"), DateTimeUtil.get_current_datetime(is_date=True)) log_file_path = WindowsUtil.convert_win_path(os.path.join(temp_dir, log_file_name)) logger.info("[开始查询] %s" % svc_input) try: # 找到本地匹配的保修历史记录 history_zip = ZipFileSVC(zip_file_path=history_zipfile, mode='a') start_time = DateTimeUtil.get_current_datetime() # 创建出所有可能查询码 svc_generator = SVCGenerator(svc_input, logger) logger.info("创建出所有可能查询码:%s" % len(svc_generator.target_svc_set)) # 根据本地匹配的非法查询码历史,筛选出目标查询码,以及非法查询码 existed_svc = history_zip.find_file_regex(svc_generator.regex) svc_generator.generate_target_svc_batch(existed_svc, invalid_history_file_path) # 调用戴尔查询API,并将API数据转化为实体类数据 output_dell_asset_list = list([]) if svc_generator.target_svc_set: batch = Batch(logger, configs) api_dell_asset_list = batch.begin(svc_generator.target_svc_set) output_dell_asset_list = api_dell_asset_list logger.info("从API中总共得到%s个结果" % (len(api_dell_asset_list))) logger.info("将实体类序列化到本地临时TXT文件") temp_text_files_path = DellAsset.serialize_txt_batch(api_dell_asset_list, temp_dir) logger.info("将序列化临时文件存到本地zip历史记录,总数:%s" % len(temp_text_files_path)) history_zip.add_new_file_batch(temp_text_files_path) logger.info("删除临时 %s 个TXT文件" % len(temp_text_files_path)) for file_path in temp_text_files_path: FileUtil.delete_file(file_path) logger.info("将API得到的实体类和历史记录实体类合并") else: logger.warn("目标查询码为空,仅从从历史记录中导出结果") for svc in svc_generator.existed_svc_set: dell_asset_content = history_zip.get_member_content(file_name="%s.txt" % svc) output_dell_asset_list.append(DellAsset.deserialize_txt(dell_asset_content)) logger.info("添加历史记录,总共得到%s个结果" % (len(output_dell_asset_list))) excel_output_path = WindowsUtil.convert_win_path(os.path.join(excel_dir, "%s.xlsx" % svc_generator.get_file_name())) DellAsset.save_as_excel_batch(output_dell_asset_list, excel_output_path) if FileUtil.is_path_existed(excel_output_path): logger.info("存为Excel文档成功") end_time = DateTimeUtil.get_current_datetime() logger.info("总用时 %s " % DateTimeUtil.datetime_diff(start_time, end_time)) logger.info("[查询结束] 总共%s个结果 保存在:%s" % (len(output_dell_asset_list), excel_output_path)) else: logger.error("[保存结果失败] %s" % excel_output_path) except Exception as e: # 若程序出现错误失败,发送邮件 logger.error("[查询失败] 已发送报告 请等待解决") logger.error("%s\n%s" % (e, traceback.format_exc())) logger.save(log_file_path) email_api_key = configs["email_api_key"] email = Email(email_api_key, subject="[查询失败] %s %s" % (DateTimeUtil.get_current_datetime(is_date=True), svc_input)) email.add_attachment(log_file_path) email.send(cc_mode=logger.has_error)
def __init__(self, batch_size, output_file, schema, validator_map): write_function = csv.writer(output_file, delimiter=',', lineterminator='\n').writerows self.batch = Batch(batch_size, write_function) self.header = None self.header_map = {} self.header_written = False self.schema = schema self.validator_map = validator_map
def export_to_batch(self, path, return_batch=False): batchable_txs = [] for account_index in self.known_accounts: batchable_tx = self.cache.load(Cache.TX, account_index) if batchable_tx: batchable_txs.append(batchable_tx) batch = Batch(self.origin_branch.master_key_names, self.destination_branch.master_key_names, batchable_txs=batchable_txs) batch.validate() batch.to_file(path) if return_batch: return batch
def close (self): os.system ("cp "+ Batch.logfile + ' ' + self.logdest) if self.isp == '4755' or self.isp == '3356' or self.isp == '7018': t = 'isp_3sizes' elif self.isp == '2914': t = 'isp' + self.isp + '_3ribs' if self.profile == True: os.system ("sudo mv "+ self.logdest + ' ' + ' /media/sf_share/ravel_plot/profile/log/') else: os.system ("sudo mv "+ self.logdest + ' ' + ' /media/sf_share/ravel_plot/' + t + '/log/') Batch.close (self)
def module_runner(module_name, serialize_queue, batch_file): module = fetch(module_name) if batch_file is None: gen = module.process() else: batch = Batch() batch.load(batch_file) # print(batch.items) gen = [transaction for item in batch.items for transaction in module.process(item)] i = 0 for transaction in gen: serialize_queue.put(transaction) i += 1
def run(self): #update local policy vars on an interval for stability if self.learner_policy.get_step() % self.update_interval == 0: self.pull_vars() #print('learner at %s, actor at %s' % ( # self.learner_policy.get_step(), # self.local_policy.get_step())) #FIXME: last obs might be from different game #since games are 4.5k steps, not a big deal n_actions = self.env.action_space.n batch = Batch() state = self.last_obs #first action in each new env is ~random lstm_state = self.local_policy.lstm_init_state done = step = 0 while not done and step < self.steps: action, value, logit, lstm_state = self.local_policy.act( state, lstm_state) next_state, reward, done, _ = self.env.step(action) #skip the specified number of frame, aggregate rewards? #FIXME: dont just skip, stack the frames #might mess things up if predicting above #using non-diff and diff frames #aggregate and non aggregate #must be constant #note the env employs frame skipping already #more skipping seems to lead to a better policy though #for _ in range(3): # if done: # break # next_state, reward_s, done, _ = self.env.step(action) #reward += reward_s #process observation data next_state = process_state(next_state) if type(action) == np.int64: action = to_onehot(action, n_actions) #add experience to batch batch.add((state, action, reward, value, done, next_state, logit, lstm_state)) #update step += 1 state = next_state self.last_obs = state return batch.get()
def batch_execute(args: dict): # get list of commands from batch file: bare_cmd_lines = [] from batch import Batch batch = Batch(args['batch_file']) batch_commands = batch.get_commands() # the not-yet-expanded commands for command in batch_commands: bare_cmd_lines.extend(command.get_cmd_lines()) # assemble command lines and experiment data output directories: cmd_lines = [] batch_offset = 1 for bare_cmd_line in bare_cmd_lines: # experiment data path: expname = '%s-%04d-%s' % (args['expname'], batch_offset, os.path.basename(bare_cmd_line[0])) # command: cmd_line = [sys.argv[0]] # recursively call ourselves: if args['dry_run']: cmd_line.extend(['--dry-run']) cmd_line.extend(['--monitor-port', '%d' % (args['monitor_port'] + batch_offset)]) cmd_line.extend(['--vnc-display', '%d' % (args['vnc_display'] + batch_offset)]) cmd_line.extend(['--build', utils.BUILD]) cmd_line.extend(['--network', args['network']]) cmd_line.extend(['--memory', args['memory']]) cmd_line.extend([args['VM[:snapshot]']]) cmd_line.extend(['sym']) cmd_line.extend(['--command-port', '%d' % (args['command_port'] + batch_offset)]) cmd_line.extend(['--expname', expname]) cmd_line.extend(['--config-file', command.config]) if args['timeout']: cmd_line.extend(['--timeout', '%d' % args['timeout']]) if args['env_var']: cmd_line.extend(['--env-var', args['env_var']]) cmd_line.extend([args['snapshot']]) cmd_line.extend(bare_cmd_line) cmd_lines.append(' '.join(cmd_line)) # counter: batch_offset += 1 utils.execute(assemble_parallel_cmd_line(args), stdin='\n'.join(cmd_lines))
class App(object): """ """ def __init__(self, args): self._bidsDir = args.bids_dir self._dicomDir = args.dicom_dir self._session = Session(args.session, args.participant, self._bidsDir) self._parser = getattr(studyparser, args.algorithm)(self._dicomDir, self._session) self._yes = args.yes self._codeDir = os.path.join(self._bidsDir, 'code') utils.make_directory_tree(self._codeDir) self._batch = Batch(self._codeDir, self._session) def run(self): utils.new_line() utils.info('Parse and group DICOM directory') self._parser.parse_acquisitions() utils.new_line() utils.info('Sort and set up acquisitions') self._parser.sort_acquisitions() #utils.new_line() #utils.ok('Acquisitions of interest:') #for _ in self._parser.caught: utils.info(_) utils.new_line() utils.warning('Acquisitions excluded:') for _ in self._parser._excluded: utils.info(_) utils.new_line() utils.info('Create YAML file for dcm2niibatch') for acq in self._parser.acquisitions: self._batch.add(acq) self._batch.write() utils.new_line() utils.ok('Batch file:') self._batch.show() if self._yes: launchBatch = True else: msg = "Do you want to launch dcm2niibatch ?" launchBatch = utils.query_yes_no(msg) if launchBatch: self._batch.launch() for acq in self._parser.acquisitions: acq.update_json() else: utils.new_line() utils.ok("To launch dcm2niibatch later:") utils.info("cd {}".format(self._codeDir)) utils.info(self._batch.command) return 0
def close(self): os.system("cp " + Batch.logfile + ' ' + self.logdest) if self.isp == '4755' or self.isp == '3356' or self.isp == '7018': t = 'isp_3sizes' elif self.isp == '2914': t = 'isp' + self.isp + '_3ribs' if self.profile == True: os.system("sudo mv " + self.logdest + ' ' + ' /media/sf_share/ravel_plot/profile/log/') else: os.system("sudo mv " + self.logdest + ' ' + ' /media/sf_share/ravel_plot/' + t + '/log/') Batch.close(self)
def test_batch_job_spawn(self): self.os_mock.listdir = mock.MagicMock() self.os_mock.listdir.return_value = ['job-dir1', 'job-dir2'] from batch_midwife import BatchMidwife from batch import Batch midwife = BatchMidwife() midwife.apprentice = mock.MagicMock() midwife.client = mock.MagicMock() midwife.batch_pub_sub = mock.MagicMock() midwife.batch_pub_sub.listen.return_value = [{'data': 'batch-lovelyhashcode'}] midwife.client.exists.return_value = True batch = Batch('uploaded') midwife.client.get.return_value = pickle.dumps(batch) midwife.client.set = mock.MagicMock() midwife.client.publish = mock.MagicMock() midwife.run() assert midwife.client.exists.call_count == 1 assert midwife.client.get.call_count == 1 assert midwife.client.set.call_count == 4 assert midwife.client.publish.call_count == 2 assert midwife.client.set.call_args_list[1][0][0] == 'job-dir1_1' assert midwife.client.set.call_args_list[2][0][0] == 'job-dir2_1' assert pickle.loads(midwife.client.set.call_args_list[3][0][1]).state == 'running' assert self.os_mock.listdir.call_count == 1
def getNext(self): "iterator" batchRange = range(self.currIdx, self.currIdx + config.BATCH_SIZE) gtTexts = [self.samples[i].gtText for i in batchRange] imgs = [] for i in batchRange: try: self.binaryImageFile.seek(self.samples[i].imageStartPosition) img = np.frombuffer( self.binaryImageFile.read(self.samples[i].imageSize), np.dtype('B')) img = img.reshape(self.samples[i].imageHeight, self.samples[i].imageWidth) img = preprocess(img, config.IMAGE_WIDTH, config.IMAGE_HEIGHT, config.RESIZE_IMAGE, config.CONVERT_IMAGE_TO_MONOCHROME, config.AUGMENT_IMAGE) imgs.append(img) except IOError as e: print("I/O error({0}): {1}".format(e.errno, e.strerror)) pass except ValueError as e: print("Value error({0}): {1}".format(e.errno, e.strerror)) pass except: print("Unexpected error:", sys.exc_info()[0]) pass self.currIdx += config.BATCH_SIZE return Batch(gtTexts, imgs)
def __init__(self, dbname, rounds): sql_script = "/home/mininet/ravel/sql_scripts/primitive.sql" #topology = 'toy' topology = 'fat' self.dbname = dbname self.create_db(sql_script) Batch.connect(self) self.load_schema(sql_script) self.load_sig_example_schema() self.load_topo() Batch.__init__(self, dbname, rounds, sql_script, topology) self.profile = False
def __init__(self,dbname, rounds): sql_script = "/home/mininet/ravel/sql_scripts/primitive.sql" #topology = 'toy' topology = 'fat' self.dbname = dbname self.create_db (sql_script) Batch.connect (self) self.load_schema (sql_script) self.load_sig_example_schema () self.load_topo () Batch.__init__(self, dbname, rounds, sql_script, topology) self.profile = False
def processUbertoolBatchRunsIntoBatchModelRuns(ubertools): logger.info("Start Ubertool Batching") batch_id = ubertools['id'] user = users.get_current_user() batchs = Batch.all() batch = None for poss_batch in batchs: logger.info(batch_id) logger.info(str(poss_batch.key())) if str(poss_batch.key()) == batch_id: batch = poss_batch logger.info(batch.to_xml()) ubertools_results = {} ubertools_data = ubertools['ubertools'] for ubertool in ubertools_data: combined_ubertool_props = combineUbertoolProperties(ubertool) ubertool_id = combined_ubertool_props["ubertool-config-name"] ubertool_result = {} #logger.info(combined_ubertool_props) ubertool_result = terrPlantRunner.runTerrPlantModel(combined_ubertool_props,ubertool_result) ubertool_result = sipRunner.runSIPModel(combined_ubertool_props,ubertool_result) #perform on all other eco models ubertools_results[ubertool_id]=ubertool_result batch.completed = db.DateTimeProperty.now() results_pickle = pickle.dumps(ubertools_results) batch.ubertool_results = results_pickle batch.put() logger.info(batch.to_xml())
def reset(self): #---------------------------------------------- # failures arrive by using poisson distribution #---------------------------------------------- if self.failure_type == 0: trace = Poisson(self.sys.num_disks, self.failure_percent, self.mtbf) if self.failure_type == 1: trace = Exponential(self.sys.num_disks, self.failure_percent, self.mtbf) if self.failure_type == 2: trace = Batch(self.sys.num_disks, self.failure_percent, self.mtbf, cascade_factor=10.0) self.trace_entry = trace.generate_failures() #------------------------------------------ # put the disk failures in the event queue #------------------------------------------ self.events_queue = [] for disk_fail_time, diskId in self.trace_entry: heappush(self.events_queue, (disk_fail_time, Disk.EVENT_FAIL, diskId)) print ">>>>> reset disk", diskId, Disk.EVENT_FAIL, "@", disk_fail_time self.mission_time = disk_fail_time print " - system mission time - ", self.mission_time #------------------------------ # initialize the system state #------------------------------ self.state = State(self.sys, self.rebuild, self.copyback, self.events_queue)
def make_monte_carlo_batch(self, nb_episodes, render, policy): """ Create a batch of episodes with a given policy Used in Monte Carlo approaches :param nb_episodes: the number of episodes in the batch :param render: whether the episode is displayed or not (True or False) :param policy: the policy controlling the agent :return: the resulting batch of episodes """ batch = Batch() self.env.set_reward_flag(False) self.env.set_duration_flag(False) for e in range(nb_episodes): episode = self.train_on_one_episode(policy, False, render) batch.add_episode(episode) return batch
def setUp(self): self.job = Job() self.batch = Batch() self.resource = Resource() rootDir = os.environ['BOLT_DIR'] self.batch.readConfig(rootDir + configDir + "/" + batchConfig) self.resource.readConfig(rootDir + configDir + "/" + resourceConfig)
def __init__(self, name, command, containerID=None, working_dir=None, image=None, endpoint=None): Batch.__init__(self, name, command, working_dir=working_dir) self.command = command self.containerID = containerID self.working_dir = working_dir self.image = image self.endpoint = endpoint self.docker_client = DockerClient() self.transfer = DataTransfer.inferDataTransportation( "127.0.0.1", self.endpoint) self.ssh_connection = None
def encode(self, G): if type(G) != list: G = [G] # encode graphs G into latent vectors b = Batch.from_data_list(G) Hg = self(b) mu, logvar = self.fc1(Hg), self.fc2(Hg) return mu, logvar
def __init__(self, name, ssh_username, keypath, command, ip=None, working_dir=None, local_working_dir=None, endpoint=None): Batch.__init__(self, name, command, working_dir=working_dir) self.ip = ip self.keypath = keypath self.command = command self.endpoint = endpoint self.working_dir = working_dir self.ssh_username = ssh_username self.local_working_dir = local_working_dir
def primitive(self): size = 10 Batch.rtm_ins(self, size) Batch.init_acl(self) Batch.init_lb(self) Batch.op_primitive(self) dbname = self.logdest.split('.')[0] self.logdest = dbname + '_primitive.log'
def primitive (self): size = 10 Batch.rtm_ins (self, size) Batch.init_acl (self) Batch.init_lb (self) Batch.op_primitive (self) dbname = self.logdest.split ('.')[0] self.logdest = dbname + '_primitive.log'
def get_experience(self, insert_dummy=True): batch = Batch(dummy=[]) for i in range(self.env_num): #append to total batch batch.append(self.buffers[i]) #set dummy key if insert_dummy: length = len(self.buffers[i]) batch.dummy += [False] * length #append dummy item at the end if not done if length: if not batch[-1].done: dummy_item = batch[-1:] dummy_item.dummy = [True] dummy_item.done = [True] dummy_item.state = dummy_item.next_state batch.append(dummy_item) return batch.to_numpy()
def data_gen(V, batch, nbatches): "Generate random data for a src-tgt copy task." # dtype = torch.FloatTensor for i in range(nbatches): data = torch.from_numpy(np.random.randint(1, V, size=(batch, 10))) data[:, 0] = 1 src = Variable(data, requires_grad=False) tgt = Variable(data, requires_grad=False) yield Batch(src, tgt, 0)
def routing_ins_acl_lb_tenant (self,hosts): cur = self.cur f = self.f [h1, h2] = random.sample(hosts, 2) Batch.update_max_fid (self) fid = self.max_fid + 1 try: t1 = time.time () cur.execute ("INSERT INTO tenant_policy VALUES ("+str (fid) +"," +str (h1) + "," + str (h2)+");") cur.execute("select max (counts) from clock;") ct = cur.fetchall () [0]['max'] cur.execute ("INSERT INTO t1 VALUES (" + str (ct+1) + ", 'on');") t2 = time.time () f.write ('----(acl+lb+rt)*tenant: route ins----' + str ((t2-t1)*1000) + '\n') f.flush () except: pass
def inferSingleImage(paraModel, paraFnImg): "recognize text in image provided by file path" img = cv2.imread(paraFnImg, cv2.IMREAD_GRAYSCALE) img = preprocess(img, config.IMAGE_WIDTH, config.IMAGE_HEIGHT, True, False, False) batch = Batch(None, [img]) #(recognized, probability) = model.inferBatch(batch) (recognized, probability) = paraModel.inferBatch(batch, True) print('Recognized:', '"' + recognized[0] + '"') print('Probability:', probability[0])
def __init__( self, package, cli_composer, deploy_status, use_package_path=False): """ Initialize an instance of Deployment """ import uuid if (use_package_path): self.batch = Batch(package.cwd) else: self.batch = Batch(tempfile.mkdtemp()) self.cli_composer = cli_composer self.cwd = self.batch.cwd self.cwd_use_package_path = use_package_path self.deployed = False self.deployment_id = '{0}'.format(uuid.uuid1()) self.deploy_status = deploy_status self.package = package self.started = False
def tenant_fullmesh (self, hosts): f = self.f cur = self.cur Batch.update_max_fid (self) fid = self.max_fid + 1 cur.execute ("select max (counts) from clock;") ct = cur.fetchall ()[0]['max'] + 1 for i in range (len (hosts)): for j in range (i+1,len (hosts)): print "tenant_fullmesh: [" + str (hosts[i]) + "," + str (hosts[j]) + "]" t1 = time.time () cur.execute ("INSERT INTO tenant_policy values (%s,%s,%s);",([str (fid) ,int (hosts[i]), int (hosts[j])])) cur.execute ("INSERT INTO p_spv values (%s,'on');",([ct])) t2 = time.time () f.write ('----rt*tenant: route ins----' + str ((t2-t1)*1000) + '\n') f.flush () ct += 1 fid += 1
def close (self): Batch.close (self)
def load_sig_example_schema (self): sql_script = '/home/mininet/ravel/xym/new.sql' Batch.load_schema(self, sql_script)
def add_flow (self, src, dst): Batch.update_max_fid(self) self.cur.execute ("INSERT INTO tm(fid,src,dst,vol,FW,LB) VALUES (%s,%s,%s,%s,%s,%s);", ([self.max_fid +1, src, dst, 0, 0, 0]))
def primitive (self): Batch.init_acl (self) print "Batch_isp.init_acl" Batch.init_lb (self) print "Batch_isp.init_lb" Batch.op_primitive (self)
def main(): def find_group_name(filepath): unix_stat_of_file = stat(fp) grp_id_of_file = unix_stat_of_file.st_gid group_name = getattr(getgrgid(grp_id_of_file), 'gr_name', None) return group_name def find_user_name(filepath): uid_of_file = unix_stat_of_file.st_uid user_name = getpwuid(uid_of_file) return user_name parser = ArgumentParser(description="{description}". \ format(description=__description__), epilog="Copyright University of Chicago; " + \ "written by {author} ". \ format(author = __author__) + \ " <{email}> University of Chicago". \ format(email = __email__)) parser.add_argument("-v", help="See the version of this program", action="version", version=__version__) parser.add_argument( \ '-b','-verbose',help="set verbose logging", action='store_const',dest='log_level', const=INFO \ ) parser.add_argument( \ '-d','--debugging',help="set debugging logging", action='store_const',dest='log_level', const=DEBUG \ ) parser.add_argument( \ '-l','--log_loc',help="save logging to a file", action="store_const",dest="log_loc", const='./{progname}.log'. \ format(progname=argv[0]) \ ) selection = parser.add_mutually_exclusive_group() selection.add_argument("--directory_path", help="Enter a directory that you need to work on ", action='store') selection.add_argument("--from_db",help="Select to create a batch " + \ "from database", action="store") parser.add_argument("--tables",help="Only use this is selecting from_db", nargs="*",action=dbBeforeTables) parser.add_argument("root",help="Enter the root of the directory path", action="store") parser.add_argument("numfiles",help="Enter the number of files you " + \ "want to check in this iteration.",action="store", type=int) args = parser.parse_args() log_format = Formatter( \ "[%(levelname)s] %(asctime)s " + \ "= %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" \ ) global logger logger = getLogger( \ "lib.uchicago.repository.logger" \ ) ch = StreamHandler() ch.setFormatter(log_format) try: logger.setLevel(args.log_level) except TypeError: logger.setLevel(INFO) if args.log_loc: fh = FileHandler(args.log_loc) fh.setFormatter(log_format) logger.addHandler(fh) logger.addHandler(ch) current_date = datetime.now() isof_current_date = current_date.strftime("%Y-%m-%dT%H:%M:%S") sixty_days_ago_date = current_date - timedelta(days=60) isof_sixty_days_ago_date = sixty_days_ago_date.strftime( \ "%Y-%m-%dT%H:%M:%S") if args.from_db: db = Database(args.from_db, ['record','file']) class Record(db.base): __table__ = Table('record', db.metadata, autoload=True) class File(db.base): __table__ = Table('file', db.metadata, autoload=True) accessions_to_check = db.session.query(Record). \ filter(or_(Record.lastFixityCheck == None, Record.lastFixityCheck \ <= isof_sixty_days_ago_date, Record.fixityCheckCompleteness \ == 'incompleted', Record.fixityCheckCompleteness \ == None)).subquery() files_to_check = db.session.query(File.accession, File.checksum, File.size, File.filepath). \ filter(File.accession== \ accessions_to_check.c.receipt, or_(File.lastFixityCheck == None, File.lastFixityCheck \ <= isof_sixty_days_ago_date)). \ order_by(func.random()).limit(args.numfiles) b = Batch(args.root, query = files_to_check) generated_output = b.find_items(from_db = True) else: b = Batch(args.root, directory = args.directory_path) generated_output = b.find_items(from_directory=True) b.set_items(generated_output) try: for n in b.items: if exists(n.filepath): sha256_fixity = n.find_hash_of_file(sha256) mime = n.find_file_mime_type() n.set_hash(sha256_fixity) n.set_file_mime_type(mime) new_hash = n.get_hash() old_hash = n.get_old_hash() if new_hash != old_hash: logger.error("{path} is corrupted".format(path=n.filepath)) else: logger.error("{path} does not exist on the filesystem".format(path=n.filepath)) return 0 except KeyboardInterrupt: logger.error("Program aborted manually") return 131
def get_all_batches(self): collection = self.batch_coll result = collection.find() batches = [] for item in result: single_batch = Batch() single_batch.set_id(item['_id']) single_batch.set_desc(item['desc']) single_batch.set_current_sem(item['current_sem']) single_batch.set_subject_array(self.construct_subject_master(item)) single_batch.set_status(item['status']) batches.append(single_batch) # print json.dumps(batches, default=Batch.__str__) return batches
def get_batch_by_id(self, batchid): collection = self.batch_coll item = collection.find_one({'_id':batchid}) if(item != None): single_batch = Batch() single_batch.set_id(item['_id']) single_batch.set_desc(item['desc']) single_batch.set_current_sem(item['current_sem']) single_batch.set_subject_array(self.construct_subject_master(item)) single_batch.set_status(item['status']) return single_batch return None
def load_sig_example_schema (self): sql_script = '/home/mininet/ravel/sql_scripts/sigcomm_example.sql' Batch.load_schema(self, sql_script)
def extractFromAggregated(self, aggregated): batch = Batch() total = sum(aggregated) entropy = batch._entropy(aggregated) return [total, entropy]
def fetch (self): Batch.fetch (self) self.cur.execute ("SELECT * FROM tm;") cs = self.cur.fetchall () self.tm = [[h['fid'], h['src'], h['dst'], h['vol'], h['fw'], h['lb']] for h in cs]