def __init__(self, output_path, prod_name, wisdom_update={}): """ Initialize postprocessor with output parameters. :param output_path: path where postprocessing files are stored :param prod_name: name of manifest json file and prefix of all output files :param wisdom_update: an optional dictionary that maps variables to modifications requested in their visualization wisdom """ logging.info("Postprocessor: output_path=%s prod_name=%s" % (output_path, prod_name)) dump(wisdom_update, "Postprocessor: wisdom_update") self.output_path = output_path self.product_name = prod_name self.manifest = {} self.wisdom_update = wisdom_update # in case the manifest exists, load the existing version mf_path = os.path.join(output_path, prod_name + '.json') if osp.exists(mf_path): self.manifest = json.load(open(mf_path)) logging.info('postprocessor: Loaded manifest at %s' % mf_path) # dump(self.manifest,"postprocessor: manifest") else: logging.info('postprocessor: manifest at %s does not exist yet' % mf_path)
def on_receive_card(self, card): if card in self.cards: return self.cards.append(card) dump(self.filename("cards"), self.cards) self.ui.update_card(self.room_id, card)
def store_user_review(observe_t=12): store_user = {} store_review = {} store = utils.load("dicts/store.p") with open("dataset/yelp_academic_dataset_review.json", "r") as f: for line in f: line = json.loads(line) business_id = line["business_id"] if business_id not in store: continue review_id = line["review_id"] user_id = line["user_id"] date = datetime.strptime(line["date"], "%Y-%m-%d") if int((date-store[business_id]["start_t"]).days/30) <= observe_t: if business_id in store_review: store_review[business_id].append(review_id) else: store_review[business_id] = [review_id] if business_id in store_user: store_user[business_id].append(user_id) else: store_user[business_id] = [user_id] utils.dump(store_user, "dicts/store_user.p") utils.dump(store_review, "dicts/store_review.p")
def find(symbols): print(utils.make_bright("<find>")) matches = [] with sqlite3.connect(utils.get_libcs_db_filepath()) as conn: conn.row_factory = sqlite3.Row for libc in conn.execute("SELECT * FROM libcs"): libc_filepath = os.path.join(utils.get_libcs_dirpath(), libc["relpath"]) with open(libc_filepath, "rb") as f: elf = elftools.elf.elffile.ELFFile(f) dynsym_section = elf.get_section_by_name(".dynsym") for symbol, address in symbols: offset = address & 0xFFF try: libc_symbol = dynsym_section.get_symbol_by_name(symbol)[0] libc_offset = libc_symbol.entry.st_value & 0xFFF if libc_offset != offset: break except (IndexError, TypeError): break else: utils.dump(dict(libc)) matches.append(dict(libc)) print(utils.make_bright("</find>")) return matches
def _log(self, query): """ Saves only the last n (100) queries """ if query not in self.queryLog: self.queryLog.append(query) dump(self.queryLog[-100:], "./obj/log.pk")
def pair_dist(): store = utils.load("dicts/store.p") store_pair = utils.load("dicts/store_pair.p") pair_d = {} for busi_1 in store_pair: l = store_pair[busi_1] for busi_2 in l: if busi_1 < busi_2: small = busi_1 large = busi_2 else: small = busi_2 large = busi_1 tup = (small, large) if tup in pair_d: continue else: x1 = store[small]['latitude'] y1 = store[small]['longitude'] x2 = store[large]['latitude'] y2 = store[large]['longitude'] first = (x1, y1) second = (x2, y2) pair_d[tup] = vincenty(first, second).miles utils.dump(pair_d, "dicts/pair_dist.p")
def show_block(filename, offset): f = DatFile(filename) f.stream.seek(offset) block_data = f.stream.read(f.block_size) dump(block_data) print "---" dump(f.stream.read(0x40))
def preprocess(raw_data, dataset): print('parsing smiles as graphs...') processed_data = {'train': [], 'valid': []} file_count = 0 for section in ['train', 'valid']: all_smiles = [] # record all smiles in training dataset for i, (smiles, QED) in enumerate([(mol['smiles'], mol['QED']) for mol in raw_data[section]]): nodes, edges = to_graph(smiles, dataset) if len(edges) <= 0: continue processed_data[section].append({ 'targets': [[(QED)]], 'graph': edges, 'node_features': nodes, 'smiles': smiles }) all_smiles.append(smiles) if file_count % 2000 == 0: print('finished processing: %d' % file_count, end='\r') file_count += 1 print('%s: 100 %% ' % (section)) # save the dataset with open('molecules_%s_%s.json' % (section, dataset), 'w') as f: json.dump(processed_data[section], f) # save all molecules in the training dataset if section == 'train': utils.dump('smiles_%s.pkl' % dataset, all_smiles)
def show_block(filename, offset): f = DatFile(filename) f.stream.seek(offset) block_data = f.stream.read(f.block_size) dump(block_data) print "---" dump(f.stream.read(0x40))
def preprocess(df): def token(text): st = LancasterStemmer() txt = nltk.word_tokenize(text.lower()) return [st.stem(word) for word in txt] top_speakers = df.groupby([c.TARGET]).size( ).loc[df.groupby([c.TARGET]).size() > 2000] main_char_lines = df.loc[df[c.TARGET].isin( top_speakers.index.values)] main_char_lines['Line'] = [line.replace( '\n', '') for line in main_char_lines['Line']] # stop = set(stopwords.words("english")) cv = CountVectorizer( # lowercase=True, tokenizer=token, # stop_words=stop, # token_pattern=u'(?u)\b\w\w+\b', analyzer=u'word', min_df=4) X = cv.fit_transform(main_char_lines['Line'].tolist()).toarray() le = LabelEncoder() y = le.fit_transform(main_char_lines[c.TARGET]) u.dump(cv, c.PATH_VECTORIZER) u.dump(le, c.PATH_ENCODER) return X, y
def dump_anim_file(entry): j, unk1, file_id, offset, size1, timestamp, version, size2, unk2 = entry print "%08X %08X %08X %s %08X | %08X %08X %08X | %08X" % (file_id, offset, size1, time.ctime(timestamp), version, size2, unk1, unk2, size2 - size1) f.stream.seek(offset) j, k, l, m, n = struct.unpack("<LLLHH", f.stream.read(0x10)) print "%08X %08X %08X %04X %04X" % (j, k, l, m, n) assert j == 0 assert k == 0 if m == 0xDA78: print "compressed" assert unk1 % 0x100 == 0x03 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[12:] content = zlib.decompress(data) assert l == len(content) else: print "uncompressed" assert unk1 % 0x100 == 0x02 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[8:] content = data dump(content[:0x100])
def get_tasks(request): utils.dump(request.GET) # date = datetime.strptime("Tue Oct 20 09:26:38 GMT 2015", '%a %b %d %H:%M:%S %Z %Y') # latest_task_list = Task.objects.all().filter(date=date).order_by('-date') latest_task_list = Task.objects.all().filter(isDeleted=False).order_by('-date') response = utils.build_obj_from_queryset(latest_task_list); return JsonResponse(response, safe=False)
def dump_anim_file(entry): j, unk1, file_id, offset, size1, timestamp, version, size2, unk2 = entry print "%08X %08X %08X %s %08X | %08X %08X %08X | %08X" % (file_id, offset, size1, time.ctime(timestamp), version, size2, unk1, unk2, size2 - size1) f.stream.seek(offset) j, k, l, m, n = struct.unpack("<LLLHH", f.stream.read(0x10)) print "%08X %08X %08X %04X %04X" % (j, k, l, m, n) assert j == 0 assert k == 0 if m == 0xDA78: print "compressed" assert unk1 % 0x100 == 0x03 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[12:] content = zlib.decompress(data) assert l == len(content) else: print "uncompressed" assert unk1 % 0x100 == 0x02 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[8:] content = data dump(content[:0x100])
def get_tasks_by_date(request): utils.dump(request.GET) #todo get with param, not Post date_filter = datetime.strptime(request.POST.get('date'), '%a %b %d %H:%M:%S %Z %Y') latest_task_list = Task.objects.all().filter(isDeleted=False, date__startswith=date_filter.date()).order_by('-date') response = utils.build_obj_from_queryset(latest_task_list); return JsonResponse(response, safe=False)
def main(run_id, data_path, score_as_pyfunc, score_as_tensorflow_lite): print("Options:") for k, v in locals().items(): print(f" {k}: {v}") utils.dump(run_id) data, _, _, _ = utils.build_data(data_path) model_uri = f"runs:/{run_id}/keras-hd5-model" predict_keras(model_uri, data) if score_as_pyfunc: predict_pyfunc(model_uri, data, "keras-hd5-model") model_name = "tensorflow-model" if artifact_exists(run_id, model_name): predict_tensorflow_model(run_id, data) else: print(f"WARNING: no model '{model_name}'") if score_as_tensorflow_lite: model_name = "tensorflow-lite-model" if artifact_exists(run_id, model_name): predict_tensorflow_lite_model(run_id, data) else: print(f"WARNING: no model '{model_name}'") model_name = "onnx-model" if artifact_exists(run_id, model_name): model_uri = f"runs:/{run_id}/'{model_name}'" predict_onnx(model_uri, data) predict_pyfunc(model_uri, data, "onnx-model") else: print(f"WARNING: no model '{model_name}'")
async def main(): call_args = get_args() config_env(call_args.random_seed, call_args.log_level) # Kaggle stores compressed files with ".zip" suffix dataset_file_path: Path = call_args.storage_path / (DATASET_FILE + ".zip") await load_dataset( ds_name=call_args.dataset_name, ds_file_name=DATASET_FILE, ds_file_path=dataset_file_path, ) embeddings_arch = call_args.storage_path / "glove.840B.300d.zip" emb_file_path = await get_embeddings(emb_arch_path=embeddings_arch) # Check if preprocessing could be skipped checksum_key = ( f"{dataset_file_path.name}_{DATASET_SIZE}_{emb_file_path.name}_{call_args.train_size}_{call_args.random_seed}" ) need_rerun = True if CHECKSUMS.get(checksum_key, None): need_rerun = False for f_name, checksum in CHECKSUMS.get(checksum_key).items(): f_path = call_args.work_store_path / f_name if not file_exists(f_path) or not verify_checksum(f_path, checksum): need_rerun = True break if need_rerun: objects_to_save = preprocess_data(dataset_file_path, DATASET_SIZE, emb_file_path, call_args.train_size) dump(dump_root=call_args.work_store_path, objects=objects_to_save) else: logging.info(f"All checksums match run configuration, skipping preprocessing.")
def get_tasks(request): utils.dump(request.GET) # date = datetime.strptime("Tue Oct 20 09:26:38 GMT 2015", '%a %b %d %H:%M:%S %Z %Y') # latest_task_list = Task.objects.all().filter(date=date).order_by('-date') latest_task_list = Task.objects.all().filter( isDeleted=False).order_by('-date') response = utils.build_obj_from_queryset(latest_task_list) return JsonResponse(response, safe=False)
def main(): i = 0 result = {} try: parse_categories() gen_embed_indexes() kv_array = specs.parse_files() while i < len(kv_array): print("----------------------------------------------") filename=kv_array[i]['filename'] #.lower().replace('.', '_') print("extracting key values from %s" %(kv_array[i]['filename'])) result[filename] = {} result[filename]['application'] = [] result[filename]['qos'] = [] result[filename]['device'] = [] for k,v in kv_array[i]['kv']: print("%s : %s" %(k, v)) if v == "": continue l = len(k.split()); korig = k if l > 1: #k = k.split()[l - 1] k = utils.k_get(k.strip(), l) try: scores = process(k.strip()) print(scores) max = 'device' if scores['device'] > scores['application'] else 'application' max = max if scores['qos'] < scores[max] else 'qos' print(max) k = utils.k_strip(korig.strip(), k.strip()) if k == None: continue print("--------") result[filename][max].append([k, v]) except Exception as e: print("Exception") print(e) pass i = i + 1 break except Exception as e: print("Exception") print(e) print("-------------") for f in result: print(f) for s in result[f]: out = s + " :" for k, v in result[f][s]: out = out + k + ":" #print(result[f][s]) utils.dump(out) print("---") create_excel_sheet(f, result[f])
def loadHead(self, stream): print "=== HEAD ===" head = GxStream(stream.getNext()) head.skip(2) # 40 00 - HEAD size s = head.getNext() while s != "": print utils.dump(s) s = head.getNext() stream.getNext() # 04 00 FF FF
def get_tasks_by_date(request): utils.dump(request.GET) #todo get with param, not Post date_filter = datetime.strptime(request.POST.get('date'), '%a %b %d %H:%M:%S %Z %Y') latest_task_list = Task.objects.all().filter( isDeleted=False, date__startswith=date_filter.date()).order_by('-date') response = utils.build_obj_from_queryset(latest_task_list) return JsonResponse(response, safe=False)
async def add_line(line: c.Line): print(line) extra_lines = pd.DataFrame([line.dict()]) global df print(df.shape) df = pd.concat([df, extra_lines], axis=0, ignore_index=True) print(df.shape) u.dump(df, c.PATH_TRAIN_EXTRA) return line
def show_file_block(filename, offset): f = DatFile(filename) f.stream.seek(offset) block_data = f.stream.read(f.block_size) zero1, zero2, file_id, size = struct.unpack("<LLLL", block_data[:0x10]) assert zero1 == 0 assert zero2 == 0 print("%08X %08X" % (file_id, size)) file_data = f.stream.read(size) dump(file_data[0x10:])
def show_file_block(filename, offset): f = DatFile(filename) f.stream.seek(offset) block_data = f.stream.read(f.block_size) zero1, zero2, file_id, size = struct.unpack("<LLLL", block_data[:0x10]) assert zero1 == 0 assert zero2 == 0 print("%08X %08X" % (file_id, size)) file_data = f.stream.read(size) dump(file_data[0x10:])
def test_formbot(): model_directory = "examples/formbot/models/nlu/current" interpreter = Interpreter.load(model_directory) text = "uh yes" result = interpreter.parse(text) utils.dump(result) text = "what about chinese food" result = interpreter.parse(text) utils.dump(result)
def load(self): # Load cards self.cards = load(self.filename("cards")) or [] self.ui.cards = self.cards[:] # Load secret key self.secret_key = load( self.filename("secret_key")) or generate_secret_key() self.secret_key = int(self.secret_key) dump(self.filename("secret_key"), self.secret_key)
def dump_image_file(entry): j, unk1, file_id, offset, size1, timestamp, version, size2, unk2 = entry f.stream.seek(offset) j, k, l, m, n = struct.unpack("<LLLHH", f.stream.read(0x10)) assert j == 0 assert k == 0 if m == 0xDA78: assert unk1 % 0x100 == 0x03 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[12:] content = zlib.decompress(data) assert l == len(content) header_id, unk1, width, height, unk2, lngth = struct.unpack( "<LLLLLL", content[:24]) assert lngth + 24 == l else: assert unk1 % 0x100 == 0x02 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[8:] header_id, unk1, width, height, unk2, lngth = struct.unpack( "<LLLLLL", data[:24]) assert lngth + 24 == size1 content = data print hex(file_id) if unk2 == 0x15: assert width * height * 4 == lngth image_0x15(header_id, width, height, content[24:]) elif unk2 == 0x14: assert width * height * 3 == lngth image_0x14(header_id, width, height, content[24:]) elif unk2 == 0x31545844: # DXT1 assert width * height == lngth * 2 image_0x31545844(header_id, width, height, content[24:]) elif unk2 == 0x33545844: # DXT3 assert width * height == lngth image_0x33545844(header_id, width, height, content[24:]) elif unk2 == 0x35545844: # DXT5 assert width * height == lngth image_0x35545844(header_id, width, height, content[24:]) elif unk2 == 0x1C: assert width * height == lngth image_0x1C(header_id, width, height, content[24:]) elif unk2 == 0x1F4: image_0x1F4(header_id, content[24:]) else: print "%08X %04X %04X" % (l, m, n) print "%08s %08s %08s %08s %08s %08s" % ("file_id", "unk1", "width", "height", "unk2", "lngth") print "%08X %08X %08X %08X %08X %08X" % (header_id, unk1, width, height, unk2, lngth) dump(content[24:])
def params(ctx): name = ctx['name'] build_root = ctx['build_root'] win_root = f'{build_root}/win' win_exe = f'{win_root}/{name}.exe' old = {k for k in ctx} ctx['win_root'] = win_root ctx['win_exe'] = win_exe dump(ctx, old)
def train_and_validate(model, X, y): X_train, X_val, y_train, y_val\ = train_test_split(X, y, test_size=0.3, random_state=42) model.fit(X_train, y_train) y_pred = model.predict(X_val) print(classification_report(y_pred, y_val)) model.fit(X, y) u.dump(model, c.PATH_MODEL) return model
async def add_lines(lines: List[c.Line]): print(lines) extra_lines = [] for line in lines: extra_lines.append(line.dict()) global df print(df.shape) df = pd.concat([df, pd.DataFrame(extra_lines)], axis=0, ignore_index=True) print(df.shape) u.dump(df, c.PATH_TRAIN_EXTRA) return line
def eval(test_loader, model, device): mse = torch.nn.MSELoss() mae = torch.nn.L1Loss() with torch.no_grad(): rmse_loss = 0 mae_loss = 0 mre_loss = 0 delta1_loss = 0 delta2_loss = 0 delta3_loss = 0 for i, (img, depth) in enumerate(test_loader): img, depth = img.to(device).float(), depth.to(device).float() depth = depth.unsqueeze(1) output = model(img) valid_mask = depth > 0 depth = depth[valid_mask] output = output[valid_mask] rmse_loss += mse(output, depth) * test_loader.batch_size mae_loss += mae(output, depth) * test_loader.batch_size mre_loss += mre(output, depth) * test_loader.batch_size delta1_loss += delta(output, depth, 1) * test_loader.batch_size delta2_loss += delta(output, depth, 2) * test_loader.batch_size delta3_loss += delta(output, depth, 3) * test_loader.batch_size image, depth_gt, depth_pred = resize_image_depth( img, depth, output) _, error_map = make_error_map(image, depth_gt.T, depth_pred.T) dump(image=image, depth=depth_pred.T, depth_gt=depth_gt.T, error_map=error_map, prefix='eval', n=i) N = len(test_loader) * test_loader.batch_size rmse_loss = torch.sqrt(rmse_loss/N) mae_loss = mae_loss / N mre_loss = mre_loss / N delta1_loss = delta1_loss / N delta2_loss = delta2_loss / N delta3_loss = delta3_loss / N print('RMSE: %f' % (rmse_loss,)) print('MAE: %f' % (mae_loss,)) print('MRE: %f' % (mre_loss,)) print('Delta1: %f' % (delta1_loss,)) print('Delta2: %f' % (delta2_loss,)) print('Delta3: %f' % (delta3_loss,))
def params(ctx): name = ctx['name'] build_root = ctx['build_root'] osx_root = f'{build_root}/osx' osx_dmg = f'{osx_root}/{name}.dmg' osx_app = f'{osx_root}/{name}.app' old = {k for k in ctx} ctx['osx_root'] = osx_root ctx['osx_dmg'] = osx_dmg ctx['osx_app'] = osx_app dump(ctx, old)
def add_steam_appid_txt(ctx): """Add steam_appid.txt for testing.""" win_root = ctx['win_root'] steam_appid = ctx['steam_appid'] win_steam_appid_txt = f'{win_root}/steam_appid.txt' with open(win_steam_appid_txt, 'wt') as f: f.write(f'{steam_appid}') old = {k for k in ctx} ctx['win_steam_appid_txt'] = win_steam_appid_txt dump(ctx, old)
def add_steam_appid_txt(ctx): """Add steam_appid.txt for testing.""" osx_app = ctx['osx_app'] steam_appid = ctx['steam_appid'] osx_steam_appid_txt = f'{osx_app}/Contents/MacOS/steam_appid.txt' with open(osx_steam_appid_txt, 'wt') as f: f.write(f'{steam_appid}') old = {k for k in ctx} ctx['osx_steam_appid_txt'] = osx_steam_appid_txt dump(ctx, old)
def dump_image_file(entry): j, unk1, file_id, offset, size1, timestamp, version, size2, unk2 = entry f.stream.seek(offset) j, k, l, m, n = struct.unpack("<LLLHH", f.stream.read(0x10)) assert j == 0 assert k == 0 if m == 0xDA78: assert unk1 % 0x100 == 0x03 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[12:] content = zlib.decompress(data) assert l == len(content) header_id, unk1, width, height, unk2, lngth = struct.unpack("<LLLLLL", content[:24]) assert lngth + 24 == l else: assert unk1 % 0x100 == 0x02 f.stream.seek(offset) data = f.stream.read(size1 + 0x08)[8:] header_id, unk1, width, height, unk2, lngth = struct.unpack("<LLLLLL", data[:24]) assert lngth + 24 == size1 content = data print hex(file_id) if unk2 == 0x15: assert width * height * 4 == lngth image_0x15(header_id, width, height, content[24:]) elif unk2 == 0x14: assert width * height * 3 == lngth image_0x14(header_id, width, height, content[24:]) elif unk2 == 0x31545844: # DXT1 assert width * height == lngth * 2 image_0x31545844(header_id, width, height, content[24:]) elif unk2 == 0x33545844: # DXT3 assert width * height == lngth image_0x33545844(header_id, width, height, content[24:]) elif unk2 == 0x35545844: # DXT5 assert width * height == lngth image_0x35545844(header_id, width, height, content[24:]) elif unk2 == 0x1C: assert width * height == lngth image_0x1C(header_id, width, height, content[24:]) elif unk2 == 0x1F4: image_0x1F4(header_id, content[24:]) else: print "%08X %04X %04X" % (l, m, n) print "%08s %08s %08s %08s %08s %08s" % ("file_id", "unk1", "width", "height", "unk2", "lngth") print "%08X %08X %08X %08X %08X %08X" % (header_id, unk1, width, height, unk2, lngth) dump(content[24:])
def train(self, train_set: pd.DataFrame, force: bool = False, save: bool = True) -> None: if not force and self.LogReg_pipeline is not None: return self.LogReg_pipeline = Pipeline([ ('tfidf', TfidfVectorizer(stop_words=self.stop_words)), ('clf', LogisticRegression(solver='sag')), ]) self.LogReg_pipeline.fit(train_set['comment_text'].map(lambda com: utils.preprocess_text(com)), train_set['bannable']) if save: utils.dump(self.LogReg_pipeline, "log_pipeline")
def predict(word_preds, stop_words=(), k=None, t='fr', query=False, lang=False, save_pred=None, load_pred=None): if load_pred: word_preds = load(load_pred) else: word_preds = list(map(list, word_preds)) for i in range(len(word_preds)): preds = list(filter(has_letter, map(clean, word_preds[i][1][:k]))) if stop_words: preds = [w for w in preds if w not in stop_words] if query: # filter src word clean_q = clean(word_preds[i][0]) preds = [p for p in preds if p != clean_q] if lang: # filter by target language preds = list(filter(is_lang(t), preds)) word_preds[i][1] = preds if save_pred: dump(word_preds, save_pred) pred_words_matrix = [ list(preds_to_words(wrd_pred, stop_words)) for _, wrd_pred in word_preds ] # src-word i -> (tfidf-word scores, tf-idf ngram scores) tfidf_results = tfidf_word_feats(pred_words_matrix) for (wrd, wrd_pred), (tfidf_words, tfidf_ngs) in \ zip(word_preds, tfidf_results): words_w_ranks = list(preds_to_words(wrd_pred, stop_words, rank=True)) yield word_row_feats( wrd, words_w_ranks, t, tfidf_words=tfidf_words, tfidf_ngrams=tfidf_ngs, )
def user(): user = {} with open("dataset/yelp_academic_dataset_user.json", "r") as f: for line in f: line = json.loads(line) user_id = line['user_id'] user[user_id] = {} # user[user_id]['review_cnt'] = line['review_count'] user[user_id]['yelp_since'] = line['yelping_since'] # user[user_id]['friends_cnt'] = len(line['friends']) # user[user_id]['fans'] = line['fans'] # user[user_id]['elite_year_cnt'] = len(line['elite']) user[user_id]['elite'] = line['elite'] user[user_id]['avg_stars'] = line['average_stars'] utils.dump(user, "dicts/user.p")
def insert(cls, *instances): """ Insert fact instances (overridden to handle Dimensions correctly) """ if instances: columns = [column for column in cls.__columns__ if not isinstance(column, AutoColumn)] sql = "%s INTO %s (\n %s\n)\n" % ( cls.INSERT, escaped(cls.__tablename__), ",\n ".join(escaped(column.name) for column in columns)) batches = cls.batch(instances) for iteration, batch in enumerate(batches, start=1): log.debug('Inserting batch %s' % (iteration), extra={"table": cls.__tablename__}) insert_statement = sql link = "VALUES" for instance in batch: values = [] for column in columns: value = instance[column.name] if isinstance(column, DimensionKey): if not value and column.optional: values.append(dump(value)) else: values.append( "(%s)" % column.dimension.__subquery__( value, instance.__dimension_selector__.timestamp(instance) # TODO This is a bit messy - shouldn't have to pass the instance back in. ) ) else: values.append(dump(value)) insert_statement += link + (" (\n %s\n)" % ",\n ".join(values)) link = "," connection = Warehouse.get() try: with closing(connection.cursor()) as cursor: cursor.execute(insert_statement) except Exception as e: classify_error(e) log.error(e) log.error(insert_statement) connection.rollback() else: connection.commit()
def search_list(self, page=1, page_size=10, query_params={}, sort_params={},pager_flag=True): if sort_params == {}: sort_params.update({"add_time": -1}) coll = self.get_coll() if pager_flag: length = coll.find(query_params).count() pager = utils.count_page(length, page, page_size) cr = coll.aggregate([ {"$match": query_params}, {"$sort": sort_params}, {"$skip": pager['skip']}, {"$limit": pager['page_size']}]) else: pager = utils.count_page(0, page, page_size) cr = coll.aggregate([ {"$match": query_params}, {"$sort": sort_params}]) objs = [] for obj in cr: obj = utils.dump(obj) try: for extra_param in self.extra_params: exec("""obj[extra_param] = self.get_%s(obj)"""%extra_param) objs.append(obj) except: objs.append(obj) return objs, pager
def prepare_fullha(config, ha_file): """ Prepare user.full_ha.file """ conf = yaml.load(ha_file) net_ip = ".".join((config['servers']['control-servers'][0]['ip'].split(".")[:3])) vipc = net_ip + ".253" conf["coe::base::controller_hostname"] = "control-server" conf["horizon::keystone_url"] = change_ip_to(conf["horizon::keystone_url"], vipc) conf["controller_names"] = [c["hostname"] for c in config['servers']['control-servers']] conf["openstack-ha::load-balancer::controller_ipaddresses"] = [c["ip"] for c in config['servers']['control-servers']] conf["openstack-ha::load-balancer::swift_proxy_ipaddresses"] = [c["ip"] for c in config['servers']['swift-proxy']] conf["openstack-ha::load-balancer::swift_proxy_names"] = [c["hostname"] for c in config['servers']['swift-proxy']] vipsw = net_ip + ".252" conf["openstack::swift::proxy::swift_proxy_net_ip"] = "%{ipaddress_eth2}" conf["openstack::swift::proxy::swift_memcache_servers"] = [i["ip"] + ":11211" for i in config['servers']['swift-proxy']] conf["nova::memcached_servers"] = [i["ip"] + ":11211" for i in config['servers']['control-servers']] conf["rabbit_hosts"] = [i["hostname"] + ":5672" for i in config['servers']['control-servers']] conf["galera::galera_servers"] = [c["ip"] for c in config['servers']['control-servers']] conf["galera::galera_master"] = config['servers']['control-servers'][0]["hostname"] + "." + DOMAIN_NAME conf["galera_master_name"] = config['servers']['control-servers'][0]["hostname"] conf["galera_master_ipaddress"] = config['servers']['control-servers'][0]["ip"] conf["galera_backup_names"] = [i["hostname"] for i in config['servers']['control-servers'][1:]] conf["galera_backup_ipaddresses"] = [i["ip"] for i in config['servers']['control-servers'][1:]] conf["openstack::swift::storage-node::storage_devices"] = ["vdb", "vdc", "vdd"] return dump(conf)
def create(self, **obj): coll = self.get_coll() curr_time = datetime.datetime.now() obj = json.loads(json.dumps(obj)) obj["add_time"] = str(curr_time) coll.insert_one(obj) return utils.dump(obj)
def __subquery__(cls, value, timestamp): """ Return a SQL SELECT query to use as a subquery within a fact INSERT. Does not append parentheses or a LIMIT clause. """ value_type = type(value) # We also check for subclasses for situations like basestring, which # matches on either str or unicode. natural_keys = [key for key in cls.__naturalkeys__ if (key.type is value_type or issubclass(value_type, key.type))] if not natural_keys: raise ValueError("Value type '%s' does not match type of any " "natural key for dimension " "'%s'" % (value_type.__name__, cls.__name__)) sql_template = ( 'SELECT {primary_key} FROM {table_name} ' 'WHERE {selector} ' 'AND `applicable_from` = (SELECT max(`applicable_from`) ' 'FROM {table_name} ' 'WHERE {selector} AND `applicable_from` <= "{timestamp}")' ) sql = sql_template.format( primary_key=escaped(cls.__primarykey__.name), table_name=escaped(cls.__tablename__), selector=" OR ".join("%s = %s" % (escaped(key.name), dump(value)) for key in natural_keys), timestamp=timestamp ) return sql
def erb(): working_dir = get_working_dir() log.setup(logging.DEBUG, path=working_dir) window_size = default_input('Window size', 3) n_nodes = default_input('N Nodes', 100) connectivity = default_input('Connectivity', 2) f = default_input('From', 0) t = default_input('To', n_nodes + 1) s = default_input('Step', n_nodes / 10) r = range(f, t, s) distribution = estimate_reservoir_distribution( 30, n_nodes, connectivity, r, window_size) name = '[NN:{}-WS:{}-K:{}]-distribution'.format(n_nodes, window_size, connectivity) dump(distribution, name, folder=working_dir)
def update(self, query_params, update_params): coll = self.get_coll() obj = coll.find_one(query_params) if obj: obj.update(update_params) ret = coll.save(obj) else: obj = {} return utils.dump(obj)
def affective(video_path, video_file, output_path): audio_path = output_path audio_file = video_file + ".wav" ret = extract_audio(video_path, video_file, audio_path, audio_file) if ret == 0: # normal return A, a = arousal.get_arousal(video_path, video_file, audio_path, audio_file) v = valence.get_valence(audio_path, audio_file, A, a) utils.dump(output_path, video_file + "_arousal.txt", a) utils.dump(output_path, video_file + "_valence.txt", v) utils.dump2(output_path + "/final/", video_file + "_final.txt" , v, a) plot_data.plot_data_pyplot(output_path + "/final/" + video_file + "_final.txt", v, a) try: os.remove(audio_path + audio_file) except OSError: print "Warning: file not removed" print "Finished"
def search(self, query_params): coll = self.get_coll() obj = coll.find_one(query_params) obj = utils.dump(obj) try: for extra_param in self.extra_params: exec ("""obj[extra_param] = self.get_%s(obj)""" % extra_param) except: pass return obj
def expression(self): s = [escaped(self.name), self.type_expression] if not self.optional: s.append("NOT NULL") default_expression = self.default_clause if default_expression: s.append(default_expression) if self.comment: s.append("COMMENT %s" % dump(self.comment)) return " ".join(s)
async def send_dispatch_event(self, event_type, guild, before=None, after=None): e = dict(ts=time(), type=event_type, producer=str(self), guild=dump(guild)) if before: if after: e['before'] = dump(before) e['after'] = dump(after) else: e['data'] = dump(before) self.log("{event}:{gid} @ {ts}".format(event=e['type'], gid=e['guild']['id'], ts=e['ts'])) await self.send('discord.events.{}'.format(e['type']), e)
def view_or_basicauth(view, request, *args, **kwargs): # Check for valid basic auth header utils.dump(request) if 'HTTP_AUTHORIZATION' in request.META: auth = request.META['HTTP_AUTHORIZATION'].split() if len(auth) == 2: if auth[0].lower() == "basic": uname, passwd = base64.b64decode(auth[1]).split(':') user = authenticate(username=uname, password=passwd) if user is not None and user.is_active: request.user = user return view(request, *args, **kwargs) # Either they did not provide an authorization header or # something in the authorization attempt failed. Send a 401 # back to them to ask them to authenticate. response = HttpResponse() response.status_code = 401 response['WWW-Authenticate'] = 'Basic realm="%s"' % "wtf" return response
def insert(cls, *instances): """ Insert one or more instances into the table as records. """ if instances: columns = [column for column in cls.__columns__ if not isinstance(column, AutoColumn)] sql = "%s INTO %s (\n %s\n)\n" % ( cls.INSERT, escaped(cls.__tablename__), ",\n ".join(escaped(column.name) for column in columns)) batches = cls.batch(instances) for iteration, batch in enumerate(batches, start=1): log.debug('Inserting batch %s' % (iteration), extra={"table": cls.__tablename__}) insert_statement = sql link = "VALUES" for instance in batch: values = [] for column in columns: value = instance[column.name] values.append(dump(value)) insert_statement += link + (" (\n %s\n)" % ",\n ".join(values)) link = "," for i in range(1, 3): connection = Warehouse.get() try: cursor = connection.cursor() cursor.execute(insert_statement) cursor.close() except Exception as e: classify_error(e) if e.__class__ == BrokenPipeError and i == 1: log.info( 'Trying once more with a fresh connection', extra={"table": cls.__tablename__} ) connection.close() else: log.error(e) return else: connection.commit() break log.debug('Finished updating %s' % cls.__tablename__, extra={"table": cls.__tablename__})
def _main(opts, args): level = logging.DEBUG if opts.debug else logging.INFO logging.basicConfig(format=FORMAT, level=level) if opts.quiet: logging.disable(logging.CRITICAL) logging.info('Imagy started') logging.debug(map(str, (args, opts))) if not opts.memorystore: store_path = opts.store_path if store_path is None: store_path = imagy_at_home = path('~').expanduser().joinpath(IMAGY_DIR_NAME) snippet = (' and backup files' if config.KEEP_ORIGINALS else '') msg = 'Using %s to store configuration%s, you can modify this path in config.py under STORE_PATH' logging.info(msg, imagy_at_home, snippet) store.load(store_path) args = [path(arg) for arg in args or FILE_PATTERNS if arg] run_daemon = opts.run if opts.clear: clear() elif opts.dump: dump(store) elif opts.revert: revert() elif opts.list: list_files() elif opts.files: do_files(*args) elif opts.deloriginals: delete_originals() elif opts.version: version() else: run_daemon = True if run_daemon: # if nothing specified so far, just run `smart mode` i.e. initialize the # directories and then run the daemon afterwards if not opts.no_init: initialize(*args) if not opts.no_watch: watch.watcher.run(*args)
def prepare_role(config, role_file): """ Prepare role_mappings file """ roles = {config["servers"]["build-server"]["hostname"]: "build"} for c in config["servers"]["control-servers"]: roles[c["hostname"]] = "controller" for c in config["servers"]["compute-servers"]: roles[c["hostname"]] = "compute" for c in config["servers"]["swift-storage"]: roles[c["hostname"]] = "swift_storage" for c in config["servers"]["swift-proxy"]: roles[c["hostname"]] = "swift_proxy" for c in config["servers"]["load-balancer"]: roles[c["hostname"]] = "load_balancer" return dump(roles)
def cache(url, path): print 'Downloading: %s' % urlparse.urlsplit(url).path page = requests.get(url, proxies=PROX) if page.status_code == 404: return {'url': url, 'uid': 'Error: 404', 'time': gmt_now()}, [] assert page.status_code == 200, 'Error: Status code error in page download, received %s @ %s' % (page.status_code, urlparse.urlsplit(url).path) tree = html.fromstring(page.content) links = list_links([str(x) for x in tree.xpath(r'//*[@href]/@href')], url=url) uid = dump(html.tostring(tree), path, verbose=True) pause() return {'url': url, 'uid': uid, 'time': gmt_now()}, links
def __init__(self, output_path, prod_name, wisdom_update = {}): """ Initialize postprocessor with output parameters. :param output_path: path where postprocessing files are stored :param prod_name: name of manifest json file and prefix of all output files :param wisdom_update: an optional dictionary that maps variables to modifications requested in their visualization wisdom """ logging.info("Postprocessor: output_path=%s prod_name=%s" % (output_path, prod_name)) dump(wisdom_update,"Postprocessor: wisdom_update") self.output_path = output_path self.product_name = prod_name self.manifest = {} self.wisdom_update = wisdom_update # in case the manifest exists, load the existing version mf_path = os.path.join(output_path, prod_name + '.json') if osp.exists(mf_path): self.manifest = json.load(open(mf_path)) logging.info('postprocessor: Loaded manifest at %s' % mf_path) # dump(self.manifest,"postprocessor: manifest") else: logging.info('postprocessor: manifest at %s does not exist yet' % mf_path)
def execute(cls, **params): database = getattr(cls, "database") query = getattr(cls, "query").format(**{key: dump(value) for key, value in params.items()}) with NamedConnection(database) as connection: with closing(connection.cursor(dictionary=True)) as cursor: cursor.execute(query) rows = [] for row in cursor: # Dump the rows immediately into memory, otherwise # the connection might timeout. rows.append(row) for row in rows: yield row
def prepare_common(config, common_file): """ Prepare user.common.file """ conf = yaml.load(common_file) net_ip = ".".join((config['servers']['control-server'][0]['ip'].split(".")[:3])) vipc = net_ip + ".253" conf["controller_public_address"] = vipc conf["controller_admin_address"] = vipc conf["controller_internal_address"] = vipc conf["coe::base::controller_hostname"] = "control-server" conf["domain_name"] = "domain.name" conf["ntp_servers"] = ["ntp.esl.cisco.com"] conf["external_interface"] = "eth4" conf["nova::compute::vncserver_proxyclient_address"] = "%{ipaddress_eth0}" conf["build_node_name"] = "build-server" conf["controller_public_url"] = change_ip_to( conf["controller_public_url"], vipc) conf["controller_admin_url"] = change_ip_to( conf["controller_admin_url"], vipc) conf["controller_internal_url"] = change_ip_to( conf["controller_internal_url"], vipc) conf["cobbler_node_ip"] = config['servers']['build-server'][0]['ip'] conf["node_subnet"] = ".".join(conf["cobbler_node_ip"].split(".")[:3]) + ".0" conf["node_gateway"] = ".".join(conf["cobbler_node_ip"].split(".")[:3]) + ".1" vipsw = ".".join((config['servers']['control-server'][0]['ip'].split(".")[:3])) + ".252" conf["swift_internal_address"] = vipsw conf["swift_public_address"] = vipsw conf["swift_admin_address"] = vipsw conf["swift_proxy_net_ip"] = "%{ipaddress_eth0}" conf['mysql::server::override_options']['mysqld']['bind-address'] = "0.0.0.0" # config['servers']['control-server'][0]['ip'] conf['swift_storage_interface'] = "eth0" conf['swift_local_net_ip'] = "%{ipaddress_eth0}" conf['internal_ip'] = "%{ipaddress_eth0}" conf['public_interface'] = "eth0" conf['private_interface'] = "eth0" conf['install_drive'] = "/dev/vda" conf['mon_initial_members'] = config['servers']['control-server'][0]["hostname"] conf['ceph_primary_mon'] = config['servers']['control-server'][0]["hostname"] conf['ceph_monitor_address'] = config['servers']['control-server'][0]["ip"] conf['ceph_cluster_interface'] = "eth0" conf['ceph_cluster_network'] = net_ip + ".0/24" conf['ceph_public_interface'] = "eth0" conf['ceph_public_network'] = net_ip + ".0/24" return dump(conf)
def cache_update(obj, path, links=[]): # headers = {'If-Modified-Since': obj['time']} headers = {} print 'Checking: %s' % urlparse.urlsplit(obj['url']).path page = requests.get(obj['url'], proxies=PROX, headers=headers) if page.status_code == 304: print 'No changes made...' else: assert page.status_code == 200, 'Error: Status code error in page download, received %s @ %s' % (page.status_code, urlparse.urlsplit(obj['url']).path) tree = html.fromstring(page.content) uid = dump(html.tostring(tree), path, verbose=True) links = list_links([str(x) for x in tree.xpath(r'//*[@href]/@href')], url=obj['url']) obj['time'] = gmt_now() return obj, links
def list(self,query_list,sort_list,use_pager=True,is_origin=False,page=1,page_size=options.page_size): length = self.get_coll().find(query_list).count() if use_pager: pager = utils.count_page(length,page,page_size) list = self.get_coll().aggregate([{"$match" : query_list}, {"$sort":sort_list}, {"$skip":pager['skip']}, {"$limit":pager['page_size']}]) else: pager = [] list = self.get_coll().aggregate([{"$match" : query_list}, {"$sort":sort_list} ]) if is_origin: return list,pager else: return utils.dump(list),pager
def prepare_new_files(config, path, use_sudo_flag): """ Prepare hostname specific files in puppet/data/hiera_data/hostname """ def write(text, path, filename, sudo): fd = StringIO(text) warn_if_fail(put(fd, os.path.join(path, filename), use_sudo=sudo)) warn_if_fail(put(fd, os.path.join(path, filename.replace("-", "_")), use_sudo=sudo)) for compute in config["servers"]["compute-servers"]: file_name = compute["hostname"] + ".yaml" ceph = {} ceph["cephdeploy::has_compute"] = True ceph["cephdeploy::osdwrapper::disks"] = ["vdb", "vdc", "vdd"] write(dump(ceph), path, file_name, use_sudo_flag) for num, lb in enumerate(config["servers"]["load-balancer"]): if num == 0: lb_text = ("openstack-ha::load-balancer::controller_state: MASTER\n" "openstack-ha::load-balancer::swift_proxy_state: BACKUP\n" ) else: lb_text = ("openstack-ha::load-balancer::controller_state: BACKUP\n" "openstack-ha::load-balancer::swift_proxy_state: MASTER\n" ) file_name = lb["hostname"] + ".yaml" write(lb_text, path, file_name, use_sudo_flag) for num, sw in enumerate(config["servers"]["swift-storage"]): sw_text = ( 'openstack::swift::storage-node::swift_zone: {num}\n' 'coe::network::interface::interface_name: "%{{swift_storage_interface}}"\n' 'coe::network::interface::ipaddress: "%{{swift_local_net_ip}}"\n' 'coe::network::interface::netmask: "%{{swift_storage_netmask}}"\n'.format(num=num+1) ) file_name = sw["hostname"] + ".yaml" write(sw_text, path, file_name, use_sudo_flag) file_name = config["servers"]["build-server"]["hostname"] + ".yaml" b_text = "apache::default_vhost: true" write(b_text, path, file_name, use_sudo_flag)
return word_counter """ Sample: Instantiate this directly and call whatever you want """ if __name__ == "__main__": toc = 'C:/data/books/game_of_thrones/A_Game_Of_Thrones_split_001.html' book = ebup_parser(table_of_contents=toc) overall = Counter() for a in range(3, book.num_chapters): text = book.get_chapter_contents(a) current_chapter_words = get_word_counter(text) overall += current_chapter_words all_words = dict(overall.most_common(200)) dump(all_words)