def stop(): """stop the workers""" for worker in workers: celery_command("stop", workers[worker]["app"], workers[ worker]["hostlist"], workers[worker]["queue"]) mq.stop() clean()
def stop(): """stop the workers""" for worker in workers: celery_command("stop", workers[worker]["app"], workers[worker]["hostlist"], workers[worker]["queue"]) mq.stop() clean()
def list_hist(source): ''' List of lists histogram. [['hello', 1], ['you', 3], ['sir', 4]] Takes text. Stores each item in text, compares each item to the rest of the words in text and keeps a running total. Used list account for no repeats. ''' histo = [] used = [] text = clean(source) # print(text) for word in text: counter = 0 if word in used: continue used.append(word) for word2 in text: if word == word2: counter += 1 instance = [word, counter] histo.append(instance) # print(histo) return histo
def newdoc(text): doc = (clean(tb(text).lower())) dataset = pd.read_csv('cleaned.csv', index_col=0).dropna() vector = TfidfVectorizer() transformer = vector.fit_transform(dataset['text']) pos = dataset['label'] #nb = MultinomialNB() #nb=nb.fit(transformer, pos) newdoc_transformed = vector.transform([doc]) # save the classifier #with open('nb_classifier.pkl', 'wb') as fid: # pickle.dump(nb, fid) # load MultinomialNB with open('nb_classifier.pkl', 'rb') as fid: nb_loaded = pickle.load(fid) #svm = SGDClassifier(loss='hinge', penalty='l2', alpha = 1e-3, n_iter = 5, random_state = 42) #svm = svm.fit(transformer, pos) #with open('svm_classifier.pkl', 'wb') as fid: # pickle.dump(nb, fid) # load svm with open('svm_classifier.pkl', 'rb') as fid: svm_loaded = pickle.load(fid) pred = nb_loaded.predict(newdoc_transformed)[0] pred1 = svm_loaded.predict(newdoc_transformed)[0] feature_names = vector.get_feature_names() doc = 0 feature_index = newdoc_transformed[doc, :].nonzero()[1] tfidf_scores = zip(feature_index, [newdoc_transformed[doc, x] for x in feature_index]) sorted_words = sorted(tfidf_scores, key=lambda x: x[1], reverse=True) if (pred == 1 or pred1 == 1): print("It's positive article") num = random.randint(1, 3) if num == 1: return ("Почему {} не оставит вас равнодушным".format(feature_names[sorted_words[0][0]])) if num == 2: return ("Как с помощью {} победить всех".format(feature_names[sorted_words[0][0]])) if num == 3: return ("Как {} вам поможет".format(feature_names[sorted_words[0][0]])) else: print("It's negative article") num = random.randint(1, 3) if num == 1: return ("Почему из-за {} так плохо".format(feature_names[sorted_words[0][0]])) if num == 2: return ("Почему мы совершаем одинаковые ошибки с {}".format(feature_names[sorted_words[0][0]])) if num == 3: return ("Как победить {} в три этапа".format(feature_names[sorted_words[0][0]]))
def run_virtual_sensor(self, inference_df): """Run virtual sensor. Args: TODO: What input format here? Currently it takes a CSV-file. input_data (): """ params = yaml.safe_load(open("params.yaml")) classification = params["clean"]["classification"] onehot_encode_target = params["clean"]["onehot_encode_target"] learning_method = params["train"]["learning_method"] input_method = params["scale"]["input"] output_method = params["scale"]["output"] window_size = params["sequentialize"]["window_size"] overlap = params["sequentialize"]["overlap"] self._check_features_existence(inference_df) df = clean(inference_df=inference_df) df = featurize(inference=True, inference_df=df) X = np.array(df) input_scaler = joblib.load(INPUT_SCALER_PATH) output_scaler = joblib.load(OUTPUT_SCALER_PATH) if input_method is not None: X = input_scaler.transform(X) X = split_X_sequences(X, window_size, overlap=overlap) if learning_method in NON_DL_METHODS: model = load(MODELS_FILE_PATH) else: model = models.load_model(MODELS_FILE_PATH) y_pred = model.predict(X) if onehot_encode_target: y_pred = np.argmax(y_pred, axis=-1) elif classification: y_pred = np.array((y_pred > 0.5), dtype=np.int) print(y_pred) # plt.figure() # plt.plot(y_pred) # plt.show() return y_pred
from CONSTANT import * from diablo3_scrapper_utils import * from clean import * from get_page import * from copy_paste import * from create_database import * from get_categorie import * from test_bd import * import sqlite3 #De-commente pour retelecharger les pages en cas de mises a jour de Diablo et de changements dans les items get_page(dir_storage) #DEBUG supprime les fichiers temporaires se trouvant dans le repertoire d'execution clean(".", True) #On copie les fichiers d'un repertoire de stockage des pages HTML pour eviter de faire trop de requetes aupres #du serveur de blizzard copy_from_data_storage(dir_storage) #On cree la base de donnees create_database() #On cree une connection vers la base de donnees pour ne pas que les #fonctions ouvrent et ferment plusieurs dizaines de fois une connection db = sqlite3.connect(DB_NAME) for element in url_extension: #On selectionne seulemnt le contenu entre les balises <tbody> et </tbody> #Le contenu entre ces lignes est le detail et le nom des items select("<tbody>\n","</tbody>\n", element)
# runs all 6 algorithms tune = 'tune' in argv # attempt multiple runs with different tuning parameters final = 'final' in argv # use best algorithm, tuning parameters, and features write = 'write' in argv # write plots if (short + tune + final) > 1: print 'COULD NOT RUN' print 'short, tune, and final are incompatible arguments' elif (short + tune + final) == 0: final = True else: if read: if shard: df_clean = pd.read_csv('../data/clean_shard.csv') else: ints_in_argv = [int(arg) for arg in argv if arg.isdigit()] if ints_in_argv: rows = ints_in_argv[0] df_clean = pd.read_csv('../data/clean.csv', nrows=rows) else: df_clean = pd.read_csv('../data/clean.csv') else: df_clean = clean(load()) model(df_clean, shard, short, tune, final, write)
details, anthem, urls, faces): self.name = name self.age = age self.college = college self.job = job self.city = city self.gender = gender self.distance = distance self.details = details self.anthem = anthem self.urls = urls self.faces = faces df = pd.read_csv('data/raw/profile_data.csv') df = clean(df) df = fill_missing_cities(df) df = add_location_values(df) def find_face(img): img = Image.open(img).convert('RGB') img = np.array(img) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) faces = facealt_cascade.detectMultiScale(gray, 1.1, 4) # print("Found {} faces".format(len(faces))) face_arr = [] for (x, y, w, h) in faces: face_arr.append(gray[y:y + h, x:x + h]) # for each in face_arr:
scoring=filtered_accuracy, n_jobs=-1, iid=False, cv=3) grid.fit(df_X_known.copy(), y_known) unknown_predictions = filtered_predict(grid, df_unknowns, filter_train=True, df_X_train=df_knowns, y_train=y_knowns) end_time(start) if __name__ == '__main__': read = 'read' in argv # reads data from .csv if read: df_knowns = pd.read_csv('../data/clean.csv') else: df_knowns = clean(load()) df_unknowns = clean(load_unknowns()) # NEED load_unknowns() function predict(df_knowns, df_unknowns)
default='instructions.yaml', help='yaml file with data generation instructions') args = parser.parse_args() cwd = os.getcwd() with open(args.instF) as f: instr = yaml.load(f, Loader=yaml.FullLoader) n = instr.get('learnNum') seg = instr.get('seg') sN = instr.get('seriesNum') ST = instr.get('seriesTrain') i = 0 for i in range( 0, n): # run the classiciation model n times for the given color setup start = datetime.datetime.now() executeOrder66(args.instF, cwd) # create the training data from the instructions file # if seg: # saveGraphs() # cleanReg(cwd) if ST: series_classification(cwd, i) cleanSeries(cwd) else: graph_classification(cwd, i) clean(cwd) print('generation and classification ' + str(i) + ' elapsed time: ', (datetime.datetime.now() - start).total_seconds())
# runs all 6 algorithms tune = 'tune' in argv # attempt multiple runs with different tuning parameters final = 'final' in argv # use best algorithm, tuning parameters, and features write = 'write' in argv # write plots if (short + tune + final) > 1: print 'COULD NOT RUN' print 'short, tune, and final are incompatible arguments' elif (short + tune + final) == 0: final = True else: if read: if shard: df_clean = pd.read_csv('../data/clean_shard.csv') else: ints_in_argv = [int(arg) for arg in argv if arg.isdigit()] if ints_in_argv: rows = ints_in_argv[0] df_clean = pd.read_csv('../data/clean.csv', nrows=rows) else: df_clean = pd.read_csv('../data/clean.csv') else: df_clean = clean(load()) model(df_clean, shard, short, tune, final, write)
def get_one_ticker(one_ticker_name): option_data = Options(one_ticker_name,data_source='yahoo').get_all_data() option_data.reset_index(inplace=True) option_data.drop('JSON', axis=1, inplace=True) r=clean(option_data) return r
def ngrams(input,n): input=clean(input) output=[] for i in range(0, len(input)-n+1): output.append(input[i:i + n]) # 切片添加 return output
def clean(self): # XXX: sice smaze vsechny data, ale pri tvorbe nove dtb uz nevytvori spravne indexy # smazeme vsechny vytvorene kolekce dtb = connect(self.db, self.application.config) clean(dtb)
def convert(code): # code += "\n" code = "\n" + code + "\n" converted = "" bold = False italic = False inlineCode = False bigCode = False link = { "isImage": False, "start": 0, "end": 0, "url": "", "titleStart": 0, "titleEnd": 0, "title": "", "urlStart": 0, "urlEnd": 0, "text": "" } # lists = [] i = 1 while i < len(code) - 1: if not code[i - 1] == "\\": if code[i] == "\n": converted += "<br/>" if within(i, len(code), 3) and code[i] + code[i + 1] + code[i + 2] == '```': i += 2 # print ("Big Code") bigCode = not bigCode if bigCode: converted += "<br/><div style='background-color: grey;'><code>" else: converted += "</code></div>" else: if i < len(code) - 4: if code[i] + code[i + 1] + code[i + 2] == "---" or code[ i] + code[i + 1] + code[i + 2] == "***": i += 3 converted += "<hr>" if (code[i] == "_" and not code[i + 1] == "_") or ( code[i] == "*" and not code[i + 1] == "*"): italic = not italic i += 1 if not italic: converted += "</i>" else: converted += "<i>" if code[i] == "`" and not code[i + 1] == "`": inlineCode = not inlineCode i += 1 if not inlineCode: converted += "</code></span>" else: converted += "<span style='color: maroon;'><code>" if code[i] == "\n": if inlineCode: converted += "</code></span>" inlineCode = False if code[i] + code[i + 1] == "**" or code[i] + code[i + 1] == "__": i += 1 bold = not bold if not bold: converted += "</b>" else: converted += "<b>" else: if getLine(code, i).split()[0][0] == "#": headernum = getLine(code, i).split()[0].count("#") tmp = "<h" + str(headernum) + ">" + getLine( code, i)[headernum:].strip() + "</h" + str( headernum) + ">" converted += tmp i += len(getLine(code, i)) if getLine(code, i).split()[0][0] == ">": # tmp = converted += "<br><span style='color: grey; background-color: grey;'>|</span>" + ( " " + convertBasic(getLine(code, i).strip())) i += (len(getLine(code, i))) if getLine(code, i).split()[0][0:2] == "-" or getLine( code, i).split()[0][0:2] == "*": tmp = convertBasic(getLine(code, i).strip()) if getLine(code, i).split()[0][0:2] == "*": converted = converted[:-3] converted += "<ul><li>" + (" " + tmp) + "</li></ul>" # print (converted) i += len(getLine(code, i)) if code[i] == "[": link["start"] = i link["isImage"] = code[i - 1] == "!" if code[i] == "]": link["end"] = i link["text"] = code[link["start"] + 1:link["end"]] # print (link["text"]) if code[i] == "\"": # if not link["start"] == 0 and link["end"] == 0: if not link["titleStart"] == 0: link["titleEnd"] = i link["title"] = code[link["titleStart"] + 1:link["titleEnd"]] # print (link["title"]) else: link["titleStart"] = i if code[i] == "(": if link["start"]: link["urlStart"] = i if code[i] == ")": if link["start"]: link["urlEnd"] = i link["url"] = code[link["urlStart"] + 1:link["urlEnd"] - (len(link["title"]))] if link["title"]: link["url"] = link["url"][:-3] # print (link["url"]) converted = converted[:0 - (len(code[link["start"]:i]))] i += 1 converted = converted[:-1] converted += processLink(link) link = { "start": 0, "end": 0, "url": "", "titleStart": 0, "titleEnd": 0, "title": "", "urlStart": 0, "urlEnd": 0, "text": "" } converted += code[i] else: converted = converted[:-1] converted += code[i] i += 1 return "<html>" + clean(converted) + "</html>"
from CONSTANT import * from diablo3_scrapper_utils import * from clean import * from get_page import * from copy_paste import * from create_database import * from get_categorie import * from test_bd import * import sqlite3 #De-commente pour retelecharger les pages en cas de mises a jour de Diablo et de changements dans les items get_page(dir_storage) #DEBUG supprime les fichiers temporaires se trouvant dans le repertoire d'execution clean(".", True) #On copie les fichiers d'un repertoire de stockage des pages HTML pour eviter de faire trop de requetes aupres #du serveur de blizzard copy_from_data_storage(dir_storage) #On cree la base de donnees create_database() #On cree une connection vers la base de donnees pour ne pas que les #fonctions ouvrent et ferment plusieurs dizaines de fois une connection db = sqlite3.connect(DB_NAME) for element in url_extension: #On selectionne seulemnt le contenu entre les balises <tbody> et </tbody> #Le contenu entre ces lignes est le detail et le nom des items select("<tbody>\n", "</tbody>\n", element)