def run(command, blocks, preprocessed): if command.parms: # Looping Array if command.parms[0].startswith("[") and command.parms[0].endswith("]"): arr = command.parms[0].replace("[", "").replace("]", "").split("|") arr = [x.lstrip(" ").strip(" ") for x in arr] var = command.parms[1] if len(command.parms) > 1 else None blocks = "\n".join([" "*block.spaces + block.text for block in blocks]) total = [] for x in arr: copy_blocks = blocks.replace(var, x) if var else blocks total.append(process(preprocess(copy_blocks, False))) return "\n".join(total) # Looping Number (Range) else: times = 0 try: times = int(command.parms[0]) except: return None var = command.parms[1] if len(command.parms) > 1 else None blocks = "\n".join([" "*block.spaces + block.text for block in blocks]) total = [] for i in range(times): copy_blocks = blocks.replace(var, str(i+1)) if var else blocks total.append(process(preprocess(copy_blocks, False))) return "\n".join(total)
def main(settings, metrics): #Begin processing validation images # troubled_ones = [3, 14, 22, 43, 66, 83, 97, 114, 161] # troubled_ones = [137] for i in range(0, len(settings['validation_files'])): # for i in troubled_ones: if 'Rink-Isbrae' in settings['validation_files'][ i] or 'Upernavik' in settings['validation_files'][ i] or 'Umiammakku' in settings['validation_files'][ i] or 'Inngia' in settings['validation_files'][i]: # if 'Inngia' in settings['validation_files'][i]: # if i == 62: preprocess(i, settings, metrics) process(settings, metrics) postprocess(settings, metrics) # break #Print statistics # print_calfin_domain_metrics(settings, metrics) # print_calfin_all_metrics(settings, metrics) # plt.show() return settings, metrics
def train_model(modelBuilder): train_df = load_dataframe('train') test_df = load_dataframe('test') X_train = process(transform_dataset(train_df), isolate) X_test = process(transform_dataset(test_df), isolate) target_train = train_df['is_iceberg'] X_train_cv, X_valid, y_train_cv, y_valid = train_test_split( X_train, target_train, random_state=1, train_size=0.75) model = modelBuilder() optimizer = Adam(lr=LEARNING_RATE, beta_1=BETA_1, beta_2=BETA_2, epsilon=EPSILON, decay=DECAY) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) model.summary() callbacks = build_save_callbacks(filepath=MODEL_PATH, patience=5) datagen = ImageDataGenerator( # featurewise_center=True, # featurewise_std_normalization=True, # rotation_range=20, # width_shift_range=0.2, # height_shift_range=0.2, # horizontal_flip=True ) datagen.fit(X_train) empty = ImageDataGenerator() empty.fit(X_valid) steps_per_epoch = len(X_train_cv) // BATCH_SIZE hist = model.fit_generator(datagen.flow(X_train_cv, y_train_cv, batch_size=BATCH_SIZE), epochs=EPOCHS, verbose=VERBOSE, validation_data=empty.flow(X_valid, y_valid), steps_per_epoch=steps_per_epoch, callbacks=callbacks) model.load_weights(filepath=MODEL_PATH) score = model.evaluate(X_valid, y_valid, verbose=1) print('Test loss:', score[0]) print('Test accuracy:', score[1]) predicted_test = model.predict_proba(X_test) save_submission(test_df, predicted_test, filename='sub.csv') save_history(hist.history, model_name=MODEL_NAME)
def train_single(cw): c, w = cw data = PennFudanDataset('dataset/PennFudanPed') data.samples = data.samples[:30] process(data, create_pipeline(threshold=c)) inputs, targets = extractor.extract(data, w=w, N=100000, threaded=False) score = train_model(inputs, targets) print('c={0} w={1} p={2:.4f} r={3:.4f} f1={4:.4f}'.format(c, w, *score)) return (cw, score)
def main(settings, metrics): #Begin processing validation images # troubled_ones = [3, 14, 22, 43, 66, 83, 97, 114, 161] # troubled_ones = [161] for i in range(0, len(settings['validation_files'])): # for i in troubled_ones: preprocess(i, settings, metrics) process(settings, metrics) postprocess(settings, metrics) #Print statistics print_calfin_domain_metrics(settings, metrics) print_calfin_all_metrics(settings, metrics) return settings, metrics
def getBoardImage(img): ''' Runs an image through processing and neural network to decode digits img: an openCV image object returns: pil_im: a PIL image object with the puzzle isolated, cropped and straightened boardString: string representing the digits and spaces of a Sudoku board (left to right, top to bottom) ''' # Process image and extract digits pil_im, numbers, parsed, missed = process(img, False) if pil_im == None: return None, None net = NetworkReader.readFrom(os.path.dirname(os.path.abspath(__file__))+'/network.xml') boardString = '' for number in numbers: if number is None: boardString += ' ' else: data=ClassificationDataSet(400, nb_classes=9, class_labels=['1','2','3','4','5','6','7','8','9']) data.appendLinked(number.ravel(),[0]) boardString += str(net.activateOnDataset(data).argmax(axis=1)[0]+1) return pil_im, boardString
def auxthread_frame (frame): global gamestate marks = processing.process (frame) gamestate = gameplay.Gamestate (marks) processing.visualize (ui, gamestate, marks)
def main(): mainFldr = r"E:\00_dataRyanParker\working" fldrList = [] outFldr = r"E:\00_dataRyanParker\output" outGdb = os.path.join(outFldr, "output.gdb") build = r"E:\00_dataRyanParker\AllOttawa_Buildings\Buildings_polygon_MTM9.shp" for f in os.listdir(mainFldr): fldrList.append(os.path.join(mainFldr, f)) if not os.path.exists(outFldr): os.makedirs(outFldr) if os.path.exists(outGdb): arcpy.Delete_management(outGdb) arcpy.CreateFileGDB_management(out_folder_path=os.path.dirname(outGdb), out_name=os.path.basename(outGdb)) start = False for fldr in fldrList: if os.path.basename(fldr) == "f29": start = True if start: print("Processing folder " + str(fldr)) tmpFldrs = processing.process(fldr, outFldr, outGdb, build) time.sleep(60) for tmp in tmpFldrs: try: shutil.rmtree(tmp) except: print("Could not remove " + tmp + " it will need to be removed manually")
def auxthread_frame (frame): global gamestate waypoints, segments = processing.process (frame) gamestate = gameplay.Gamestate (waypoints, segments) processing.visualize (ui, gamestate, waypoints, segments)
def auxthread_frame(frame): if not ui.in_tutorial: frame = processing.process(frame) gamestate.add_frame(frame) gamestate.visualize(ui)
def main(args, cmd=False): if cmd: del args[0] if not args: if cmd: print("You have to specify the file/s") return setup() for arg in args: try: html = "" with open(arg, "r") as file: html = process(preprocess(file.read())) filename = arg.split(".")[0] with open(f"{filename}.html", "w") as file: file.write("<!DOCTYPE html>\n" + html) if cmd: print(f"The file {arg} compiled.") except FileNotFoundError as e: if cmd: print( f"Hey! One of the files ({arg}) doesn't exist. Continuing with all of the other files..." ) else: warning(f"One of the files ({arg}) is not found. ")
def pipeline(host_image, payload_image, encoding_significant_digits, significant_digit_interval): # type: ( # PIL.Image, PIL.Image, int, Tuple[int, int] # ) -> Tuple[PIL.Image, Dict[int, PIL.Image]] """Encodes an image using some params, returning encoded and decoded images. Args: host_image: Host image. payload_image: Payload image. (Yeah, I know, and I don't care. These values are documented elsewhere.) encoding_significant_digits: The significant digits to use in the encoding. significant_digit_interval: The significant digits to scan over when decoding the image. Returns: A 2-tuple; the first item is the encoded image, the second item is a dictionary mapping from significant_digit to decoded PIL.Image. """ encoded = encoding.encode(host_image, payload_image, encoding_significant_digits) return (encoded, processing.process( image=encoded, significant_digit_interval=significant_digit_interval))
def auxthread_frame(frame): global gamestate waypoints, segments = processing.process(frame) gamestate = gameplay.Gamestate(waypoints, segments) processing.visualize(ui, gamestate, waypoints, segments)
def entryPage(): errors = [] results = {} if request.method == "POST": # get url that the user has entered try: datepicker = request.form['datepicker'] processing.process(datepicker,False) #r = requests.get(datepicker) #print(r.text) except: errors.append( "Unable to get URL. Please make sure it's valid and try again." ) print("UNAVBLELE") return render_template('index.html', errors=errors, results=results)
def auxthread_frame (frame): if not ui.in_tutorial: frame = processing.process (frame) gamestate.add_frame (frame) gamestate.visualize (ui)
def hello(): req = request if request.method == 'GET': return "Hello world!" req = request env = os.environ headers = request.headers sk = request.json['secret_key'] if sk != os.getenv("secret_key"): return "Hello world!" service = google_service_api.get_service() processing.process(service) return "Completed"
def process_file(filename): """ Upload and process file """ if request.method == 'POST': try: rotate_angle = int(request.form['rotateby']) if rotate_angle**2 > 16: rotate_angle = (rotate_angle % 4) rotate_angle = rotate_angle * 90 except: rotate_angle = 0 processing.process(os.path.join(app.config['UPLOAD_FOLDER'], filename), rotate_angle) print "ran?" ## Redirect to reading view return redirect( url_for('uploaded_file', filename=filename) ) #"upload success" #redirect(url_for('uploaded_file',filename=filename))
def _analyse_full(env_dict): # info = raw_input("key info") logger.warning("Info key: %s", info) if "logs_file_runs" in env_dict: log_file = "unknown file" _log = logger while _log.parent is not None: _log = _log.parent for h in _log.handlers: if isinstance(h, logging.FileHandler): log_file = h.baseFilename break with codecs.open(env_dict["logs_file_runs"], encoding="utf-8", mode="a+") as fout: fout.write(u"<%s>: %s\n" % (log_file, info)) # delete everything _delete_index(env_dict, logger) # solr info stats_url = "/select?q=*:*&stats=true&stats.field=category&stats.field=citations_count&stats.field=math_count&rows=0&indent=true" utils.info_solr_home(env_dict, logger, stats_url=stats_url) # process - index import processing s = time.time() processing.process(env_dict, exported_process, None) e = time.time() logger.info(u"Processing method took [%s]s ([%s]m)", e - s, (e - s) / 60.0) # commit env_dict["indexer"]["optimise"] = True _commit_to_index(env_dict) logger.info(u"Committed to index") # solr info info = {} utils.info_solr_home(env_dict, logger, stats_url=stats_url, info_dict=info) # time info["index_time"] = u"%s:%s:%s\n" % (u"type", u"index time", (e - s)) lgr_str = u"\n" for v in info.values(): lgr_str += v + "\n" logger.info(lgr_str) # do performance testing _test_queries(env_dict)
def run(command, blocks, preprocessed): temp = [] for parm in command.parms: with open(parm, "r") as file: html = process(preprocess(file.read(), False)) temp.append(html) temp = [x for x in temp if x] return "\n".join(temp) if temp else None
def dataset(**kwargs): data_dir = path filepaths = glob.glob(path + "/**/*.txt", recursive=True) filepaths = filepaths[:3] # kwargs = return [ processing.process(1, [file], labels=file.split('/')[-2], **kwargs) for file in filepaths ]
def recognize_visitor(self, picture): result = [] faces = processing.process(picture.path) for face in faces: prediction, confidence = self.model.predict(face) if prediction >= 0: result.append((prediction, confidence)) return result, len(faces)
def _analyse_full( env_dict ): # info = raw_input("key info") logger.warning( "Info key: %s", info ) if "logs_file_runs" in env_dict: log_file = "unknown file" _log = logger while _log.parent is not None: _log = _log.parent for h in _log.handlers: if isinstance( h, logging.FileHandler ): log_file = h.baseFilename break with codecs.open( env_dict["logs_file_runs"], encoding="utf-8", mode="a+" ) as fout: fout.write( u"<%s>: %s\n" % (log_file, info) ) # delete everything _delete_index( env_dict, logger ) # solr info stats_url = "/select?q=*:*&stats=true&stats.field=category&stats.field=citations_count&stats.field=math_count&rows=0&indent=true" utils.info_solr_home( env_dict, logger, stats_url=stats_url ) # process - index import processing s = time.time() processing.process(env_dict, exported_process, None) e = time.time() logger.info( u"Processing method took [%s]s ([%s]m)", e - s, (e - s) / 60.0 ) # commit env_dict["indexer"]["optimise"] = True _commit_to_index(env_dict) logger.info( u"Committed to index" ) # solr info info = {} utils.info_solr_home( env_dict, logger, stats_url=stats_url, info_dict=info ) # time info["index_time"] = u"%s:%s:%s\n" % ( u"type", u"index time", (e - s) ) lgr_str = u"\n" for v in info.values(): lgr_str += v + "\n" logger.info( lgr_str ) # do performance testing _test_queries( env_dict )
def submit(): file = request.files['file'] if file and file.content_type in app.config['ALLOWED_TYPES']: result = process(file) return jsonify(result.serialize()) else: return Response( response="Only the following mimetypes are accepted: {}".format( str(app.config['ALLOWED_TYPES'])), status=404)
def process_image(self): importlib.reload( processing) # Reload the processing file to make testing easier mostCommon = processing.process(self, copy( self.originImage)) # Run the function\ for i in mostCommon: item = QtWidgets.QListWidgetItem(i) self.plateList.addItem(item) QtWidgets.QListWidgetItem()
def prepare_dataset(video_path, frame_folder, flow_folder, name, speeds=None): tqdm.set_lock(Lock()) # manually set internal lock #Step 1, Extract frames and speed dataframe_dict = {} if not os.path.exists(frame_folder): os.makedirs(frame_folder) print("Reading the video file") video_sk = skvideo.io.vread(video_path) print("Extracting the frames") for index, frame in enumerate(tqdm(video_sk)): saving_path = os.path.join(frame_folder, str(index)+'.jpg') if speeds is None: speed = 0 else: speed = speeds[index] dataframe_dict[index] = [saving_path, index, speed] skvideo.io.vwrite(saving_path, frame) processed_dataframe = pd.DataFrame.from_dict(dataframe_dict, orient='index') processed_dataframe.columns = ['frame_path', 'frame_index', 'speed'] print("Saving the dataframe") processed_dataframe.to_csv(os.path.join(PREPARED_DATA_PATH, name +'_meta.csv'), index=False) #Step 2, compute optical flow between frames and average the speed flow_dict = {} if not os.path.exists(flow_folder): os.makedirs(flow_folder) print("Computing the optical flow") for index in tqdm(range(len(processed_dataframe ) - 1)): idx1 = index idx2 = index + 1 frame1 = processed_dataframe.iloc[[idx1]] frame2 = processed_dataframe.iloc[[idx2]] assert(frame2['frame_index'].values[0] - frame1['frame_index'].values[0] == 1) assert(frame2['frame_index'].values[0] > frame1['frame_index'].values[0]) frame1_path = frame1['frame_path'].values[0] frame1_speed = frame1['speed'].values[0] frame2_path = frame2['frame_path'].values[0] frame2_speed = frame2['speed'].values[0] flow = process(frame1_path, frame2_path, SIZE) flow_path = os.path.join(flow_folder, str(index) + '.png') cv2.imwrite(flow_path, flow) speed = np.mean([frame1_speed, frame2_speed]) flow_dict[index] = [flow_path, speed] flow_dataframe = pd.DataFrame.from_dict(flow_dict, orient='index') flow_dataframe.columns = ['flow_path', 'speed'] print("Saving the flow dataframe") flow_dataframe.to_csv(os.path.join(PREPARED_DATA_PATH, name +'_flow_meta.csv'), index=False)
def auxthread_frame (frame): global maze, n_arrows if maze == None or maze.w != ui.maze_size or ui.refresh: ui.refresh = False maze = gameplay.Maze (ui.maze_size, ui.maze_size, ui.seed) arrows = processing.process (frame) n_arrows = len (arrows) maze.trace (arrows) processing.visualize (ui, arrows) maze.visualize (ui)
def start_process(self, req): self.update_state(state='PROGRESS', meta={ "client": req['uid'], "output": 'PROGRESS' }) user = getUserSignatures(req['uid']) meta = {"client": req['uid']} if (user is None): meta = { "client": req['uid'], "output": 'FAILURE', "isAuthValid": False, "msg": "invalid user id" } else: try: meta["isAuthValid"] = proc.process(req, user) except: meta = { "client": req['uid'], "output": 'FAILURE', "isAuthValid": False, "msg": "Error while computing values", } try: requests.post(f'{SERVER_GO_URL}/authAnswer', data=json.dumps({ "client": req['uid'], "isAuthValid": meta["isAuthValid"] }), headers={'Content-Type': 'application/json'}) meta["output"] = "SUCCESS" except: meta = { "client": req['uid'], "output": 'FAILURE', "isAuthValid": False, "msg": "communication with auth server failed" } finally: return meta
def main(settings, metrics): #Begin processing validation images # troubled_ones = [3, 14, 22, 43, 66, 83, 97, 114, 161] # troubled_ones = [10234] # 10302-10405 # for i in range(10233, 10234): # for i in troubled_ones: for i in range(21142, len(settings['validation_files'])): # if 'Rink-Isbrae' in settings['validation_files'][i]: if 'Upernavik' in settings['validation_files'][ i] or 'Umiammakku' in settings['validation_files'][ i] or 'Inngia' in settings['validation_files'][i]: # preprocess(i, settings, metrics) process(settings, metrics) postprocess(settings, metrics) #Print statistics # print_calfin_domain_metrics(settings, metrics) # print_calfin_all_metrics(settings, metrics) return settings, metrics
def main(): args = parse_args() loaded_data = load_data() (X_train, y_train, X_test) = process(loaded_data) X = pd.concat([X_train, X_test]) trained_models = [] for target in TARGET_COLUMNS: model = make_simple_model(target) model = model.fit(X_train, y_train[target]) trained_models.append(model) y_fits = [] y_preds = [] for m in trained_models: result = m.predict(X) result_df = pd.DataFrame(result, index=X.index) y_fitted = result_df.loc[:"2020-04-30 23:30:00", :] y_pred = result_df.loc["2020-05-01 00:00:00":"2020-07-22 23:30:00", :] y_fits.append(y_fitted) y_preds.append(y_pred) y_fits_df = pd.concat(y_fits, axis=1) y_fits_df.columns = [f"{c}_fitted" for c in TARGET_COLUMNS] fits = pd.concat([y_fits_df, y_train.loc[:"2020-04-30 23:30:00":, :]], axis=1) sub = pd.concat(y_preds, axis=1) sub.columns = TARGET_COLUMNS sub.to_csv(f'submission.csv') if args.verbose: show_results(trained_models, fits) for trained_model in trained_models: print( pd.DataFrame(trained_model.cv_results_)[[ 'mean_train_score', 'std_train_score', 'mean_test_score', 'std_test_score' ]]) fits.plot() sub.plot() plt.show()
def process(self, connection): super(Graph, self).process(connection) query = self.recv() query = Query.parse(query) cnx = self.open() root = cnx.root() try: result = list(process(query, root)) except Exception: transaction.abort() cnx.close() self.send({ 'type': 'exception', 'data': traceback.format_exc() }) else: transaction.commit() self.send({'type': 'result', 'data': result}) cnx.close()
def main_loop(training_file, testing_file, use_ff=False): # Parse the CSV file images_gray, labels = parse_csv(training_file) test_images, test_labels = parse_csv(testing_file) # Create the appropriate model for recognition if use_ff: model = cv2.createFisherFaceRecognizer() else: model = cv2.createLBPHFaceRecognizer(threshold=LBP_RECOGNITION_THRESHOLD) # Train the model with the gray-scale images and the labels print("Training the recognizer..."), sys.stdout.flush() model.train(numpy.array(images_gray), numpy.array(labels)) print("OK.") # Statistics recognized_faces_good = 0 unknowns = 0 for s in range(0, len(test_labels)): image = processing.process(test_images[s]) for i in image: prediction, confidence = model.predict(i) print("PREDICTED: %d, REAL: %d, CONFIDENCE: %s" % (prediction, test_labels[s], confidence)) if prediction == test_labels[s]: recognized_faces_good += 1 if prediction < 0: unknowns += 1 total = len(test_labels) print("Well predicted faces: %s of %s total." % (recognized_faces_good, total)) print("Desconocidos: %d" % unknowns)
from processing import process from data import getDataFromFolder import cv2 from network import loadSavedData, trainNetwork, saveNetwork from pybrain.tools.xml.networkwriter import NetworkWriter from imagesolver import getBoardImage from stringsolver import solveString # Process single photo img = cv2.imread("sampleimage.jpg") pil_im, numbers, parsedcheck, missedcheck = process(img, True) # # Create dataset # data = getDataFromFolder('/Users/kdelaney/Downloads/Photos/', '/Users/kdelaney/Downloads/Photos/data.txt') # # Train neural network # trndata, tstdata = loadSavedData('trainingdata', 'testdata') # network = trainNetwork(10, .0014, trndata, tstdata, 'network.xml') # NetworkWriter.writeToFile(network, 'network') # Decode board # img = cv2.imread("sampleimage.jpg") # croppedimg, boardString = getBoardImage(img) # croppedimg.show() # print "Decoded board: ", boardString # # # Solve board # solvedString = solveString(boardString)
#coding: utf-8 from os import path from processing import read, process s = process(read(path.join('testcase', '5.example')))
def generate_sets(dataset, w=11, N=5000): process(dataset, create_pipeline()) return extractor.extract(dataset, w=w, N=N)
# Compute list difference with files in db and to ignore rawFiles = diff(rawFiles, sampleFiles) rawFiles = diff(rawFiles, ignoreFiles) # Log list of files print strftime("%Y-%m-%d %H:%M:%S", localtime()) + ' -- ' + \ str(len(rawFiles)) + " files to process: " + \ str(rawFiles) + '\n' logFile_fh.flush() # Process new files time.sleep(60) # Sleep 1 min to be sure that RAW file are completely copied to the network drive. for raw_file_fullPath in rawFiles: processing.process(raw_file_fullPath, logFile_fh) # Add to file in the list of files stored in the database sampleFiles.append(raw_file_fullPath) #sys.exit() except (EnvironmentError, IOError), e: # parent of IOError, OSError *and* WindowsError where available print('MSQC error:' + str(e)) except OperationalError, e: print('MSQC error:' + str(e)) # Dummy query to DB to keep connection alive instruments = list(Instrument.objects.all()) # Sleep for the specified amount of time
def getDataFromSudokuDataset(): ''' Creates pybrain ClassificationDataSet from folder of images from Sudoku dataset found at https://github.com/wichtounet/sudoku_dataset ''' data = ClassificationDataSet(400, nb_classes=9, class_labels=['1','2','3','4','5','6','7','8','9']) path = '/Users/kdelaney/Downloads/sudoku_dataset-master/images/' dirs = os.listdir(path) parsed = 0 missed = 0 missednumbers = 0 # count of numbers in image that were not parsed falsenumbers = 0 # count of false number parsings from empty squares gatherednumbers = 0 correctspaces = 0 for p in range(0,len(dirs), 2): img = cv2.imread((path + dirs[p+1])) print((path + dirs[p+1])) dat = [] with open((path + dirs[p])) as f: next(f) next(f) for line in f: dat += line.split() pil_im, numbers, parsedcheck, missedcheck = process(img, False) parsed += parsedcheck missed += missedcheck ind = 0 if numbers is not None: for number in numbers: if number is None: if dat[ind] == '0': correctspaces += 1 ind += 1 else: missednumbers += 1 ind += 1 else: if dat[ind] == '0': falsenumbers += 1 ind += 1 else: gatherednumbers += 1 data.appendLinked(number.ravel(), [int(dat[ind])-1]) ind += 1 print ("\nprocessed: ") print("\n Test images processed successfully: " + str(parsed)) print(" Test images not processed successfully : " + str(missed)+"\n") print(" Number of digit samples gathered (true positives): " + str(gatherednumbers)) print(" Number of spaces confirmed (true negatives): " + str(correctspaces)+"\n") print(" Number of digit samples missed in a processed image (false negatives): " + str(missednumbers)) print(" Number of digit samples removed (false positives): " + str(falsenumbers)) return data
def _main(): #print process(_read()).solve ret = process(_read()) print ret.solve
facts = [] for sentence in scenario.split("\n"): try: if not sentence: continue if sentence[0] == "#": continue #IGNORE THOSE LINES, FOR THEY ARE COMMENTS if "." in sentence: sentence = sentence[:-1] #REMOVE "." at end of line print("\nSENTENCE: {0}".format(sentence)) sentence = processing.clean_negation(sentence) tokens = sentence.split() trees = list( cp.parse(tokens)) # arbre base sur les mots de la grammaire if (len(trees) > 1): print('Is Ambiguous !!!') for tree in trees: # draw.tree.draw_trees(tree) # Draw tree on python window sem = tree.label()["SEM"] facts = processing.process(str(sem)) for fact in facts: print(fact) with open("facts.clp", "a") as text: text.write(fact + '\n') except: continue
from sklearn.externals import joblib from dataset import PennFudanDataset from processing import process from classifier import extractor from filteropt import create_pipeline from sklearn.metrics import confusion_matrix import matplotlib.pyplot as plt dataset = PennFudanDataset('dataset/PennFudanPed') pipeline = create_pipeline(threshold=20) process(dataset, pipeline) inputs, targets = extractor.extract(dataset, w=11, N=20000) model = joblib.load('trained/quick_2.pkl') predicted = model.predict(inputs) cm = confusion_matrix(targets, predicted) print(cm) plt.matshow(cm) plt.title('Confusion matrix') plt.colorbar() plt.ylabel('True label') plt.xlabel('Predicted label') plt.show()
if input_from_cmd: env["input"] = env["input_base"] + input_from_cmd # special case function wrappers # exported_name = '%s%s' % (env["datasets"]["export_prefix"], what_to_do) # call default if what_to_do == env["datasets"]["default_method"]: try: exported_process = getattr(module, exported_name) if not hasattr(module, "exported_commit"): raise #assert getattr( module, exported_name ), "export_process must be present in module" what_to_do = lambda x: processing.process(x, exported_process, module.exported_commit) except: logger.error(u"Invalid operation name [%s]", exported_name) return lambda x: _print_with_options(dir(module)) # call in parallel or call only fnc else: try: exported_fnc = getattr(module, exported_name) if parallel_process: what_to_do = lambda x: processing.process(x, exported_fnc, None) else: what_to_do = getattr(module, exported_name) except: logger.error(u"Invalid operation name [%s]", exported_name) return lambda x: _print_with_options(dir(module))
'_treetagger_out.txt', '_raw.conll' ] tmp_fnames = [ os.path.join(tmp_path, fname_clean + fsuffix) for fsuffix in tmp_fsuffixes ] # output file and folder out_path = get_path_from_config(config, 'OUT_PATH', 'out') if out_fname is None: out_fname = os.path.join(out_path, fname_clean + '.conll') else: out_fname = os.path.join(out_path, out_fname) # create output and temp folder if needed for path in [tmp_path, out_path]: if not os.path.exists(path): os.makedirs(path) # rock'n'roll process(in_fname, out_fname, config['DEFAULT']['APP_ROOT'], config['mystem']['MYSTEM_PATH'], config['malt']['MALT_ROOT'], config['malt']['MALT_NAME'], config['malt']['MODEL_NAME'], config['dicts']['COMP_DICT_PATH'], config['treetagger']['TREETAGGER_BIN'], config['treetagger']['TREETAGGER_PAR'], *tmp_fnames) # remove temp files for fname in tmp_fnames: os.remove(fname)
def getDataFromFolder(folderpath, datapath): ''' Creates pybrain ClassificationDataSet from folder of iPhone images and .txt file of data ''' # Setup Dataset for PyBrain data = ClassificationDataSet(400, nb_classes=9, class_labels=['1','2','3','4','5','6','7','8','9']) # Get photos dirs = os.listdir(folderpath) # Get data dat = [] with open(datapath) as f: dat = f.read().splitlines() # Set variables parsed = 0 missed = 0 missednumbers = 0 # count of numbers in image that were not parsed falsenumbers = 0 # count of false number parsings from empty squares gatherednumbers = 0 correctspaces = 0 # Process each photo/data pairing for p in range(1,len(dirs)-1): # Create image img = cv2.imread((folderpath + dirs[p])) print(folderpath + dirs[p]) # Get all the digits in the image pil_im, numbers, parsedcheck, missedcheck = process(img, False) # Board located successfully parsed += parsedcheck # Board not located missed += missedcheck ind = 0 # Match digits photos with data for number in numbers: if (number is None): # True negative if dat[p-1][ind] == '0': correctspaces += 1 ind += 1 # False negative else: missednumbers += 1 ind += 1 else: # False positive if dat[p-1][ind] == '0': falsenumbers += 1 ind += 1 # True positive else: gatherednumbers += 1 data.appendLinked(number.ravel(), [int(dat[p-1][ind])-1]) ind += 1 # Print results print ("\nData processed: ") print("\n Puzzles located successfully: " + str(parsed)) print(" Puzzles not located successfully: " + str(missed)+"\n") print(" Number of digit samples gathered (true positives): " + str(gatherednumbers)) print(" Number of spaces confirmed (true negatives): " + str(correctspaces)+"\n") print(" Number of digit samples missed in a processed image (false negatives): " + str(missednumbers)) print(" Number of digit samples that needed to be removed (false positives): " + str(falsenumbers)) return data
async def read_item(input: int): result = process(input) return {"Result": str(result)}
config.read('config.ini') ifname_list, ofname_list = get_paths(ifolder, ofolder) # temporary files and folder tmp_fnames = ['mystem_in.txt', 'mystem_out.txt', 'treetagger_in.txt', 'treetagger_out.txt', 'raw.conll'] tmp_fnames = [os.path.join(config['DEFAULT']['TMP_PATH'], fname) for fname in tmp_fnames] total = len(ifname_list) for i, (ifname, ofname) in enumerate(zip(ifname_list, ofname_list)): print('{}/{} Processing {}'.format(i+1, total, ifname)) # check if we have already got this file # during the previous run of the script if os.path.exists(ofname): print('Already processed') else: print('Destination {}'.format(ofname)) start_time = time_counter() process(ifname, ofname, config['DEFAULT']['APP_ROOT'], config['mystem']['MYSTEM_PATH'], config['malt']['MALT_ROOT'], config['malt']['MALT_NAME'], config['malt']['MODEL_NAME'], config['dicts']['COMP_DICT_PATH'], config['treetagger']['TREETAGGER_BIN'], config['treetagger']['TREETAGGER_PAR'], *tmp_fnames) print('Performed in {:.3f} sec'.format(time_counter() - start_time))