def put(self, request, book, name): flags = json.loads(request.body).get('flags', DatabaseBookPermissionFlag.NONE) permissions = DatabaseBook( book).get_permissions().get_or_add_group_permissions( name, BookPermissionFlags(flags)) return Response(permissions.to_json(), status.HTTP_201_CREATED)
def get(self, request, book): book = DatabaseBook(book) user_permissions = book.resolve_user_permissions(request.user) if not user_permissions or user_permissions.flags == DatabaseBookPermissionFlag.NONE: raise Response(status=status.HTTP_404_NOT_FOUND) return Response(user_permissions.to_json())
def post(self, request, book): flags = json.loads(request.body)['flags'] flags = flags & DatabaseBookPermissionFlag.ALLOWED_OTHER_PERMISSIONS book = DatabaseBook(book) permissions = book.get_permissions().permissions permissions.default.flags = flags book.get_permissions().write() return Response(permissions.default.to_json())
def read_all_page_stats(books: List[str], ignore_page: List[str] = []): all_stats = [] for book in books: book = DatabaseBook(book) for page in book.pages(): if any([s in page.page for s in ignore_page]): continue all_stats.append(page.page_statistics()) return all_stats
def output_book_2(self, test_pcgts_files, predictions, all_tp_staves, all_fp_staves, all_fn_staves): global_args = self.args.global_args output_tp = True output_fp = True output_fn = True if global_args.output_only_tp: output_fp = False output_fn = False if global_args.output_only_fp: output_tp = False output_fn = False logger.info("Outputting music lines to {}".format( global_args.output_book)) assert (predictions is not None) pred_book = DatabaseBook(global_args.output_book) output_pcgts_by_page_name = {} for pcgts in test_pcgts_files: o_pcgts = PcGts.from_file( pred_book.page(pcgts.page.location.page).file('pcgts')) output_pcgts_by_page_name[pcgts.page.location.page] = o_pcgts o_pcgts.page.music_regions.clear() for p, tp_staves, fp_staves, fn_staves in zip(predictions, all_tp_staves, all_fp_staves, all_fn_staves): o_pcgts = output_pcgts_by_page_name[ p.line.operation.page.location.page] if output_tp: for ml, gt_ml in [(ml, gt_ml) for ml, gt_ml, _ in tp_staves if ml in p.music_lines]: if global_args.output_symbols: ml.symbols = gt_ml.symbols[:] o_pcgts.page.music_regions.append( Block(block_type=BlockType.MUSIC, lines=[ml])) if output_fp: for ml in [ml for ml in fp_staves if ml in p.music_lines]: ml.symbols.clear() o_pcgts.page.music_regions.append( Block(block_type=BlockType.MUSIC, lines=[ml])) if output_fn: for gt_ml in fn_staves: if not global_args.output_symbols: gt_ml.symbols.clear() o_pcgts.page.music_regions.append( Block(block_type=BlockType.MUSIC, lines=[gt_ml])) for _, o_pcgts in output_pcgts_by_page_name.items(): o_pcgts.to_file(o_pcgts.page.location.file('pcgts').local_path())
def test_pages_with_lock(self): book = DatabaseBook('demo') pages = book.pages_with_lock([LockState(Locks.STAFF_LINES, True)]) self.assertListEqual([p.local_path() for p in pages], [book.page('page_test_lock').local_path()]) pages = book.pages_with_lock([ LockState(Locks.STAFF_LINES, False), LockState(Locks.SYMBOLS, True) ]) self.assertListEqual([p.local_path() for p in pages], [])
def put(self, request, book): book = DatabaseBook(book) if not book.exists(): return Response(status=status.HTTP_400_BAD_REQUEST) file = book.local_path("virtual_keyboard.json") json.dump(json.loads(request.body, encoding="utf-8"), open(file, 'w'), indent=4) return Response()
def get(self, request, book): book = DatabaseBook(book) if not book.exists(): return Response(status=status.HTTP_400_BAD_REQUEST) file = book.local_path("virtual_keyboard.json") if not os.path.exists(file): file = book.local_default_virtual_keyboards_path('default.json') with open(file) as f: return Response(json.load(f))
def post(self, request, book, operation): body = json.loads(request.body, encoding='utf-8') book = DatabaseBook(book) algorithm = Step.predictor(AlgorithmTypes(operation)) page_selection = PageSelection.from_params(PageSelectionParams.from_dict(body), book) pages = page_selection.get_pages(algorithm.unprocessed) return Response({ 'pages': [p.page for p in pages], 'pageCount': page_selection.page_count.value, 'singlePage': page_selection.single_page, 'book': book.book, 'totalPages': len(book.pages()), })
def test_single_line_001(self): try: book = DatabaseBook('demo') file = book.page("page_test_monodi_export_001") pcgts = file.pcgts() root = PcgtsToMeiConverter(pcgts) self.assertTrue(root.is_valid) # test to string and write root.to_string() buffer = BytesIO() root.write(buffer, pretty_print=True) except Exception as e: logging.exception(e) raise e
def remove_invalid_files(apps, schema_editor): books = DatabaseBook.list_available() for book in tqdm(books, "Removing old files"): for page in book.pages(): obsolete_files = [ 'annotation.json', 'binary_cropped.png', 'binary_cropped_preview.jpg', 'binary_deskewed.png', 'binary_deskewed_preview.jpg', 'binary_original.png', 'binary_original_preview.jpg', 'color_cropped.jpg', 'color_cropped_preview.jpg', 'color_deskewed.jpg', 'color_deskewed_preview.jpg', 'gray_cropped.jpg', 'gray_cropped_preview.jpg', 'gray_deskewed.jpg', 'gray_deskewed_preview.jpg', 'gray_original.jpg', 'gray_original_preview.jpg', 'connected_components_deskewed.pkl', ] for f in obsolete_files: f = page.local_file_path(f) if os.path.exists(f): os.remove(f)
def get(self, request, book, operation): book = DatabaseBook(book) body = json.loads(request.body, encoding='utf-8') if request.body else {} task_runner = BookOperationView.op_to_task_runner(operation, book, body) models = task_runner.list_available_models_for_book(book) return Response(models.to_dict())
def dataset_by_locked_pages( n_train, locks: List[LockState], shuffle: bool = True, datasets: List[DatabaseBook] = None ) -> Tuple[List[PcGts], List[PcGts]]: logger.info("Finding PcGts files with valid ground truth") pcgts = [] for dataset in (datasets if datasets else DatabaseBook.list_available()): logger.debug("Listing files of dataset '{}'".format(dataset.book)) if not dataset.exists(): raise ValueError("Dataset '{}' does not exist at '{}'".format( dataset.book, dataset.local_path())) for page in dataset.pages_with_lock(locks): pcgts.append(PcGts.from_file(page.file('pcgts'))) if len(pcgts) == 0: raise EmptyDataSetException() if shuffle: random.shuffle(pcgts) train_pcgts = pcgts[:int(len(pcgts) * n_train)] val_pcgts = pcgts[len(train_pcgts):] if 0 < n_train < 1 and (len(train_pcgts) == 0 or len(val_pcgts) == 0): raise EmptyDataSetException() return train_pcgts, val_pcgts
def test_single_line_001(self): try: book = DatabaseBook('demo') file = book.page("page_test_monodi_export_001") pcgts = file.pcgts() root = PcgtsToMonodiConverter([pcgts]).root j = root.to_json() drop_all_attributes(j, 'uuid') with open(file.local_file_path('monodi.json'), 'r') as f: ref = json.load(f) self.maxDiff = None self.assertEqual(ref, j) except Exception as e: logging.exception(e) raise e
def remove_word_and_neume_connector_layer(apps, schema_editor): books = DatabaseBook.list_available() for book in books: for page in book.pages(): pcgts_file = page.file('pcgts') try: if not pcgts_file.exists(): continue with open(pcgts_file.local_path(), 'r') as f: pcgts = json.load(f) page = pcgts['page'] if not page: continue text_regions = page.get('textRegions', []) for text_region in text_regions: text_lines = text_region.get('textLines', []) for text_line in text_lines: words = text_line.get('words', []) text_line['syllables'] = text_line.get('syllables', []) if not words: continue for word in words: text_line['syllables'] += word.get('syllables', []) annotations = page.get('annotations', {}) for connection in annotations.get('connections', []): for syllable_connector in connection.get( 'syllableConnectors', []): if 'refID' in syllable_connector: syllable_connector[ 'syllableID'] = syllable_connector['refID'] neume_connectors = syllable_connector.get( 'neumeConnectors', []) if len(neume_connectors) == 0: continue elif len(neume_connectors) == 1: syllable_connector['neumeID'] = neume_connectors[ 0]['refID'] else: raise ValueError( "Cannot convert {}. Neume connector has {} neume connectors. " "You need to manually convert this file. " "".format(pcgts_file.local_path(), len(neume_connectors))) with open(pcgts_file.local_path(), 'w') as f: json.dump(pcgts, f) except Exception as e: logger.error( "Exception occurred during processing of page {}".format( pcgts_file.local_path())) raise e
def to_train_val( self, locks: List[LockState], shuffle: bool = True, books: List[DatabaseBook] = None ) -> Tuple[List[PcGts], List[PcGts]]: if self.includeAllTrainingData: books = DatabaseBook.list_available() return dataset_by_locked_pages(self.nTrain, locks, shuffle, books)
def post(self, request, book, operation): book = DatabaseBook(book) body = json.loads(request.body, encoding='utf-8') if request.body else {} task_runner = BookOperationView.op_to_task_runner(operation, book, body) if task_runner is not None: task_id = operation_worker.id_by_task_runner(task_runner) if task_id: return Response({'task_id': task_id}) return Response(status=status.HTTP_404_NOT_FOUND)
def create_new_model(cls, book: DatabaseBook, id: Optional[str] = None) -> Model: import datetime id = id if id else str(uuid.uuid4()) time = datetime.datetime.now() models = ModelsId.from_external(book.book, cls.type()) return Model(MetaId(models, time.strftime("%Y-%m-%dT%H:%M:%S")), ModelMeta(id, time, style=book.get_meta().notationStyle))
def list_available_models_for_style(cls, style: str) -> DatabaseAvailableModels: default_style_model = cls.default_model_for_style( style).meta() if cls.default_model_for_style(style) else None return DatabaseAvailableModels( selected_model=default_style_model, default_book_style_model=default_style_model, models_of_same_book_style=[ (b.get_meta(), cls.newest_model_for_book(b).meta()) for b in DatabaseBook.list_available_of_style(style) if cls.newest_model_for_book(b) ])
def delete(self, request, book, page, operation): page = DatabasePage(DatabaseBook(book), page) if operation == 'clean': for key, _ in DatabaseFile.file_definitions().items(): if key != 'color_original': DatabaseFile(page, key).delete() return Response() elif operation == 'delete': page.delete() return Response()
def post(self, request, book, page, operation): body = json.loads(request.body, encoding='utf-8') page = DatabasePage(DatabaseBook(book), page) task_runner = OperationView.op_to_task_runner(operation, page, body) if task_runner: try: id = operation_worker.id_by_task_runner(task_runner) return Response({'task_id': id}, status=status.HTTP_202_ACCEPTED) except Exception as e: logger.error(e) return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def get(self, request, book, operation): book = DatabaseBook(book) body = json.loads(request.body, encoding='utf-8') if request.body else {} task_runner = BookOperationView.op_to_task_runner(operation, book, body) if task_runner is not None: task_id = operation_worker.id_by_task_runner(task_runner) op_status = operation_worker.status(task_id) if op_status: return Response({'status': op_status.to_dict()}) else: return Response(status.HTTP_204_NO_CONTENT) return Response(status=status.HTTP_204_NO_CONTENT)
def put(self, request, book, operation): body = json.loads(request.body, encoding='utf-8') book = DatabaseBook(book) task_runner = BookOperationView.op_to_task_runner(operation, book, body) if task_runner: try: id = operation_worker.put(task_runner, request.user) return Response({'task_id': id}, status=status.HTTP_202_ACCEPTED) except TaskAlreadyQueuedException as e: return Response({'task_id': e.task_id}, status=status.HTTP_303_SEE_OTHER) except Exception as e: logger.error(e) return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def pcgts_to_relative_coords(apps, schema_editor): books = DatabaseBook.list_available() for book in tqdm(books, "Converting to relative coords"): for page in book.pages(): pcgts_file = page.file('pcgts') size = Image.open(page.file('color_original').local_path()).size if not pcgts_file.exists(): continue with open(pcgts_file.local_path()) as f: j = json.load(f) was_local = to_relative_coords(j, size) if not was_local: with open(pcgts_file.local_path(), 'w') as f: json.dump(j, f)
def delete(self, request, book, operation, model): book = DatabaseBook(book) task_runner = BookOperationView.op_to_task_runner(operation, book, {}) # check that the model is really part of the model for m in task_runner.algorithm_meta().models_for_book(book).list_models(): if m.id() == model: m.delete() return Response() return APIError(status.HTTP_404_NOT_FOUND, "Model with id '{}' was not found in book '{}'. Available books {}.".format( model, book.book, [m.id() for m in task_runner.algorithm_meta().models_for_book(book).list_models()]), "Model not found.", ErrorCodes.MODEL_NOT_FOUND, ).response()
def pcgts_update_version(apps, schema_editor): books = DatabaseBook.list_available() version = 1 for book in tqdm(books, "Converting to pcgts version {}".format(version)): for page in book.pages(): pcgts_file = page.file('pcgts') if not pcgts_file.exists(): continue with open(pcgts_file.local_path()) as f: j = json.load(f) upgraded = update_pcgts(j, target_version=version) if upgraded: with open(pcgts_file.local_path(), 'w') as f: json.dump(j, f, indent=2)
def load(book: DatabaseBook): path = book.local_path('book_dictionary.json') try: with open(path) as f: d = DatabaseDictionary.from_book_json(book, json.load(f)) except FileNotFoundError: try: path = os.path.join(settings.BASE_DIR, 'internal_storage', 'default_dictionary', 'default_dictionary.json') with open(path) as f: d = DatabaseDictionary.from_book_json(book, json.load(f)) except FileNotFoundError: d = DatabaseDictionary(b_id=book.book) return d
def fix_dataset_params(apps, schema_editor): # book models for book in DatabaseBook.list_available(): if not os.path.exists(book.local_models_path()): continue for alg in os.listdir(book.local_models_path()): alg_dir = os.path.join(book.local_models_path(alg)) for model in os.listdir(alg_dir): path = os.path.join(alg_dir, model, 'dataset_params.json') fix_file(path) # default models default_models = os.path.join(BASE_DIR, 'internal_storage', 'default_models') if os.path.exists(default_models): for t in os.listdir(default_models): t_dir = os.path.join(default_models, t) for alg in os.listdir(t_dir): path = os.path.join(t_dir, alg, 'dataset_params.json') fix_file(path)
def op_to_task_runner(operation: str, book: DatabaseBook, body: dict) -> TaskRunner: from omr.steps.algorithmtypes import AlgorithmTypes for at in AlgorithmTypes: if at.value == operation: from restapi.operationworker.taskrunners.taskrunnerprediction import TaskRunnerPrediction, AlgorithmPredictorParams, Settings r = AlgorithmRequest.from_dict(body) meta = book.get_meta() meta.algorithmPredictorParams[at] = r.params return TaskRunnerPrediction(at, PageSelection.from_params(r.selection, book), Settings(meta.algorithm_predictor_params(at), store_to_pcgts=r.store_to_pcgts) ) # check if operation is linked to a task if operation == 'train_symbols': from restapi.operationworker.taskrunners.taskrunnersymboldetectiontrainer import TaskRunnerSymbolDetectionTrainer, TaskTrainerParams return TaskRunnerSymbolDetectionTrainer(book, TaskTrainerParams.from_dict(body.get('trainParams', {}))) elif operation == 'train_staff_line_detector': from restapi.operationworker.taskrunners.taskrunnertrainer import TaskRunnerTrainer, TaskTrainerParams return TaskRunnerTrainer(book, TaskTrainerParams.from_dict(body.get('trainParams', {})), AlgorithmTypes.STAFF_LINES_PC) elif operation == 'train_character_recognition': from restapi.operationworker.taskrunners.taskrunnertrainer import TaskRunnerTrainer, TaskTrainerParams return TaskRunnerTrainer(book, TaskTrainerParams.from_dict(body.get('trainParams', {})), AlgorithmTypes.OCR_CALAMARI) else: raise NotImplementedError()
self.dict: typing.Dict[str, Entry] = { x.value: x for x in MonodiXlsxConfig.default_config } self.entries = MonodiXlsxConfig.default_config if __name__ == '__main__': from matplotlib import pyplot as plt from PIL import Image import numpy as np from database import DatabaseBook, DatabaseFile from database.file_formats.pcgts import PageScaleReference book = DatabaseBook('Annotation___Square_Notation') pages = book.pages()[0] pcgts = [ DatabaseFile(page, 'pcgts', create_if_not_existing=True).page.pcgts() for page in [pages] ] file = pages.file('color_norm_x2').local_path() orig = Image.open(file) orig = np.array(orig) lines = pcgts[0].page.all_music_lines() page = pcgts[0].page for p in lines: # page = p.line.operation.page def p2i(l):