def put(self, request, book, name):
flags = json.loads(request.body).get('flags',
DatabaseBookPermissionFlag.NONE)
permissions = DatabaseBook(
book).get_permissions().get_or_add_group_permissions(
name, BookPermissionFlags(flags))
return Response(permissions.to_json(), status.HTTP_201_CREATED)
def get(self, request, book):
book = DatabaseBook(book)
user_permissions = book.resolve_user_permissions(request.user)
if not user_permissions or user_permissions.flags == DatabaseBookPermissionFlag.NONE:
raise Response(status=status.HTTP_404_NOT_FOUND)
return Response(user_permissions.to_json())
def post(self, request, book):
flags = json.loads(request.body)['flags']
flags = flags & DatabaseBookPermissionFlag.ALLOWED_OTHER_PERMISSIONS
book = DatabaseBook(book)
permissions = book.get_permissions().permissions
permissions.default.flags = flags
book.get_permissions().write()
return Response(permissions.default.to_json())
def read_all_page_stats(books: List[str], ignore_page: List[str] = []):
all_stats = []
for book in books:
book = DatabaseBook(book)
for page in book.pages():
if any([s in page.page for s in ignore_page]):
continue
all_stats.append(page.page_statistics())
return all_stats
def output_book_2(self, test_pcgts_files, predictions, all_tp_staves,
all_fp_staves, all_fn_staves):
global_args = self.args.global_args
output_tp = True
output_fp = True
output_fn = True
if global_args.output_only_tp:
output_fp = False
output_fn = False
if global_args.output_only_fp:
output_tp = False
output_fn = False
logger.info("Outputting music lines to {}".format(
global_args.output_book))
assert (predictions is not None)
pred_book = DatabaseBook(global_args.output_book)
output_pcgts_by_page_name = {}
for pcgts in test_pcgts_files:
o_pcgts = PcGts.from_file(
pred_book.page(pcgts.page.location.page).file('pcgts'))
output_pcgts_by_page_name[pcgts.page.location.page] = o_pcgts
o_pcgts.page.music_regions.clear()
for p, tp_staves, fp_staves, fn_staves in zip(predictions,
all_tp_staves,
all_fp_staves,
all_fn_staves):
o_pcgts = output_pcgts_by_page_name[
p.line.operation.page.location.page]
if output_tp:
for ml, gt_ml in [(ml, gt_ml) for ml, gt_ml, _ in tp_staves
if ml in p.music_lines]:
if global_args.output_symbols:
ml.symbols = gt_ml.symbols[:]
o_pcgts.page.music_regions.append(
Block(block_type=BlockType.MUSIC, lines=[ml]))
if output_fp:
for ml in [ml for ml in fp_staves if ml in p.music_lines]:
ml.symbols.clear()
o_pcgts.page.music_regions.append(
Block(block_type=BlockType.MUSIC, lines=[ml]))
if output_fn:
for gt_ml in fn_staves:
if not global_args.output_symbols:
gt_ml.symbols.clear()
o_pcgts.page.music_regions.append(
Block(block_type=BlockType.MUSIC, lines=[gt_ml]))
for _, o_pcgts in output_pcgts_by_page_name.items():
o_pcgts.to_file(o_pcgts.page.location.file('pcgts').local_path())
def test_pages_with_lock(self):
book = DatabaseBook('demo')
pages = book.pages_with_lock([LockState(Locks.STAFF_LINES, True)])
self.assertListEqual([p.local_path() for p in pages],
[book.page('page_test_lock').local_path()])
pages = book.pages_with_lock([
LockState(Locks.STAFF_LINES, False),
LockState(Locks.SYMBOLS, True)
])
self.assertListEqual([p.local_path() for p in pages], [])
def put(self, request, book):
book = DatabaseBook(book)
if not book.exists():
return Response(status=status.HTTP_400_BAD_REQUEST)
file = book.local_path("virtual_keyboard.json")
json.dump(json.loads(request.body, encoding="utf-8"),
open(file, 'w'),
indent=4)
return Response()
def get(self, request, book):
book = DatabaseBook(book)
if not book.exists():
return Response(status=status.HTTP_400_BAD_REQUEST)
file = book.local_path("virtual_keyboard.json")
if not os.path.exists(file):
file = book.local_default_virtual_keyboards_path('default.json')
with open(file) as f:
return Response(json.load(f))
def post(self, request, book, operation):
body = json.loads(request.body, encoding='utf-8')
book = DatabaseBook(book)
algorithm = Step.predictor(AlgorithmTypes(operation))
page_selection = PageSelection.from_params(PageSelectionParams.from_dict(body), book)
pages = page_selection.get_pages(algorithm.unprocessed)
return Response({
'pages': [p.page for p in pages],
'pageCount': page_selection.page_count.value,
'singlePage': page_selection.single_page,
'book': book.book,
'totalPages': len(book.pages()),
})
def test_single_line_001(self):
try:
book = DatabaseBook('demo')
file = book.page("page_test_monodi_export_001")
pcgts = file.pcgts()
root = PcgtsToMeiConverter(pcgts)
self.assertTrue(root.is_valid)
# test to string and write
root.to_string()
buffer = BytesIO()
root.write(buffer, pretty_print=True)
except Exception as e:
logging.exception(e)
raise e
def remove_invalid_files(apps, schema_editor):
books = DatabaseBook.list_available()
for book in tqdm(books, "Removing old files"):
for page in book.pages():
obsolete_files = [
'annotation.json',
'binary_cropped.png',
'binary_cropped_preview.jpg',
'binary_deskewed.png',
'binary_deskewed_preview.jpg',
'binary_original.png',
'binary_original_preview.jpg',
'color_cropped.jpg',
'color_cropped_preview.jpg',
'color_deskewed.jpg',
'color_deskewed_preview.jpg',
'gray_cropped.jpg',
'gray_cropped_preview.jpg',
'gray_deskewed.jpg',
'gray_deskewed_preview.jpg',
'gray_original.jpg',
'gray_original_preview.jpg',
'connected_components_deskewed.pkl',
]
for f in obsolete_files:
f = page.local_file_path(f)
if os.path.exists(f):
os.remove(f)
def get(self, request, book, operation):
book = DatabaseBook(book)
body = json.loads(request.body, encoding='utf-8') if request.body else {}
task_runner = BookOperationView.op_to_task_runner(operation, book, body)
models = task_runner.list_available_models_for_book(book)
return Response(models.to_dict())
def dataset_by_locked_pages(
n_train,
locks: List[LockState],
shuffle: bool = True,
datasets: List[DatabaseBook] = None
) -> Tuple[List[PcGts], List[PcGts]]:
logger.info("Finding PcGts files with valid ground truth")
pcgts = []
for dataset in (datasets if datasets else DatabaseBook.list_available()):
logger.debug("Listing files of dataset '{}'".format(dataset.book))
if not dataset.exists():
raise ValueError("Dataset '{}' does not exist at '{}'".format(
dataset.book, dataset.local_path()))
for page in dataset.pages_with_lock(locks):
pcgts.append(PcGts.from_file(page.file('pcgts')))
if len(pcgts) == 0:
raise EmptyDataSetException()
if shuffle:
random.shuffle(pcgts)
train_pcgts = pcgts[:int(len(pcgts) * n_train)]
val_pcgts = pcgts[len(train_pcgts):]
if 0 < n_train < 1 and (len(train_pcgts) == 0 or len(val_pcgts) == 0):
raise EmptyDataSetException()
return train_pcgts, val_pcgts
def test_single_line_001(self):
try:
book = DatabaseBook('demo')
file = book.page("page_test_monodi_export_001")
pcgts = file.pcgts()
root = PcgtsToMonodiConverter([pcgts]).root
j = root.to_json()
drop_all_attributes(j, 'uuid')
with open(file.local_file_path('monodi.json'), 'r') as f:
ref = json.load(f)
self.maxDiff = None
self.assertEqual(ref, j)
except Exception as e:
logging.exception(e)
raise e
def remove_word_and_neume_connector_layer(apps, schema_editor):
books = DatabaseBook.list_available()
for book in books:
for page in book.pages():
pcgts_file = page.file('pcgts')
try:
if not pcgts_file.exists():
continue
with open(pcgts_file.local_path(), 'r') as f:
pcgts = json.load(f)
page = pcgts['page']
if not page:
continue
text_regions = page.get('textRegions', [])
for text_region in text_regions:
text_lines = text_region.get('textLines', [])
for text_line in text_lines:
words = text_line.get('words', [])
text_line['syllables'] = text_line.get('syllables', [])
if not words:
continue
for word in words:
text_line['syllables'] += word.get('syllables', [])
annotations = page.get('annotations', {})
for connection in annotations.get('connections', []):
for syllable_connector in connection.get(
'syllableConnectors', []):
if 'refID' in syllable_connector:
syllable_connector[
'syllableID'] = syllable_connector['refID']
neume_connectors = syllable_connector.get(
'neumeConnectors', [])
if len(neume_connectors) == 0:
continue
elif len(neume_connectors) == 1:
syllable_connector['neumeID'] = neume_connectors[
0]['refID']
else:
raise ValueError(
"Cannot convert {}. Neume connector has {} neume connectors. "
"You need to manually convert this file. "
"".format(pcgts_file.local_path(),
len(neume_connectors)))
with open(pcgts_file.local_path(), 'w') as f:
json.dump(pcgts, f)
except Exception as e:
logger.error(
"Exception occurred during processing of page {}".format(
pcgts_file.local_path()))
raise e
def to_train_val(
self,
locks: List[LockState],
shuffle: bool = True,
books: List[DatabaseBook] = None
) -> Tuple[List[PcGts], List[PcGts]]:
if self.includeAllTrainingData:
books = DatabaseBook.list_available()
return dataset_by_locked_pages(self.nTrain, locks, shuffle, books)
def post(self, request, book, operation):
book = DatabaseBook(book)
body = json.loads(request.body, encoding='utf-8') if request.body else {}
task_runner = BookOperationView.op_to_task_runner(operation, book, body)
if task_runner is not None:
task_id = operation_worker.id_by_task_runner(task_runner)
if task_id:
return Response({'task_id': task_id})
return Response(status=status.HTTP_404_NOT_FOUND)
def create_new_model(cls,
book: DatabaseBook,
id: Optional[str] = None) -> Model:
import datetime
id = id if id else str(uuid.uuid4())
time = datetime.datetime.now()
models = ModelsId.from_external(book.book, cls.type())
return Model(MetaId(models, time.strftime("%Y-%m-%dT%H:%M:%S")),
ModelMeta(id, time, style=book.get_meta().notationStyle))
def list_available_models_for_style(cls,
style: str) -> DatabaseAvailableModels:
default_style_model = cls.default_model_for_style(
style).meta() if cls.default_model_for_style(style) else None
return DatabaseAvailableModels(
selected_model=default_style_model,
default_book_style_model=default_style_model,
models_of_same_book_style=[
(b.get_meta(), cls.newest_model_for_book(b).meta())
for b in DatabaseBook.list_available_of_style(style)
if cls.newest_model_for_book(b)
])
def delete(self, request, book, page, operation):
page = DatabasePage(DatabaseBook(book), page)
if operation == 'clean':
for key, _ in DatabaseFile.file_definitions().items():
if key != 'color_original':
DatabaseFile(page, key).delete()
return Response()
elif operation == 'delete':
page.delete()
return Response()
def post(self, request, book, page, operation):
body = json.loads(request.body, encoding='utf-8')
page = DatabasePage(DatabaseBook(book), page)
task_runner = OperationView.op_to_task_runner(operation, page, body)
if task_runner:
try:
id = operation_worker.id_by_task_runner(task_runner)
return Response({'task_id': id}, status=status.HTTP_202_ACCEPTED)
except Exception as e:
logger.error(e)
return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def get(self, request, book, operation):
book = DatabaseBook(book)
body = json.loads(request.body, encoding='utf-8') if request.body else {}
task_runner = BookOperationView.op_to_task_runner(operation, book, body)
if task_runner is not None:
task_id = operation_worker.id_by_task_runner(task_runner)
op_status = operation_worker.status(task_id)
if op_status:
return Response({'status': op_status.to_dict()})
else:
return Response(status.HTTP_204_NO_CONTENT)
return Response(status=status.HTTP_204_NO_CONTENT)
def put(self, request, book, operation):
body = json.loads(request.body, encoding='utf-8')
book = DatabaseBook(book)
task_runner = BookOperationView.op_to_task_runner(operation, book, body)
if task_runner:
try:
id = operation_worker.put(task_runner, request.user)
return Response({'task_id': id}, status=status.HTTP_202_ACCEPTED)
except TaskAlreadyQueuedException as e:
return Response({'task_id': e.task_id}, status=status.HTTP_303_SEE_OTHER)
except Exception as e:
logger.error(e)
return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def pcgts_to_relative_coords(apps, schema_editor):
books = DatabaseBook.list_available()
for book in tqdm(books, "Converting to relative coords"):
for page in book.pages():
pcgts_file = page.file('pcgts')
size = Image.open(page.file('color_original').local_path()).size
if not pcgts_file.exists():
continue
with open(pcgts_file.local_path()) as f:
j = json.load(f)
was_local = to_relative_coords(j, size)
if not was_local:
with open(pcgts_file.local_path(), 'w') as f:
json.dump(j, f)
def delete(self, request, book, operation, model):
book = DatabaseBook(book)
task_runner = BookOperationView.op_to_task_runner(operation, book, {})
# check that the model is really part of the model
for m in task_runner.algorithm_meta().models_for_book(book).list_models():
if m.id() == model:
m.delete()
return Response()
return APIError(status.HTTP_404_NOT_FOUND,
"Model with id '{}' was not found in book '{}'. Available books {}.".format(
model, book.book,
[m.id() for m in task_runner.algorithm_meta().models_for_book(book).list_models()]),
"Model not found.",
ErrorCodes.MODEL_NOT_FOUND,
).response()
def pcgts_update_version(apps, schema_editor):
books = DatabaseBook.list_available()
version = 1
for book in tqdm(books, "Converting to pcgts version {}".format(version)):
for page in book.pages():
pcgts_file = page.file('pcgts')
if not pcgts_file.exists():
continue
with open(pcgts_file.local_path()) as f:
j = json.load(f)
upgraded = update_pcgts(j, target_version=version)
if upgraded:
with open(pcgts_file.local_path(), 'w') as f:
json.dump(j, f, indent=2)
def load(book: DatabaseBook):
path = book.local_path('book_dictionary.json')
try:
with open(path) as f:
d = DatabaseDictionary.from_book_json(book, json.load(f))
except FileNotFoundError:
try:
path = os.path.join(settings.BASE_DIR,
'internal_storage',
'default_dictionary',
'default_dictionary.json')
with open(path) as f:
d = DatabaseDictionary.from_book_json(book, json.load(f))
except FileNotFoundError:
d = DatabaseDictionary(b_id=book.book)
return d
def fix_dataset_params(apps, schema_editor):
# book models
for book in DatabaseBook.list_available():
if not os.path.exists(book.local_models_path()):
continue
for alg in os.listdir(book.local_models_path()):
alg_dir = os.path.join(book.local_models_path(alg))
for model in os.listdir(alg_dir):
path = os.path.join(alg_dir, model, 'dataset_params.json')
fix_file(path)
# default models
default_models = os.path.join(BASE_DIR, 'internal_storage',
'default_models')
if os.path.exists(default_models):
for t in os.listdir(default_models):
t_dir = os.path.join(default_models, t)
for alg in os.listdir(t_dir):
path = os.path.join(t_dir, alg, 'dataset_params.json')
fix_file(path)
def op_to_task_runner(operation: str, book: DatabaseBook, body: dict) -> TaskRunner:
from omr.steps.algorithmtypes import AlgorithmTypes
for at in AlgorithmTypes:
if at.value == operation:
from restapi.operationworker.taskrunners.taskrunnerprediction import TaskRunnerPrediction, AlgorithmPredictorParams, Settings
r = AlgorithmRequest.from_dict(body)
meta = book.get_meta()
meta.algorithmPredictorParams[at] = r.params
return TaskRunnerPrediction(at,
PageSelection.from_params(r.selection, book),
Settings(meta.algorithm_predictor_params(at), store_to_pcgts=r.store_to_pcgts)
)
# check if operation is linked to a task
if operation == 'train_symbols':
from restapi.operationworker.taskrunners.taskrunnersymboldetectiontrainer import TaskRunnerSymbolDetectionTrainer, TaskTrainerParams
return TaskRunnerSymbolDetectionTrainer(book, TaskTrainerParams.from_dict(body.get('trainParams', {})))
elif operation == 'train_staff_line_detector':
from restapi.operationworker.taskrunners.taskrunnertrainer import TaskRunnerTrainer, TaskTrainerParams
return TaskRunnerTrainer(book, TaskTrainerParams.from_dict(body.get('trainParams', {})), AlgorithmTypes.STAFF_LINES_PC)
elif operation == 'train_character_recognition':
from restapi.operationworker.taskrunners.taskrunnertrainer import TaskRunnerTrainer, TaskTrainerParams
return TaskRunnerTrainer(book, TaskTrainerParams.from_dict(body.get('trainParams', {})), AlgorithmTypes.OCR_CALAMARI)
else:
raise NotImplementedError()
self.dict: typing.Dict[str, Entry] = {
x.value: x
for x in MonodiXlsxConfig.default_config
}
self.entries = MonodiXlsxConfig.default_config
if __name__ == '__main__':
from matplotlib import pyplot as plt
from PIL import Image
import numpy as np
from database import DatabaseBook, DatabaseFile
from database.file_formats.pcgts import PageScaleReference
book = DatabaseBook('Annotation___Square_Notation')
pages = book.pages()[0]
pcgts = [
DatabaseFile(page, 'pcgts', create_if_not_existing=True).page.pcgts()
for page in [pages]
]
file = pages.file('color_norm_x2').local_path()
orig = Image.open(file)
orig = np.array(orig)
lines = pcgts[0].page.all_music_lines()
page = pcgts[0].page
for p in lines:
# page = p.line.operation.page
def p2i(l):
