class DownloadCDTask(TaskBase):
def __init__(self, **kwargs):
super(DownloadCDTask, self).__init__(**kwargs)
self._cd = None
self._priority = 'medium'
self._library = kwargs['library']
self.identifier = kwargs['identifier']
self.logger.info('Download CD: Downloading {}'.format(self.identifier))
self.notifications_manager = NotificationManager()
self._download_type = None
self.start_time = None
self.total_time = None
def create_pipeline(self):
return [
self._begin,
self._get_ia_session,
self._load_item,
self._verify_is_CD,
self._verify_is_stub_item,
self._create_stub_CD,
self._create_files,
self._create_scandata,
self._set_states,
self._release_lock,
self._send_stats,
]
def _begin(self):
self.start_time = time.time()
def handle_event(self, event_name, *args, **kwargs):
if event_name == 'on_state' and self.state == CANCELLED_WITH_ERROR:
if self._cd:
self._cd.do_move_to_trash()
self._cd.do_delete_anyway()
def _get_ia_session(self):
self.dispatch_progress('Getting IA session')
self._ia_session = get_ia_session()
def _load_item(self):
self.dispatch_progress('Loading item')
self.item = self._ia_session.get_item(self.identifier)
self.logger.info('Download CD: target item: {}'.format(
self.item.identifier))
def _verify_is_CD(self):
self.dispatch_progress('Verifying this is an ArchiveCD item')
mediatype = self.item.metadata.get('mediatype')
software_version = self.item.metadata.get('software_version')
assert mediatype == 'audio', 'This is not an audio item. It is {}.'.format(
mediatype)
assert software_version is not None, 'This item was not created with ArchiveCD'
assert 'ArchiveCD' in software_version, 'This item was not created with ArchiveCD'
def _verify_is_stub_item(self):
self.dispatch_progress('Verifying this is a stub item')
stub_file = self.item.get_file('stub.txt')
#if not stub_file.exists:
# raise Exception('No stub file found!')
def _create_stub_CD(self):
self.dispatch_progress('Creating local CD')
message = "This CD is being downloaded and no actions are available just yet."
cd_id = str(uuid4())
self._cd = self._library.new_cd(cd_id,
status='download_incomplete',
error=message)
self._cd.set_lock()
self._cd.logger.info('Download CD: Created stub CD {}'.format(
self._cd))
def _create_files(self):
self.dispatch_progress('Downloading files')
ret = []
with open(os.path.join(self._cd.path, 'identifier.txt'), 'w+') as fp:
fp.write(self.item.identifier)
ret.append(fp.name)
self._cd.logger.info('Download CD: Created {}'.format(fp.name))
with open(os.path.join(self._cd.path, 'downloaded'), 'w+') as fp:
fp.write('True')
ret.append(fp.name)
self._cd.logger.info('Download CD: Created {}'.format(fp.name))
with open(os.path.join(self._cd.path, 'uuid'), 'w+') as fp:
fp.write(self._cd.uuid)
ret.append(fp.name)
self._cd.logger.info('Download CD: Created {}'.format(fp.name))
self.item.get_file(self.item.identifier + '_meta.xml') \
.download(file_path=self._cd.path + '/metadata.xml')
ret.append('{}'.format(self._cd.path + '/metadata.xml'))
self._cd.logger.info('Download CD: Created metadata.xml')
self._cd.reload_metadata()
if not os.path.exists(os.path.join(self._cd.path, 'thumbs')):
os.makedirs(os.path.join(self._cd.path, 'thumbs'))
ret.append('{}'.format(self._cd.path + '/thumbs'))
self._cd.logger.info('Download CD: Created thumbs directory')
self.item.get_file(self.item.identifier + '_itemimage.png') \
.download(file_path=self._cd.path + '/cover.png')
ret.append('{}'.format(self._cd.path + '/cover.png'))
self._cd.logger.info('Download CD: Downloaded cover')
self._files = ret
self._cd.logger.info('Download CD: Created files.')
def _create_scandata(self):
self.dispatch_progress('Creating scandata')
self._scandata=\
ScanData(self._cd.path)
self._scandata.save()
self._cd.reload_scandata()
self._cd.logger.info('Download CD: Created scandata.')
def _set_states(self):
self.dispatch_progress('Setting states')
self._cd.do_finish_download()
def _send_stats(self):
self.dispatch_progress('Notifying iabdash')
payload = {
'files': self._files,
'total_time': self.total_time,
}
push_event('tts-cd-downloaded', payload, 'cd', self.identifier,
os.path.join(self._cd.path, "iabdash.log"))
self.notifications_manager.add_notification(
title='Downloaded',
message="CD {} has been downloaded.".format(self.identifier),
show_system_tile=False,
book=self._cd)
def _release_lock(self):
self.total_time = time.time() - self.start_time
self._cd.logger.info('Download CD: ------ DONE. Downloaded {0} in '
'{1}s ----------'.format(self.identifier,
self.total_time))
self._cd.release_lock()
class ImportFolderTask(TaskBase):
def __init__(self, **kwargs):
super(ImportFolderTask, self).__init__(**kwargs)
self.source_path = kwargs['path']
self.library = kwargs['library']
self.book_obj = None
self.image_stack = None
self.scandata = None
self.metadata = None
self.DEFAULT_FIELDS_AND_VALUES = [
('operator', get_sc_metadata()['operator']),
('scanningcenter', get_sc_metadata()['scanningcenter']),
('ppi', Scribe3Configuration().get_numeric_or_none('ppi')),
]
self.do_not_rotate = True
def create_pipeline(self):
return [
self._load_directory,
self._verify_preconditions,
self._make_book_object,
self._load_metadata,
self._augment_metadata,
self._write_metadata,
self._load_image_stack,
self._make_scandata,
self._check_for_missing_images,
self._move_image_stack,
self._generate_thumbs,
]
def handle_event(self, event_name, *args, **kwargs):
if event_name == 'on_state' and self.state == CANCELLED_WITH_ERROR:
if self.book_obj:
self.book_obj.do_move_to_trash()
self.book_obj.do_delete_anyway()
def _load_directory(self):
self.dispatch_progress('Loading directory')
if not [
f
for f in os.listdir(self.source_path) if not f.startswith('.')
]:
raise Exception('The folder you selected is empty')
self.directory_list = list(os.walk(os.path.join(self.source_path)))[0]
def _verify_preconditions(self):
self.dispatch_progress('Verifying preconditions')
if '0000.jpg' not in self.directory_list[2]:
raise Exception('No image stack provided')
def _make_book_object(self):
self.dispatch_progress('Making book object')
generated_uuid = str(uuid4())
self.book_obj = self.library.new_book(generated_uuid)
def _load_metadata(self):
self.dispatch_progress('Loading metadata')
if 'metadata.xml' in self.directory_list[2]:
self.metadata = get_metadata(self.source_path)
else:
self.metadata = {}
def _augment_metadata(self):
for field, default_value in self.DEFAULT_FIELDS_AND_VALUES:
if field not in self.metadata and default_value is not None:
self.metadata[field] = default_value
def _load_image_stack(self):
self.dispatch_progress('Loading image stack')
self.image_stack = sorted([
k for k in self.directory_list[2]
if re.match('\d{4}\.jpg$', os.path.basename(k))
])
# consider .*[^\d]\d{4}.jpg
def _make_scandata(self):
self.dispatch_progress('Generating scandata')
self.scandata = ScanData(self.book_obj.path)
for image in self.image_stack:
if image == '0000.jpg':
leaf_number = 0
else:
leaf_number = self.__extract_number_from_file(image)
side = 'left' if leaf_number % 2 == 0 else 'right'
page_type = 'Normal'
if image == '0000.jpg':
page_type = 'Color Card'
elif image == '0001.jpg':
page_type = 'Cover'
elif leaf_number == len(self.image_stack) - 1:
page_type = 'Color Card'
self.scandata.insert(leaf_number, side, page_type)
if self.do_not_rotate:
self.scandata.update_rotate_degree(leaf_number, 0)
def _check_for_missing_images(self):
self.dispatch_progress('Checking for image stack integrity')
if not (self.source_path and self.scandata):
raise Exception('Cover image is missing!')
max_leaf_number = self.scandata.get_max_leaf_number()
if max_leaf_number is None or max_leaf_number < 1:
raise Exception('Cover image is missing!')
for leaf_number in range(max_leaf_number + 1):
leaf_data = self.scandata.get_page_data(leaf_number)
image_path = os.path.join(self.source_path,
'{:04d}.jpg'.format(leaf_number))
if not (leaf_data and os.path.exists(image_path)):
if leaf_number == 0 or leaf_number == 1:
raise Exception('Cover image is missing!')
raise Exception('Image #{} is missing'.format(leaf_number))
self.scandata.save()
self.book_obj.reload_scandata()
def _write_metadata(self):
self.dispatch_progress('Writing metadata')
set_metadata(self.metadata, self.book_obj.path)
self.book_obj.reload_metadata()
self.book_obj.do_create_metadata()
def _move_image_stack(self):
self.dispatch_progress('Relocating image stack')
for image in self.image_stack:
source = os.path.join(self.source_path, image)
destination = os.path.join(self.book_obj.path, image)
shutil.copy(source, destination)
def _generate_thumbs(self):
self.dispatch_progress('Generating thumbs')
for n, image in enumerate(self.image_stack):
self.dispatch_progress('Generating thumbs [{}/{}]'.format(
n, len(self.image_stack)))
source_image = os.path.join(self.book_obj.path, image)
target_image = os.path.join(self.book_obj.path, 'thumbnails',
image)
current_degree = int(
self.scandata.get_page_data(n).get('rotateDegree', 0))
rotate_by = convert_scandata_angle_to_thumbs_rotation(
current_degree, None)
thumbnail_size = (1500, 1000)
if Scribe3Configuration().is_true('low_res_proxies'):
thumbnail_size = (750, 500)
image = Image.open(source_image)
image.thumbnail(thumbnail_size)
image = image.rotate(rotate_by, expand=True)
image.save(target_image, 'JPEG', quality=90)
@staticmethod
def __extract_number_from_file(filename):
number = filename.split('.jpg')[0]
ret = number.lstrip('0')
return int(ret)
class ReShootScreenBackend(WidgetBackend):
EVENT_CAPTURE_LEAF = 'on_capture_leaf'
EVENT_CURRENT_LEAF = 'on_current_leaf'
EVENT_ROTATE_LEAF = 'on_rotate_leaf'
EVENT_PAGE_TYPE = 'on_page_type'
EVENT_SHOW_ORIGINAL_FILE = 'on_show_original_file'
EVENT_SHOW_RESHOOT_FILE = 'on_show_reshoot_file'
EVENT_SHOW_PAGE_TYPE_FORM_POPUP = 'on_show_page_type_form_popup'
EVENT_GO_BACK = 'on_go_back'
__events__ = (EVENT_CAPTURE_LEAF, EVENT_CURRENT_LEAF, EVENT_ROTATE_LEAF,
EVENT_PAGE_TYPE, EVENT_GO_BACK, EVENT_SHOW_ORIGINAL_FILE,
EVENT_SHOW_RESHOOT_FILE, EVENT_SHOW_PAGE_TYPE_FORM_POPUP)
def __init__(self, **kwargs):
super(ReShootScreenBackend, self).__init__(**kwargs)
self._note_leafs = []
self._reverse_cams = False
self._cameras_count = 0
self._current_leaf_index = 0
self._capture_running = False
self._keyboard_action_handler = ReShootScreenKeyboardHandler(self)
self.keyboard_detector = None
self.book = None
self.reopen_at = 0
self.scandata = None
self.camera_system = None
self.window = None
def init(self):
if not self.scandata:
self.scandata = ScanData(self.book['path'], downloaded=True)
self._note_leafs[:] = self.scandata.iter_flagged_leafs()
try:
leaf_index = self._note_leafs.index(self.reopen_at)
except ValueError:
leaf_index = 0
self.set_current_leaf_index(leaf_index)
if not self.keyboard_detector:
detector = ReShootActionDetector(RESHOOT_ACTION_BINDINGS)
self.keyboard_detector = detector
self._keyboard_action_handler.detector = self.keyboard_detector
self._cameras_count = self.camera_system.cameras.get_num_cameras()
self._capture_running = False
self._reverse_cams = False
self.config = Scribe3Configuration()
super(ReShootScreenBackend, self).init()
def reset(self):
self.book = None
self.reopen_at = None
self.scandata = None
self.camera_system = None
self.window = None
del self._note_leafs[:]
self._current_leaf_index = 0
self._reverse_cams = False
self._cameras_count = 0
self._capture_running = False
super(ReShootScreenBackend, self).reset()
def is_capture_running(self):
return self._capture_running
def is_reshoot_leaf_ready(self):
path, thumb_path = self.get_current_reshoot_paths()
return exists(path) and exists(thumb_path)
def can_switch_cameras(self):
return self._cameras_count > 1
def can_capture_spread(self):
return cradle_closed and not self._capture_running and self._cameras_count > 0
def get_current_leaf_number(self):
return self._note_leafs[self._current_leaf_index]
def get_current_leaf_index(self):
return self._current_leaf_index
def set_current_leaf_index(self, index):
max_index = max(0, len(self._note_leafs) - 1)
if 0 <= index <= max_index and self._current_leaf_index != index:
self._current_leaf_index = index
self.dispatch(self.EVENT_CURRENT_LEAF)
def get_leafs_count(self):
return len(self._note_leafs)
def get_book_metadata(self):
md = get_metadata(self.book['path'])
return {
'identifier': self.book.get('identifier', None),
'path': self.book['path'],
'title': md.get('title', None),
'creator': md.get('creator', md.get('author', None)),
'language': md.get('language', None)
}
def get_leaf_data(self):
leaf_number = self.get_current_leaf_number()
leaf_data = self.scandata.get_page_data(leaf_number)
page_number_data = leaf_data.get('pageNumber', None)
page_number = self._get_page_number(page_number_data)
return {
'hand_side': leaf_data.get('handSide', None),
'page_number': page_number,
'page_type': leaf_data['pageType'],
'note': leaf_data.get('note', None)
}
def get_current_reshoot_paths(self):
leaf_number = self.get_current_leaf_number()
image_name = '{:04d}.jpg'.format(leaf_number)
book_path = self.book['path']
path = join(book_path, 'reshooting', image_name)
thumb_path = join(book_path, 'reshooting', 'thumbnails', image_name)
ensure_dir_exists(join(book_path, 'reshooting'))
ensure_dir_exists(join(book_path, 'reshooting', 'thumbnails'))
return path, thumb_path
def get_current_original_paths(self):
leaf_number = self.get_current_leaf_number()
image_name = '{:04d}.jpg'.format(leaf_number)
book_path = self.book['path']
path = join(book_path, image_name)
thumb_path = join(book_path, 'thumbnails', image_name)
return path, thumb_path
def _get_page_number(self, page_number_data):
# TODO: Remove this method when scandata structure becomes the same
# for reshooting mode and otherwise
if page_number_data:
if isinstance(page_number_data, dict):
page_number = page_number_data.get('num', None)
return None if page_number is None else int(page_number)
elif isinstance(page_number_data, str):
return int(page_number_data)
return None
def goto_previous_leaf(self, *args):
self.set_current_leaf_index(self._current_leaf_index - 1)
def goto_next_leaf(self, *args):
self.set_current_leaf_index(self._current_leaf_index + 1)
def goto_first_leaf(self, *args):
self.set_current_leaf_index(0)
def goto_last_leaf(self, *args):
max_index = max(0, len(self._note_leafs) - 1)
self.set_current_leaf_index(max_index)
def goto_rescribe_screen(self, *args):
self.dispatch(self.EVENT_GO_BACK)
def show_original_file(self, *args):
self.dispatch(self.EVENT_SHOW_ORIGINAL_FILE)
def show_reshoot_file(self, *args):
if self.is_reshoot_leaf_ready():
self.dispatch(self.EVENT_SHOW_RESHOOT_FILE)
def show_page_type_form_popup(self, *args):
if self.is_reshoot_leaf_ready():
self.dispatch(self.EVENT_SHOW_PAGE_TYPE_FORM_POPUP)
def save_leaf_note(self, note):
scandata = self.scandata
leaf_number = self.get_current_leaf_number()
if scandata.get_note(leaf_number) != note:
scandata.set_note(leaf_number, note)
scandata.save()
if note:
self.logger.info(
'ReShootScreenBackend: Updated leaf %d with note: %s' %
(leaf_number, '\n%s' % note if '\n' in note else note))
else:
self.logger.info(
'ReShootScreenBackend: Removed note from leaf {}'.format(
leaf_number))
def update_page_type(self, leaf_number, page_type):
scandata = self.scandata
scandata.update_page_type(leaf_number, page_type)
scandata.save()
self.dispatch(self.EVENT_PAGE_TYPE, page_type)
def update_leaf_rotation_if_necessary(self, leaf_number):
if self._cameras_count == 1:
self.logger.info(
'ReShootScreenBackend: Reshooting in single-camera mode, will '
'rotate by system default of {} degrees'.format(
self.config.get_integer('default_single_camera_rotation',
180)))
new_degree = self.config.get_integer(
'default_single_camera_rotation', 180)
self.scandata.update_rotate_degree(leaf_number, new_degree)
self.scandata.save()
self.logger.info(
'ReShootScreenBackend: Set leaf {} rotation to {} degree(s)'.
format(leaf_number, new_degree))
def enable_keyboard_actions(self, *args):
self._keyboard_action_handler.enable()
def disable_keyboard_actions(self, *args):
self._keyboard_action_handler.disable()
def are_keyboard_actions_enabled(self):
return self._keyboard_action_handler.is_enabled()
def switch_cameras(self, *args):
if not self._capture_running and self.can_switch_cameras():
self._reverse_cams = not self._reverse_cams
self.logger.info('ReShootScreen: Switched cameras')
self.capture_spread()
def are_cameras_switched(self):
return self._reverse_cams
def capture_spread(self, *args):
if not self.can_capture_spread():
return
if not has_free_disk_space(self.book['path']):
self.logger.info('capture_spread: the disk is full!')
report = {camera_system.KEY_ERROR: DiskFullError()}
self.dispatch(self.EVENT_CAPTURE_LEAF, report)
return
leaf_number = self.get_current_leaf_number()
side = self._get_capture_camera_side(leaf_number)
path, thumb_path = self.get_current_reshoot_paths()
camera_kwargs = self._create_camera_kwargs(side, leaf_number)
self.logger.info(
'ReShootScreen: Capturing new image for leaf {}, camera side {}, '
'{}using reversed cameras'.format(
leaf_number, side, '' if self._reverse_cams else 'not '))
self._capture_running = True
self.delete_current_spread()
self.update_leaf_rotation_if_necessary(leaf_number)
report = {camera_system.KEY_CAPTURE_START: True}
self.dispatch(self.EVENT_CAPTURE_LEAF, report)
self.camera_system.left_queue.put(camera_kwargs)
def _get_capture_camera_side(self, leaf_number):
if self._cameras_count == 1:
camera_side = 'foldout'
else:
camera_side = 'left' if leaf_number % 2 == 0 else 'right'
if self._reverse_cams:
camera_side = 'left' if camera_side == 'right' else 'right'
return camera_side
def _capture_spread_end(self, report, *args):
self._capture_running = False
report[camera_system.KEY_CAPTURE_END] = True
if self.is_initialized():
stats = report[camera_system.KEY_STATS]
leaf_number = report[camera_system.KEY_EXTRA]['leaf_number']
self.scandata.set_capture_time(leaf_number, stats['capture_time'])
self.dispatch(self.EVENT_CAPTURE_LEAF, report)
def delete_current_spread(self, *args):
path, thumb_path = self.get_current_reshoot_paths()
self._delete_file(path)
self._delete_file(thumb_path)
def _delete_file(self, path):
if exists(path):
os.remove(path)
self.logger.info('ReShootScreenBackend: Removed: {}'.format(path))
def rotate_reshoot_leaf(self, *args):
scandata_rotation_angle = 90
path, thumb_path = self.get_current_reshoot_paths()
if not self.is_reshoot_leaf_ready():
self.logger.error(
'ReShootScreen: Failed to rotate. Image not found: {}'.format(
thumb_path))
return
leaf_number = self.get_current_leaf_number()
leaf_data = self.scandata.get_page_data(leaf_number)
current_degree = int(leaf_data.get('rotateDegree', 0))
new_degree = (current_degree + scandata_rotation_angle) % 360
self.scandata.update_rotate_degree(leaf_number, new_degree)
self.scandata.save()
rotate_by = convert_scandata_angle_to_thumbs_rotation(
new_degree, scandata_rotation_angle)
image = Image.open(path)
size = (1500, 1000) # (6000,4000)/4
image.thumbnail(size)
image = image.rotate(rotate_by, expand=True)
image.save(thumb_path, 'JPEG', quality=90)
self.logger.info(
'ReShootScreenBackend: Set leaf {} rotation to {} degree(s) in scandata ( {} thumbs-equivalent) from {}'
.format(leaf_number, new_degree, rotate_by, current_degree))
self.dispatch(self.EVENT_ROTATE_LEAF)
def _create_camera_kwargs(self, camera_side, leaf_number):
path, thumb_path = self.get_current_reshoot_paths()
return {
camera_system.KEY_CALLBACK: self._capture_spread_end,
camera_system.KEY_SIDE: camera_side,
camera_system.KEY_PATH: path,
camera_system.KEY_THUMB_PATH: thumb_path,
camera_system.KEY_EXTRA: {
'leaf_number': leaf_number
}
}
def on_capture_leaf(self, report):
pass
def on_current_leaf(self, *args):
pass
def on_rotate_leaf(self, *args):
pass
def on_page_type(self, *args):
pass
def on_show_original_file(self, *args):
pass
def on_show_reshoot_file(self, *args):
pass
def on_show_page_type_form_popup(self, *args):
pass
def on_go_back(self, *args):
pass
class DownloadBookTask(TaskBase):
def __init__(self, **kwargs):
super(DownloadBookTask, self).__init__(**kwargs)
self._book = None
self._priority = 'medium'
self._library = kwargs['library']
self.identifier = kwargs['identifier']
self.logger.info('Download books: Downloading {}'.format(
self.identifier))
self.notifications_manager = NotificationManager()
self._download_type = None
def create_pipeline(self):
return [
self._get_ia_session,
self._load_item,
self._validate_repub_state,
self._create_stub_book,
self._load_book_metadata,
self._create_files,
self._create_scandata,
self._get_checkout_information,
self._write_claimer_file,
self._download_proxies,
self._set_states,
self._release_lock,
self._send_stats,
]
def handle_event(self, event_name, *args, **kwargs):
if event_name == 'on_state' and self.state == CANCELLED_WITH_ERROR:
if self._book:
self._book.do_move_to_trash()
self._book.do_delete_anyway()
def _get_ia_session(self):
self.dispatch_progress('Getting IA session')
self._ia_session = get_ia_session()
def _load_item(self):
self.dispatch_progress('Loading item')
try:
self.item = self._ia_session.get_item(self.identifier)
assert self.item.metadata['repub_state'] is not None
except Exception as e:
self.logger.error('No repub_state or item darkened. Skipping...')
raise e
self.logger.info(
'Download book: target item: {} (repub_state = {})'.format(
self.item.identifier, self.item.metadata['repub_state']))
def _validate_repub_state(self):
self.dispatch_progress('Validating repub state')
is_repub_state_valid = lambda x: int(x.metadata[
'repub_state']) in scribe_globals.ALLOWED_DOWNLOAD_REPUB_STATES
self.logger.info('Validating repub_state {}'.format(
int(self.item.metadata['repub_state'])))
if not is_repub_state_valid(self.item):
msg = 'Download book: Repub state is not 31 or 34 or 41' \
'(is {1}), refusing to download item {0}' \
.format(self.item.identifier, self.item.metadata['repub_state'])
self.logger.error(msg)
raise Exception(msg)
def _create_stub_book(self):
self.dispatch_progress('Creating local book')
message = "This books is being downloaded and no actions are available just yet."
book_id = str(uuid4())
self._book = self._library.new_book(book_id,
status='download_incomplete',
error=message)
self._book.set_lock()
self._book.logger.info('Download book: Created stub book {}'.format(
self._book))
def _load_book_metadata(self):
self.dispatch_progress('Loading metadata')
md_url = ('https://{}/RePublisher/RePublisher-viewScanData.php'
'?id={}'.format(self.item.d1, self.identifier))
self._md = requests.get(md_url, timeout=5)
self._book.logger.info(
'Download book: Fetch scandata from cluster: {}'.format(
self._md.status_code))
def _create_files(self):
self.dispatch_progress('Downloading files')
ret = []
with open(os.path.join(self._book.path, 'identifier.txt'), 'w+') as fp:
fp.write(self.item.identifier)
ret.append(fp.name)
self._book.logger.info('Download book: Created {}'.format(fp.name))
with open(os.path.join(self._book.path, 'downloaded'), 'w+') as fp:
fp.write('True')
ret.append(fp.name)
self._book.logger.info('Download book: Created {}'.format(fp.name))
with open(os.path.join(self._book.path, 'uuid'), 'w+') as fp:
fp.write(self._book.uuid)
ret.append(fp.name)
self._book.logger.info('Download book: Created {}'.format(fp.name))
with open(os.path.join(self._book.path, 'scandata.xml'), 'w+') as fp:
fp.write(self._md.content.decode())
ret.append(fp.name)
self._book.logger.info('Download book: Created {}'.format(fp.name))
self.item.get_file(self.item.identifier + '_meta.xml') \
.download(file_path=self._book.path + '/metadata.xml')
ret.append('{}'.format(self._book.path + '/metadata.xml'))
self._book.logger.info('Download book: Created metadata.xml')
self._book.reload_metadata()
if not os.path.exists(os.path.join(self._book.path, 'reshooting')):
os.makedirs(os.path.join(self._book.path, 'reshooting'))
ret.append('{}'.format(self._book.path + '/reshooting'))
self._book.logger.info('Download book: Created reshooting directory')
self._files = ret
self._book.logger.info('Download book: Created files, now converting '
'scandata from RePublisher XML to Scribe3 JSON')
def _create_scandata(self):
self.dispatch_progress('Creating scandata')
sc_path = os.path.join(self._book.path, 'scandata.xml')
tree = book_helpers.validate_scandata_xml(sc_path, self._book)
scandata_xml = book_helpers.create_normalized_scandata(
tree, self._book)
json_data = book_helpers.convert_normalized_scandata_to_json(
scandata_xml)
json_new = {}
self._book.logger.info('Download book: Now converting to Scribe3 JSON')
json_new['bookData'] = book_helpers.build_bookdata(
json_data, self._book)
json_new['pageData'] = book_helpers.build_pagedata(
json_data, self._book)
with open(os.path.join(self._book.path, 'scandata.json'),
'w') as outfile:
json.dump(json_new, outfile)
self._book.logger.info('Download book: Created {}'.format(
outfile.name))
self._scandata = ScanData(self._book.path)
self._scandata.save()
self._book.reload_scandata()
self._book.logger.info('Download book: Created scandata.')
def _get_checkout_information(self):
self.dispatch_progress('Pulling checkout information')
book_checkout_url = ('https://{}/RePublisher/RePublisher-'
'checkoutBook.php?peek=true&id={}'.format(
self.item.d1, self._book.identifier))
self._book.logger.info(
'Getting checkout information from {}'.format(book_checkout_url))
ret = self._ia_session.get(book_checkout_url)
self._book.logger.info('Got {} ({})'.format(ret.text, ret.status_code))
self._checkout_info = json.loads(ret.text)
def _write_claimer_file(self):
self.dispatch_progress('Writing claimer file')
if 'claimed_by' in self._checkout_info and self._checkout_info[
'claimed_by'] != False:
claimer = self._checkout_info['claimed_by']
else:
claimer = '-'
with open(os.path.join(self._book.path, 'claimer'), 'w+') as fp:
fp.write(claimer)
self._claimer = claimer
self._book.logger.info('This book was claimed by {}'.format(claimer))
def _download_proxies(self):
self.dispatch_progress('Downloading proxies')
all_ok = True
counter = 0
page_data = self._scandata.dump_raw()['pageData']
for i, page in enumerate(page_data):
self.dispatch_progress('Downloading proxies [{}/{}]'.format(
i, len(page_data)))
if int(page) != i:
self._book.logger.error('Download book: Download Proxies: '
'CRITICAL MISMATCH')
break
short_msg = 'Download pics | {percent:.1f}% | {n}/{total}'.format(
percent=i * 100 / len(page_data),
n=i,
total=len(page_data),
)
self._book.update_message(short_msg)
url = book_helpers.get_cluster_proxy_url_by_leaf(
self._scandata, self.item, page)
res = self.download_proxy_image(page, self._book, url)
all_ok = all_ok and res
counter += 1
if res:
self._book.logger.debug(
'Download book: Got proxies for leaf #{0}'.format(page))
else:
self._book.logger.error(
'Download book: Error downloading leaf #{0}'.format(page))
try:
leafnr = self._scandata.get_page_num(page)['num']
except Exception:
pass
self._book.logger.info(
'Download book: Downloaded {} proxy images.'.format(counter))
return all_ok
def _set_states(self):
self.dispatch_progress('Setting states')
self._book.error = None
if int(self.item.metadata['repub_state']) == 31:
book_final_repub_state = 32
self._download_type = 'corrections'
self._book.do_end_download_correction()
elif int(self.item.metadata['repub_state']) == 41:
book_final_repub_state = 42
self._download_type = 'foldouts'
self._book.do_end_download_foldout()
else:
self._book.logger(
'Error while processing item in repub_state {}'.format(
self.item.metadata['repub_state']))
raise Exception(
'remote repub state in inconsistent with book download')
self._book.logger.info(
'Setting remote repub_state to {}'.format(book_final_repub_state))
mdapi_response = self.item.modify_metadata(
{'repub_state': book_final_repub_state})
self._book.logger.info(
'Response from MDAPI: {}'.format(mdapi_response))
if mdapi_response:
self._mdapi_response_text = mdapi_response.text
self._book.logger.info('Body of MDAPI: {}'.format(
self._mdapi_response_text))
if mdapi_response.status_code != 200:
raise Exception(
'MDAPI response was not OK! - Got this instead: {} - {}'.
format(mdapi_response.status_code, mdapi_response.text))
self._book.logger.info(
'Download book: Set book repub_state to {}'.format(
book_final_repub_state))
self._book_final_repub_state = book_final_repub_state
else:
raise Exception('No response from MDAPI. Aborting download.')
def _send_stats(self):
self.dispatch_progress('Notifying iabdash')
payload = {
'repub_state': self._book_final_repub_state,
'checkout_info': self._checkout_info,
'claimer': self._claimer,
'files': self._files,
}
push_event('tts-book-downloaded', payload, 'book', self.identifier,
os.path.join(self._book.path, "iabdash.log"))
self.notifications_manager.add_notification(
title='Downloaded',
message="{} has been downloaded and is ready for {}.".format(
self.identifier, self._download_type),
show_system_tile=False,
book=self._book)
def _release_lock(self):
total_time = 100
self._book.logger.info('Download book: ------ DONE. Downloaded {0} in '
'{1}s ----------'.format(
self.identifier, total_time))
self._book.release_lock()
def download_proxy_image(
self,
page,
book,
url,
):
def is_proxy_valid(proxy_path):
return True
file_target = '{n:04d}.jpg'.format(n=int(page))
dest = os.path.os.path.join(book.path, "thumbnails", file_target)
if url is not None:
image = self._ia_session.get(url).content
with open(dest, 'wb+') as proxy:
book.logger.debug('Writing {}'.format(dest))
proxy.write(image)
book.logger.info('Download book: Written {}'.format(
proxy.name))
else:
import shutil
book.logger.debug('Page {} has no proxy, adding missing '
'image at {}'.format(page, dest))
shutil.copyfile(scribe_globals.MISSING_IMAGE, dest)
ret = is_proxy_valid(dest)
return ret