def common_setup(self): # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_ext() self._set_model() self._set_greatest()
def common_setup(self): """ Set up common pdf processing steps """ # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_pdf_filename()
def common_setup(self): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False
def common_setup(self): self.video_config = mgg \ .global_config['media_type:mediagoblin.media_types.video'] # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False
def common_setup(self): self.ascii_config = mgg.global_config['plugins'][ 'mediagoblin.media_types.ascii'] # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'conversions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.charset = None
def common_setup(self): """ Setup the workbench directory and pull down the original file, add the audio_config, transcoder, thumbnailer and spectrogram_tmp path """ self.audio_config = mgg \ .global_config['plugins']['mediagoblin.media_types.audio'] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = AudioTranscoder() self.thumbnailer = AudioThumbnailer()
def process_pdf(proc_state): """Code to process a pdf file. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench queued_filename = proc_state.get_queued_filename() name_builder = FilenameBuilder(queued_filename) # Copy our queued local workbench to its final destination original_dest = name_builder.fill('{basename}{ext}') proc_state.copy_original(original_dest) # Create a pdf if this is a different doc, store pdf for viewer ext = queued_filename.rsplit('.', 1)[-1].lower() if ext == 'pdf': pdf_filename = queued_filename else: pdf_filename = queued_filename.rsplit('.', 1)[0] + '.pdf' unoconv = where('unoconv') Popen(executable=unoconv, args=[unoconv, '-v', '-f', 'pdf', queued_filename]).wait() if not os.path.exists(pdf_filename): _log.debug('unoconv failed to convert file to pdf') raise BadMediaFail() proc_state.store_public(keyname=u'pdf', local_file=pdf_filename) pdf_info_dict = pdf_info(pdf_filename) for name, width, height in [ (u'thumb', mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']), (u'medium', mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']), ]: filename = name_builder.fill('{basename}.%s.png' % name) path = workbench.joinpath(filename) create_pdf_thumb(pdf_filename, path, width, height) assert (os.path.exists(path)) proc_state.store_public(keyname=name, local_file=path) proc_state.delete_queue_file() entry.media_data_init(**pdf_info_dict) entry.save()
def process_pdf(proc_state): """Code to process a pdf file. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench queued_filename = proc_state.get_queued_filename() name_builder = FilenameBuilder(queued_filename) # Copy our queued local workbench to its final destination original_dest = name_builder.fill('{basename}{ext}') proc_state.copy_original(original_dest) # Create a pdf if this is a different doc, store pdf for viewer ext = queued_filename.rsplit('.', 1)[-1].lower() if ext == 'pdf': pdf_filename = queued_filename else: pdf_filename = queued_filename.rsplit('.', 1)[0] + '.pdf' unoconv = where('unoconv') call(executable=unoconv, args=[unoconv, '-v', '-f', 'pdf', queued_filename]) if not os.path.exists(pdf_filename): _log.debug('unoconv failed to convert file to pdf') raise BadMediaFail() proc_state.store_public(keyname=u'pdf', local_file=pdf_filename) pdf_info_dict = pdf_info(pdf_filename) for name, width, height in [ (u'thumb', mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']), (u'medium', mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']), ]: filename = name_builder.fill('{basename}.%s.png' % name) path = workbench.joinpath(filename) create_pdf_thumb(pdf_filename, path, width, height) assert(os.path.exists(path)) proc_state.store_public(keyname=name, local_file=path) proc_state.delete_queue_file() entry.media_data_init(**pdf_info_dict) entry.save()
def process_image(proc_state): """Code to process an image. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench # Conversions subdirectory to avoid collisions conversions_subdir = os.path.join( workbench.dir, 'conversions') os.mkdir(conversions_subdir) queued_filename = proc_state.get_queued_filename() name_builder = FilenameBuilder(queued_filename) # EXIF extraction exif_tags = extract_exif(queued_filename) gps_data = get_gps_data(exif_tags) # Always create a small thumbnail resize_tool(proc_state, True, 'thumb', name_builder.fill('{basename}.thumbnail{ext}'), conversions_subdir, exif_tags) # Possibly create a medium resize_tool(proc_state, False, 'medium', name_builder.fill('{basename}.medium{ext}'), conversions_subdir, exif_tags) # Copy our queued local workbench to its final destination proc_state.copy_original(name_builder.fill('{basename}{ext}')) # Remove queued media file from storage and database proc_state.delete_queue_file() # Insert exif data into database exif_all = clean_exif(exif_tags) if len(exif_all): entry.media_data_init(exif_all=exif_all) if len(gps_data): for key in list(gps_data.keys()): gps_data['gps_' + key] = gps_data.pop(key) entry.media_data_init(**gps_data)
def process_image(proc_state): """Code to process an image. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench # Conversions subdirectory to avoid collisions conversions_subdir = os.path.join(workbench.dir, 'conversions') os.mkdir(conversions_subdir) queued_filename = proc_state.get_queued_filename() name_builder = FilenameBuilder(queued_filename) # EXIF extraction exif_tags = extract_exif(queued_filename) gps_data = get_gps_data(exif_tags) # Always create a small thumbnail resize_tool(proc_state, True, 'thumb', name_builder.fill('{basename}.thumbnail{ext}'), conversions_subdir, exif_tags) # Possibly create a medium resize_tool(proc_state, False, 'medium', name_builder.fill('{basename}.medium{ext}'), conversions_subdir, exif_tags) # Copy our queued local workbench to its final destination proc_state.copy_original(name_builder.fill('{basename}{ext}')) # Remove queued media file from storage and database proc_state.delete_queue_file() # Insert exif data into database exif_all = clean_exif(exif_tags) if len(exif_all): entry.media_data_init(exif_all=exif_all) if len(gps_data): for key in list(gps_data.keys()): gps_data['gps_' + key] = gps_data.pop(key) entry.media_data_init(**gps_data)
def common_setup(self): """ Set up common pdf processing steps """ # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_pdf_filename()
def common_setup(self, resolution=None): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False if resolution: self.curr_file = 'webm_' + str(resolution) self.part_filename = (self.name_builder.fill('{basename}.' + str(resolution) + '.webm')) else: self.curr_file = 'webm_video' self.part_filename = self.name_builder.fill('{basename}.medium.webm')
def common_setup(self): self.video_config = mgg \ .global_config['media_type:mediagoblin.media_types.video'] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False
def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.image_config = mgg.global_config[ 'media_type:mediagoblin.media_types.image'] ## @@: Should this be two functions? # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join(self.workbench.dir, 'convirsions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction self.exif_tags = extract_exif(self.process_filename)
def common_setup(self): """ Setup the workbench directory and pull down the original file, add the audio_config, transcoder, thumbnailer and spectrogram_tmp path """ self.audio_config = mgg.global_config["plugins"]["mediagoblin.media_types.audio"] # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = AudioTranscoder() self.thumbnailer = AudioThumbnailer()
def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.svg_config = mgg.global_config['plugins']['mediagoblin_svg'] # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'conversions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename)
def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.image_config = mgg.global_config[ 'media_type:mediagoblin.media_types.image'] ## @@: Should this be two functions? # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'convirsions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction self.exif_tags = extract_exif(self.process_filename)
def common_setup(self): """ Pull out a full-size JPEG-preview """ super().common_setup() self._original_raw = self.process_filename # Read EXIF data md = pyexiv2.ImageMetadata(self._original_raw) md.read() self.process_filename = os.path.join(self.conversions_subdir, self.entry.queued_media_file[-1]) # Extract the biggest preview and write it as our working image md.previews[-1].write_to_file(self.process_filename.encode('utf-8')) self.process_filename += '.jpg' _log.debug('Wrote new file from {} to preview (jpg) {}'.format( self._original_raw, self.process_filename)) # Override the namebuilder with our new jpg-based name self.name_builder = FilenameBuilder(self.process_filename)
class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ acceptable_files = ['original', 'best_quality', 'webm_video'] def common_setup(self): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False def copy_original(self): # If we didn't transcode, then we need to keep the original if not self.did_transcode or \ (self.video_config['keep_original'] and self.did_transcode): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _keep_best(self): """ If there is no original, keep the best file that we have """ if not self.entry.media_files.get('best_quality'): # Save the best quality file if no original? if not self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_video'): self.entry.media_files['best_quality'] = self.entry \ .media_files['webm_video'] def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) if not file_metadata: return False skip = True if keyname == 'webm_video': if kwargs.get('medium_size') != file_metadata.get('medium_size'): skip = False elif kwargs.get('vp8_quality') != file_metadata.get('vp8_quality'): skip = False elif kwargs.get('vp8_threads') != file_metadata.get('vp8_threads'): skip = False elif kwargs.get('vorbis_quality') != \ file_metadata.get('vorbis_quality'): skip = False elif keyname == 'thumb': if kwargs.get('thumb_size') != file_metadata.get('thumb_size'): skip = False return skip def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.medium.webm')) if not medium_size: medium_size = ( mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if not vp8_quality: vp8_quality = self.video_config['vp8_quality'] if not vp8_threads: vp8_threads = self.video_config['vp8_threads'] if not vorbis_quality: vorbis_quality = self.video_config['vorbis_quality'] file_metadata = {'medium_size': medium_size, 'vp8_threads': vp8_threads, 'vp8_quality': vp8_quality, 'vorbis_quality': vorbis_quality} if self._skip_processing('webm_video', **file_metadata): return # Extract metadata and keep a record of it metadata = self.transcoder.discover(self.process_filename) store_metadata(self.entry, metadata) # Figure out whether or not we need to transcode this video or # if we can skip it if skip_transcode(metadata, medium_size): _log.debug('Skipping transcoding') dst_dimensions = metadata['videowidth'], metadata['videoheight'] # If there is an original and transcoded, delete the transcoded # since it must be of lower quality then the original if self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_video'): self.entry.media_files['webm_video'].delete() else: self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) dst_dimensions = self.transcoder.dst_data.videowidth,\ self.transcoder.dst_data.videoheight self._keep_best() # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, 'webm_video', tmp_dst, self.name_builder.fill('{basename}.medium.webm')) _log.debug('Saved medium') self.entry.set_file_metadata('webm_video', **file_metadata) self.did_transcode = True # Save the width and height of the transcoded video self.entry.media_data_init( width=dst_dimensions[0], height=dst_dimensions[1]) def generate_thumb(self, thumb_size=None): # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}.thumbnail.jpg')) if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'],) if self._skip_processing('thumb', thumb_size=thumb_size): return # We will only use the width so that the correct scale is kept transcoders.VideoThumbnailerMarkII( self.process_filename, tmp_thumb, thumb_size[0]) # Checking if the thumbnail was correctly created. If it was not, # then just give up. if not os.path.exists (tmp_thumb): return # Push the thumbnail to public storage _log.debug('Saving thumbnail...') store_public(self.entry, 'thumb', tmp_thumb, self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size)
class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ acceptable_files = ['original', 'best_quality', 'webm_video'] def common_setup(self): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False def copy_original(self): # If we didn't transcode, then we need to keep the original if not self.did_transcode or \ (self.video_config['keep_original'] and self.did_transcode): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _keep_best(self): """ If there is no original, keep the best file that we have """ if not self.entry.media_files.get('best_quality'): # Save the best quality file if no original? if not self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_video'): self.entry.media_files['best_quality'] = self.entry \ .media_files['webm_video'] def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) if not file_metadata: return False skip = True if keyname == 'webm_video': if kwargs.get('medium_size') != file_metadata.get('medium_size'): skip = False elif kwargs.get('vp8_quality') != file_metadata.get('vp8_quality'): skip = False elif kwargs.get('vp8_threads') != file_metadata.get('vp8_threads'): skip = False elif kwargs.get('vorbis_quality') != \ file_metadata.get('vorbis_quality'): skip = False elif keyname == 'thumb': if kwargs.get('thumb_size') != file_metadata.get('thumb_size'): skip = False return skip def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.medium.webm')) if not medium_size: medium_size = ( mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if not vp8_quality: vp8_quality = self.video_config['vp8_quality'] if not vp8_threads: vp8_threads = self.video_config['vp8_threads'] if not vorbis_quality: vorbis_quality = self.video_config['vorbis_quality'] file_metadata = {'medium_size': medium_size, 'vp8_threads': vp8_threads, 'vp8_quality': vp8_quality, 'vorbis_quality': vorbis_quality} if self._skip_processing('webm_video', **file_metadata): return # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) # metadata's stream info here is a DiscovererContainerInfo instance, # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; # metadata itself has container-related data in tags, like video-codec store_metadata(self.entry, metadata) orig_dst_dimensions = (metadata.get_video_streams()[0].get_width(), metadata.get_video_streams()[0].get_height()) # Figure out whether or not we need to transcode this video or # if we can skip it if skip_transcode(metadata, medium_size): _log.debug('Skipping transcoding') dst_dimensions = orig_dst_dimensions # If there is an original and transcoded, delete the transcoded # since it must be of lower quality then the original if self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_video'): self.entry.media_files['webm_video'].delete() else: self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) if self.transcoder.dst_data: video_info = self.transcoder.dst_data.get_video_streams()[0] dst_dimensions = (video_info.get_width(), video_info.get_height()) self._keep_best() # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, 'webm_video', tmp_dst, self.name_builder.fill('{basename}.medium.webm')) _log.debug('Saved medium') self.entry.set_file_metadata('webm_video', **file_metadata) self.did_transcode = True else: dst_dimensions = orig_dst_dimensions # Save the width and height of the transcoded video self.entry.media_data_init( width=dst_dimensions[0], height=dst_dimensions[1]) def generate_thumb(self, thumb_size=None): # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}.thumbnail.jpg')) if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'],) if self._skip_processing('thumb', thumb_size=thumb_size): return # We will only use the width so that the correct scale is kept transcoders.capture_thumb( self.process_filename, tmp_thumb, thumb_size[0]) # Checking if the thumbnail was correctly created. If it was not, # then just give up. if not os.path.exists (tmp_thumb): return # Push the thumbnail to public storage _log.debug('Saving thumbnail...') store_public(self.entry, 'thumb', tmp_thumb, self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size)
class CommonImageProcessor(MediaProcessor): """ Provides a base for various media processing steps """ # list of acceptable file keys in order of prefrence for reprocessing acceptable_files = ['original', 'medium'] def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.image_config = mgg.global_config[ 'media_type:mediagoblin.media_types.image'] ## @@: Should this be two functions? # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'convirsions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction self.exif_tags = extract_exif(self.process_filename) def generate_medium_if_applicable(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, False, 'medium', self.process_filename, self.name_builder.fill('{basename}.medium{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def generate_thumb(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, True, 'thumb', self.process_filename, self.name_builder.fill('{basename}.thumbnail{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def copy_original(self): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def extract_metadata(self): # Is there any GPS data gps_data = get_gps_data(self.exif_tags) # Insert exif data into database exif_all = clean_exif(self.exif_tags) if len(exif_all): self.entry.media_data_init(exif_all=exif_all) if len(gps_data): for key in list(gps_data.keys()): gps_data['gps_' + key] = gps_data.pop(key) self.entry.media_data_init(**gps_data)
def process_stl(proc_state): """Code to process an stl or obj model. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench queued_filepath = entry.queued_media_file queued_filename = workbench.localized_file( mgg.queue_store, queued_filepath, 'source') name_builder = FilenameBuilder(queued_filename) ext = queued_filename.lower().strip()[-4:] if ext.startswith("."): ext = ext[1:] else: ext = None # Attempt to parse the model file and divine some useful # information about it. with open(queued_filename, 'rb') as model_file: model = model_loader.auto_detect(model_file, ext) # generate preview images greatest = [model.width, model.height, model.depth] greatest.sort() greatest = greatest[-1] def snap(name, camera, width=640, height=640, project="ORTHO"): filename = name_builder.fill(name) workbench_path = workbench.joinpath(filename) shot = { "model_path": queued_filename, "model_ext": ext, "camera_coord": camera, "camera_focus": model.average, "camera_clip": greatest*10, "greatest": greatest, "projection": project, "width": width, "height": height, "out_file": workbench_path, } blender_render(shot) # make sure the image rendered to the workbench path assert os.path.exists(workbench_path) # copy it up! with open(workbench_path, 'rb') as rendered_file: public_path = create_pub_filepath(entry, filename) with mgg.public_store.get_file(public_path, "wb") as public_file: public_file.write(rendered_file.read()) return public_path thumb_path = snap( "{basename}.thumb.jpg", [0, greatest*-1.5, greatest], mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height'], project="PERSP") perspective_path = snap( "{basename}.perspective.jpg", [0, greatest*-1.5, greatest], project="PERSP") topview_path = snap( "{basename}.top.jpg", [model.average[0], model.average[1], greatest*2]) frontview_path = snap( "{basename}.front.jpg", [model.average[0], greatest*-2, model.average[2]]) sideview_path = snap( "{basename}.side.jpg", [greatest*-2, model.average[1], model.average[2]]) ## Save the public file stuffs model_filepath = create_pub_filepath( entry, name_builder.fill('{basename}{ext}')) with mgg.public_store.get_file(model_filepath, 'wb') as model_file: with open(queued_filename, 'rb') as queued_file: model_file.write(queued_file.read()) # Remove queued media file from storage and database. # queued_filepath is in the task_id directory which should # be removed too, but fail if the directory is not empty to be on # the super-safe side. mgg.queue_store.delete_file(queued_filepath) # rm file mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir entry.queued_media_file = [] # Insert media file information into database media_files_dict = entry.setdefault('media_files', {}) media_files_dict[u'original'] = model_filepath media_files_dict[u'thumb'] = thumb_path media_files_dict[u'perspective'] = perspective_path media_files_dict[u'top'] = topview_path media_files_dict[u'side'] = sideview_path media_files_dict[u'front'] = frontview_path # Put model dimensions into the database dimensions = { "center_x" : model.average[0], "center_y" : model.average[1], "center_z" : model.average[2], "width" : model.width, "height" : model.height, "depth" : model.depth, "file_type" : ext, } entry.media_data_init(**dimensions)
class CommonStlProcessor(MediaProcessor): """ Provides a common base for various stl processing steps """ acceptable_files = ['original'] def common_setup(self): # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_ext() self._set_model() self._set_greatest() def _set_ext(self): ext = self.name_builder.ext[1:] if not ext: ext = None self.ext = ext def _set_model(self): """ Attempt to parse the model file and divine some useful information about it. """ with open(self.process_filename, 'rb') as model_file: self.model = model_loader.auto_detect(model_file, self.ext) def _set_greatest(self): greatest = [self.model.width, self.model.height, self.model.depth] greatest.sort() self.greatest = greatest[-1] def copy_original(self): copy_original(self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _snap(self, keyname, name, camera, size, project="ORTHO"): filename = self.name_builder.fill(name) workbench_path = self.workbench.joinpath(filename) shot = { "model_path": self.process_filename, "model_ext": self.ext, "camera_coord": camera, "camera_focus": self.model.average, "camera_clip": self.greatest * 10, "greatest": self.greatest, "projection": project, "width": size[0], "height": size[1], "out_file": workbench_path, } blender_render(shot) # make sure the image rendered to the workbench path assert os.path.exists(workbench_path) # copy it up! store_public(self.entry, keyname, workbench_path, filename) def generate_thumb(self, thumb_size=None): if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']) self._snap("thumb", "{basename}.thumb.jpg", [0, self.greatest * -1.5, self.greatest], thumb_size, project="PERSP") def generate_perspective(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap("perspective", "{basename}.perspective.jpg", [0, self.greatest * -1.5, self.greatest], size, project="PERSP") def generate_topview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "top", "{basename}.top.jpg", [self.model.average[0], self.model.average[1], self.greatest * 2], size) def generate_frontview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "front", "{basename}.front.jpg", [self.model.average[0], self.greatest * -2, self.model.average[2]], size) def generate_sideview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "side", "{basename}.side.jpg", [self.greatest * -2, self.model.average[1], self.model.average[2]], size) def store_dimensions(self): """ Put model dimensions into the database """ dimensions = { "center_x": self.model.average[0], "center_y": self.model.average[1], "center_z": self.model.average[2], "width": self.model.width, "height": self.model.height, "depth": self.model.depth, "file_type": self.ext, } self.entry.media_data_init(**dimensions)
class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ acceptable_files = ['original, best_quality', 'webm_144p', 'webm_360p', 'webm_480p', 'webm_720p', 'webm_1080p', 'webm_video'] def common_setup(self, resolution=None): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False if resolution: self.curr_file = 'webm_' + str(resolution) self.part_filename = (self.name_builder.fill('{basename}.' + str(resolution) + '.webm')) else: self.curr_file = 'webm_video' self.part_filename = self.name_builder.fill('{basename}.medium.webm') def copy_original(self): # If we didn't transcode, then we need to keep the original self.did_transcode = False for each_res in self.video_config['available_resolutions']: if 'webm_{}'.format(each_res) in self.entry.media_files: self.did_transcode = True break if not self.did_transcode or self.video_config['keep_original']: copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) self.entry.save() def keep_best(self): """ If there is no original, keep the best file that we have """ best_file = None best_file_dim = (0, 0) for each_res in self.video_config['available_resolutions']: curr_dim = ACCEPTED_RESOLUTIONS[each_res] if curr_dim[0] >= best_file_dim[0] and curr_dim[1] >= best_file_dim[1]: best_file = each_res best_file_dim = curr_dim if not self.entry.media_files.get('best_quality'): # Save the best quality file if no original? if not self.entry.media_files.get('original') and \ self.entry.media_files.get(str(best_file)): self.entry.media_files['best_quality'] = self.entry \ .media_files[str(best_file)] def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) if not file_metadata: return False skip = True if 'webm' in keyname: if kwargs.get('medium_size') != file_metadata.get('medium_size'): skip = False elif kwargs.get('vp8_quality') != file_metadata.get('vp8_quality'): skip = False elif kwargs.get('vp8_threads') != file_metadata.get('vp8_threads'): skip = False elif kwargs.get('vorbis_quality') != \ file_metadata.get('vorbis_quality'): skip = False elif keyname == 'thumb': if kwargs.get('thumb_size') != file_metadata.get('thumb_size'): skip = False return skip def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.part_filename) if not medium_size: medium_size = ( mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if not vp8_quality: vp8_quality = self.video_config['vp8_quality'] if not vp8_threads: vp8_threads = self.video_config['vp8_threads'] if not vorbis_quality: vorbis_quality = self.video_config['vorbis_quality'] file_metadata = {'medium_size': medium_size, 'vp8_threads': vp8_threads, 'vp8_quality': vp8_quality, 'vorbis_quality': vorbis_quality} if self._skip_processing(self.curr_file, **file_metadata): return metadata = transcoders.discover(self.process_filename) orig_dst_dimensions = (metadata.get_video_streams()[0].get_width(), metadata.get_video_streams()[0].get_height()) # Figure out whether or not we need to transcode this video or # if we can skip it if skip_transcode(metadata, medium_size): _log.debug('Skipping transcoding') # If there is an original and transcoded, delete the transcoded # since it must be of lower quality then the original if self.entry.media_files.get('original') and \ self.entry.media_files.get(self.curr_file): self.entry.media_files[self.curr_file].delete() else: _log.debug('Entered transcoder') video_config = (mgg.global_config['plugins'] ['mediagoblin.media_types.video']) num_res = len(video_config['available_resolutions']) default_res = video_config['default_resolution'] self.transcoder.transcode(self.process_filename, tmp_dst, default_res, num_res, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) if self.transcoder.dst_data: # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, self.curr_file, tmp_dst, self.part_filename) _log.debug('Saved medium') self.entry.set_file_metadata(self.curr_file, **file_metadata) self.did_transcode = True def generate_thumb(self, thumb_size=None): _log.debug("Enter generate_thumb()") # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}.thumbnail.jpg')) if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'],) if self._skip_processing('thumb', thumb_size=thumb_size): return # We will only use the width so that the correct scale is kept transcoders.capture_thumb( self.process_filename, tmp_thumb, thumb_size[0]) # Checking if the thumbnail was correctly created. If it was not, # then just give up. if not os.path.exists (tmp_thumb): return # Push the thumbnail to public storage _log.debug('Saving thumbnail...') store_public(self.entry, 'thumb', tmp_thumb, self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size) def store_orig_metadata(self): # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) # metadata's stream info here is a DiscovererContainerInfo instance, # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; # metadata itself has container-related data in tags, like video-codec store_metadata(self.entry, metadata) _log.debug("Stored original video metadata")
class CommonPdfProcessor(MediaProcessor): """ Provides a base for various pdf processing steps """ acceptable_files = ["original", "pdf"] def common_setup(self): """ Set up common pdf processing steps """ # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_pdf_filename() def _set_pdf_filename(self): if self.name_builder.ext == ".pdf": self.pdf_filename = self.process_filename elif self.entry.media_files.get("pdf"): self.pdf_filename = self.workbench.localized_file(mgg.public_store, self.entry.media_files["pdf"]) else: self.pdf_filename = self._generate_pdf() def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) skip = True if not file_metadata: return False if keyname == "thumb": if kwargs.get("thumb_size") != file_metadata.get("thumb_size"): skip = False elif keyname == "medium": if kwargs.get("size") != file_metadata.get("size"): skip = False return skip def copy_original(self): copy_original(self.entry, self.process_filename, self.name_builder.fill("{basename}{ext}")) def generate_thumb(self, thumb_size=None): if not thumb_size: thumb_size = (mgg.global_config["media:thumb"]["max_width"], mgg.global_config["media:thumb"]["max_height"]) if self._skip_processing("thumb", thumb_size=thumb_size): return # Note: pdftocairo adds '.png', so don't include an ext thumb_filename = os.path.join(self.workbench.dir, self.name_builder.fill("{basename}.thumbnail")) executable = where("pdftocairo") args = [executable, "-scale-to", str(min(thumb_size)), "-singlefile", "-png", self.pdf_filename, thumb_filename] _log.debug("calling {0}".format(repr(" ".join(args)))) Popen(executable=executable, args=args).wait() # since pdftocairo added '.png', we need to include it with the # filename store_public(self.entry, "thumb", thumb_filename + ".png", self.name_builder.fill("{basename}.thumbnail.png")) self.entry.set_file_metadata("thumb", thumb_size=thumb_size) def _generate_pdf(self): """ Store the pdf. If the file is not a pdf, make it a pdf """ tmp_pdf = self.process_filename unoconv = where("unoconv") Popen(executable=unoconv, args=[unoconv, "-v", "-f", "pdf", self.process_filename]).wait() if not os.path.exists(tmp_pdf): _log.debug("unoconv failed to convert file to pdf") raise BadMediaFail() store_public(self.entry, "pdf", tmp_pdf, self.name_builder.fill("{basename}.pdf")) return self.workbench.localized_file(mgg.public_store, self.entry.media_files["pdf"]) def extract_pdf_info(self): pdf_info_dict = pdf_info(self.pdf_filename) self.entry.media_data_init(**pdf_info_dict) def generate_medium(self, size=None): if not size: size = (mgg.global_config["media:medium"]["max_width"], mgg.global_config["media:medium"]["max_height"]) if self._skip_processing("medium", size=size): return # Note: pdftocairo adds '.png', so don't include an ext filename = os.path.join(self.workbench.dir, self.name_builder.fill("{basename}.medium")) executable = where("pdftocairo") args = [executable, "-scale-to", str(min(size)), "-singlefile", "-png", self.pdf_filename, filename] _log.debug("calling {0}".format(repr(" ".join(args)))) Popen(executable=executable, args=args).wait() # since pdftocairo added '.png', we need to include it with the # filename store_public(self.entry, "medium", filename + ".png", self.name_builder.fill("{basename}.medium.png")) self.entry.set_file_metadata("medium", size=size)
def process_video(proc_state): """ Process a video entry, transcode the queued media files (originals) and create a thumbnail for the entry. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench video_config = mgg.global_config['media_type:mediagoblin.media_types.video'] queued_filepath = entry.queued_media_file queued_filename = proc_state.get_queued_filename() name_builder = FilenameBuilder(queued_filename) medium_basename = name_builder.fill('{basename}-640p.webm') medium_filepath = create_pub_filepath(entry, medium_basename) thumbnail_basename = name_builder.fill('{basename}.thumbnail.jpg') thumbnail_filepath = create_pub_filepath(entry, thumbnail_basename) # Create a temporary file for the video destination (cleaned up with workbench) tmp_dst = os.path.join(workbench.dir, medium_basename) # Transcode queued file to a VP8/vorbis file that fits in a 640x640 square progress_callback = ProgressCallback(entry) dimensions = ( mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) # Extract metadata and keep a record of it metadata = transcoders.VideoTranscoder().discover(queued_filename) store_metadata(entry, metadata) # Figure out whether or not we need to transcode this video or # if we can skip it if skip_transcode(metadata): _log.debug('Skipping transcoding') dst_dimensions = metadata['videowidth'], metadata['videoheight'] # Push original file to public storage _log.debug('Saving original...') proc_state.copy_original(queued_filepath[-1]) did_transcode = False else: transcoder = transcoders.VideoTranscoder() transcoder.transcode(queued_filename, tmp_dst, vp8_quality=video_config['vp8_quality'], vp8_threads=video_config['vp8_threads'], vorbis_quality=video_config['vorbis_quality'], progress_callback=progress_callback, dimensions=dimensions) dst_dimensions = transcoder.dst_data.videowidth,\ transcoder.dst_data.videoheight # Push transcoded video to public storage _log.debug('Saving medium...') mgg.public_store.copy_local_to_storage(tmp_dst, medium_filepath) _log.debug('Saved medium') entry.media_files['webm_640'] = medium_filepath did_transcode = True # Save the width and height of the transcoded video entry.media_data_init( width=dst_dimensions[0], height=dst_dimensions[1]) # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(workbench.dir, thumbnail_basename) # Create a thumbnail.jpg that fits in a 180x180 square transcoders.VideoThumbnailerMarkII( queued_filename, tmp_thumb, 180) # Push the thumbnail to public storage _log.debug('Saving thumbnail...') mgg.public_store.copy_local_to_storage(tmp_thumb, thumbnail_filepath) entry.media_files['thumb'] = thumbnail_filepath # save the original... but only if we did a transcoding # (if we skipped transcoding and just kept the original anyway as the main # media, then why would we save the original twice?) if video_config['keep_original'] and did_transcode: # Push original file to public storage _log.debug('Saving original...') proc_state.copy_original(queued_filepath[-1]) # Remove queued media file from storage and database proc_state.delete_queue_file()
def process_audio(proc_state): """Code to process uploaded audio. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench audio_config = mgg.global_config['media_type:mediagoblin.media_types.audio'] queued_filepath = entry.queued_media_file queued_filename = workbench.localized_file( mgg.queue_store, queued_filepath, 'source') name_builder = FilenameBuilder(queued_filename) webm_audio_filepath = create_pub_filepath( entry, '{original}.webm'.format( original=os.path.splitext( queued_filepath[-1])[0])) if audio_config['keep_original']: with open(queued_filename, 'rb') as queued_file: original_filepath = create_pub_filepath( entry, name_builder.fill('{basename}{ext}')) with mgg.public_store.get_file(original_filepath, 'wb') as \ original_file: _log.debug('Saving original...') original_file.write(queued_file.read()) entry.media_files['original'] = original_filepath transcoder = AudioTranscoder() with NamedTemporaryFile(dir=workbench.dir) as webm_audio_tmp: progress_callback = ProgressCallback(entry) transcoder.transcode( queued_filename, webm_audio_tmp.name, quality=audio_config['quality'], progress_callback=progress_callback) transcoder.discover(webm_audio_tmp.name) _log.debug('Saving medium...') mgg.public_store.get_file(webm_audio_filepath, 'wb').write( webm_audio_tmp.read()) entry.media_files['webm_audio'] = webm_audio_filepath # entry.media_data_init(length=int(data.audiolength)) if audio_config['create_spectrogram']: spectrogram_filepath = create_pub_filepath( entry, '{original}-spectrogram.jpg'.format( original=os.path.splitext( queued_filepath[-1])[0])) with NamedTemporaryFile(dir=workbench.dir, suffix='.ogg') as wav_tmp: _log.info('Creating OGG source for spectrogram') transcoder.transcode( queued_filename, wav_tmp.name, mux_string='vorbisenc quality={0} ! oggmux'.format( audio_config['quality'])) thumbnailer = AudioThumbnailer() with NamedTemporaryFile(dir=workbench.dir, suffix='.jpg') as spectrogram_tmp: thumbnailer.spectrogram( wav_tmp.name, spectrogram_tmp.name, width=mgg.global_config['media:medium']['max_width'], fft_size=audio_config['spectrogram_fft_size']) _log.debug('Saving spectrogram...') mgg.public_store.get_file(spectrogram_filepath, 'wb').write( spectrogram_tmp.read()) entry.media_files['spectrogram'] = spectrogram_filepath with NamedTemporaryFile(dir=workbench.dir, suffix='.jpg') as thumb_tmp: thumbnailer.thumbnail_spectrogram( spectrogram_tmp.name, thumb_tmp.name, (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height'])) thumb_filepath = create_pub_filepath( entry, '{original}-thumbnail.jpg'.format( original=os.path.splitext( queued_filepath[-1])[0])) mgg.public_store.get_file(thumb_filepath, 'wb').write( thumb_tmp.read()) entry.media_files['thumb'] = thumb_filepath else: entry.media_files['thumb'] = ['fake', 'thumb', 'path.jpg'] # Remove queued media file from storage and database. # queued_filepath is in the task_id directory which should # be removed too, but fail if the directory is not empty to be on # the super-safe side. mgg.queue_store.delete_file(queued_filepath) # rm file mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir entry.queued_media_file = []
class CommonPdfProcessor(MediaProcessor): """ Provides a base for various pdf processing steps """ acceptable_files = ['original', 'pdf'] def common_setup(self): """ Set up common pdf processing steps """ # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_pdf_filename() def _set_pdf_filename(self): if self.name_builder.ext == '.pdf': self.pdf_filename = self.process_filename elif self.entry.media_files.get('pdf'): self.pdf_filename = self.workbench.localized_file( mgg.public_store, self.entry.media_files['pdf']) else: self.pdf_filename = self._generate_pdf() def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) skip = True if not file_metadata: return False if keyname == 'thumb': if kwargs.get('thumb_size') != file_metadata.get('thumb_size'): skip = False elif keyname == 'medium': if kwargs.get('size') != file_metadata.get('size'): skip = False return skip def copy_original(self): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def generate_thumb(self, thumb_size=None): if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']) if self._skip_processing('thumb', thumb_size=thumb_size): return # Note: pdftocairo adds '.png', so don't include an ext thumb_filename = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}.thumbnail')) executable = where('pdftocairo') args = [executable, '-scale-to', str(min(thumb_size)), '-singlefile', '-png', self.pdf_filename, thumb_filename] _log.debug('calling {0}'.format(repr(' '.join(args)))) Popen(executable=executable, args=args).wait() # since pdftocairo added '.png', we need to include it with the # filename store_public(self.entry, 'thumb', thumb_filename + '.png', self.name_builder.fill('{basename}.thumbnail.png')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size) def _generate_pdf(self): """ Store the pdf. If the file is not a pdf, make it a pdf """ tmp_pdf = os.path.splitext(self.process_filename)[0] + '.pdf' unoconv = where('unoconv') args = [unoconv, '-v', '-f', 'pdf', self.process_filename] _log.debug('calling %s' % repr(args)) Popen(executable=unoconv, args=args).wait() if not os.path.exists(tmp_pdf): _log.debug('unoconv failed to convert file to pdf') raise BadMediaFail() store_public(self.entry, 'pdf', tmp_pdf, self.name_builder.fill('{basename}.pdf')) return self.workbench.localized_file( mgg.public_store, self.entry.media_files['pdf']) def extract_pdf_info(self): pdf_info_dict = pdf_info(self.pdf_filename) self.entry.media_data_init(**pdf_info_dict) def generate_medium(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if self._skip_processing('medium', size=size): return # Note: pdftocairo adds '.png', so don't include an ext filename = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.medium')) executable = where('pdftocairo') args = [executable, '-scale-to', str(min(size)), '-singlefile', '-png', self.pdf_filename, filename] _log.debug('calling {0}'.format(repr(' '.join(args)))) Popen(executable=executable, args=args).wait() # since pdftocairo added '.png', we need to include it with the # filename store_public(self.entry, 'medium', filename + '.png', self.name_builder.fill('{basename}.medium.png')) self.entry.set_file_metadata('medium', size=size)
class CommonSvgProcessor(MediaProcessor): """ Provides a base for various svg processing steps """ acceptable_files = ['original'] def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.svg_config = mgg.global_config['plugins']['mediagoblin_svg'] # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'conversions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) def generate_preview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if self.svg_config['svg_previews']: # delete existing thumbnail, if it doesn't match the original if self.entry.media_files.has_key('preview') and \ self.entry.media_files['preview'] != self.entry.media_files['original']: mgg.public_store.delete_file(self.entry.media_files['preview']) self.entry.media_files['preview'] = self.entry.media_files.get('original') else: preview_filename = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.preview.png')) render_preview(self.process_filename, preview_filename, size) store_public(self.entry, 'preview', preview_filename, self.name_builder.fill('{basename}.preview.png')) def generate_thumb(self, size=None): if not size: size = (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']) if self.svg_config['svg_thumbnails']: # delete existing thumbnail, if it doesn't match the original if self.entry.media_files.has_key('thumb') and \ self.entry.media_files['thumb'] != self.entry.media_files['original']: mgg.public_store.delete_file(self.entry.media_files['thumb']) self.entry.media_files['thumb'] = self.entry.media_files.get('original') else: thumb_filename = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.thumbnail.png')) render_preview(self.process_filename, thumb_filename, size) store_public(self.entry, 'thumb', thumb_filename) def copy_original(self): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}'))
def process_image(entry): """ Code to process an image """ workbench = mgg.workbench_manager.create_workbench() # Conversions subdirectory to avoid collisions conversions_subdir = os.path.join( workbench.dir, 'conversions') os.mkdir(conversions_subdir) queued_filepath = entry.queued_media_file queued_filename = workbench.localized_file( mgg.queue_store, queued_filepath, 'source') name_builder = FilenameBuilder(queued_filename) # EXIF extraction exif_tags = extract_exif(queued_filename) gps_data = get_gps_data(exif_tags) # Always create a small thumbnail thumb_filepath = create_pub_filepath( entry, name_builder.fill('{basename}.thumbnail{ext}')) resize_image(entry, queued_filename, thumb_filepath, exif_tags, conversions_subdir, (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height'])) # If the size of the original file exceeds the specified size of a `medium` # file, a `.medium.jpg` files is created and later associated with the media # entry. medium = Image.open(queued_filename) if medium.size[0] > mgg.global_config['media:medium']['max_width'] \ or medium.size[1] > mgg.global_config['media:medium']['max_height'] \ or exif_image_needs_rotation(exif_tags): medium_filepath = create_pub_filepath( entry, name_builder.fill('{basename}.medium{ext}')) resize_image( entry, queued_filename, medium_filepath, exif_tags, conversions_subdir, (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height'])) else: medium_filepath = None # we have to re-read because unlike PIL, not everything reads # things in string representation :) queued_file = file(queued_filename, 'rb') with queued_file: original_filepath = create_pub_filepath( entry, name_builder.fill('{basename}{ext}')) with mgg.public_store.get_file(original_filepath, 'wb') \ as original_file: original_file.write(queued_file.read()) # Remove queued media file from storage and database mgg.queue_store.delete_file(queued_filepath) entry.queued_media_file = [] # Insert media file information into database media_files_dict = entry.setdefault('media_files', {}) media_files_dict[u'thumb'] = thumb_filepath media_files_dict[u'original'] = original_filepath if medium_filepath: media_files_dict[u'medium'] = medium_filepath # Insert exif data into database exif_all = clean_exif(exif_tags) if len(exif_all): entry.media_data_init(exif_all=exif_all) if len(gps_data): for key in list(gps_data.keys()): gps_data['gps_' + key] = gps_data.pop(key) entry.media_data_init(**gps_data) # clean up workbench workbench.destroy_self()
def process_audio(entry): audio_config = mgg.global_config['media_type:mediagoblin.media_types.audio'] workbench = mgg.workbench_manager.create_workbench() queued_filepath = entry.queued_media_file queued_filename = workbench.localized_file( mgg.queue_store, queued_filepath, 'source') name_builder = FilenameBuilder(queued_filename) webm_audio_filepath = create_pub_filepath( entry, '{original}.webm'.format( original=os.path.splitext( queued_filepath[-1])[0])) if audio_config['keep_original']: with open(queued_filename, 'rb') as queued_file: original_filepath = create_pub_filepath( entry, name_builder.fill('{basename}{ext}')) with mgg.public_store.get_file(original_filepath, 'wb') as \ original_file: _log.debug('Saving original...') original_file.write(queued_file.read()) entry.media_files['original'] = original_filepath transcoder = AudioTranscoder() with tempfile.NamedTemporaryFile() as webm_audio_tmp: progress_callback = ProgressCallback(entry) transcoder.transcode( queued_filename, webm_audio_tmp.name, quality=audio_config['quality'], progress_callback=progress_callback) transcoder.discover(webm_audio_tmp.name) _log.debug('Saving medium...') mgg.public_store.get_file(webm_audio_filepath, 'wb').write( webm_audio_tmp.read()) entry.media_files['webm_audio'] = webm_audio_filepath # entry.media_data_init(length=int(data.audiolength)) if audio_config['create_spectrogram']: spectrogram_filepath = create_pub_filepath( entry, '{original}-spectrogram.jpg'.format( original=os.path.splitext( queued_filepath[-1])[0])) with tempfile.NamedTemporaryFile(suffix='.ogg') as wav_tmp: _log.info('Creating OGG source for spectrogram') transcoder.transcode( queued_filename, wav_tmp.name, mux_string='vorbisenc quality={0} ! oggmux'.format( audio_config['quality'])) thumbnailer = AudioThumbnailer() with tempfile.NamedTemporaryFile(suffix='.jpg') as spectrogram_tmp: thumbnailer.spectrogram( wav_tmp.name, spectrogram_tmp.name, width=mgg.global_config['media:medium']['max_width'], fft_size=audio_config['spectrogram_fft_size']) _log.debug('Saving spectrogram...') mgg.public_store.get_file(spectrogram_filepath, 'wb').write( spectrogram_tmp.read()) entry.media_files['spectrogram'] = spectrogram_filepath with tempfile.NamedTemporaryFile(suffix='.jpg') as thumb_tmp: thumbnailer.thumbnail_spectrogram( spectrogram_tmp.name, thumb_tmp.name, (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height'])) thumb_filepath = create_pub_filepath( entry, '{original}-thumbnail.jpg'.format( original=os.path.splitext( queued_filepath[-1])[0])) mgg.public_store.get_file(thumb_filepath, 'wb').write( thumb_tmp.read()) entry.media_files['thumb'] = thumb_filepath else: entry.media_files['thumb'] = ['fake', 'thumb', 'path.jpg'] mgg.queue_store.delete_file(queued_filepath) # clean up workbench workbench.destroy_self()
class CommonPdfProcessor(MediaProcessor): """ Provides a base for various pdf processing steps """ acceptable_files = ['original', 'pdf'] def common_setup(self): """ Set up common pdf processing steps """ # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_pdf_filename() def _set_pdf_filename(self): if self.name_builder.ext == '.pdf': self.pdf_filename = self.process_filename elif self.entry.media_files.get('pdf'): self.pdf_filename = self.workbench.localized_file( mgg.public_store, self.entry.media_files['pdf']) else: self.pdf_filename = self._generate_pdf() def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) skip = True if not file_metadata: return False if keyname == 'thumb': if kwargs.get('thumb_size') != file_metadata.get('thumb_size'): skip = False elif keyname == 'medium': if kwargs.get('size') != file_metadata.get('size'): skip = False return skip def copy_original(self): copy_original(self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def generate_thumb(self, thumb_size=None): if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']) if self._skip_processing('thumb', thumb_size=thumb_size): return # Note: pdftocairo adds '.png', so don't include an ext thumb_filename = os.path.join( self.workbench.dir, self.name_builder.fill('{basename}.thumbnail')) executable = where('pdftocairo') args = [ executable, '-scale-to', str(min(thumb_size)), '-singlefile', '-png', self.pdf_filename, thumb_filename ] _log.debug('calling {0}'.format(repr(' '.join(args)))) Popen(executable=executable, args=args).wait() # since pdftocairo added '.png', we need to include it with the # filename store_public(self.entry, 'thumb', thumb_filename + '.png', self.name_builder.fill('{basename}.thumbnail.png')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size) def _generate_pdf(self): """ Store the pdf. If the file is not a pdf, make it a pdf """ tmp_pdf = os.path.splitext(self.process_filename)[0] + '.pdf' unoconv = where('unoconv') args = [unoconv, '-v', '-f', 'pdf', self.process_filename] _log.debug('calling %s' % repr(args)) Popen(executable=unoconv, args=args).wait() if not os.path.exists(tmp_pdf): _log.debug('unoconv failed to convert file to pdf') raise BadMediaFail() store_public(self.entry, 'pdf', tmp_pdf, self.name_builder.fill('{basename}.pdf')) return self.workbench.localized_file(mgg.public_store, self.entry.media_files['pdf']) def extract_pdf_info(self): pdf_info_dict = pdf_info(self.pdf_filename) self.entry.media_data_init(**pdf_info_dict) def generate_medium(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if self._skip_processing('medium', size=size): return # Note: pdftocairo adds '.png', so don't include an ext filename = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.medium')) executable = where('pdftocairo') args = [ executable, '-scale-to', str(min(size)), '-singlefile', '-png', self.pdf_filename, filename ] _log.debug('calling {0}'.format(repr(' '.join(args)))) Popen(executable=executable, args=args).wait() # since pdftocairo added '.png', we need to include it with the # filename store_public(self.entry, 'medium', filename + '.png', self.name_builder.fill('{basename}.medium.png')) self.entry.set_file_metadata('medium', size=size)
class CommonAsciiProcessor(MediaProcessor): """ Provides a base for various ascii processing steps """ acceptable_files = ['original', 'unicode'] def common_setup(self): self.ascii_config = mgg.global_config['plugins'][ 'mediagoblin.media_types.ascii'] # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'conversions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.charset = None def copy_original(self): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _detect_charset(self, orig_file): d_charset = chardet.detect(orig_file.read()) # Only select a non-utf-8 charset if chardet is *really* sure # Tested with "Feli\x0109an superjaron", which was detected if d_charset['confidence'] < 0.9: self.charset = 'utf-8' else: self.charset = d_charset['encoding'] _log.info('Charset detected: {0}\nWill interpret as: {1}'.format( d_charset, self.charset)) # Rewind the file orig_file.seek(0) def store_unicode_file(self): with open(self.process_filename, 'rb') as orig_file: self._detect_charset(orig_file) unicode_filepath = create_pub_filepath(self.entry, 'ascii-portable.txt') with mgg.public_store.get_file(unicode_filepath, 'wb') \ as unicode_file: # Decode the original file from its detected charset (or UTF8) # Encode the unicode instance to ASCII and replace any # non-ASCII with an HTML entity (&# unicode_file.write( six.text_type(orig_file.read().decode( self.charset)).encode( 'ascii', 'xmlcharrefreplace')) self.entry.media_files['unicode'] = unicode_filepath def generate_thumb(self, font=None, thumb_size=None): with open(self.process_filename, 'rb') as orig_file: # If no font kwarg, check config if not font: font = self.ascii_config.get('thumbnail_font', None) if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']) if self._skip_resizing(font, thumb_size): return tmp_thumb = os.path.join( self.conversions_subdir, self.name_builder.fill('{basename}.thumbnail.png')) ascii_converter_args = {} # If there is a font from either the config or kwarg, update # ascii_converter_args if font: ascii_converter_args.update( {'font': self.ascii_config['thumbnail_font']}) converter = asciitoimage.AsciiToImage( **ascii_converter_args) thumb = converter._create_image( orig_file.read()) with open(tmp_thumb, 'w') as thumb_file: thumb.thumbnail( thumb_size, Image.ANTIALIAS) thumb.save(thumb_file) thumb_info = {'font': font, 'width': thumb_size[0], 'height': thumb_size[1]} _log.debug('Copying local file to public storage') store_public(self.entry, 'thumb', tmp_thumb, self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', **thumb_info) def _skip_resizing(self, font, thumb_size): thumb_info = self.entry.get_file_metadata('thumb') if not thumb_info: return False skip = True if thumb_info.get('font') != font: skip = False elif thumb_info.get('width') != thumb_size[0]: skip = False elif thumb_info.get('height') != thumb_size[1]: skip = False return skip
class CommonAudioProcessor(MediaProcessor): """ Provides a base for various audio processing steps """ acceptable_files = ['original', 'best_quality', 'webm_audio'] def common_setup(self): """ Setup the workbench directory and pull down the original file, add the audio_config, transcoder, thumbnailer and spectrogram_tmp path """ self.audio_config = mgg \ .global_config['plugins']['mediagoblin.media_types.audio'] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = AudioTranscoder() self.thumbnailer = AudioThumbnailer() def copy_original(self): if self.audio_config['keep_original']: copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _keep_best(self): """ If there is no original, keep the best file that we have """ if not self.entry.media_files.get('best_quality'): # Save the best quality file if no original? if not self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_audio'): self.entry.media_files['best_quality'] = self.entry \ .media_files['webm_audio'] def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) skip = True if not file_metadata: return False if keyname == 'webm_audio': if kwargs.get('quality') != file_metadata.get('quality'): skip = False elif keyname == 'spectrogram': if kwargs.get('max_width') != file_metadata.get('max_width'): skip = False elif kwargs.get('fft_size') != file_metadata.get('fft_size'): skip = False elif keyname == 'thumb': if kwargs.get('size') != file_metadata.get('size'): skip = False return skip def transcode(self, quality=None): if not quality: quality = self.audio_config['quality'] if self._skip_processing('webm_audio', quality=quality): return progress_callback = ProgressCallback(self.entry) webm_audio_tmp = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}{ext}')) self.transcoder.transcode( self.process_filename, webm_audio_tmp, quality=quality, progress_callback=progress_callback) self._keep_best() _log.debug('Saving medium...') store_public(self.entry, 'webm_audio', webm_audio_tmp, self.name_builder.fill('{basename}.medium.webm')) self.entry.set_file_metadata('webm_audio', **{'quality': quality}) def create_spectrogram(self, max_width=None, fft_size=None): if not max_width: max_width = mgg.global_config['media:medium']['max_width'] if not fft_size: fft_size = self.audio_config['spectrogram_fft_size'] if self._skip_processing('spectrogram', max_width=max_width, fft_size=fft_size): return wav_tmp = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}.ogg')) _log.info('Creating OGG source for spectrogram') self.transcoder.transcode(self.process_filename, wav_tmp, mux_name='oggmux') spectrogram_tmp = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}-spectrogram.jpg')) self.thumbnailer.spectrogram( wav_tmp, spectrogram_tmp, width=max_width, fft_size=fft_size) _log.debug('Saving spectrogram...') store_public(self.entry, 'spectrogram', spectrogram_tmp, self.name_builder.fill('{basename}.spectrogram.jpg')) file_metadata = {'max_width': max_width, 'fft_size': fft_size} self.entry.set_file_metadata('spectrogram', **file_metadata) def generate_thumb(self, size=None): if not size: max_width = mgg.global_config['media:thumb']['max_width'] max_height = mgg.global_config['media:thumb']['max_height'] size = (max_width, max_height) if self._skip_processing('thumb', size=size): return thumb_tmp = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}-thumbnail.jpg')) # We need the spectrogram to create a thumbnail spectrogram = self.entry.media_files.get('spectrogram') if not spectrogram: _log.info('No spectrogram found, we will create one.') self.create_spectrogram() spectrogram = self.entry.media_files['spectrogram'] spectrogram_filepath = mgg.public_store.get_local_path(spectrogram) self.thumbnailer.thumbnail_spectrogram( spectrogram_filepath, thumb_tmp, tuple(size)) store_public(self.entry, 'thumb', thumb_tmp, self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', **{'size': size})
class CommonStlProcessor(MediaProcessor): """ Provides a common base for various stl processing steps """ acceptable_files = ['original'] def common_setup(self): # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self._set_ext() self._set_model() self._set_greatest() def _set_ext(self): ext = self.name_builder.ext[1:] if not ext: ext = None self.ext = ext def _set_model(self): """ Attempt to parse the model file and divine some useful information about it. """ with open(self.process_filename, 'rb') as model_file: self.model = model_loader.auto_detect(model_file, self.ext) def _set_greatest(self): greatest = [self.model.width, self.model.height, self.model.depth] greatest.sort() self.greatest = greatest[-1] def copy_original(self): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _snap(self, keyname, name, camera, size, project="ORTHO"): filename = self.name_builder.fill(name) workbench_path = self.workbench.joinpath(filename) shot = { "model_path": self.process_filename, "model_ext": self.ext, "camera_coord": camera, "camera_focus": self.model.average, "camera_clip": self.greatest*10, "greatest": self.greatest, "projection": project, "width": size[0], "height": size[1], "out_file": workbench_path, } blender_render(shot) # make sure the image rendered to the workbench path assert os.path.exists(workbench_path) # copy it up! store_public(self.entry, keyname, workbench_path, filename) def generate_thumb(self, thumb_size=None): if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height']) self._snap( "thumb", "{basename}.thumb.jpg", [0, self.greatest*-1.5, self.greatest], thumb_size, project="PERSP") def generate_perspective(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "perspective", "{basename}.perspective.jpg", [0, self.greatest*-1.5, self.greatest], size, project="PERSP") def generate_topview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "top", "{basename}.top.jpg", [self.model.average[0], self.model.average[1], self.greatest*2], size) def generate_frontview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "front", "{basename}.front.jpg", [self.model.average[0], self.greatest*-2, self.model.average[2]], size) def generate_sideview(self, size=None): if not size: size = (mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) self._snap( "side", "{basename}.side.jpg", [self.greatest*-2, self.model.average[1], self.model.average[2]], size) def store_dimensions(self): """ Put model dimensions into the database """ dimensions = { "center_x": self.model.average[0], "center_y": self.model.average[1], "center_z": self.model.average[2], "width": self.model.width, "height": self.model.height, "depth": self.model.depth, "file_type": self.ext, } self.entry.media_data_init(**dimensions)
class CommonImageProcessor(MediaProcessor): """ Provides a base for various media processing steps """ # list of acceptable file keys in order of prefrence for reprocessing acceptable_files = ['original', 'medium'] def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.image_config = mgg.global_config[ 'media_type:mediagoblin.media_types.image'] ## @@: Should this be two functions? # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join(self.workbench.dir, 'convirsions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction self.exif_tags = extract_exif(self.process_filename) def generate_medium_if_applicable(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, False, 'medium', self.process_filename, self.name_builder.fill('{basename}.medium{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def generate_thumb(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, True, 'thumb', self.process_filename, self.name_builder.fill('{basename}.thumbnail{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def copy_original(self): copy_original(self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def extract_metadata(self): # Is there any GPS data gps_data = get_gps_data(self.exif_tags) # Insert exif data into database exif_all = clean_exif(self.exif_tags) if len(exif_all): self.entry.media_data_init(exif_all=exif_all) if len(gps_data): for key in list(gps_data.keys()): gps_data['gps_' + key] = gps_data.pop(key) self.entry.media_data_init(**gps_data)
class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ acceptable_files = ['original', 'best_quality', 'webm_video'] def common_setup(self): self.video_config = mgg \ .global_config['media_type:mediagoblin.media_types.video'] # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False def copy_original(self): # If we didn't transcode, then we need to keep the original if not self.did_transcode or \ (self.video_config['keep_original'] and self.did_transcode): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _keep_best(self): """ If there is no original, keep the best file that we have """ if not self.entry.media_files.get('best_quality'): # Save the best quality file if no original? if not self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_video'): self.entry.media_files['best_quality'] = self.entry \ .media_files['webm_video'] def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.medium.webm')) if not medium_size: medium_size = ( mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) if not vp8_quality: vp8_quality = self.video_config['vp8_quality'] if not vp8_threads: vp8_threads = self.video_config['vp8_threads'] if not vorbis_quality: vorbis_quality = self.video_config['vorbis_quality'] # Extract metadata and keep a record of it metadata = self.transcoder.discover(self.process_filename) store_metadata(self.entry, metadata) # Figure out whether or not we need to transcode this video or # if we can skip it if skip_transcode(metadata, medium_size): _log.debug('Skipping transcoding') dst_dimensions = metadata['videowidth'], metadata['videoheight'] # If there is an original and transcoded, delete the transcoded # since it must be of lower quality then the original if self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_video'): self.entry.media_files['webm_video'].delete() else: self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) dst_dimensions = self.transcoder.dst_data.videowidth,\ self.transcoder.dst_data.videoheight self._keep_best() # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, 'webm_video', tmp_dst, self.name_builder.fill('{basename}.medium.webm')) _log.debug('Saved medium') self.did_transcode = True # Save the width and height of the transcoded video self.entry.media_data_init( width=dst_dimensions[0], height=dst_dimensions[1]) def generate_thumb(self, thumb_size=None): # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( '{basename}.thumbnail.jpg')) if not thumb_size: thumb_size = (mgg.global_config['media:thumb']['max_width'],) # We will only use the width so that the correct scale is kept transcoders.VideoThumbnailerMarkII( self.process_filename, tmp_thumb, thumb_size[0]) # Push the thumbnail to public storage _log.debug('Saving thumbnail...') store_public(self.entry, 'thumb', tmp_thumb, self.name_builder.fill('{basename}.thumbnail.jpg'))
def process_video(proc_state): """ Process a video entry, transcode the queued media files (originals) and create a thumbnail for the entry. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench video_config = mgg.global_config['media_type:mediagoblin.media_types.video'] queued_filepath = entry.queued_media_file queued_filename = proc_state.get_queued_filename() name_builder = FilenameBuilder(queued_filename) medium_filepath = create_pub_filepath( entry, name_builder.fill('{basename}-640p.webm')) thumbnail_filepath = create_pub_filepath( entry, name_builder.fill('{basename}.thumbnail.jpg')) # Create a temporary file for the video destination (cleaned up with workbench) tmp_dst = NamedTemporaryFile(dir=workbench.dir, delete=False) with tmp_dst: # Transcode queued file to a VP8/vorbis file that fits in a 640x640 square progress_callback = ProgressCallback(entry) dimensions = ( mgg.global_config['media:medium']['max_width'], mgg.global_config['media:medium']['max_height']) # Extract metadata and keep a record of it metadata = transcoders.VideoTranscoder().discover(queued_filename) store_metadata(entry, metadata) # Figure out whether or not we need to transcode this video or # if we can skip it if skip_transcode(metadata): _log.debug('Skipping transcoding') dst_dimensions = metadata['videowidth'], metadata['videoheight'] # Push original file to public storage _log.debug('Saving original...') proc_state.copy_original(queued_filepath[-1]) did_transcode = False else: transcoder = transcoders.VideoTranscoder() transcoder.transcode(queued_filename, tmp_dst.name, vp8_quality=video_config['vp8_quality'], vp8_threads=video_config['vp8_threads'], vorbis_quality=video_config['vorbis_quality'], progress_callback=progress_callback, dimensions=dimensions) dst_dimensions = transcoder.dst_data.videowidth,\ transcoder.dst_data.videoheight # Push transcoded video to public storage _log.debug('Saving medium...') mgg.public_store.copy_local_to_storage(tmp_dst.name, medium_filepath) _log.debug('Saved medium') entry.media_files['webm_640'] = medium_filepath did_transcode = True # Save the width and height of the transcoded video entry.media_data_init( width=dst_dimensions[0], height=dst_dimensions[1]) # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = NamedTemporaryFile(dir=workbench.dir, suffix='.jpg', delete=False) with tmp_thumb: # Create a thumbnail.jpg that fits in a 180x180 square transcoders.VideoThumbnailerMarkII( queued_filename, tmp_thumb.name, 180) # Push the thumbnail to public storage _log.debug('Saving thumbnail...') mgg.public_store.copy_local_to_storage(tmp_thumb.name, thumbnail_filepath) entry.media_files['thumb'] = thumbnail_filepath # save the original... but only if we did a transcoding # (if we skipped transcoding and just kept the original anyway as the main # media, then why would we save the original twice?) if video_config['keep_original'] and did_transcode: # Push original file to public storage _log.debug('Saving original...') proc_state.copy_original(queued_filepath[-1]) # Remove queued media file from storage and database proc_state.delete_queue_file()
def process_audio(proc_state): """Code to process uploaded audio. Will be run by celery. A Workbench() represents a local tempory dir. It is automatically cleaned up when this function exits. """ entry = proc_state.entry workbench = proc_state.workbench audio_config = mgg.global_config[ 'media_type:mediagoblin.media_types.audio'] queued_filepath = entry.queued_media_file queued_filename = workbench.localized_file(mgg.queue_store, queued_filepath, 'source') name_builder = FilenameBuilder(queued_filename) webm_audio_filepath = create_pub_filepath( entry, '{original}.webm'.format( original=os.path.splitext(queued_filepath[-1])[0])) if audio_config['keep_original']: with open(queued_filename, 'rb') as queued_file: original_filepath = create_pub_filepath( entry, name_builder.fill('{basename}{ext}')) with mgg.public_store.get_file(original_filepath, 'wb') as \ original_file: _log.debug('Saving original...') original_file.write(queued_file.read()) entry.media_files['original'] = original_filepath transcoder = AudioTranscoder() with NamedTemporaryFile(dir=workbench.dir) as webm_audio_tmp: progress_callback = ProgressCallback(entry) transcoder.transcode(queued_filename, webm_audio_tmp.name, quality=audio_config['quality'], progress_callback=progress_callback) transcoder.discover(webm_audio_tmp.name) _log.debug('Saving medium...') mgg.public_store.get_file(webm_audio_filepath, 'wb').write(webm_audio_tmp.read()) entry.media_files['webm_audio'] = webm_audio_filepath # entry.media_data_init(length=int(data.audiolength)) if audio_config['create_spectrogram']: spectrogram_filepath = create_pub_filepath( entry, '{original}-spectrogram.jpg'.format( original=os.path.splitext(queued_filepath[-1])[0])) with NamedTemporaryFile(dir=workbench.dir, suffix='.ogg') as wav_tmp: _log.info('Creating OGG source for spectrogram') transcoder.transcode( queued_filename, wav_tmp.name, mux_string='vorbisenc quality={0} ! oggmux'.format( audio_config['quality'])) thumbnailer = AudioThumbnailer() with NamedTemporaryFile(dir=workbench.dir, suffix='.jpg') as spectrogram_tmp: thumbnailer.spectrogram( wav_tmp.name, spectrogram_tmp.name, width=mgg.global_config['media:medium']['max_width'], fft_size=audio_config['spectrogram_fft_size']) _log.debug('Saving spectrogram...') mgg.public_store.get_file(spectrogram_filepath, 'wb').write(spectrogram_tmp.read()) entry.media_files['spectrogram'] = spectrogram_filepath with NamedTemporaryFile(dir=workbench.dir, suffix='.jpg') as thumb_tmp: thumbnailer.thumbnail_spectrogram( spectrogram_tmp.name, thumb_tmp.name, (mgg.global_config['media:thumb']['max_width'], mgg.global_config['media:thumb']['max_height'])) thumb_filepath = create_pub_filepath( entry, '{original}-thumbnail.jpg'.format( original=os.path.splitext(queued_filepath[-1])[0])) mgg.public_store.get_file(thumb_filepath, 'wb').write(thumb_tmp.read()) entry.media_files['thumb'] = thumb_filepath else: entry.media_files['thumb'] = ['fake', 'thumb', 'path.jpg'] # Remove queued media file from storage and database. # queued_filepath is in the task_id directory which should # be removed too, but fail if the directory is not empty to be on # the super-safe side. mgg.queue_store.delete_file(queued_filepath) # rm file mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir entry.queued_media_file = []
def process_video(entry): """ Process a video entry, transcode the queued media files (originals) and create a thumbnail for the entry. """ video_config = mgg.global_config['media_type:mediagoblin.media_types.video'] workbench = mgg.workbench_manager.create_workbench() queued_filepath = entry.queued_media_file queued_filename = workbench.localized_file( mgg.queue_store, queued_filepath, 'source') name_builder = FilenameBuilder(queued_filename) medium_filepath = create_pub_filepath( entry, name_builder.fill('{basename}-640p.webm')) thumbnail_filepath = create_pub_filepath( entry, name_builder.fill('{basename}.thumbnail.jpg')) # Create a temporary file for the video destination tmp_dst = tempfile.NamedTemporaryFile() with tmp_dst: # Transcode queued file to a VP8/vorbis file that fits in a 640x640 square progress_callback = ProgressCallback(entry) transcoder = transcoders.VideoTranscoder() transcoder.transcode(queued_filename, tmp_dst.name, vp8_quality=video_config['vp8_quality'], vp8_threads=video_config['vp8_threads'], vorbis_quality=video_config['vorbis_quality'], progress_callback=progress_callback) # Push transcoded video to public storage _log.debug('Saving medium...') mgg.public_store.get_file(medium_filepath, 'wb').write( tmp_dst.read()) _log.debug('Saved medium') entry.media_files['webm_640'] = medium_filepath # Save the width and height of the transcoded video entry.media_data_init( width=transcoder.dst_data.videowidth, height=transcoder.dst_data.videoheight) # Create a temporary file for the video thumbnail tmp_thumb = tempfile.NamedTemporaryFile(suffix='.jpg') with tmp_thumb: # Create a thumbnail.jpg that fits in a 180x180 square transcoders.VideoThumbnailerMarkII( queued_filename, tmp_thumb.name, 180) # Push the thumbnail to public storage _log.debug('Saving thumbnail...') mgg.public_store.get_file(thumbnail_filepath, 'wb').write( tmp_thumb.read()) _log.debug('Saved thumbnail') entry.media_files['thumb'] = thumbnail_filepath if video_config['keep_original']: # Push original file to public storage queued_file = file(queued_filename, 'rb') with queued_file: original_filepath = create_pub_filepath( entry, queued_filepath[-1]) with mgg.public_store.get_file(original_filepath, 'wb') as \ original_file: _log.debug('Saving original...') original_file.write(queued_file.read()) _log.debug('Saved original') entry.media_files['original'] = original_filepath mgg.queue_store.delete_file(queued_filepath)
class CommonImageProcessor(MediaProcessor): """ Provides a base for various media processing steps """ # list of acceptable file keys in order of prefrence for reprocessing acceptable_files = ['original', 'medium'] def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.image_config = mgg.global_config['plugins'][ 'mediagoblin.media_types.image'] ## @@: Should this be two functions? # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join( self.workbench.dir, 'conversions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction self.exif_tags = extract_exif(self.process_filename) def generate_medium_if_applicable(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, False, 'medium', self.process_filename, self.name_builder.fill('{basename}.medium{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def generate_thumb(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, True, 'thumb', self.process_filename, self.name_builder.fill('{basename}.thumbnail{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def copy_original(self): copy_original( self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def extract_metadata(self, file): """ Extract all the metadata from the image and store """ # Extract GPS data and store in Location gps_data = get_gps_data(self.exif_tags) if len(gps_data): Location.create({"position": gps_data}, self.entry) # Insert exif data into database exif_all = clean_exif(self.exif_tags) if len(exif_all): self.entry.media_data_init(exif_all=exif_all) # Extract file metadata try: im = Image.open(self.process_filename) except IOError: raise BadMediaFail() metadata = { "width": im.size[0], "height": im.size[1], } self.entry.set_file_metadata(file, **metadata)
class CommonAudioProcessor(MediaProcessor): """ Provides a base for various audio processing steps """ acceptable_files = ["original", "best_quality", "webm_audio"] def common_setup(self): """ Setup the workbench directory and pull down the original file, add the audio_config, transcoder, thumbnailer and spectrogram_tmp path """ self.audio_config = mgg.global_config["plugins"]["mediagoblin.media_types.audio"] # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = AudioTranscoder() self.thumbnailer = AudioThumbnailer() def copy_original(self): if self.audio_config["keep_original"]: copy_original(self.entry, self.process_filename, self.name_builder.fill("{basename}{ext}")) def _keep_best(self): """ If there is no original, keep the best file that we have """ if not self.entry.media_files.get("best_quality"): # Save the best quality file if no original? if not self.entry.media_files.get("original") and self.entry.media_files.get("webm_audio"): self.entry.media_files["best_quality"] = self.entry.media_files["webm_audio"] def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) skip = True if not file_metadata: return False if keyname == "webm_audio": if kwargs.get("quality") != file_metadata.get("quality"): skip = False elif keyname == "spectrogram": if kwargs.get("max_width") != file_metadata.get("max_width"): skip = False elif kwargs.get("fft_size") != file_metadata.get("fft_size"): skip = False elif keyname == "thumb": if kwargs.get("size") != file_metadata.get("size"): skip = False return skip def transcode(self, quality=None): if not quality: quality = self.audio_config["quality"] if self._skip_processing("webm_audio", quality=quality): return progress_callback = ProgressCallback(self.entry) webm_audio_tmp = os.path.join(self.workbench.dir, self.name_builder.fill("{basename}{ext}")) self.transcoder.transcode( self.process_filename, webm_audio_tmp, quality=quality, progress_callback=progress_callback ) self._keep_best() _log.debug("Saving medium...") store_public(self.entry, "webm_audio", webm_audio_tmp, self.name_builder.fill("{basename}.medium.webm")) self.entry.set_file_metadata("webm_audio", **{"quality": quality}) def create_spectrogram(self, max_width=None, fft_size=None): if not max_width: max_width = mgg.global_config["media:medium"]["max_width"] if not fft_size: fft_size = self.audio_config["spectrogram_fft_size"] if self._skip_processing("spectrogram", max_width=max_width, fft_size=fft_size): return wav_tmp = os.path.join(self.workbench.dir, self.name_builder.fill("{basename}.ogg")) _log.info("Creating OGG source for spectrogram") self.transcoder.transcode(self.process_filename, wav_tmp, mux_name="oggmux") spectrogram_tmp = os.path.join(self.workbench.dir, self.name_builder.fill("{basename}-spectrogram.jpg")) self.thumbnailer.spectrogram(wav_tmp, spectrogram_tmp, width=max_width, fft_size=fft_size) _log.debug("Saving spectrogram...") store_public(self.entry, "spectrogram", spectrogram_tmp, self.name_builder.fill("{basename}.spectrogram.jpg")) file_metadata = {"max_width": max_width, "fft_size": fft_size} self.entry.set_file_metadata("spectrogram", **file_metadata) def generate_thumb(self, size=None): if not size: max_width = mgg.global_config["media:thumb"]["max_width"] max_height = mgg.global_config["media:thumb"]["max_height"] size = (max_width, max_height) if self._skip_processing("thumb", size=size): return thumb_tmp = os.path.join(self.workbench.dir, self.name_builder.fill("{basename}-thumbnail.jpg")) # We need the spectrogram to create a thumbnail spectrogram = self.entry.media_files.get("spectrogram") if not spectrogram: _log.info("No spectrogram found, we will create one.") self.create_spectrogram() spectrogram = self.entry.media_files["spectrogram"] spectrogram_filepath = mgg.public_store.get_local_path(spectrogram) self.thumbnailer.thumbnail_spectrogram(spectrogram_filepath, thumb_tmp, tuple(size)) store_public(self.entry, "thumb", thumb_tmp, self.name_builder.fill("{basename}.thumbnail.jpg")) self.entry.set_file_metadata("thumb", **{"size": size})
class CommonAudioProcessor(MediaProcessor): """ Provides a base for various audio processing steps """ acceptable_files = ['original', 'best_quality', 'webm_audio'] def common_setup(self): """ Setup the workbench directory and pull down the original file, add the audio_config, transcoder, thumbnailer and spectrogram_tmp path """ self.audio_config = mgg \ .global_config['plugins']['mediagoblin.media_types.audio'] # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) self.transcoder = AudioTranscoder() self.thumbnailer = AudioThumbnailer() def copy_original(self): if self.audio_config['keep_original']: copy_original(self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def _keep_best(self): """ If there is no original, keep the best file that we have """ if not self.entry.media_files.get('best_quality'): # Save the best quality file if no original? if not self.entry.media_files.get('original') and \ self.entry.media_files.get('webm_audio'): self.entry.media_files['best_quality'] = self.entry \ .media_files['webm_audio'] def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) skip = True if not file_metadata: return False if keyname == 'webm_audio': if kwargs.get('quality') != file_metadata.get('quality'): skip = False elif keyname == 'spectrogram': if kwargs.get('max_width') != file_metadata.get('max_width'): skip = False elif kwargs.get('fft_size') != file_metadata.get('fft_size'): skip = False elif keyname == 'thumb': if kwargs.get('size') != file_metadata.get('size'): skip = False return skip def transcode(self, quality=None): if not quality: quality = self.audio_config['quality'] if self._skip_processing('webm_audio', quality=quality): return progress_callback = ProgressCallback(self.entry) webm_audio_tmp = os.path.join( self.workbench.dir, self.name_builder.fill('{basename}{ext}')) self.transcoder.transcode(self.process_filename, webm_audio_tmp, quality=quality, progress_callback=progress_callback) self.transcoder.discover(webm_audio_tmp) self._keep_best() _log.debug('Saving medium...') store_public(self.entry, 'webm_audio', webm_audio_tmp, self.name_builder.fill('{basename}.medium.webm')) self.entry.set_file_metadata('webm_audio', **{'quality': quality}) def create_spectrogram(self, max_width=None, fft_size=None): if not max_width: max_width = mgg.global_config['media:medium']['max_width'] if not fft_size: fft_size = self.audio_config['spectrogram_fft_size'] if self._skip_processing('spectrogram', max_width=max_width, fft_size=fft_size): return wav_tmp = os.path.join(self.workbench.dir, self.name_builder.fill('{basename}.ogg')) _log.info('Creating OGG source for spectrogram') self.transcoder.transcode( self.process_filename, wav_tmp, mux_string='vorbisenc quality={0} ! oggmux'.format( self.audio_config['quality'])) spectrogram_tmp = os.path.join( self.workbench.dir, self.name_builder.fill('{basename}-spectrogram.jpg')) self.thumbnailer.spectrogram(wav_tmp, spectrogram_tmp, width=max_width, fft_size=fft_size) _log.debug('Saving spectrogram...') store_public(self.entry, 'spectrogram', spectrogram_tmp, self.name_builder.fill('{basename}.spectrogram.jpg')) file_metadata = {'max_width': max_width, 'fft_size': fft_size} self.entry.set_file_metadata('spectrogram', **file_metadata) def generate_thumb(self, size=None): if not size: max_width = mgg.global_config['media:thumb']['max_width'] max_height = mgg.global_config['media:thumb']['max_height'] size = (max_width, max_height) if self._skip_processing('thumb', size=size): return thumb_tmp = os.path.join( self.workbench.dir, self.name_builder.fill('{basename}-thumbnail.jpg')) # We need the spectrogram to create a thumbnail spectrogram = self.entry.media_files.get('spectrogram') if not spectrogram: _log.info('No spectrogram found, we will create one.') self.create_spectrogram() spectrogram = self.entry.media_files['spectrogram'] spectrogram_filepath = mgg.public_store.get_local_path(spectrogram) self.thumbnailer.thumbnail_spectrogram(spectrogram_filepath, thumb_tmp, tuple(size)) store_public(self.entry, 'thumb', thumb_tmp, self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', **{'size': size})
class CommonImageProcessor(MediaProcessor): """ Provides a base for various media processing steps """ # list of acceptable file keys in order of prefrence for reprocessing acceptable_files = ['original', 'medium'] def common_setup(self): """ Set up the workbench directory and pull down the original file """ self.image_config = mgg.global_config['plugins'][ 'mediagoblin.media_types.image'] ## @@: Should this be two functions? # Conversions subdirectory to avoid collisions self.conversions_subdir = os.path.join(self.workbench.dir, 'conversions') os.mkdir(self.conversions_subdir) # Pull down and set up the processing file self.process_filename = get_process_filename(self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) # Exif extraction self.exif_tags = extract_exif(self.process_filename) def generate_medium_if_applicable(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, False, 'medium', self.process_filename, self.name_builder.fill('{basename}.medium{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def generate_thumb(self, size=None, quality=None, filter=None): if not quality: quality = self.image_config['quality'] if not filter: filter = self.image_config['resize_filter'] resize_tool(self.entry, True, 'thumb', self.process_filename, self.name_builder.fill('{basename}.thumbnail{ext}'), self.conversions_subdir, self.exif_tags, quality, filter, size) def copy_original(self): copy_original(self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) def extract_metadata(self, file): """ Extract all the metadata from the image and store """ # Extract GPS data and store in Location gps_data = get_gps_data(self.exif_tags) if len(gps_data): Location.create({"position": gps_data}, self.entry) # Insert exif data into database exif_all = clean_exif(self.exif_tags) if len(exif_all): self.entry.media_data_init(exif_all=exif_all) # Extract file metadata try: im = Image.open(self.process_filename) except IOError: raise BadMediaFail() metadata = { "width": im.size[0], "height": im.size[1], } self.entry.set_file_metadata(file, **metadata)