def action_test(self): """ Test if ikaaro instances of this Python environment are alive""" print "**********************************************************" print " TEST" print "**********************************************************" for ikaaro in config.get_sections_by_type("ikaaro"): if ikaaro.options["pyenv"] == self.name: uri = ikaaro.options["uri"] try: vfs.open("%s/;_ctrl" % uri) except GError: print "[ERROR] ", uri else: print "[OK]", uri
def action_test(self): """ Test if ikaaro instances of this Python environment are alive""" print '**********************************************************' print ' TEST' print '**********************************************************' for ikaaro in config.get_sections_by_type('ikaaro'): if ikaaro.options['pyenv'] == self.name: uri = ikaaro.options['uri'] try: vfs.open('%s/;_ctrl' % uri) except GError: print '[ERROR] ', uri else: print '[OK]', uri
def test_copy_folder_to_folder(self): vfs.copy('tests', 'tmp') file = vfs.open('tmp/tests/hello.txt') try: self.assertEqual(file.read(), 'hello world\n') finally: file.close()
def check_image(filename, context): if vfs.exists(filename) is False: print u"(WW) The filename '%s' doesn't exist" % filename filename = context.image_not_found_path im = None if filename.startswith('http://'): # Remote file # If the image is a remote file, we create a StringIO # object contains the image data to avoid reportlab problems ... data = vfs.open(filename).read() my_file = context.get_tmp_file() filename = my_file.name my_file.write(data) my_file.close() im = Image(string=data) if im is None: im = ro_database.get_handler(filename, Image) x, y = im.get_size() if not (x or y): print u'image not valid : %s' % filename filename = context.image_not_found_path im = ro_database.get_handler(filename, Image) x, y = im.get_size() return filename, (x, y)
def test_copy_file(self): vfs.copy('tests/hello.txt', 'tmp/hello.txt.bak') file = vfs.open('tmp/hello.txt.bak') try: self.assertEqual(file.read(), 'hello world\n') finally: file.close()
def is_running_in_rw_mode(self): address = self.config.get_value('listen-address').strip() if address == '*': address = '127.0.0.1' port = self.config.get_value('listen-port') url = 'http://%s:%s/;_ctrl' % (address, port) try: h = vfs.open(url) except GError: # The server is not running return False data = h.read() return json.loads(data)['read-only'] is False
def test20_append(self): # Initialize file = vfs.make_file('tests/toto.txt') try: file.write('hello\n') finally: file.close() # Test file = vfs.open('tests/toto.txt', APPEND) try: file.write('bye\n') finally: file.close() self.assertEqual(open('tests/toto.txt').read(), 'hello\nbye\n') # Remove temporary file vfs.remove('tests/toto.txt')
def test21_write_and_truncate(self): # Initialize file = vfs.make_file('tests/toto.txt') try: file.write('hello\n') finally: file.close() # Test file = vfs.open('tests/toto.txt', WRITE) try: file.write('bye\n') finally: file.close() self.assertEqual(open('tests/toto.txt').read(), 'bye\n') # Remove temporary file vfs.remove('tests/toto.txt')
def encode_avi_to_flv(self, tmpfolder, inputfile, name, width): """ Take a *inputfile* video, tafking is *name*, encode it in flv inside the *tmpfolder*, at given *width*. """ flv_filename = "%s.flv" % name ratio = self.get_ratio(tmpfolder, inputfile) height = int(round(float(width)/ratio)) ffmpeg = ['ffmpeg', '-i', '%s' % inputfile, '-acodec', 'libfaac', '-ar', '22050', '-ab', '32k', '-f', 'flv', '-s', '%sx%s' % (width, height), '-sameq', flv_filename] get_pipe(ffmpeg, cwd=tmpfolder) self.add_metadata_to_flv(tmpfolder, flv_filename) self.make_flv_thumbnail(tmpfolder, name, width) tmpdir = vfs.open(tmpfolder) # Copy the flv content to a data variable flv_file = tmpdir.open(flv_filename) try: flv_data = flv_file.read() finally: flv_file.close() # Copy the thumb content thumb_file = tmpdir.open('%s.png' % name) try: thumb_data = thumb_file.read() finally: thumb_file.close() # Need to add the PNG to ikaaro # Return a FLV file and a PNG thumbnail flvfile = [flv_filename, 'video/x-flv', flv_data, 'flv'] flvthumb = ['thumb_%s' % name, 'image/png', thumb_data, 'png'] if((len(flv_data) == 0) or (len(thumb_data) == 0)): #exit encoded = None else: encoded = {'flvfile':flvfile, 'flvthumb':flvthumb} return encoded
def is_running_in_rw_mode(self, mode='running'): is_running = self.is_running() if not is_running: return False if mode == 'request': address = self.config.get_value('listen-address').strip() if address == '*': address = '127.0.0.1' port = self.config.get_value('listen-port') url = 'http://%s:%s/;_ctrl' % (address, port) try: h = vfs.open(url) except GError: # The server is not running return False data = h.read() return json.loads(data)['read-only'] is False elif mode == 'running': kw = self.get_running_informations() return not kw.get('read_only', False)
def make_thumbnail_only(self, tmpfolder, inputfile, name, width): """ Take a *inputfile* video, using is *name*, create a thumbnail as a PNG file in the *tmpfolder*, at given *width*. """ flv_filename = "%s" % name ratio = self.get_ratio(tmpfolder, inputfile) height = int(round(float(width)/ratio)) self.add_metadata_to_flv(tmpfolder, flv_filename) self.make_thumbnail(tmpfolder, name, width) tmpdir = vfs.open(tmpfolder) # Copy the flv content to a data variable flv_file = tmpdir.open(flv_filename) try: flv_data = flv_file.read() finally: flv_file.close() # Copy the thumb content thumb_file = tmpdir.open('%s.png' % name) try: thumb_data = thumb_file.read() finally: thumb_file.close() # Need to add the PNG to ikaaro # Return a PNG thumbnail flvthumb = ['%s_thumb' % name, 'image/png', thumb_data, 'png'] if(len(thumb_data) == 0): #exit thumbnail = None else: thumbnail = {'flvthumb':flvthumb} return thumbnail
def read_file(context, uri, file): uri = get_reference(uri) # Shortcuts elements = context['elements'] references = context['references'] # XML stream stream = XMLParser(file.read()) # Parse ! stack = [] for type, value, line in stream: # Set the good encoding if type == XML_DECL: context['encoding'] = value[1] elif type == START_ELEMENT: tag_uri, tag_name, attrs = value # Set your context variable read_common_attrs(tag_uri, attrs, context) # Only RNG ! if tag_uri == rng_uri: # <element> if tag_name == 'element': # Create a new element qnames = read_qnames(attrs, context, stream) element = { 'qnames': qnames, 'attributes': [], 'data': None, 'is_empty': True, 'refs': [] } # My father has at least a child, me! if stack: stack[-1]['is_empty'] = False # And push it elements.append(element) stack.append(element) # <attribute> elif tag_name == 'attribute': # Create a new attribute qnames = read_qnames(attrs, context, stream) attribute = {'qnames': qnames, 'data': None, 'refs': []} # And push it stack[-1]['attributes'].append(attribute) stack.append(attribute) # <data> elif tag_name == 'data': type = attrs[None, 'type'] last = stack[-1] if last['data'] is not None: last['data'] = '' else: last['data'] = type # <ref> elif tag_name == 'ref': name = attrs[None, 'name'] if stack: stack[-1]['refs'].append(name) # <define> elif tag_name == 'define': name = attrs[None, 'name'] # New or merge ? if name in references and (None, 'combine') in attrs: ref = references[name] else: ref = { 'attributes': [], 'data': None, 'is_empty': True, 'refs': [] } references[name] = ref stack.append(ref) # <text> elif tag_name == 'text': last = stack[-1] if last['data'] is not None: last['data'] = '' else: last['data'] = 'string' # <value> elif tag_name == 'value': stack[-1]['data'] = '' # <include> elif tag_name == 'include': href = attrs[None, 'href'] include_uri = uri.resolve(href) include_uri = str(include_uri) include_file = vfs.open(include_uri) read_file(context, include_uri, include_file) # Ignored tags elif tag_name in [ 'grammar', 'start', 'choice', 'optional', 'zeroOrMore', 'oneOrMore', 'group', 'empty', 'interleave', 'param', 'list', 'mixed' ]: continue # Tags not implemented else: raise NotImplementedError, ('relax NG: <%s> not ' 'implemented') % tag_name elif type == END_ELEMENT: tag_uri, tag_name = value if (tag_uri == rng_uri and tag_name in ['element', 'attribute', 'define']): stack.pop()
def test15_open_file(self): file = vfs.open('tests/hello.txt') self.assertEqual(file.read(), 'hello world\n')
def load_state_from_uri(self, uri): file = vfs.open(uri) try: self.load_state_from_file(file) finally: file.close()
def encode_video_to_flv(self, tmpfolder, inputfile, name, width, encode=False): """ Take a *inputfile* video, taking is *name*, encode it in flv inside the *tmpfolder*, at given *width*. Return *video_low* and a *video_low_thumb* files of the original file """ flv_filename = "%s.flv" % name ratio = self.get_ratio(tmpfolder, inputfile) height = int(round(float(width)/ratio)) original_W, original_H = self.get_size_frompath(tmpfolder, inputfile) print("original_W = %s, original_H = %s" % (original_W, original_H)) if original_W is not None: if self.check_resize_height_is_even(width, original_W, original_H): even = True crop = False else: even = self.get_next_even_height( width, original_W, original_H) crop = even * float(original_W) / float(width) crop = crop - int(original_H) height = even if not even: msg = u"The ratio width/height is not even, so the video cannot be encoded!" raise NotImplementedError, "%s" % msg else: if not encode: #TWO PASS # Pass one ffmpeg = ['ffmpeg', '-i', '%s' % inputfile, '-sameq', '-s', '%sx%s' % (width, height), '-pass', '1', '-vcodec', 'libx264', '-fpre', '/usr/share/ffmpeg/libx264-normal.ffpreset', '-b', '512k', '-bt', '512k', '-threads', '0', '-f', 'rawvideo', '-f', 'mp4', '-an', '-y', '/dev/null'] print("Pass 1 : ffmpeg = %s" % ffmpeg) get_pipe(ffmpeg, cwd=tmpfolder) # Pass two ffmpeg = ['ffmpeg', '-i', '%s' % inputfile, '-sameq', '-s', '%sx%s' % (width, height), '-f', 'mp4', '-pass', '2', '-acodec', 'libfaac', '-ab', '128k', '-ac', '2', '-vcodec', 'libx264', '-fpre', '/usr/share/ffmpeg/libx264-normal.ffpreset', '-b', '512k', '-bt', '512k', '-threads', '0', '-metadata', 'author="Tchack"', '-metadata', 'copyright="Tous droits réservés - Tchack/ALUMA Productions"', flv_filename] print("Pass 2 : ffmpeg = %s" % ffmpeg) get_pipe(ffmpeg, cwd=tmpfolder) elif encode == 'one': #ONE PASS ffmpeg = ['ffmpeg', '-i', '%s' % inputfile, '-acodec', 'libfaac', '-ar', '22050', '-ab', '32k', '-f', 'flv', '-s', '%sx%s' % (width, height), '-sameq', flv_filename] get_pipe(ffmpeg, cwd=tmpfolder) elif encode == 'one_chroma_faststart': tmp_flv_filename = "tmp_%s.flv" % name ffmpeg= ['ffmpeg', '-y', '-i', '%s' % inputfile, #'-fpre', '/usr/share/ffmpeg/libx264-hq.ffpreset', '-preset', 'fast', '-s', '%sx%s' % (width, height), '-vcodec', 'libx264', '-b', '512k', '-acodec', 'libfaac', '-ar', '44100', '-ab', '128k', '-coder', 'ac', '-me_method', 'full', '-me_range', '16', '-subq', '5', '-sc_threshold', '40', '-cmp', '+chroma', '-partitions', '+parti4x4+partp8x8+partb8x8', '-i_qfactor', '0.71', '-keyint_min', '25', '-b_strategy', '1', '-g', '250', '-r', '20', '-metadata', 'author="Tchack"', '-metadata', 'copyright="Tous droits réservés - Tchack/ALUMA Productions"', #'-vf', 'crop=in_w:in_h+%s' % int(crop), #tmp_flv_filename] flv_filename] if crop: ffmpeg.insert(-1, '-vf') ffmpeg.insert(-1, 'crop=in_w:in_h+%s' % int(crop)) print("Pass : ffmpeg = %s" % ffmpeg) get_pipe(ffmpeg, cwd=tmpfolder) #ffmpeg = ['qt-faststart', "\"./%s\"" % tmp_flv_filename, "\"./%s\"" % flv_filename] #print("Pass : faststart = %s" % ffmpeg) #get_pipe(ffmpeg, cwd=tmpfolder) self.add_metadata_to_flv(tmpfolder, flv_filename) self.make_flv_thumbnail(tmpfolder, name, width) tmpdir = vfs.open(tmpfolder) # Copy the flv content to a data variable flv_file = tmpdir.open(flv_filename) try: flv_data = flv_file.read() finally: flv_file.close() # Copy the thumb content thumb_file = tmpdir.open('%s.png' % name) try: thumb_data = thumb_file.read() finally: thumb_file.close() # Need to add the PNG to ikaaro # Return a FLV file and a PNG thumbnail flvfile = ["%s" % name, 'video/x-flv', flv_data, 'flv', width, height] flvthumb = ['%s_thumb' % name, 'image/png', thumb_data, 'png'] if((len(flv_data) == 0) or (len(thumb_data) == 0)): #exit encoded = None else: encoded = {'flvfile':flvfile, 'flvthumb':flvthumb} return encoded
def get_test_filenames(test_path, force_download): """Return the test file names If the test files does'nt exists, we download it """ uris = {'http://download.wikimedia.org/qualitywiki/latest': [('qualitywiki-latest-stub-articles.xml', '.gz'), #~ 3.1 KB ('qualitywiki-latest-stub-meta-current.xml', '.gz'), #~ 11.0 KB ('qualitywiki-latest-stub-meta-history.xml', '.gz')], #~ 28.9 KB 'http://download.wikimedia.org/tawiki/latest': [('tawiki-latest-stub-articles.xml', '.gz'), #~ 1.2 MB ('tawiki-latest-stub-meta-history.xml', '.gz')], #~ 7.3 MB 'http://www.w3.org/XML/Test/': [('xmlts20080205', '.tar.gz')] } compressed_dir_path = join(test_path, 'compressed_files') if force_download is True: if lfs.exists(compressed_dir_path): print 'Remove compressed directory ', compressed_dir_path lfs.remove(compressed_dir_path) for names in uris.itervalues(): for (name, ext) in names: path = join(test_path, name) if lfs.exists(path): print 'Remove %s file' % path lfs.remove(path) # test directory if lfs.exists(test_path) is False: lfs.make_folder(test_path) # compressed directory if lfs.exists(compressed_dir_path) is False: lfs.make_folder(compressed_dir_path) else: lfs.open(compressed_dir_path) test_dir_filenames = lfs.get_names(test_path) for base_uri, names in uris.iteritems(): for (name, ext) in names: if test_dir_filenames.count(name): continue compressed_dest = join(compressed_dir_path, '%s%s' % (name, ext)) # check if tarball already exists if lfs.exists(compressed_dest) is False: src = join(base_uri, '%s%s' % (name, ext)) print 'GET %s file' % src dest = join(test_path, name) if vfs.exists(src) is False: print "%s uri does not exists" % src continue src_file = vfs.open(src) # save Gzip file compressed_dest_file = lfs.make_file(compressed_dest) compressed_dest_file.write(src_file.read()) compressed_dest_file.close() src_file.close() print 'Extract file %s' % compressed_dest # Uncompressed File Path if name == 'xmlts20080205': # uncompress only xmlconf.xml file tar = open_tar(compressed_dest) xmlconf_file = tar.extractfile('xmlconf/xmlconf.xml') ucf_path = join(test_path, name) ucf_file = lfs.make_file(ucf_path) ucf_file.write(xmlconf_file.read()) ucf_file.close() else: # untar Gzip file compressed_dest_file = lfs.open(compressed_dest) gzip_file = GzipFile(compressed_dest) ucf_path = join(test_path, name) ucf_file = lfs.make_file(ucf_path) ucf_file.write(gzip_file.read()) compressed_dest_file.close() gzip_file.close() ucf_file.close() tests = [] # update test dir name test_dir_filenames = lfs.get_names(test_path) for filename in test_dir_filenames: real_path = join(test_path, filename) if lfs.is_file(real_path): bytes = lfs.get_size(real_path) tests.append((real_path, filename, bytes, get_string_size(bytes))) tests.sort(key=lambda x: x[2]) return tests
def test17_move_file(self): vfs.copy('tests/hello.txt', 'tests/hello.txt.bak') vfs.move('tests/hello.txt.bak', 'tests/hello.txt.old') file = vfs.open('tests/hello.txt.old') self.assertEqual(file.read(), 'hello world\n') self.assertEqual(vfs.exists('tests/hello.txt.bak'), False)
def setUp(self): self.tests = vfs.open('tests/') vfs.make_folder('tests/toto.txt')
print '* Error:', path print '*' raise relative_path = Path('..').resolve2(path) for source, context, line in units: po.add_unit(relative_path, source, context, line) print # Update locale.pot if not vfs.exists('locale/locale.pot'): vfs.make_file('locale/locale.pot') write('* Update PO template ') data = po.to_str() file = vfs.open('locale/locale.pot', WRITE) try: file.write(data) finally: file.close() print # Update PO files folder = vfs.open('locale') filenames = set([x for x in folder.get_names() if x[-3:] == '.po']) filenames.add('%s.po' % src_language) for language in config.get_value('target_languages'): filenames.add('%s.po' % language) filenames = list(filenames) filenames.sort()
def safe_open(self, reference, mode=None): if self.database is None: return vfs.open(reference, mode=mode) return self.database.safe_open(reference, mode=mode)
def read_file(context, uri, file): uri = get_reference(uri) # Shortcuts elements = context['elements'] references = context['references'] # XML stream stream = XMLParser(file.read()) # Parse ! stack = [] for type, value, line in stream: # Set the good encoding if type == XML_DECL: context['encoding'] = value[1] elif type == START_ELEMENT: tag_uri, tag_name, attrs = value # Set your context variable read_common_attrs(tag_uri, attrs, context) # Only RNG ! if tag_uri == rng_uri: # <element> if tag_name == 'element': # Create a new element qnames = read_qnames(attrs, context, stream) element = {'qnames': qnames, 'attributes': [], 'data': None, 'is_empty': True, 'refs': []} # My father has at least a child, me! if stack: stack[-1]['is_empty'] = False # And push it elements.append(element) stack.append(element) # <attribute> elif tag_name == 'attribute': # Create a new attribute qnames = read_qnames(attrs, context, stream) attribute = {'qnames': qnames, 'data': None, 'refs': []} # And push it stack[-1]['attributes'].append(attribute) stack.append(attribute) # <data> elif tag_name == 'data': type = attrs[None, 'type'] last = stack[-1] if last['data'] is not None: last['data'] = '' else: last['data'] = type # <ref> elif tag_name == 'ref': name = attrs[None, 'name'] if stack: stack[-1]['refs'].append(name) # <define> elif tag_name == 'define': name = attrs[None, 'name'] # New or merge ? if name in references and (None, 'combine') in attrs: ref = references[name] else: ref = {'attributes': [], 'data': None, 'is_empty': True, 'refs': []} references[name] = ref stack.append(ref) # <text> elif tag_name == 'text': last = stack[-1] if last['data'] is not None: last['data'] = '' else: last['data'] = 'string' # <value> elif tag_name == 'value': stack[-1]['data'] = '' # <include> elif tag_name == 'include': href = attrs[None, 'href'] include_uri = uri.resolve(href) include_uri = str(include_uri) include_file = vfs.open(include_uri) read_file(context, include_uri, include_file) # Ignored tags elif tag_name in ['grammar', 'start', 'choice', 'optional', 'zeroOrMore', 'oneOrMore', 'group', 'empty', 'interleave', 'param', 'list', 'mixed']: continue # Tags not implemented else: raise NotImplementedError, ('relax NG: <%s> not ' 'implemented') % tag_name elif type == END_ELEMENT: tag_uri, tag_name = value if (tag_uri == rng_uri and tag_name in ['element', 'attribute', 'define']): stack.pop()
print '* Error:', path print '*' raise relative_path = Path('..').resolve2(path) for source, context, line in units: po.add_unit(relative_path, source, context, line) print # Update locale.pot if not vfs.exists('locale/locale.pot'): vfs.make_file('locale/locale.pot') write('* Update PO template ') data = po.to_str() file = vfs.open('locale/locale.pot', WRITE) try: file.write(data) finally: file.close() print # Update PO files folder = vfs.open('locale') filenames = set([ x for x in folder.get_names() if x[-3:] == '.po' ]) filenames.add('%s.po' % src_language) for language in config.get_value('target_languages'): filenames.add('%s.po' % language) filenames = list(filenames) filenames.sort()