def test_remove(self): folder = self.root.get_handler('tests') folder.del_handler('toto.txt') self.assertEqual(lfs.exists('tests/toto.txt'), True) self.assertEqual(folder.has_handler('toto.txt'), False) # Save self.database.save_changes() self.assertEqual(lfs.exists('tests/toto.txt'), False) self.assertEqual(folder.has_handler('toto.txt'), False)
def action_build(self): cwd = self.get_path() local.chdir(cwd) # itools package: build if lfs.exists('%s/setup.conf' % cwd): build = '%s/bin/ipkg-build.py' % prefix if not lfs.exists(build): build = 'ipkg-build.py' local.run([build]) local.run([executable, 'setup.py', '--quiet', 'sdist'])
def test_remove_add(self): folder = self.root.get_handler('tests') folder.del_handler('toto.txt') folder.set_handler('toto.txt', TextFile()) self.assertEqual(lfs.exists('tests/toto.txt'), True) self.assertEqual(folder.get_handler('toto.txt', soft=True) is not None, True) # Save self.database.save_changes() self.assertEqual(lfs.exists('tests/toto.txt'), True) self.assertEqual(folder.get_handler('toto.txt', soft=True) is not None, True)
def test_remove_add(self): fables = self.root # Firstly add 31.txt fables.set_handler('31.txt', TextFile()) self.database.save_changes() fables.del_handler('31.txt') fables.set_handler('31.txt', TextFile()) self.assertEqual(lfs.exists('fables/31.txt'), True) self.assertEqual(fables.has_handler('31.txt'), True) # Save self.database.save_changes() self.assertEqual(lfs.exists('fables/31.txt'), True) self.assertEqual(fables.has_handler('31.txt'), True)
def test_broken_commit(self): # Changes (copy&paste) fables = self.root fable = fables.get_handler('30.txt') fable = fable.clone() fables.set_handler('31.txt', fable) # Add broken handler broken = BrokenHandler() fables.set_handler('broken.txt', broken) # Commit self.assertRaises(NameError, self.database.save_changes) # Test self.assertEqual(lfs.exists('fables/database/31.txt'), False) self.assertEqual(lfs.exists('fables/database/broken.txt'), False)
def test_broken_commit(self): # Changes (copy&paste) fables = self.root fable = fables.get_handler("30.txt") fable = fable.clone() fables.set_handler("31.txt", fable) # Add broken handler broken = BrokenHandler() fables.set_handler("broken.txt", broken) # Commit self.assertRaises(NameError, self.database.save_changes) # Test self.assertEqual(lfs.exists("fables/database/31.txt"), False) self.assertEqual(lfs.exists("fables/database/broken.txt"), False)
def test_stop_server(self): server = SERVER server.stop() # Remove database path = DATABASE_TEST_PATH if lfs.exists(path): lfs.remove(path)
def test_move_file(self): lfs.copy('tests/hello.txt', 'tests/hello.txt.bak') lfs.move('tests/hello.txt.bak', 'tests/hello.txt.old') file = lfs.open('tests/hello.txt.old') self.assertEqual(file.read(), 'hello world\n') self.assertEqual(lfs.exists('tests/hello.txt.bak'), False) lfs.remove('tests/hello.txt.old')
def __init__(self, path, fields, read_only=False): self.nb_transactions = 0 self.last_transaction_dtime = None self.path = abspath(path) + '/' self.fields = fields self.read_only = read_only # Open database self.path_data = '%s/database/' % self.path # Check if is a folder self.path_data = '%s/database/' % self.path if not lfs.is_folder(self.path_data): error = '"{0}" should be a folder, but it is not'.format(self.path_data) raise ValueError(error) # New interface to Git self.worktree = open_worktree(self.path_data) # Initialize the database, but chrooted self.fs = lfs.open(self.path_data) # Static FS database_static_path = '{0}/database_static'.format(path) if not lfs.exists(database_static_path): self.init_backend_static(path) self.static_fs = lfs.open(database_static_path) # Patchs backend self.patchs_backend = PatchsBackend(path, self.fs, read_only) # Catalog self.catalog = self.get_catalog()
def load(self): path = expanduser('~/.usine') if lfs.is_file(path): return 'ERROR: %s is a file, remove it first' % path # Make the user configuration file if needed if not lfs.exists(path): print 'Making the configuration folder:', path lfs.make_folder(path) return 'Now add the INI files within the folder' # Read the user configuration file ini = [ '%s/%s' % (path, x) for x in lfs.get_names(path) if x[-4:] == '.ini' ] if len(ini) == 0: return 'ERROR: zero INI files found in %s/' % path # Read the ini file cfg = RawConfigParser() cfg.read(ini) # Get the data for section in cfg._sections: options = cfg._sections[section] type, name = section.split() module = modules[type] obj = module(options) # Keep the data unit self.by_type.setdefault(type, []).append(obj) self.by_type_and_name[(type, name)] = obj # Sort for type in self.by_type: self.by_type[type].sort(key=lambda x: x.name)
def tearDown(self): folder = lfs.open('tests') for name in 'toto.txt', 'fofo.txt', 'fofo2.txt', 'empty': if folder.exists(name): folder.remove(name) if lfs.exists("test_dir"): lfs.remove("test_dir")
def test_remove_folder(self): # Create hierarchy lfs.make_folder('tests/folder') lfs.make_folder('tests/folder/a') lfs.make_file('tests/folder/a/hello.txt') # Remove and test lfs.remove('tests/folder') self.assertEqual(lfs.exists('tests/folder'), False)
def test_add_copy(self): database = self.database folder = self.root.get_handler('tests') folder.set_handler('fofo.txt', TextFile()) folder.copy_handler('fofo.txt', 'fofo2.txt') # Save database.save_changes() self.assertEqual(lfs.exists('tests/fofo2.txt'), True)
def action_sync(self): # Case 1: Fetch folder = self.get_path() if lfs.exists(folder): local.run('git fetch origin', cwd=folder) return # Case 2: Clone local.run(['git', 'clone', self.get_url(), folder])
def _init_test(self, value): # Init data if not lfs.exists(self.config_path): lfs.make_file(self.config_path) # Write data config = ConfigFile(self.config_path) config.set_value("test", value) config.save_state()
def _init_test(self, value): # Init data if not lfs.exists(self.config_path): lfs.make_file(self.config_path) # Write data config = rw_database.get_handler(self.config_path, ConfigFile) config.set_value("test", value) config.save_state()
def test_abort(self): # Changes (copy&paste) fables = self.root fable = fables.get_handler("30.txt") fable = fable.clone() fables.set_handler("31.txt", fable) # Abort self.database.abort_changes() # Test self.assertEqual(lfs.exists("fables/database/31.txt"), False)
def test_commit(self): # Changes (copy&paste) fables = self.root fable = fables.get_handler('30.txt') fable = fable.clone() fables.set_handler('31.txt', fable) # Commit self.database.save_changes() # Test self.assertEqual(lfs.exists('fables/database/31.txt'), True)
def test_abort(self): # Changes (copy&paste) fables = self.root fable = fables.get_handler('30.txt') fable = fable.clone() fables.set_handler('31.txt', fable) # Abort self.database.abort_changes() # Test self.assertEqual(lfs.exists('fables/database/31.txt'), False)
def action_update(self): """ If config folder is a GIT repository, rebase it """ path = expanduser('~/.usine') for x in lfs.get_names(path): folder = '{}/{}'.format(path, x) if lfs.exists('{}/.git'.format(folder)): local.run(['git', 'fetch', 'origin'], cwd=folder) local.run(['git', 'reset', '--hard', 'origin/master'], cwd=folder)
def test_commit(self): # Changes (copy&paste) fables = self.root fable = fables.get_handler("30.txt") fable = fable.clone() fables.set_handler("31.txt", fable) # Commit self.database.save_changes() # Test self.assertEqual(lfs.exists("fables/database/31.txt"), True)
def test_add_abort(self): database = self.database folder = self.root.get_handler('tests') self.assertEqual(folder.has_handler('fofo.txt'), False) folder.set_handler('fofo.txt', TextFile()) self.assertEqual(folder.has_handler('fofo.txt'), True) database.abort_changes() self.assertEqual(folder.has_handler('fofo.txt'), False) # Save database.save_changes() self.assertEqual(lfs.exists('tests/fofo.txt'), False)
def tearDown(self): # Restore logging register_logger(None, 'itools.database') # Clean file-system paths = [ 'fables/catalog', 'fables/database/.git', 'fables/database/31.txt', 'fables/database/agenda', 'fables/database/broken.txt' ] for path in paths: if lfs.exists(path): lfs.remove(path)
def __init__(self, db_path, db_fs, read_only): self.db_fs = db_fs self.db_path = db_path # Init patchs folder self.patchs_path = '{0}/database/.git/patchs'.format(db_path) if not lfs.exists(self.patchs_path): lfs.make_folder(self.patchs_path) self.patchs_fs = lfs.open(self.patchs_path) # Launch rotate on patchs folder (only one time, so only on RW database) if not read_only: self.launch_rotate()
def test_remove_add(self): fables = self.root # Firstly add 31.txt fables.set_handler("31.txt", TextFile()) self.database.save_changes() fables.del_handler("31.txt") fables.set_handler("31.txt", TextFile()) self.assertEqual(fables.get_handler("31.txt", soft=True) is not None, True) # Save self.database.save_changes() self.assertEqual(lfs.exists("fables/database/31.txt"), True) self.assertEqual(fables.get_handler("31.txt", soft=True) is not None, True)
def get_environment_key(self, server): """ In development environment we can use '/ui_dev/skin/' directory for skin (to avoid build JS/CSS at every changes) """ if server and server.is_development_environment(): build_path = str(server.environment['build_path']) base_path = self.key.split('/ui/')[0] new_key = self.key.replace(base_path, build_path) new_key = new_key.replace('/ui/', '/ui_dev/') if lfs.exists(new_key): # Use local skin return new_key return self.key
def action_build(self): """Make a source distribution for every required Python package. """ path = expanduser('~/.usine/cache') if not lfs.exists(path): lfs.make_folder(path) print '**********************************************************' print ' BUILD' print '**********************************************************' for name, branch in self.get_packages(): config.options.branch = branch source = self.get_source(name) source.action_dist()
def test_empty_folder(self): """Empty folders do not exist. """ database = self.database root = self.root # Setup root.set_handler('tests/empty/sub/toto.txt', TextFile()) database.save_changes() root.del_handler('tests/empty/sub/toto.txt') database.save_changes() self.assertEqual(lfs.exists('tests/empty'), True) # Test self.assertRaises(RuntimeError, root.set_handler, 'tests/empty', TextFile())
def action_build(self): """Make a source distribution for every required Python package. """ path = expanduser("~/.usine/cache") if not lfs.exists(path): lfs.make_folder(path) print "**********************************************************" print " BUILD" print "**********************************************************" for name, version in self.get_packages(): config.options.version = version source = self.get_source(name) source.action_dist()
def test_remove_abort(self): database = self.database folder = self.root.get_handler('tests') self.assertEqual(folder.get_handler('toto.txt', soft=True) is not None, True) folder.del_handler('toto.txt') self.assertEqual(folder.get_handler('toto.txt', soft=True) is not None, False) database.abort_changes() self.assertEqual(folder.get_handler('toto.txt', soft=True) is not None, True) # Save database.save_changes() self.assertEqual(lfs.exists('tests/toto.txt'), True)
def make(worktree, rules, manifest): for source in worktree.get_filenames(): # Exclude if 'docs/' in source: continue # Apply rules for source_ext, target_ext, f in rules: if source.endswith(source_ext): target = source[:-len(source_ext)] + target_ext if not lfs.exists(target) or \ lfs.get_mtime(source) > lfs.get_mtime(target): f(source, target) # 1. Compile manifest.add(target) # 2. Update manifest print target # 3. Print
def tearDown(self): # Restore logging register_logger(None, "itools.database") # Clean file-system paths = [ "fables/catalog", "fables/database/.git", "fables/database/31.txt", "fables/database/agenda", "fables/database/broken.txt", ] for path in paths: if lfs.exists(path): lfs.remove(path)
def action_build(self): """Make a source distribution for every required Python package. """ # Get .cache folder path = expanduser('~/.usine/cache') if not lfs.exists(path): lfs.make_folder(path) for name, version in self.get_packages(): config.options.version = version source = self.get_source(name) if self.is_local: source.action_sync() source.action_checkout() else: # If we build for remote we want to build a dist source.action_dist()
def __init__(self, path): self.nb_transactions = 0 self.last_transaction_dtime = None self.path = abspath(path) + '/' # Open database self.path_data = '%s/database/' % self.path # Check if is a folder self.path_data = '%s/database/' % self.path if not lfs.is_folder(self.path_data): error = '"{0}" should be a folder, but it is not'.format( self.path_data) raise ValueError(error) # New interface to Git self.worktree = open_worktree(self.path_data) # Initialize the database, but chrooted self.fs = lfs.open(self.path_data) # Static FS database_static_path = '{0}/database_static'.format(path) if not lfs.exists(database_static_path): self.init_backend_static(path) self.static_fs = lfs.open(database_static_path)
def make_git_database(path, size_min, size_max): """Create a new empty Git database if the given path does not exists or is a folder. If the given path is a folder with content, the Git archive will be initialized and the content of the folder will be added to it in a first commit. """ if not lfs.exists(path): mkdir(path) # Init command = ['git', 'init', '-q'] call(command, cwd=path, stdout=PIPE) # Add command = ['git', 'add', '.'] error = call(command, cwd=path, stdout=PIPE, stderr=PIPE) # Commit if error == 0: command = ['git', 'commit', '-q', '-m', 'Initial commit'] call(command, cwd=path, stdout=PIPE) # Ok return GitDatabase(path, size_min, size_max)
def __init__(self, path, fields, read_only=False): self.nb_transactions = 0 self.last_transaction_dtime = None self.path = abspath(path) + '/' self.fields = fields self.read_only = read_only # Open database self.path_data = '%s/database/' % self.path # Check if is a folder self.path_data = '%s/database/' % self.path if not lfs.is_folder(self.path_data): error = '"{0}" should be a folder, but it is not'.format(self.path_data) raise ValueError(error) # New interface to Git self.worktree = open_worktree(self.path_data) # Initialize the database, but chrooted self.fs = lfs.open(self.path_data) # Static FS database_static_path = '{0}/database_static'.format(path) if not lfs.exists(database_static_path): self.init_backend_static(path) self.static_fs = lfs.open(database_static_path) # Catalog self.catalog = self.get_catalog()
def reindex_catalog(self, quiet=False, quick=False, as_test=False): if self.is_running_in_rw_mode(): print 'Cannot proceed, the server is running in read-write mode.' return # Check for database consistency if quick is False and check_database(self.target) is False: return False # Create a temporary new catalog catalog_path = '%s/catalog.new' % self.target if lfs.exists(catalog_path): lfs.remove(catalog_path) catalog = make_catalog(catalog_path, get_register_fields()) # Get the root root = self.root # Build a fake context context = self.get_fake_context() # Update t0, v0 = time(), vmsize() doc_n = 0 error_detected = False if as_test: log = open('%s/log/update-catalog' % self.target, 'w').write for obj in root.traverse_resources(): if not isinstance(obj, Resource): continue if not quiet: print doc_n, obj.abspath doc_n += 1 context.resource = obj # Index the document try: catalog.index_document(obj) except Exception: if as_test: error_detected = True log('*** Error detected ***\n') log('Abspath of the resource: %r\n\n' % str(obj.abspath)) log(format_exc()) log('\n') else: raise # Free Memory del obj self.database.make_room() if not error_detected: if as_test: # Delete the empty log file remove('%s/log/update-catalog' % self.target) # Update / Report t1, v1 = time(), vmsize() v = (v1 - v0)/1024 print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v) # Commit print '[Commit]', sys.stdout.flush() catalog.save_changes() # Commit / Replace old_catalog_path = '%s/catalog' % self.target if lfs.exists(old_catalog_path): lfs.remove(old_catalog_path) lfs.move(catalog_path, old_catalog_path) # Commit / Report t2, v2 = time(), vmsize() v = (v2 - v1)/1024 print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v) return True else: print '[Update] Error(s) detected, the new catalog was NOT saved' print ('[Update] You can find more infos in %r' % join(self.target, 'log/update-catalog')) return False
def __init__(self, target, read_only=False, cache_size=None, profile_space=False): target = lfs.get_absolute_path(target) self.target = target self.read_only = read_only # Set timestamp self.timestamp = str(int(time() / 2)) # Load the config config = get_config(target) self.config = config load_modules(config) self.modules = config.get_value('modules') # Contact Email self.smtp_from = config.get_value('smtp-from') # Full-text indexing self.index_text = config.get_value('index-text', type=Boolean, default=True) # Accept cors self.accept_cors = config.get_value( 'accept-cors', type=Boolean, default=False) # Profile Memory if profile_space is True: import guppy.heapy.RM # The database if cache_size is None: cache_size = config.get_value('database-size') if ':' in cache_size: size_min, size_max = cache_size.split(':') else: size_min = size_max = cache_size size_min, size_max = int(size_min), int(size_max) read_only = read_only or config.get_value('database-readonly') database = get_database(target, size_min, size_max, read_only) self.database = database # Find out the root class root = get_root(database) # Load environment file root_file_path = inspect.getfile(root.__class__) environement_path = str(get_reference(root_file_path).resolve('environment.json')) if vfs.exists(environement_path): with open(environement_path, 'r') as f: data = f.read() self.environment = json.loads(data) # Init fake context context = get_fake_context(database, root.context_cls) context.server = self # Initialize access_log = '%s/log/access' % target super(Server, self).__init__(root, access_log=access_log) # Email service self.spool = lfs.resolve2(self.target, 'spool') spool_failed = '%s/failed' % self.spool if not lfs.exists(spool_failed): lfs.make_folder(spool_failed) # Configuration variables get_value = config.get_value self.smtp_host = get_value('smtp-host') self.smtp_login = get_value('smtp-login', default='').strip() self.smtp_password = get_value('smtp-password', default='').strip() # Email is sent asynchronously self.flush_spool() # Logging log_file = '%s/log/events' % target log_level = config.get_value('log-level') if log_level not in log_levels: msg = 'configuraion error, unexpected "%s" value for log-level' raise ValueError, msg % log_level log_level = log_levels[log_level] logger = Logger(log_file, log_level, rotate=timedelta(weeks=3)) register_logger(logger, None) logger = WebLogger(log_file, log_level) register_logger(logger, 'itools.web') # Session timeout self.session_timeout = get_value('session-timeout') # Register routes self.register_dispatch_routes()
def get_test_filenames(test_path, force_download): """Return the test file names If the test files does'nt exists, we download it """ uris = {'http://download.wikimedia.org/qualitywiki/latest': [('qualitywiki-latest-stub-articles.xml', '.gz'), #~ 3.1 KB ('qualitywiki-latest-stub-meta-current.xml', '.gz'), #~ 11.0 KB ('qualitywiki-latest-stub-meta-history.xml', '.gz')], #~ 28.9 KB 'http://download.wikimedia.org/tawiki/latest': [('tawiki-latest-stub-articles.xml', '.gz'), #~ 1.2 MB ('tawiki-latest-stub-meta-history.xml', '.gz')], #~ 7.3 MB 'http://www.w3.org/XML/Test/': [('xmlts20080205', '.tar.gz')] } compressed_dir_path = join(test_path, 'compressed_files') if force_download is True: if lfs.exists(compressed_dir_path): print 'Remove compressed directory ', compressed_dir_path lfs.remove(compressed_dir_path) for names in uris.itervalues(): for (name, ext) in names: path = join(test_path, name) if lfs.exists(path): print 'Remove %s file' % path lfs.remove(path) # test directory if lfs.exists(test_path) is False: lfs.make_folder(test_path) # compressed directory if lfs.exists(compressed_dir_path) is False: lfs.make_folder(compressed_dir_path) else: lfs.open(compressed_dir_path) test_dir_filenames = lfs.get_names(test_path) for base_uri, names in uris.iteritems(): for (name, ext) in names: if test_dir_filenames.count(name): continue compressed_dest = join(compressed_dir_path, '%s%s' % (name, ext)) # check if tarball already exists if lfs.exists(compressed_dest) is False: src = join(base_uri, '%s%s' % (name, ext)) print 'GET %s file' % src dest = join(test_path, name) if vfs.exists(src) is False: print "%s uri does not exists" % src continue src_file = vfs.open(src) # save Gzip file compressed_dest_file = lfs.make_file(compressed_dest) compressed_dest_file.write(src_file.read()) compressed_dest_file.close() src_file.close() print 'Extract file %s' % compressed_dest # Uncompressed File Path if name == 'xmlts20080205': # uncompress only xmlconf.xml file tar = open_tar(compressed_dest) xmlconf_file = tar.extractfile('xmlconf/xmlconf.xml') ucf_path = join(test_path, name) ucf_file = lfs.make_file(ucf_path) ucf_file.write(xmlconf_file.read()) ucf_file.close() else: # untar Gzip file compressed_dest_file = lfs.open(compressed_dest) gzip_file = GzipFile(compressed_dest) ucf_path = join(test_path, name) ucf_file = lfs.make_file(ucf_path) ucf_file.write(gzip_file.read()) compressed_dest_file.close() gzip_file.close() ucf_file.close() tests = [] # update test dir name test_dir_filenames = lfs.get_names(test_path) for filename in test_dir_filenames: real_path = join(test_path, filename) if lfs.is_file(real_path): bytes = lfs.get_size(real_path) tests.append((real_path, filename, bytes, get_string_size(bytes))) tests.sort(key=lambda x: x[2]) return tests
parser = OptionParser('%prog', version=version, description=description) parser.add_option('-s', '--srx', help='Use an other SRX file than the default one.') options, args = parser.parse_args() if len(args) != 0: parser.error('incorrect number of arguments') # Read configuration for languages config = get_config() src_language = config.get_value('source_language', default='en') # Get local folder package_root = config.get_value('package_root') if lfs.exists(package_root): locale_folder_path = Path('{0}/locale'.format(package_root)) else: locale_folder_path = Path('locale/') locale_folder = lfs.open(locale_folder_path) # The SRX file if options.srx is not None: srx_handler = ro_database.get_handler(options.srx) else: srx_handler = None # Initialize message catalog po = POFile() lines = [] for line in open('MANIFEST').readlines():
def tearDown(self): if lfs.exists(self.config_path): lfs.remove(self.config_path)
def setUp(self): self.config_path = "tests/setup.conf.test" if lfs.exists(self.config_path): lfs.remove(self.config_path)
def tearDown(self): for name in ['agenda', 'books']: name = 'tests/%s' % name if lfs.exists(name): lfs.remove(name)