Ejemplo n.º 1
0
def build_lib(location, archive=None, dependancies=None, ignore_errors=False):
    """Build a project with a single lib (for testing)"""
    if archive is None:
        archive = Archive()

    if location.startswith('py:'):
        py = location.split(':', 1)[1]
        __import__(py)
        module = sys.modules[py]
        location = dirname(abspath(module.__file__))

    with fsopendir(location) as import_fs:
        lib = archive.load_library(import_fs)
        #lib.finalize()

    #require_libs = lib._cfg.get_list('tests', 'import', '') or []
    if dependancies:
        for require_lib in dependancies:
            if require_lib.startswith('py:'):
                py = require_lib.split(':', 1)[1]
                __import__(py)
                module = sys.modules[py]
                location = dirname(abspath(module.__file__))
            else:
                location = require_lib
            with fsopendir(location) as import_fs:
                _lib = archive.load_library(import_fs)

    archive.finalize(ignore_errors=ignore_errors)

    return archive, lib
Ejemplo n.º 2
0
def main():
    args = parse_args_or_exit()

    logging.basicConfig(level=logging.ERROR if args.quiet else logging.DEBUG)

    rpm_lib = rpm_adapter.SimpleRPM()

    templates = spec_template.templates_from_dir(
        fsopendir(args.config_dir),
        rpm_lib)

    if args.print_only:
        for template in templates:
            print template.main_source
        sys.exit(0)

    target_dir = fsopendir(args.target_dir)

    if args.dry_run:
        executor = executors.PrintExecutor(sys.stdout)
    else:
        executor = executors.RealExecutor()

    for template in templates:
        source = sources.GitHubSource(template.main_source)
        commands = source.clone_commands(target_dir)
        log.info(commands)
        result = executor.run(commands)

        if result.return_code != 0:
            log.warning("FAILED: %s", commands)
        if result.stdout:
            log.warning("STDOUT: %s", result.stdout)
        if result.stderr:
            log.warning("STDERR: %s", result.stderr)
Ejemplo n.º 3
0
    def setup_bundle(self, name, source_url=None, build_url=None, library=None):
        """Configure a bundle from existing sources"""
        from test import bundles
        from os.path import dirname, join
        from fs.opener import fsopendir
        import yaml

        if not library:
            library = self.library()

        if not source_url:
            source_url = 'mem://source'.format(name)

        if not build_url:
            build_url = 'mem://build'.format(name)

        for fs_url in (source_url, build_url):
            d = parse_url_to_dict(fs_url)

            # For persistent fs types, make sure it is empty before the test.
            if d['scheme'] not in ('temp', 'mem'):
                assert fsopendir(fs_url).isdirempty('/')

        test_source_fs = fsopendir(join(dirname(bundles.__file__), 'example.com', name))

        config = yaml.load(test_source_fs.getcontents('bundle.yaml'))

        b = library.new_from_bundle_config(config)
        b.set_file_system(source_url=source_url, build_url=build_url)

        self.copy_bundle_files(test_source_fs, b.source_fs)
        return b
Ejemplo n.º 4
0
def _refdocs_init():
    if REFDOC_ROOT.startswith("rpc://"):
        try:
            l_refdoc_fs = RPCFS("http://" + REFDOC_ROOT[len("rpc://") :])
        except RemoteConnectionError:
            lpath = os.path.join(INSTANCE_ROOT, "refdocs")
            logger.error("Could not connect to configured REFDOCS. " "Using {0}.".format(lpath))
            l_refdoc_fs = fsopendir(lpath, create_dir=True)
    else:
        l_refdoc_fs = fsopendir(REFDOC_ROOT, create_dir=True)
    return l_refdoc_fs
Ejemplo n.º 5
0
def _docstore_init():
    if DOCSTORE_ROOT.startswith("rpc://"):
        try:
            l_docstore_fs = RPCFS("http://" + DOCSTORE_ROOT[len("rpc://") :])
        except RemoteConnectionError:
            lpath = os.path.join(INSTANCE_ROOT, "docstore")
            logger.error("Could not connect to configured DOCSTORE. " "Using {0}.".format(lpath))
            l_docstore_fs = fsopendir(lpath, create_dir=True)
    else:
        l_docstore_fs = fsopendir(DOCSTORE_ROOT, create_dir=True)
    return l_docstore_fs
Ejemplo n.º 6
0
def _wallet_init():
    if DOCUMENT_WALLET_ROOT.startswith('rpc://'):
        try:
            l_wallet_fs = RPCFS('http://' + DOCUMENT_WALLET_ROOT[len('rpc://'):])
        except RemoteConnectionError:
            lpath = os.path.join(INSTANCE_ROOT, 'wallet')
            logger.error('Could not connect to configured DOCUMENT_WALLET. '
                         'Using {0}.'.format(lpath))
            l_wallet_fs = fsopendir(lpath, create_dir=True)
    else:
        l_wallet_fs = fsopendir(DOCUMENT_WALLET_ROOT, create_dir=True)
    return l_wallet_fs
Ejemplo n.º 7
0
    def test_type_intuit(self):
        from ambry_sources.intuit import TypeIntuiter

        cache_fs = fsopendir(self.setup_temp_dir())
        spec = self.sources['simple_fixed']
        s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))

        f = MPRowsFile(cache_fs, spec.name)

        with f.writer as w:
            w.load_rows(s)

        with f.reader as r:
            ti = TypeIntuiter().process_header(r.headers).run(r.rows, r.n_rows)

        with f.writer as w:
            w.set_types(ti)

        columns = []
        with f.reader as w:
            for col in w.columns:
                columns.append((col.pos, col.name, col.type))
        expected_columns = [
            (1, u'id', u'int'),
            (2, u'uuid', u'str'),
            (3, u'int', u'int'),
            (4, u'float', u'float')
        ]
        self.assertEqual(columns, expected_columns)
Ejemplo n.º 8
0
    def get_rc(cls, rewrite=True):
        """Create a new config file for test and return the RunConfig.

        This method will start with the user's default Ambry configuration, but will replace the
        library.filesystem_root with the value of filesystem.test, then depending on the value of the AMBRY_TEST_DB
        environmental variable, it will set library.database to the DSN of either database.test-sqlite or
        database.test-postgres

        """

        config = ambry.run.load()  # not cached; get_config is

        orig_root = config.library.filesystem_root
        root_dir = config.filesystem.test.format(root=orig_root)

        if config.library.database == cls.library_test_dsn:
            raise Exception('production database and test database can not be the same.')

        config.library.filesystem_root = root_dir
        config.library.database = cls.library_test_dsn
        config.accounts = None

        test_root = fsopendir(root_dir, create_dir=True)

        if rewrite:
            with test_root.open('config.yaml', 'w', encoding='utf-8', ) as f:
                config.loaded = None
                config.dump(f)

        return ambry.run.load(test_root.getsyspath('/'))
Ejemplo n.º 9
0
 def send_file(self, filename):
     """Send file to the client."""
     try:
         fs = fsopendir(self.uri)
         return send_file(fs.open(filename))
     except Exception as e:
         raise StorageError('Could not send file: {}'.format(e))
Ejemplo n.º 10
0
    def test_row_intuit(self):
        """Check that the sources can be loaded and analyzed without exceptions and that the
        guesses for headers and start are as expected"""

        from ambry_sources.intuit import RowIntuiter

        cache_fs = fsopendir('temp://')
        # cache_fs = fsopendir('/tmp/ritest/')

        sources = self.load_sources('sources-non-std-headers.csv')

        for source_name, spec in sources.items():

            s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))

            rows = list(s)
            l = len(rows)

            # the files are short, so the head and tail overlap
            ri = RowIntuiter(debug=False).run(rows[:int(l*.75)], rows[int(l*.25):], len(rows))

            print source_name, ri.start_line, ri.header_lines

            self.assertEqual(
                spec.expect_headers,
                ','.join(str(e) for e in ri.header_lines),
                'Headers of {} source does not match to row intuiter'.format(spec.name))

            self.assertEqual(
                spec.expect_start, ri.start_line,
                'Start line of {} source does not match to row intuiter start line.'.format(spec.name))
Ejemplo n.º 11
0
def upload_files():
    if 'file' not in request.files:
        return jsonify(success=False, message='No file part'), 400

    attachment = request.files['file']
    if not attachment:
        return

    filedir = current_app.config['RECORD_EDITOR_FILE_UPLOAD_FOLDER']
    fs = fsopendir(filedir, create_dir=True)
    filename = secure_filename(attachment.filename)
    base_filename = filename
    count = 1
    while fs.isfile(filename) and count < MAX_UNIQUE_KEY_COUNT:
        filename = '%s_%s' % (base_filename, count)
        count += 1

    with fs.open(filename, mode='wb') as remote_file:
        copy_file(attachment.stream, remote_file)

    full_url = fs.getpathurl(filename, allow_none=True)
    if not full_url:
        full_url = fs.getsyspath(filename)

    return jsonify({'path': full_url})
Ejemplo n.º 12
0
def make_ramfs():
    def getsyspath(fname):
        return 'SYSPATH:' + fname

    fs = fsopendir('ram:///')
    fs.getsyspath = mock.Mock(side_effect=getsyspath)
    return fs
Ejemplo n.º 13
0
    def run_upload(self):
        args = self.args

        path = os.path.abspath(os.path.join(args.location, 'lib.ini'))
        try:
            with io.open(path, 'rt') as f:
                lib_settings = settings.SettingsContainer.read_from_file(f)
        except IOError:
            self.console.error("unable to read library settings from '{}'".format(path))
            return -1

        lib_name = lib_settings.get("lib", "name")
        lib_version = args.version or lib_settings.get("lib", "version")

        if args.docs:
            return self.upload_docs(lib_name, lib_version)

        package_name = "{}-{}".format(lib_name, lib_version)
        package_filename = "{}.zip".format(package_name)

        upload_info = self.call('package.get-upload-info')
        upload_url = upload_info['url']

        lib_fs = fsopendir(args.location)
        package_destination_fs = lib_fs.makeopendir('__moyapackage__')

        if not package_destination_fs.exists(package_filename):
            raise CommandError("package '{}' does not exist, run 'moya-pm build'".format(package_filename))

        self.upload(upload_url,
                    lib_name,
                    lib_version,
                    package_destination_fs,
                    package_filename,
                    overwrite=args.overwrite)
Ejemplo n.º 14
0
    def __init__(self, name, namespace, fs=None, compress=False, compress_min=1024):
        super(FileCache, self).__init__(name,
                                        namespace,
                                        compress=compress,
                                        compress_min=compress_min,
                                        thread_safe=True)
        if fs is None:
            fs = fsopendir('filecache', create_dir=True)
        elif isinstance(fs, string_types):
            fs = fsopendir(fs, create_dir=True)

        if namespace:
            sub_dir = namespace.replace('/', '-').replace(' ', '-')
            fs = fs.makeopendir(sub_dir)
        self.fs = fs
        self.max_key_length = 80
Ejemplo n.º 15
0
    def test_generator(self):
        from ambry_sources.sources import GeneratorSource, SourceSpec
        from ambry_sources import head, tail
        cache_fs = fsopendir(self.setup_temp_dir())

        def gen():

            yield list('abcde')

            for i in range(10):
                yield [i, i + 1, i + 2, i + 3, i + 4]

        f = HDFPartition(cache_fs, 'foobar')

        s = GeneratorSource(SourceSpec('foobar'), gen())

        ri = RowIntuiter().run(head(s, 100), tail(s, 100))
        row_spec = self._row_intuiter_to_dict(ri)
        ti = TypeIntuiter().process_header(ri.headers).run(GeneratorSource(SourceSpec('foobar'), gen()))
        with f.writer as w:
            w.set_row_spec(row_spec, ri.headers)
            w.set_types(ti)

        f.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))

        self.assertEqual(f.headers, list('abcde'))
        rows = []

        for row in f.select():
            rows.append(row.dict)
        self.assertEqual(len(rows), 10)
        self.assertEqual(rows[0], {'a': 0, 'b': 1, 'c': 2, 'd': 3, 'e': 4})
        self.assertEqual(rows[-1], {'a': 9, 'b': 10, 'c': 11, 'd': 12, 'e': 13})
Ejemplo n.º 16
0
    def test_creates_virtual_tables_for_partition_with_segment_without_errors(self):

        fs = fsopendir('temp://')

        def gen():
            # generate header
            yield ['col1', 'col2']

            # generate rows
            yield [0, 0]
            yield [1, 1]

        mprows = MPRowsFile(fs, 'example.com/simple-0.1.3/1.mpr')
        mprows.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))

        # create virtual tables. This should not raise an error.
        #
        connection = apsw.Connection(':memory:')
        try:
            add_partition(connection, mprows, 'vid1')
        except Exception as exc:
            raise AssertionError('partition adding unexpectadly failed with {} error.'.format(exc))

        # check selected rows
        #
        cursor = connection.cursor()
        result = cursor.execute('SELECT * FROM {}'.format('vid1')).fetchall()
        self.assertEqual(result, [(0, 0), (1, 1)])
Ejemplo n.º 17
0
 def test_created_source_has_zip_filesystem(self):
     # FIXME: Optimize to use local file instead of downloading it all the time.
     cache_fs = fsopendir(self.setup_temp_dir())
     sources = self.load_sources(file_name='geo_sources.csv')
     spec = sources['community_plan']
     source = get_source(spec, cache_fs, callback=lambda x, y: (x, y))
     self.assertIsInstance(source._fstor._fs, ZipFS)
Ejemplo n.º 18
0
    def test_library(self):

        db_path = '/tmp/foo/bar/library.db'
        if os.path.exists(db_path):
            os.remove(db_path)

        config_root = fsopendir('temp://')
        config_root.createfile(CONFIG_FILE)
        config_file_syspath = config_root.getsyspath(CONFIG_FILE)

        with open(config_file_syspath, 'w') as f:
            f.write("""
library:
    category: development
    filesystem_root: /tmp/foo/bar
    remotes:
        census: s3://test.library.civicknowledge.com/census
        public: s3://test.library.civicknowledge.com/public
        restricted: s3://test.library.civicknowledge.com/restricted
        test: s3://test.library.civicknowledge.com/test""")

        config = load(config_root.getsyspath('/'))

        lf = LibraryFilesystem(config)

        self.assertTrue('/tmp/foo/bar', lf.root)

        l = Library(config)
        l.sync_config()

        self.assertEqual(
            sorted(['test', 'restricted', 'census', 'public']),
            sorted([x.short_name for x in l.remotes]))
Ejemplo n.º 19
0
    def test_all(self):
        """ Test all sources from geo_sources.csv """
        cache_fs = fsopendir(self.setup_temp_dir())

        sources = self.load_sources(file_name='geo_sources.csv')
        for name, spec in sources.items():
            if name == 'highways':
                # it is already tested. Skip.
                continue

            source = get_source(spec, cache_fs, callback=lambda x, y: (x, y))

            # now check its load to MPRows
            mpr = MPRowsFile(cache_fs, spec.name).load_rows(source)
            first_row = next(iter(mpr.reader))

            # Are columns recognized properly?

            NAME_INDEX = 1  # which element of the column description contains name.
            # Collect all names from column descriptors. Skip first elem of the schema because
            # it's descriptor of column descriptor elements.
            columns = [x[NAME_INDEX] for x in mpr.meta['schema'][1:]]
            self.assertIn('id', columns)
            self.assertIn('geometry', columns)

            # Is first row valid?
            self.assertEqual(len(columns), len(first_row))
Ejemplo n.º 20
0
    def run(self):
        args = self.args
        fs = fsopendir(args.fs)

        from ...command.sub import project_serve
        location = os.path.dirname(project_serve.__file__)

        init_logging(pathjoin(location, 'logging.ini'))

        if args.serve_templates:
            ini = 'templatesettings.ini'
        else:
            ini = 'settings.ini'

        application = WSGIApplication(location, ini, 'main', disable_autoreload=True)
        application.archive.filesystems['static'] = fs

        server = make_server(args.host,
                             int(args.port),
                             application,
                             handler_class=RequestHandler)
        log.info("server started on http://{}:{}".format(args.host, args.port))

        def handle_error(request, client_address):
            _type, value, tb = sys.exc_info()
            if isinstance(value, KeyboardInterrupt):
                interrupt_main()
        server.handle_error = handle_error

        try:
            server.serve_forever()
        finally:
            application.close()
Ejemplo n.º 21
0
Archivo: start.py Proyecto: esaye/moya
    def templatize(self, path):
        from fs.opener import fsopendir
        from fs.path import splitext, split
        fs = fsopendir(path)
        text_ext = ['', '.py', '.ini', '.xml', '.html', '.txt', '.json']
        bin_ext = ['.png', '.jpg', '.ico', '.gif']

        def check_path(path):
            dirname, filename = split(path)
            return filename not in [".svn", ".hg"]

        for path in fs.walkfiles(wildcard=check_path):
            _, ext = splitext(path)
            ext = ext.lower()
            if ext in text_ext:
                print('@TEXT {}'.format(path))
                for line in fs.open(path, 'rt'):
                    print(line.rstrip())
            elif ext in bin_ext:
                print('@BIN {}'.format(path))
                with fs.open(path, 'rb') as f:
                    chunk = f.read(64)
                    while chunk:
                        print(''.join('%02x' % ord(b) for b in chunk))
                        chunk = f.read(64)
Ejemplo n.º 22
0
    def test_ctor(self):

        d = '/tmp/socrata'

        from os import makedirs
        from os.path import exists
        from shutil import rmtree

        if exists(d):
            print "Make", d
            rmtree(d)

        makedirs(d)

        cache_fs = fsopendir(d)  # fsopendir(self.setup_temp_dir())


        sources = self.load_sources(file_name='sources.csv')
        spec = sources['facilities']
        source = get_source(spec, cache_fs)

        def cb(*args):
            print args

        mpr = MPRowsFile(cache_fs, spec.name).load_rows(source, callback = cb, limit = 10)
Ejemplo n.º 23
0
 def test_writes_rows_is_cache_is_large(self, fake_write_rows):
     temp_fs = fsopendir("temp://")
     parent = MagicMock()
     writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
     writer.cache = [[] for i in range(10000)]
     writer.insert_row(["row1"])
     fake_write_rows.assert_called_once_with()
Ejemplo n.º 24
0
    def test_file_does_not_exist(self):
        fs = fsopendir('ram:///')

        self.assertRaises(
            exceptions.NoSuchFile,
            lambda: spec_template.template_from_file(
                'xenops-cli.spec.in', fs, rpm_adapter.SimpleRPM()))
Ejemplo n.º 25
0
    def test_non_spec_ins_are_excluded(self):
        fs = fsopendir('ram:///')
        fs.createfile('somefile')

        templates = spec_template.templates_from_dir(
            fs, rpm_adapter.SimpleRPM())
        self.assertEquals([], templates)
Ejemplo n.º 26
0
    def test_executes_select_query_without_any_error(self, fake_shares):
        fake_shares.return_value = True

        def gen():
            # generate header
            yield ['col1', 'col2']

            # generate first row
            yield [0, 0]

        fs = fsopendir('temp://')
        datafile = MPRowsFile(fs, 'vid1')
        datafile.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))
        connection = None
        try:
            PostgreSQLTestBase._create_postgres_test_db()
            connection = psycopg2.connect(**PostgreSQLTestBase.pg_test_db_data)

            # create foreign table for partition
            with connection.cursor() as cursor:
                # we have to close opened transaction.
                cursor.execute('COMMIT;')
                add_partition(cursor, datafile, 'vid1')

            # query just created foreign table.
            with connection.cursor() as cursor:
                cursor.execute('SELECT * FROM partitions.vid1;')
        finally:
            if connection:
                connection.close()
            PostgreSQLTestBase._drop_postgres_test_db()
Ejemplo n.º 27
0
 def test_returns_dict_with_description(self):
     temp_fs = fsopendir("temp://")
     reader = MagicMock()
     hdf_partition = HDFPartition(temp_fs, path="temp.h5")
     ret = hdf_partition._info(reader)
     self.assertIn("version", ret)
     self.assertIn("data_start_pos", ret)
Ejemplo n.º 28
0
    def test_returns_self(self):
        temp_fs = fsopendir("temp://")
        parent = MagicMock()
        filename = temp_fs.getsyspath("temp.h5")
        _create_h5(filename)

        reader = HDFReader(parent, filename)
        self.assertEqual(reader, reader.__enter__())
Ejemplo n.º 29
0
    def test_removes_files(self):
        temp_fs = fsopendir("temp://")
        temp_fs.createfile("temp.h5")
        hdf_partition = HDFPartition(temp_fs, path="temp.h5")

        self.assertTrue(temp_fs.exists("temp.h5"))
        hdf_partition.remove()
        self.assertFalse(temp_fs.exists("temp.h5"))
Ejemplo n.º 30
0
    def test_returns_stat_from_meta(self):
        temp_fs = fsopendir("temp://")

        hdf_partition = HDFPartition(temp_fs, path="temp.h5")

        with patch.object(HDFPartition, "meta", new_callable=PropertyMock) as fake_meta:
            fake_meta.return_value = {"stats": 22}
            self.assertEqual(hdf_partition.stats, 22)
Ejemplo n.º 31
0
    def run(self):
        log.setLevel(logging.ERROR)
        args = self.args

        conf_path = self.app.conf

        dataplicity_path = dirname(conf_path)

        if args.build:
            do_build(dataplicity_path)

        with fsopendir(dataplicity_path) as src_fs:
            version = firmware.get_version(src_fs)

            filename = "firmware-{}.zip".format(version)
            firmware_path = join('__firmware__', filename)
            try:
                firmware_contents = src_fs.getcontents(firmware_path, 'rb')
            except ResourceNotFoundError:
                print "{} is missing, you can build firmware with 'dataplicity build'".format(firmware_path)
                return -1

        firmware_b64 = b64encode(firmware_contents)

        client = self.app.make_client(log)
        conf = client.conf
        remote = client.remote

        device_class_name = conf.get('device', 'class')
        #serial = conf.get('device', 'serial')

        ui = firmware.get_ui(fsopendir(dataplicity_path))

        print "uploading firmware..."
        with remote.batch() as batch:
            batch.call_with_id('auth_result',
                               'device.check_auth',
                               device_class=device_class_name,
                               serial=client.serial,
                               auth_token=client.auth_token)
            batch.call_with_id("publish_result",
                               "device.publish",
                               device_class=device_class_name,
                               version=version,
                               firmware_b64=firmware_b64,
                               ui=ui,
                               replace=args.replace)

        batch.get_result('auth_result')
        try:
            publish_result = batch.get_result('publish_result')
        except JSONRPCError as e:
            if e.code == ErrorCodes.FIRMWARE_EXISTS:
                print "Firmware {:010} exists!\nBump the version number in firmware.conf or use --replace to overwrite".format(version)
                return -1
            raise

        print "visit {} to manage firmware".format(publish_result['url'])

        if args.bump:
            with fsopendir(dataplicity_path) as src_fs:
                firmware.bump(src_fs)
Ejemplo n.º 32
0
 def get_url(fs_path):
     """ Get a URL for the file """
     (dirurl, filename) = opener.pathsplit(fs_path)
     fs = opener.fsopendir(dirurl)
     return fs.getpathurl(filename)
Ejemplo n.º 33
0
 def get_syspath(fs_path):
     """ Get a local system path to the file """
     (dirurl, filename) = opener.pathsplit(fs_path)
     fs = opener.fsopendir(dirurl)
     return fs.getsyspath(filename)
Ejemplo n.º 34
0
if tempfile.tempdir is None:
    tempfile.tempdir = tempfile.mkdtemp()

#: The path to the temporary directory which all application code can import,
#: and create whatever temporary files it needs within it.
#:
#: This directory will be removed by Tendril at clean application exit or
#: by the Operating System as per it's policies.
#:
#: Every execution of tendril in a separate process owns it's own temporary
#: directory.
#:
#: .. seealso:: :func:`fsutils_cleanup`
#:
TEMPDIR = tempfile.gettempdir()
temp_fs = fsopendir(TEMPDIR)


def get_tempname():
    """
    Gets a random string for use as a temporary filename.

    :return: A filename that can be used.
    """
    return next(tempfile._get_candidate_names())


def zipdir(path, zfpath):
    """
    Creates a zip file at ``zfpath`` containing all the files in ``path``.
    This function is simple wrapper around python's :mod:`zipfile` module.
Ejemplo n.º 35
0
    def logic(self, context):
        start = time()
        (name, _location, py, priority,
         template_priority) = self.get_parameters(context, 'name', 'location',
                                                  'py', 'priority',
                                                  'templatepriority')
        archive = self.document.archive
        absolute = False
        if _location is not None:
            location = _location
        else:
            if py in sys.modules:
                reload(sys.modules[py])
            try:
                __import__(py)
            except ImportError as e:
                raise errors.ElementError(
                    "unable to import Python module '{}'".format(py),
                    element=self,
                    diagnosis=text_type(e))
            module = sys.modules[py]
            location = dirname(abspath(module.__file__))
            absolute = True

        if '::/' in location:
            import_fs = fsopendir(location)
        else:
            if absolute:
                import_fs = fsopendir(location)
            else:
                project_fs = context['fs']
                if project_fs.hassyspath('/'):
                    project_path = project_fs.getsyspath('/')
                    import_path = pathjoin(project_path, location)
                    import_fs = fsopendir(import_path)
                else:
                    import_fs = context['fs'].opendir(location)
        lib = archive.load_library(import_fs,
                                   priority=priority,
                                   template_priority=template_priority,
                                   long_name=name,
                                   rebuild=context.root.get('_rebuild', False))
        if lib.failed_documents:
            if _location is not None:
                msg = "Failed to load library '{}' from location '{}'"
                raise errors.StartupFailedError(
                    msg.format(name or lib.long_name, _location))
            elif py:
                msg = "Failed to load library '{}' from Python module '{}'"
                raise errors.StartupFailedError(
                    msg.format(name or lib.long_name, py))
            else:
                raise errors.StartupFailedError(
                    "Failed to load library '{}'".format(name
                                                         or lib.long_name))
        startup_log.debug("%s imported %.1fms", lib, (time() - start) * 1000.0)
        if lib.priority:
            startup_log.debug("%s priority is %s", lib, lib.priority)
        if lib.template_priority:
            startup_log.debug("%s template priority is %s", lib,
                              lib.template_priority)
Ejemplo n.º 36
0
    def test_csv_time(self):
        """ Time writing rows with a PartitionDataFile. """
        from ambry.etl.partition import new_partition_data_file

        fs = fsopendir('mem://test')

        data = []
        ncols = 30

        types = (int, float, str)

        schema = [(i, types[i % 3]) for i in range(ncols)]

        #
        # Two different mungers, one uses a loop, one unrolls the loop in a lambda

        def munger1(schema):
            """Create a function to call casters on a row. Using an eval is about 11% faster than
            using a loop """

            funcs = []

            for name, t in schema:
                funcs.append('cast_{}(row[{}])'.format(t.__name__, name))

            return eval('lambda row: [{}]'.format(','.join(funcs)))

        row_munger1 = munger1(schema)

        def row_munger2(row):
            out = []
            for name, t in schema:
                if t == str:
                    out.append(cast_str(row[name]))
                elif t == int:
                    out.append(cast_int(row[name]))
                else:
                    out.append(cast_float(row[name]))

            return out

        data.append([str(j) for j in range(ncols)])

        def randdata(t):
            import random
            if t == str:
                return '%020x' % random.randrange(16 ** 20)
            elif t == int:
                return random.randint(0, 100000)
            else:
                return random.random()

        for i in range(100):
            data.append([str(randdata(schema[i][1])) for i in range(ncols)])

        cdf = new_partition_data_file(fs, 'foo.msg')

        import time
        n = 30000
        s = time.time()
        for i in range(n):
            row = data[i % 100]
            row = row_munger1(row)
            cdf.insert(row)

        print('Munger 1', round(float(n)/(time.time() - s), 3), 'rows/s')

        s = time.time()
        for i in range(n):
            row = data[i % 100]
            row = row_munger2(row)
            cdf.insert(row)

        print('Munger 2', round(float(n) / (time.time() - s), 3), 'rows/s')
Ejemplo n.º 37
0
    def test_basic_config(self):
        config_root = fsopendir('temp://')
        config_root.createfile(CONFIG_FILE)
        config_file_syspath = config_root.getsyspath(CONFIG_FILE)

        with open(config_file_syspath, 'w') as f:
            f.write("""
library:
    category: development
    remotes:
        census: s3://test.library.civicknowledge.com/census
        public: s3://test.library.civicknowledge.com/public
        restricted: s3://test.library.civicknowledge.com/restricted
        test: s3://test.library.civicknowledge.com/test
                    """)

        with self.assertRaises(ConfigurationError):
            config = load(config_root.getsyspath('/'))

        if 'AMBRY_DB' in os.environ:
            del os.environ['AMBRY_DB']

        with open(config_file_syspath, 'w') as f:
            f.write("""
library:
    category: development
    filesystem_root: /tmp/foo/bar
    database: postgres://foo:bar@baz:5432/ambry
    remotes:
        census: s3://test.library.civicknowledge.com/census
        public: s3://test.library.civicknowledge.com/public
        restricted: s3://test.library.civicknowledge.com/restricted
        test: s3://test.library.civicknowledge.com/test
            """)

        config = load(config_root.getsyspath('/'))
        config.account = None

        self.assertEquals('postgres://*****:*****@baz:5432/ambry',
                          config.library.database)
        self.assertEquals('/tmp/foo/bar', config.library.filesystem_root)

        self.assertEqual(2, len(config.loaded))
        self.assertEqual(config_file_syspath, config.loaded[0])

        with open(config_file_syspath, 'w') as f:
            f.write("""
library:
    filesystem_root: /foo/root
            """)

        os.environ['AMBRY_DB'] = 'sqlite:////library.db'

        with open(config_file_syspath, 'w') as f:
            f.write("""""")

        os.environ['AMBRY_DB'] = 'sqlite:////{root}/library.db'
        os.environ['AMBRY_ROOT'] = '/tmp/foo/bar'

        config = load(config_root.getsyspath('/'))

        lf = LibraryFilesystem(config)

        self.assertEqual('sqlite://///tmp/foo/bar/library.db', lf.database_dsn)
        self.assertEqual('/tmp/foo/bar/downloads/a/b', lf.downloads('a', 'b'))
Ejemplo n.º 38
0
    def start_library(self):
        console = self.console

        from ...tools import get_moya_dir
        from os.path import join, abspath
        project_path = None
        if self.args.location is not None:
            library_path = self.args.location
        else:
            try:
                project_path = get_moya_dir(self.args.project_location)
            except:
                console.error(
                    "Please run 'moya start library' inside your project directory, or specifiy the -o switch"
                )
                return False
            library_path = abspath(join(project_path, './local/'))

        cfg = None
        if not self.args.location and project_path:
            from ... import build
            cfg = build.read_config(project_path, self.get_settings())

        if not self.args.acceptdefaults:
            console.table([
                [
                    Cell("Moya Library Wizard",
                         bold=True,
                         fg="green",
                         center=True)
                ],
                [
                    """This will ask you a few questions, then create a new library in your Moya project based on your answers.

Default values are shown in grey (simply hit return to accept defaults). Some defaults may be taken from your ".bashrc" file, if it exists.
"""
                ]
            ])
        author = self.get_author_details()
        library = {}
        library["title"] = LibraryTitle.ask(console, default=self.args.title)
        longname = self.args.longname or make_name(author["organization"],
                                                   library["title"])
        longname = library["longname"] = LibraryLongName.ask(console,
                                                             default=longname)
        library["url"] = LibraryURL.ask(console, default="")
        library["namespace"] = LibraryNamespace.ask(console, default="")
        mount = None
        appname = None

        do_mount = DoMount.ask(console, default="yes")
        if do_mount:
            mount = Mount.ask(console,
                              default=self.args.mount
                              or "/{}/".format(make_name(library["title"])))
            appname = AppName.ask(console,
                                  default=self.args.name
                                  or make_name(library["title"]))

        data = dict(author=author,
                    library=library,
                    timezone=self.get_timezone())

        actions = []

        from ...command.sub import library_template
        from fs.memoryfs import MemoryFS
        from fs.opener import fsopendir
        memfs = MemoryFS()
        templatebuilder.compile_fs_template(memfs,
                                            library_template.template,
                                            data=data)
        dest_fs = fsopendir(join(library_path, library["longname"]),
                            create_dir=True,
                            writeable=True)

        continue_overwrite = 'overwrite'
        if not dest_fs.isdirempty('.'):
            if self.args.force:
                continue_overwrite = 'overwrite'
            elif self.args.new:
                continue_overwrite = 'new'
            else:
                continue_overwrite = DirNotEmpty.ask(console, default="cancel")

        if continue_overwrite != 'cancel':
            if continue_overwrite == 'overwrite':
                from fs.utils import copydir
                copydir(memfs, dest_fs)
                actions.append("Written library files to {}".format(
                    dest_fs.getsyspath('.')))
            elif continue_overwrite == 'new':
                files_copied = copy_new(memfs, dest_fs)
                table = [[
                    Cell("{} new file(s) written".format(len(files_copied)),
                         fg="green",
                         bold=True,
                         center=True),
                ]]
                for path in files_copied:
                    table.append(
                        [Cell(dest_fs.desc(path), bold=True, fg="black")])
                console.table(table)
                return 0

            if cfg:
                project_cfg = cfg['project']
                location = project_cfg['location']
                server_name = "main"

                if location:
                    with fsopendir(project_path) as project_fs:
                        with project_fs.opendir(location) as server_fs:
                            from lxml.etree import fromstring, ElementTree, parse
                            from lxml.etree import XML, Comment
                            server_xml_path = server_fs.getsyspath(
                                project_cfg['startup'])
                            root = parse(server_xml_path)
                            import_tag = XML(
                                '<import location="./local/{longname}" />\n\n'.
                                format(**library))
                            import_tag.tail = "\n"
                            install_tag = None

                            if mount:
                                tag = '<install name="{appname}" lib="{longname}" mount="{mount}" />'
                            else:
                                tag = '<install name="{appname}" lib="{longname}" />'
                            install_tag = XML(
                                tag.format(appname=appname,
                                           longname=longname,
                                           mount=mount))
                            install_tag.tail = "\n\n"

                            def has_child(node, tag, **attribs):
                                for el in node.findall(tag):
                                    #items = dict(el.items())
                                    if all(
                                            el.get(k, None) == v
                                            for k, v in attribs.items()):
                                        return True
                                return False

                            for server in root.findall(
                                    "{{http://moyaproject.com}}server[@docname='{}']"
                                    .format(server_name)):
                                add_import_tag = not has_child(
                                    server,
                                    "{http://moyaproject.com}import",
                                    location="./local/{}".format(longname))
                                add_install_tag = not has_child(
                                    server,
                                    "{http://moyaproject.com}install",
                                    lib=longname) and install_tag is not None

                                if add_import_tag or add_install_tag:
                                    comment = Comment(
                                        "Added by 'moya start library'")
                                    comment.tail = "\n"
                                    server.append(comment)
                                if add_import_tag:
                                    server.append(import_tag)
                                    actions.append("Added <import> tag")
                                if add_install_tag:
                                    server.append(install_tag)
                                    actions.append("Added <install> tag")
                                    if mount:
                                        actions.append(
                                            "Mounted application on {}".format(
                                                mount))

                            root.write(server_xml_path)

            table = [[
                Cell("Library files written successfully!",
                     fg="green",
                     bold=True,
                     center=True)
            ]]

            actions_text = "\n".join(" * " + action for action in actions)
            table.append([Cell(actions_text, fg="blue", bold=True)])
            table.append([
                """A new library has been added to the project, containing some simple example functionality.\nSee http://moyaproject.com/docs/creatinglibraries/ for more information."""
            ])
            console.table(table)

            return 0

        console.text("No project files written.", fg="red", bold=True).nl()
        return -1
Ejemplo n.º 39
0
    def start_project(self):
        console = self.console

        if not self.args.acceptdefaults:
            console.table([
                [
                    Cell("Moya Project Wizard",
                         bold=True,
                         fg="green",
                         center=True)
                ],
                [
                    """This will ask you a few questions, then create a new Moya project based on your answers.

Default values are shown in blue (hit return to accept defaults). Some defaults may be taken from your ".moyarc" file, if it exists."""
                ]
            ])

        author = self.get_author_details()
        project = {}
        project["title"] = ProjectTitle.ask(console, default=self.args.title)
        longname = make_name(author["organization"], project["title"])
        project["database"] = Database.ask(console, default='y')
        if project["database"]:
            project["auth"] = Auth.ask(console, default='y')
            project['signup'] = Signup.ask(console, default='y')
            project["pages"] = Pages.ask(console, default='y')
            project["feedback"] = Feedback.ask(console, default='y')
            project["blog"] = Blog.ask(console, default='y')
        project["comments"] = project.get("feedback", False) or project.get(
            "pages", False)
        project["wysihtml5"] = project.get("feedback", False) or project.get(
            "pages", False)
        project['jsonrpc'] = JSONRPC.ask(console, default='y')

        dirname = longname.split('.', 1)[-1].replace('.', '_')
        dirname = ProjectDirName.ask(console, default="./" + dirname)

        data = dict(author=author,
                    project=project,
                    timezone=self.get_timezone(),
                    secret=make_secret())

        from ...command.sub import project_template
        from fs.memoryfs import MemoryFS
        from fs.opener import fsopendir
        memfs = MemoryFS()
        templatebuilder.compile_fs_template(memfs,
                                            project_template.template,
                                            data=data)

        dest_fs = fsopendir(self.args.location or dirname,
                            create_dir=True,
                            writeable=True)
        continue_overwrite = 'overwrite'
        if not dest_fs.isdirempty('.'):
            if self.args.force:
                continue_overwrite = 'overwrite'
            elif self.args.new:
                continue_overwrite = 'new'
            else:
                continue_overwrite = DirNotEmpty.ask(console, default="cancel")

        if continue_overwrite == 'overwrite':
            from fs.utils import copydir
            copydir(memfs, dest_fs)
            console.table([
                [
                    Cell("Project files written successfully!",
                         fg="green",
                         bold=True,
                         center=True)
                ],
                [
                    """See readme.txt in the project directory for the next steps.\n\nBrowse to http://moyaproject.com/gettingstarted/ if you need further help."""
                ]
            ])
            return 0
        elif continue_overwrite == 'new':
            files_copied = copy_new(memfs, dest_fs)
            table = [[
                Cell("{} new file(s) written".format(len(files_copied)),
                     fg="green",
                     bold=True,
                     center=True),
            ]]
            for path in files_copied:
                table.append([Cell(dest_fs.desc(path), bold=True, fg="black")])
            console.table(table)
            return 0

        console.text("No project files written.", fg="red", bold=True).nl()
        return -1
Ejemplo n.º 40
0
def copy_file(fn, fp, tmppath):
    path = join(tmppath, fp)
    fn_new = fn + '_copy'
    this_fs = fsopendir(path)
    this_fs.copy(fn, fn_new)
    return fn_new
Ejemplo n.º 41
0
    def test_datafile_read_write(self):
        from ambry_sources.mpf import MPRowsFile
        from fs.opener import fsopendir
        import datetime
        from random import randint, random
        from uuid import uuid4

        fs = fsopendir('temp://')  # fs = fsopendir('/tmp/hdf5/')

        # fs = fsopendir('/tmp/pmpf')

        N = 50000

        # Basic read/ write tests.

        row = lambda: [
            None, 1,
            random(),
            str(uuid4()),
            datetime.date(randint(2000, 2015), randint(1, 12), 10),
            datetime.date(randint(2000, 2015), randint(1, 12), 10)
        ]
        headers = list('abcdefghi')[:len(row())]

        rows = [row() for i in range(N)]

        with Timer() as t:
            df = MPRowsFile(fs, 'foobar')
            w = df.writer

            w.headers = headers

            w.meta['source']['url'] = 'blah blah'

            for i in range(N):
                w.insert_row(rows[i])

            w.close()

        print('MSGPack write ', float(N) / t.elapsed, w.n_rows)

        with Timer() as t:
            count = 0
            i = 0
            s = 0

            r = df.reader

            for i, row in enumerate(r):
                count += 1

            r.close()

        print('MSGPack read  ', float(N) / t.elapsed, i, count, s)

        with Timer() as t:
            count = 0

            r = df.reader

            for row in r.rows:
                count += 1

            r.close()

        print('MSGPack rows  ', float(N) / t.elapsed)

        with Timer() as t:
            count = 0

            r = df.reader

            for row in r.raw:
                count += 1

            r.close()

        print('MSGPack raw   ', float(N) / t.elapsed)
Ejemplo n.º 42
0
 def is_local(fs_path):
     """ Determine if file is a local file """
     (dirurl, filename) = opener.pathsplit(fs_path)
     fs = opener.fsopendir(dirurl)
     return fs.hassyspath(filename)
Ejemplo n.º 43
0
    def run(self):
        parser = self.get_argparse()
        args = parser.parse_args(sys.argv[1:])

        if args.version is None:
            major, minor = __version__.split('.')[:2]
            version = "{}.{}".format(major, minor)
        else:
            version = args.version

        try:
            with open(expanduser(args.settings), 'rt') as f_ini:
                cfg = SettingsContainer.read_from_file(f_ini)
                print("Read settings from {}".format(args.settings))
        except IOError:
            cfg = SettingsContainer()

        from ..docgen.extracter import Extracter
        from ..docgen.builder import Builder
        from ..command import doc_project
        location = dirname(doc_project.__file__)

        try:
            base_docs_fs = OSFS('text')
        except FSError:
            sys.stderr.write('run me from moya/docs directory\n')
            return -1
        extract_fs = OSFS(join('doccode', version), create=True)
        languages = [
            d for d in base_docs_fs.listdir(dirs_only=True) if len(d) == 2
        ]

        def do_extract():
            print("Extracting docs v{}".format(version))
            utils.remove_all(extract_fs, '/')
            try:
                archive, context, doc = moya_build.build_server(
                    location, 'settings.ini')
            except Exception:
                raise
                return -1

            extract_fs.makedir("site/docs", recursive=True)
            extract_fs.makedir("site/tags", recursive=True)
            #extract_fs.makedir("libs")

            with extract_fs.opendir('site/tags') as tags_fs:
                extracter = Extracter(archive, tags_fs)
                const_data = {}
                builtin_tags = []
                for namespace in self.builtin_namespaces:
                    xmlns = getattr(namespaces, namespace, None)
                    if xmlns is None:
                        raise ValueError(
                            "XML namespace '{}' is not in namespaces.py".
                            format(namespace))
                    namespace_tags = archive.registry.get_elements_in_xmlns(
                        xmlns).values()
                    builtin_tags.extend(namespace_tags)

                extracter.extract_tags(builtin_tags, const_data=const_data)

            for language in languages:
                with extract_fs.makeopendir("site/docs") as language_fs:
                    doc_extracter = Extracter(None, language_fs)
                    docs_fs = base_docs_fs.opendir(language)
                    doc_extracter.extract_site_docs(docs_fs, dirname=language)

        if args.extract:
            do_extract()

        if args.build:
            theme_path = cfg.get('paths', 'theme', None)
            dst_path = join('html', version)
            if theme_path is None:
                theme_fs = OSFS('theme')
            else:
                theme_fs = fsopendir(theme_path)

            output_path = cfg.get('paths', 'output', None)

            if output_path is None:
                output_base_fs = OSFS(dst_path, create=True)
            else:
                output_root_base_fs = fsopendir(output_path)
                output_base_fs = output_root_base_fs.makeopendir(
                    dst_path, recursive=True)

            #output_base_fs = OSFS(join('html', version), create=True)
            utils.remove_all(output_base_fs, '/')

            def do_build():
                print("Building docs v{}".format(version))
                lib_info = {}
                lib_paths = {}
                for long_name, lib in self.document_libs:
                    lib_info[long_name] = moya_build.get_lib_info(lib)
                    lib_paths[long_name] = output_base_fs.getsyspath(
                        join('libs', long_name, 'index.html'))
                for language in languages:
                    docs_fs = base_docs_fs.makeopendir(language)
                    output_fs = output_base_fs.makeopendir(language)
                    utils.remove_all(output_fs, '/')

                    with extract_fs.opendir("site") as extract_site_fs:
                        builder = Builder(extract_site_fs, output_fs, theme_fs)
                        from ..tools import timer
                        with timer('render time'):
                            builder.build({
                                "libs": lib_info,
                                "lib_paths": lib_paths
                            })

                    # output_base_fs.makedir("libs", allow_recreate=True)
                    # for long_name, lib in self.document_libs:
                    #     source_path = extract_fs.getsyspath(join("libs", long_name))
                    #     output_path = output_base_fs.getsyspath('libs')
                    #     cmd_template = 'moya --debug doc build {} --theme libtheme --source "{}" --output "{}"'
                    #     cmd = cmd_template.format(lib, source_path, output_path)
                    #     os.system(cmd)

            def extract_build():
                do_extract()
                do_build()

            do_build()

            if not args.nobrowser:
                import webbrowser
                index_url = "file://" + output_base_fs.getsyspath(
                    'en/index.html')
                print(index_url)
                webbrowser.open(index_url)

            if args.watch:
                print("Watching for changes...")
                observer = Observer()
                path = base_docs_fs.getsyspath('/')

                reload_watcher = ReloadChangeWatcher(base_docs_fs,
                                                     extract_build)
                observer.schedule(reload_watcher, path, recursive=True)
                observer.start()

                while 1:
                    try:
                        time.sleep(0.1)
                    except:
                        break

        return 0
Ejemplo n.º 44
0
 def delete(fs_path):
     """ Delete the file on storage """
     (dirurl, filename) = opener.pathsplit(fs_path)
     fs = opener.fsopendir(dirurl)
     fs.remove(filename)
Ejemplo n.º 45
0
    def test_datafile_read_write(self):
        import datetime
        from random import randint, random
        from contexttimer import Timer
        from uuid import uuid4

        fs = fsopendir('temp://')

        N = 50000

        # Basic read/write tests.

        def rand_date_a():
            return datetime.date(randint(2000, 2015), randint(1, 12), 10)

        epoch = datetime.date(1970, 1, 1)

        def rand_date_b():
            return (datetime.date(randint(2000, 2015), randint(1, 12), 10) -
                    epoch).total_seconds()

        row = lambda: (0, 1, random(), str(uuid4()), rand_date_b(),
                       rand_date_b())

        headers = list('abcdefghi')[:len(row())]

        rows = [row() for i in range(N)]

        def write_large_blocks():

            df = HDFPartition(fs, path='foobar')

            if df.exists:
                df.remove()
            with Timer() as t, df.writer as w:
                w.headers = headers
                type_index = w.meta['schema'][0].index('type')
                pos_index = w.meta['schema'][0].index('pos')
                columns = w.meta['schema'][1:]
                for column in columns:
                    column[type_index] = type(rows[0][column[pos_index] -
                                                      1]).__name__
                w.insert_rows(rows)

            print('HDF write large', float(N) / t.elapsed, w.n_rows)

        def write_small_blocks():
            df = HDFPartition(fs, path='foobar')

            if df.exists:
                df.remove()

            with Timer() as t, df.writer as w:
                w.headers = headers
                type_index = w.meta['schema'][0].index('type')
                pos_index = w.meta['schema'][0].index('pos')
                columns = w.meta['schema'][1:]
                for column in columns:
                    column[type_index] = type(rows[0][column[pos_index] -
                                                      1]).__name__
                for i in range(N):
                    w.insert_row(rows[i])
            print('HDF write small', float(N) / t.elapsed, w.n_rows)

        write_large_blocks()

        write_small_blocks()

        # timing reader.
        df = HDFPartition(fs, 'foobar')

        with Timer() as t:
            count = 0
            i = 0
            s = 0
            r = df.reader
            for i, row in enumerate(r):
                count += 1
            r.close()

        print('HDFPartition iter  ', float(N) / t.elapsed, i, count, s)

        with Timer() as t:
            count = 0
            r = df.reader
            for row in r.rows:
                count += 1
            r.close()

        print('HDFPartition rows  ', float(N) / t.elapsed)

        with Timer() as t:
            count = 0
            r = df.reader
            for row in r.raw:
                count += 1
            r.close()
        print('HDFPartition raw   ', float(N) / t.elapsed)
Ejemplo n.º 46
0
 def test_fixed(self):
     cache_fs = fsopendir(self.setup_temp_dir())
     spec = self.sources['simple_fixed']
     s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))
     f = MPRowsFile(cache_fs, spec.name).load_rows(s)
     self.assertEqual(f.headers, ['id', 'uuid', 'int', 'float'])
Ejemplo n.º 47
0
def build_server(fs,
                 settings_path,
                 server_element="main",
                 no_console=False,
                 rebuild=False,
                 validate_db=False,
                 breakpoint=False,
                 master_settings=None):
    """Build a server"""
    start = time()
    archive = Archive()
    console = archive.console
    try:
        archive, context, doc = build(fs,
                                      settings_path,
                                      rebuild=rebuild,
                                      master_settings=master_settings)
        console = archive.console
    except errors.ParseError as e:
        if not no_console:
            line, col = e.position
            console.document_error(text_type(e), e.path, e.code, line, col)
        return None
    except errors.ElementError as element_error:
        if not no_console:
            line = element_error.source_line
            col = 0
            console.document_error(text_type(element_error),
                                   element_error.element._location,
                                   element_error.element._code, line, col)
        raise errors.StartupFailedError('Failed to build project')

    if isinstance(fs, string_types):
        if '://' in fs:
            fs = fsopendir(fs)
        else:
            fs = OSFS(fs)
    archive.project_fs = fs

    try:
        app, server = doc.get_element(server_element)
    except errors.ElementNotFoundError:
        raise errors.StartupFailedError(
            "no <server> element called '{}' found in the project (check setting [project]/startup)"
            .format(server_element))
    error_msg = None
    docs_location = archive.cfg.get('project', 'location')
    try:
        server.startup(archive,
                       context,
                       fs.opendir(docs_location),
                       breakpoint=breakpoint)
    except errors.StartupFailedError as error:
        error_msg = text_type(error)
        #raise
    except errors.ElementError as e:
        raise
    except Exception as e:
        failed = render_failed_documents(archive,
                                         console,
                                         no_console=no_console)
        if failed:
            raise errors.StartupFailedError(
                "{} document(s) failed to build".format(failed))

        if hasattr(e, '__moyaconsole__'):
            e.__moyaconsole__(console)
        error_msg = text_type(e)
        raise errors.StartupFailedError(error_msg or 'Failed to build project')

    failed = render_failed_documents(archive, console, no_console=no_console)
    if failed:
        raise errors.StartupFailedError(error_msg or 'Failed to build project')

# archive.finalize()
    archive.init_media()
    archive.init_data()

    if validate_db:
        from . import db
        if db.validate_all(archive, console) == 0:
            startup_log.debug('models validated successfully')
        else:
            msg = "Models failed to validate, see 'moya db validate' for more information"
            raise errors.StartupFailedError(msg)

    startup_log.info("%s built %.1fms", server, (time() - start) * 1000.0)
    return ServerBuildResult(archive=archive, context=context, server=server)
Ejemplo n.º 48
0
    def test_datafile_read_write(self):
        from fs.opener import fsopendir
        import datetime
        from random import randint, random
        from contexttimer import Timer
        from uuid import uuid4

        fs = fsopendir('mem://')

        # fs = fsopendir('/tmp/pmpf')

        N = 50000

        # Basic read/ write tests.

        def rand_date():
            return datetime.date(randint(2000, 2015), randint(1, 12), 10)

        def rand_datetime():
            return datetime.datetime(randint(2000, 2015), randint(1, 12), 10)

        def rand_time():
            return datetime.time(randint(0, 23), randint(0, 59), 10)

        row = lambda: (None, 1, random(), str(uuid4()), rand_date(),
                       rand_datetime(), rand_time())

        headers = list('abcdefghi')[:len(row())]

        rows = [row() for i in range(N)]

        def write_large_blocks():

            df = MPRowsFile(fs, 'foobar')

            if df.exists:
                df.remove()

            with Timer() as t, df.writer as w:
                w.headers = headers
                w.insert_rows(rows)

            print('MSGPack write L', float(N) / t.elapsed, w.n_rows)

        def write_small_blocks():
            df = MPRowsFile(fs, 'foobar')

            if df.exists:
                df.remove()

            with Timer() as t, df.writer as w:

                for i in range(N):
                    w.headers = headers
                    w.insert_row(rows[i])

            print('MSGPack write S', float(N) / t.elapsed, w.n_rows)

        print()
        # Write the whole file with insert_rows() which writes all of the rows at once.
        write_large_blocks()

        # Write the file in blocks, with insert_rows collecting rows into a cache, then writting the
        # cached blocks.
        write_small_blocks()

        df = MPRowsFile(fs, 'foobar')

        with Timer() as t:
            count = 0
            i = 0
            s = 0

            r = df.reader

            for i, row in enumerate(r):
                count += 1
            r.close()

        print('MSGPack read   ', float(N) / t.elapsed, i, count, s)

        with Timer() as t:
            count = 0

            r = df.reader

            for row in r.rows:

                count += 1

            r.close()

        print('MSGPack rows   ', float(N) / t.elapsed)

        with Timer() as t:
            count = 0

            r = df.reader

            for row in r.raw:
                count += 1

            r.close()

        print('MSGPack raw    ', float(N) / t.elapsed)
Ejemplo n.º 49
0
    def test_load_check_headers(self):
        """Just check that all of the sources can be loaded without exceptions"""

        cache_fs = fsopendir('temp://')

        headers = {
            'mz_with_zip_xl': [
                u('id'),
                u('gvid'),
                u('renter_cost_gt_30'),
                u('renter_cost_gt_30_cv'),
                u('owner_cost_gt_30_pct'),
                u('owner_cost_gt_30_pct_cv')
            ],
            'mz_no_zip': [u('id'), u('uuid'),
                          u('int'), u('float')],
            'namesu8': [
                u('origin_english'),
                u('name_english'),
                u('origin_native'),
                u('name_native')
            ],
            'sf_zip': [u('id'), u('uuid'),
                       u('int'), u('float')],
            'simple': [u('id'), u('uuid'),
                       u('int'), u('float')],
            'csv_no_csv': [u('id'), u('uuid'),
                           u('int'), u('float')],
            'mz_with_zip': [u('id'), u('uuid'),
                            u('int'), u('float')],
            'rpeople': [u('name'), u('size')],
            'rent07': [
                u('id'),
                u('gvid'),
                u('renter_cost_gt_30'),
                u('renter_cost_gt_30_cv'),
                u('owner_cost_gt_30_pct'),
                u('owner_cost_gt_30_pct_cv')
            ],
            'simple_fixed': [u('id'), u('uuid'),
                             u('int'),
                             u('float')],
            'altname': [u('id'), u('foo'),
                        u('bar'), u('baz')],
            'rentcsv': [
                u('id'),
                u('gvid'),
                u('renter_cost_gt_30'),
                u('renter_cost_gt_30_cv'),
                u('owner_cost_gt_30_pct'),
                u('owner_cost_gt_30_pct_cv')
            ],
            'renttab': [
                u('id'),
                u('gvid'),
                u('renter_cost_gt_30'),
                u('renter_cost_gt_30_cv'),
                u('owner_cost_gt_30_pct'),
                u('owner_cost_gt_30_pct_cv')
            ],
            'multiexcel': [
                u('id'),
                u('gvid'),
                u('renter_cost_gt_30'),
                u('renter_cost_gt_30_cv'),
                u('owner_cost_gt_30_pct'),
                u('owner_cost_gt_30_pct_cv')
            ],
            'rent97': [
                u('id'),
                u('gvid'),
                u('renter_cost_gt_30'),
                u('renter_cost_gt_30_cv'),
                u('owner_cost_gt_30_pct'),
                u('owner_cost_gt_30_pct_cv')
            ]
        }

        for source_name, spec in self.sources.items():
            print(source_name)
            s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))

            f = MPRowsFile(cache_fs, spec.name)

            if f.exists:
                f.remove()

            f.load_rows(s)

            with f.reader as r:
                if spec.name in headers:
                    self.assertEqual(headers[spec.name], r.headers)
Ejemplo n.º 50
0
 def get_fs(self, path):
     if path is None:
         path = join(get_moya_dir(), './documentation')
     fs = fsopendir(path, create_dir=True)
     fs.dir_mode = int('777', 8)
     return fs
Ejemplo n.º 51
0
    if REFDOC_ROOT.startswith('rpc://'):
        try:
            l_refdoc_fs = RPCFS('http://' + REFDOC_ROOT[len('rpc://'):])
        except RemoteConnectionError:
            lpath = os.path.join(INSTANCE_ROOT, 'refdocs')
            logger.error('Could not connect to configured REFDOCS. '
                         'Using {0}.'.format(lpath))
            l_refdoc_fs = fsopendir(lpath, create_dir=True)
    else:
        l_refdoc_fs = fsopendir(REFDOC_ROOT, create_dir=True)
    return l_refdoc_fs


refdoc_fs = _refdocs_init()

workspace_fs = fsopendir(os.path.join(INSTANCE_ROOT, 'scratch'),
                         create_dir=True)
local_fs = fsopendir('/')


class ExposedDocument(object):
    def __init__(self, desc, fspath, fs, ts=None, efield=None):
        self.desc = desc
        self.path = fspath
        self.fs = fs
        self.ts = ts
        self.efield = efield
        self._get_fs_prefix()

    def _get_fs_prefix(self):
        if self.fs == refdoc_fs:
            self._prefix = os.path.join('/expose', REFDOC_PREFIX)
Ejemplo n.º 52
0
    def run(self):
        args = self.args
        application = WSGIApplication(self.location, self.get_settings(),
                                      args.server)
        archive = application.archive

        filesystems = archive.filesystems

        fs = None
        if args.fs:
            try:
                fs = filesystems[args.fs]
            except KeyError:
                self.console.error("No filesystem called '%s'" % args.fs)
                return -1

        if args.tree is not None:
            if fs is None:
                self.console.error("Filesystem required")
                return -1
            with fs.opendir(args.tree) as tree_fs:
                tree_fs.tree()
            return

        if args.listdir:
            if fs is None:
                self.console.error("Filesystem required")
                return -1
            for path in fs.opendir(args.listdir).listdir():
                if fs.isdir(path):
                    self.console(path, fg="cyan", bold=True).nl()
                else:
                    self.console(path).nl()

        elif args.cat:
            if fs is None:
                self.console.error("Filesystem required")
                return -1
            contents = fs.getcontents(args.cat)
            self.console.cat(contents, args.cat)

        elif args.open:
            if fs is None:
                self.console.error("Filesystem required")
                return -1

            filepath = fs.getsyspath(args.open, allow_none=True)
            if filepath is None:
                self.console.error(
                    "No system path for '%s' in filesystem '%s'" %
                    (args.open, args.fs))
                return -1

            import subprocess
            if os.name == 'mac':
                subprocess.call(('open', filepath))
            elif os.name == 'nt':
                subprocess.call(('start', filepath), shell=True)
            elif os.name == 'posix':
                subprocess.call(('xdg-open', filepath))
            else:
                self.console.error(
                    "Don't know how to open files on this platform (%s)" %
                    os.name)

        elif args.syspath:
            if fs is None:
                self.console.error("Filesystem required (use -cat FILESYSTEM)")
                return -1
            if not fs.exists(args.syspath):
                self.console.error(
                    "No file called '%s' found in filesystem '%s'" %
                    (args.syspath, args.fs))
                return -1
            syspath = fs.getsyspath(args.syspath, allow_none=True)
            if syspath is None:
                self.console.error(
                    "No system path for '%s' in filesystem '%s'" %
                    (args.syspath, args.fs))
            else:
                self.console(syspath).nl()

        elif args.copy:
            if len(args.copy) == 1:
                src = '/'
                dst = args.copy[0]
            elif len(args.copy) == 2:
                src, dst = args.copy
            else:
                self.console.error("--copy requires 1 or 2 arguments")
                return -1

            if fs.isdir(src):
                src_fs = fs.opendir(src)
                dst_fs = fsopendir(dst, create_dir=True)

                if not args.force and not dst_fs.isdirempty('/'):
                    response = raw_input(
                        "'%s' is not empty. Copying may overwrite directory contents. Continue? "
                        % dst)
                    if response.lower() not in ('y', 'yes'):
                        return 0

                from fs.utils import copydir
                copydir(src_fs, dst_fs)
            else:
                with fs.open(src, 'rb') as read_f:
                    with open(dst, 'wb') as write_f:
                        while 1:
                            chunk = read_f.read(16384)
                            if not chunk:
                                break
                            write_f.write(chunk)

        else:
            table = [[
                Cell("Name", bold=True),
                Cell("Type", bold=True),
                Cell("Location", bold=True)
            ]]

            if fs is None:
                list_filesystems = filesystems.items()
            else:
                list_filesystems = [(args.fs, fs)]

            for name, fs in sorted(list_filesystems):

                if isinstance(fs, MultiFS):
                    location = '\n'.join(
                        mount_fs.desc('/') for mount_fs in fs.fs_sequence)
                    fg = "yellow"
                elif isinstance(fs, MountFS):
                    mount_desc = []
                    for path, dirmount in fs.mount_tree.items():
                        mount_desc.append('%s->%s' %
                                          (path, dirmount.fs.desc('/')))
                    location = '\n'.join(mount_desc)
                    fg = "magenta"
                else:
                    syspath = fs.getsyspath('/', allow_none=True)
                    if syspath is not None:
                        location = syspath
                        fg = "green"
                    else:
                        try:
                            location = fs.desc('/')
                        except FSError as e:
                            location = text_type(e)
                            fg = "red"
                        else:
                            fg = "blue"
                table.append([
                    Cell(name),
                    Cell(type(fs).__name__),
                    Cell(location, bold=True, fg=fg)
                ])
            self.console.table(table, header=True)
Ejemplo n.º 53
0
    def load(self, fs, settings_path=None):
        self.loaded = True
        self.load_fs = fs
        self.loaded_ini = fs.desc('lib.ini')
        try:
            self._cfg = cfg = SettingsContainer.read(fs, 'lib.ini')
        except FSError as e:
            raise errors.LibraryLoadError(
                'failed to load lib.ini from "{path}" ({exc})',
                path=fs.desc("lib.ini"),
                exc=e,
                lib=self)

        def cfgget(section, key, bool=False, default=Ellipsis):
            try:
                if bool:
                    value = cfg[section][key].strip().lower() in ('yes',
                                                                  'true')
                else:
                    value = cfg[section][key]
            except KeyError:
                if default is Ellipsis:
                    raise errors.LibraryLoadError(
                        "required key [{}]/{} not found in lib.ini".format(
                            section, key),
                        lib=self)
                return default
            else:
                return value

        self.long_name = cfgget('lib', 'name')

        if self.long_name in self.archive.libs:
            raise errors.LibraryLoadError(
                "already loaded this library from '{}'".format(
                    self.loaded_ini),
                lib=self,
                diagnosis=
                "Check for a previous <import> that loads this library")

        py_requires = cfgget('lib', 'pyrequires', default=None)
        if py_requires:
            try:
                version_ok = pyversion.check(py_requires)
            except ValueError as e:
                raise errors.LibraryLoadError(
                    "bad Py version specification in [lib]/pyrequires ({})".
                    format(text_type(e)),
                    lib=self)
            if not version_ok:
                versions = ", ".join(
                    "Python {}.{}".format(*v)
                    for v in pyversion.list_compatible(py_requires))
                raise errors.LibraryLoadError(
                    "one of the following Python versions required: {versions}",
                    lib=self.long_name,
                    versions=versions)

        self.title = cfgget('lib', 'title', default=None)
        self.url = cfgget('lib', 'url', default=None)
        try:
            self.version = Version(cfgget('lib', 'version'))
        except ValueError as e:
            raise errors.LibraryLoadError(text_type(e), lib=self.long_name)

        self.namespace = cfgget('lib', 'namespace')
        self.docs_location = cfgget('lib', 'location', default=None)
        self.tests_location = cfgget('tests', 'location', default=None)

        self.system_settings = {
            'templates_directory': self.long_name or '',
            'data_directory': self.long_name or ''
        }

        project_cfg = self.archive.cfg

        settings = SettingsSectionContainer()

        def update_settings(section):
            settings.update(
                (k, SettingContainer(v)) for k, v in iteritems(cfg[section]))

        if 'author' in cfg:
            self.author.update(cfg['author'])

        if 'lib' in cfg:
            self.libinfo.update(cfg['lib'])

        if 'settings' in cfg:
            update_settings('settings')

        if 'templates' in cfg:
            self.templates_info = cfg['templates']

        if 'data' in cfg:
            self.data_info = cfg['data']
            location = cfgget('data', 'location')
            try:
                self.data_fs = fs.opendir(location)
            except FSError as e:
                raise errors.LibraryLoadError(
                    "Unable to read data from {path} ({exc})",
                    path=location,
                    exc=e,
                    lib=self)

        if 'documentation' in cfg:
            self.documentation_location = cfg['documentation'].get(
                'location', './docs')

        if 'translations' in cfg:
            i18n = cfg['translations']
            self.translations_location = i18n.get('location', './translations')
            self.default_language = i18n.get('default_language', 'en')
            self.languages = split_commas(i18n.get('languages', 'en'))
            self._localedir = self.load_fs.getsyspath(
                self.translations_location)
            if self.languages:
                startup_log.debug('%s reading translations %s', self,
                                  textual_list(self.languages, 'and'))
                self.translations.read('messages', self._localedir,
                                       self.languages)

        if project_cfg and ('lib:' + self.long_name) in project_cfg:
            update_settings("lib:" + self.long_name)

        self.settings = settings

        for section_name, section in iteritems(cfg):
            if ':' in section_name:
                what, name = section_name.split(':', 1)
            else:
                continue

            if what.startswith('py'):
                if self.no_py:
                    continue
                try:
                    version_ok = pyversion.check(what)
                except ValueError as e:
                    raise errors.LibraryLoadError(
                        "Bad Py version specification ({})".format(
                            text_type(e)),
                        lib=self)
                if version_ok:
                    location = cfgget(section_name, 'location')
                    py_fs = fs.opendir(location)
                    try:
                        fs_import(self, py_fs, name or self.long_name)
                    except errors.StartupFailedError as e:
                        raise errors.LibraryLoadError(text_type(e),
                                                      exc=e,
                                                      lib=self)
                    except Exception as e:
                        raise
                        #console.exception(e, tb=True).div()
                        raise errors.LibraryLoadError(
                            "Error in Python extension",
                            py_exception=e,
                            lib=self)

            elif what == 'media':
                location = cfgget(section_name, 'location')
                try:
                    media_fs = fs.opendir(location)
                except FSError as e:
                    raise errors.LibraryLoadError(
                        "Unable to read media from {path} ({exc})",
                        path=location,
                        exc=e,
                        lib=self)
                if media_fs.hassyspath('/'):
                    self.media[name] = fsopendir(media_fs.getsyspath('/'))
                else:
                    self.media[name] = media_fs

        if self.docs_location:
            with self.load_fs.opendir(self.docs_location) as docs_fs:
                self.import_documents(docs_fs, recurse=True)