Exemple #1
0
def url(context, link_url):
    """
    Get the path for a page in the Cactus build.
    We'll need this because paths can be rewritten with prettifying.
    """
    site = context['__CACTUS_SITE__']
    page = context['__CACTUS_CURRENT_PAGE__']

    url = site.get_url_for_page(link_url)

    if url is None:

        # See if we're trying to link to an /subdir/index.html with /subdir
        link_url_index = os.path.join(link_url, "index.html")
        url_link_url_index = site.get_url_for_page(link_url_index)

        if url_link_url_index is None:
            logger.warn('%s: page resource does not exist: %s', page.link_url, link_url)

        url = link_url

    if site.prettify_urls:
        return relpath(urllib.parse.urljoin(site.url,  url.rsplit('index.html', 1)[0]), dirname(page.absolute_final_url))


    return relpath(urllib.parse.urljoin(site.url, url), dirname(page.absolute_final_url))
Exemple #2
0
def walk_prefix(prefix, ignore_predefined_files=True, windows_forward_slashes=True):
    """
    Return the set of all files in a given prefix directory.
    """
    res = set()
    prefix = abspath(prefix)
    ignore = {'pkgs', 'envs', 'conda-bld', 'conda-meta', '.conda_lock',
              'users', 'LICENSE.txt', 'info', 'conda-recipes', '.index',
              '.unionfs', '.nonadmin'}
    binignore = {'conda', 'activate', 'deactivate'}
    if sys.platform == 'darwin':
        ignore.update({'python.app', 'Launcher.app'})
    for fn in os.listdir(prefix):
        if ignore_predefined_files and fn in ignore:
            continue
        if isfile(join(prefix, fn)):
            res.add(fn)
            continue
        for root, dirs, files in os.walk(join(prefix, fn)):
            should_ignore = ignore_predefined_files and root == join(prefix, 'bin')
            for fn2 in files:
                if should_ignore and fn2 in binignore:
                    continue
                res.add(relpath(join(root, fn2), prefix))
            for dn in dirs:
                path = join(root, dn)
                if islink(path):
                    res.add(relpath(path, prefix))

    if on_win and windows_forward_slashes:
        return {path.replace('\\', '/') for path in res}
    else:
        return res
Exemple #3
0
def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    config = Configuration('statsmodels', parent_package, top_path)

    # these are subpackages because they have Cython code
    config.add_subpackage('nonparametric')
    config.add_subpackage('tsa')

    #TODO: delegate the non-test stuff to subpackages
    config.add_data_files('sandbox/panel/test_data.txt')

    curdir = os.path.abspath(os.path.dirname(__file__))

    extradatafiles = [relpath(os.path.join(r,d),start=curdir)
                      for r,ds,f in os.walk(os.path.join(curdir, 'datasets'))
                      for d in f if not os.path.splitext(d)[1] in
                          ['.py', '.pyc']]
    for f in extradatafiles:
        config.add_data_files(f)

    # add all the test and results directories for non *.py files
    for root, dirnames, filenames in os.walk(curdir):
        for dir_name in dirnames:
            if dir_name in ['tests', 'results'] and root != 'sandbox':
                config.add_data_dir(relpath(
                                    os.path.join(root, dir_name),
                                    start = curdir)
                                    )

    return config
Exemple #4
0
def static(context, link_url):
    """
    Get the path for a static file in the Cactus build.
    We'll need this because paths can be rewritten with fingerprinting.
    """
    #TODO: Support URLS that don't start with `/static/`
    site = context['__CACTUS_SITE__']
    page = context['__CACTUS_CURRENT_PAGE__']

    url = site.get_url_for_static(link_url)

    if url is None:

        # For the static method we check if we need to add a prefix
        helper_keys = [
            "/static/" + link_url,
            "/static"  + link_url,
            "static/"  + link_url
        ]

        for helper_key in helper_keys:

            url_helper_key = site.get_url_for_static(helper_key)

            if url_helper_key is not None:
                return relpath(urllib.parse.urljoin( site.static_path, helper_key), dirname(page.absolute_final_url))

        url = link_url

    return relpath(urllib.parse.urljoin( site.static_path, link_url), dirname(page.absolute_final_url))
Exemple #5
0
    def __iter__(self):
        """Seek static files and result full and relative paths.

        :return generator: Walk files
        """
        app_and_blueprints = self.collect.filter(
            [self.collect.app] + list(self.collect.blueprints.values()))

        destination_list = set()

        for bp in app_and_blueprints:
            if bp.has_static_folder and op.isdir(bp.static_folder):
                for root, _, files in walk(bp.static_folder):
                    for f in files:
                        spath = op.join(root, f)
                        tpath = op.relpath(spath, bp.static_folder.rstrip('/'))
                        relative = (bp.static_url_path and
                                    self.collect.static_url and
                                    bp.static_url_path.startswith(op.join(
                                        self.collect.static_url, '')))  # noqa
                        if relative:
                            tpath = op.join(op.relpath(
                                bp.static_url_path, self.collect.static_url
                            ), tpath)

                        if tpath in destination_list:
                            self.log("{0} already sourced".format(tpath))
                            continue

                        destination_list.add(tpath)
                        yield bp, spath, tpath
Exemple #6
0
    def _handle_and_return_installed_items(ds, installed_items, failed_items, save):
        if save and ds is not None:
            _save_installed_datasets(ds, installed_items)
        if failed_items:
            msg = ''
            for act, l in (("succeeded", installed_items), ("failed", failed_items)):
                if not l:
                    continue
                if msg:
                    msg += ', and '
                msg += "%s %s" % (
                  single_or_plural("dataset", "datasets", len(l),
                                   include_count=True),
                  act)
                if ds:
                    paths = [relpath(i.path, ds.path)
                             if hasattr(i, 'path')
                             else i if not i.startswith(ds.path) else relpath(i, ds.path)
                             for i in l]
                else:
                    paths = l
                msg += " (%s)" % (", ".join(map(str, paths)))
            msg += ' to install'

            # we were asked for multiple installations
            if installed_items or len(failed_items) > 1:
                raise IncompleteResultsError(
                    results=installed_items, failed=failed_items, msg=msg)
            else:
                raise InstallFailedError(msg=msg)

        return installed_items[0] \
            if len(installed_items) == 1 else installed_items
Exemple #7
0
    def merge_related(self, path_, d, n, merged, content_types):
        path_ = path.abspath(path.join('/', path_))
        if path_ in merged:
            return

        head, tail = path.split(path_)
        new_path = path.join(head, '%s%s' % (n, tail))

        merged[path_] = new_path


        try:
            os.makedirs(path.dirname(path.join(self.main_dir, path.relpath(new_path, '/'))))
        except OSError:
            pass
        shutil.copy(
            path.join(d, path.relpath(path_, '/')),
            path.join(self.main_dir, path.relpath(new_path, '/')),
        )

        rels_path = path.relpath(path.join(head, '_rels', '%s.rels' % tail), '/')
        if path.exists(path.join(d, rels_path)):
            new_rels_path = path.relpath(path.join(head, '_rels', '%s%s.rels' % (n, tail)), '/')
            non_follow_types = [
                'http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster',
                'http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesMaster',
                'http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout',
                'http://schemas.openxmlformats.org/officeDocument/2006/relationships/oleObject',
            ]
            rels = etree.parse(path.join(d, rels_path))
            for e in rels.getroot():
                if e.get('Type') in non_follow_types:
                    continue
                self.merge_related(path.join(path_, '..', e.get('Target')), d, n, merged, content_types)

            for e in rels.getroot():
                if e.get('Type') in non_follow_types:
                    continue
                new_abs_path = path.join('/', merged[path.normpath(path.join(path_, '..', e.get('Target')))])
                ext = e.get('Target').rsplit('.', 1)[-1]
                if ext == 'xml' or ext not in self.default_content_types:
                    for c in content_types.getroot():
                        if c.get('PartName') == path.abspath(path.join('/', path_, '..', e.get('Target'))):
                            if not any(c.get('PartName') == new_abs_path for c in self.content_types.tree.getroot()):
                                c = copy.deepcopy(c)
                                c.set('PartName', new_abs_path)
                                self.content_types.tree.getroot().append(c)
                            break
                    else:
                        raise Exception
                e.set(
                    'Target',
                    new_abs_path,
                )
            try:
                os.makedirs(path.dirname(path.join(self.main_dir, new_rels_path)))
            except OSError:
                pass
            rels.write(path.join(self.main_dir, new_rels_path))
            path.join(self.main_dir, new_rels_path)
	def mutate(self):
		"""
		Performs the mutation.  Applies mutation operator to each source file,
		then stores a diff between the original and mutated file.

		# mutants = # source files x # mutation operators
		"""
		count = 0
		start = time()

		for src_file in self.project.source_files():
			original_path = join(self.project.settings["source_path"], src_file)
			mutant_path = join(out_dir, src_file)
			mkdir_p(dirname(mutant_path))

			for (op, invoke) in self.project.settings["mutants"].items():
				if invoke:
					p = Popen(["txl", original_path, join("vendor", "conman", "%s.Txl" % op)], stdout=open(mutant_path, "w"), stderr=open("/dev/null"))
					self.store.put(diff(relpath(original_path), relpath(mutant_path)), op, src_file)
					count += 1

					if count % 1000 == 0:
						print("Generated %d mutants.  Elapsed time %.02f seconds." % (count, (time() - start)))

		stop = time()
		print("Generated %d mutants in %d seconds." % (count, (stop - start)))
Exemple #9
0
def compile(conf, env):
    """Copy or compile assets to output directory.  If an asset is used as template, it
    won't be copied to the output directory."""

    global __writers, __default

    ext_map = dict((cls.ext, cls) for cls in (
        SASSWriter, SCSSWriter, LESSWriter, HTMLWriter, XMLWriter
    ))

    other = [(prefix, filelist(prefix, conf['static_ignore'])) for prefix in conf['static']]
    other = [((relpath(path, prefix), prefix) for path in generator)
        for prefix, generator in other]

    files = ((path, conf['theme']) for path in filelist(conf['theme'], conf['theme_ignore']))
    files = ((relpath(path, prefix), prefix) for path, prefix in files)
    files = ((path, prefix) for path, prefix in files if path not in env.engine.templates)

    for path, directory in chain(files, chain(*other)):

        # initialize writer for extension if not already there
        _, ext = splitext(path)
        if ext in ext_map and ext not in __writers:
            __writers[ext] = ext_map[ext](conf, env)

        src, dest = join(directory, path), join(conf['output_dir'], path)
        writer = __writers.get(ext, __defaultwriter)
        writer.write(src, dest, force=env.options.force, dryrun=env.options.dryrun)
Exemple #10
0
def vcs_ignore(config, args):
    """Create a VCS ignore file for a wiki."""
    
    log = logging.getLogger('markdoc.vcs-ignore')
    log.debug('Creating ignore file for %s' % args.vcs)
    wiki_root = config['meta.root'] # shorter local alias.
    
    ignore_file_lines = []
    ignore_file_lines.append(p.relpath(config.html_dir, start=wiki_root))
    ignore_file_lines.append(p.relpath(config.temp_dir, start=wiki_root))
    if args.vcs == 'hg':
        ignore_file_lines.insert(0, 'syntax: glob')
        ignore_file_lines.insert(1, '')
    
    if args.output == '-':
        log.debug('Writing ignore file to stdout')
        fp = sys.stdout
    else:
        if not args.output:
            filename = p.join(wiki_root, '.%signore' % args.vcs)
        else:
            filename = p.join(wiki_root, args.output)
        log.info('Writing ignore file to %s' % p.relpath(filename, start=wiki_root))
        fp = open(filename, 'w')
    
    try:
        fp.write('\n'.join(ignore_file_lines) + '\n')
    finally:
        if fp is not sys.stdout:
            fp.close()
    
    log.debug('Ignore file written.')
Exemple #11
0
    def apply(self,
              result,
              original_file_dict,
              file_diff_dict,
              colored: bool=True):
        '''
        Print a diff of the patch that would be applied.

        :param colored: Whether or not to use colored output.
        '''
        printer = ConsolePrinter(colored)

        for filename, this_diff in sorted(result.diffs.items()):
            to_filename = this_diff.rename if this_diff.rename else filename
            to_filename = "/dev/null" if this_diff.delete else to_filename
            original_file = original_file_dict[filename]
            try:
                current_file = file_diff_dict[filename].modified
                new_file = (file_diff_dict[filename] + this_diff).modified
            except KeyError:
                current_file = original_file
                new_file = this_diff.modified

            if tuple(current_file) != tuple(new_file):
                print_beautified_diff(difflib.unified_diff(current_file,
                                                           new_file,
                                                           fromfile=filename,
                                                           tofile=to_filename),
                                      printer)
            elif filename != to_filename:
                print_from_name(printer, join('a', relpath(filename)))
                print_to_name(printer, join('b', relpath(to_filename)))

        return file_diff_dict
 def get_simple_state_json(self, configuration):
     state_data = {
         'id': self._id,
         'url': self._url,
         'dom_path': posixpath.join(
                         posixpath.join(
                             *(relpath(
                                 configuration.get_path('dom'),
                                 configuration.get_path('root')
                                 ).split(os.sep) ) ), self._id + '.txt' ),
         'img_path': posixpath.join(
                         posixpath.join(
                             *(relpath(
                                 configuration.get_path('state'),
                                 configuration.get_path('root')
                                 ).split(os.sep) ) ), self._id  + '.png' ),
         'log_path': posixpath.join(
                         posixpath.join(
                             *(relpath(
                                 configuration.get_path('log'),
                                 configuration.get_path('root')
                                 ).split(os.sep) ) ), self._id ,'browser_'+ str( configuration._browserID)+ '.json' ),
         'depth': self._depth
     }
     return state_data
Exemple #13
0
def _collect_infos(dirname):

    """ Utility function used by ExplodedZipFile to generate ZipInfo
    entries for all of the files and directories under dirname """

    for r, _ds, fs in walk(dirname):
        if not islink(r) and r != dirname:
            i = ZipInfo()
            i.filename = join(relpath(r, dirname), "")
            i.file_size = 0
            i.compress_size = 0
            i.CRC = 0
            yield i.filename, i

        for f in fs:
            df = join(r, f)
            relfn = relpath(join(r, f), dirname)

            if islink(df):
                pass

            elif isfile(df):
                i = ZipInfo()
                i.filename = relfn
                i.file_size = getsize(df)
                i.compress_size = i.file_size
                i.CRC = file_crc32(df)
                yield i.filename, i

            else:
                # TODO: is there any more special treatment?
                pass
Exemple #14
0
 def _read_rule(self, filename):
     filename = path.realpath(filename)
     logging.info("Read %s" % path.relpath(filename))
     try:
         rule = {u"_filename": filename}
         isData = False
         for line in codecs.open(filename, "r", "utf-8"):
             if isData:
                 if u"content" not in rule:
                     rule[u"content"] = u""
                 rule[u"content"] += line
             elif line.strip() == "":
                 isData = True
             elif line.startswith("#"):
                 continue
             else:
                 key, value = line.split(":", 1)
                 rule[key.lower().strip()] = value.strip()
         if u"content" in rule:
             rule[u"content"] = rule[u"content"].strip()
         self._rules[filename] = rule
         self._update_rules_list()
         return rule
     except Exception, e:
         logging.warning("Failed reading {0}: {1}".format(path.relpath(filename), e))
    def get_state_json(self, configuration):
        state_data = {
            'id': self._id,
            'url': self._url,
            'depth': self._depth,
            # output unix style path for website: first unpack dirs in get_path('dom'),
            # and then posixpath.join them with the filename
            'dom_path': posixpath.join(
                            posixpath.join(
                                *(relpath(
                                    configuration.get_path('dom'),
                                    configuration.get_path('root')
                                    ).split(os.sep) ) ), self._id, self._id + '.txt' ),
            'img_path': posixpath.join(
                            posixpath.join(
                                *(relpath(
                                    configuration.get_path('state'),
                                    configuration.get_path('root')
                                    ).split(os.sep) ) ), self._id  + '.png' ),
            'log_path': posixpath.join(
                            posixpath.join(
                                *(relpath(
                                    configuration.get_path('log'),
                                    configuration.get_path('root')
                                    ).split(os.sep) ) ), self._id, 'browser_'+str(configuration._browserID) + '.json' ),

            'clickable': self.get_all_clickables_json(),
            'inputs': self.get_all_inputs_json(),
            'selects': self.get_all_selects_json(),
            'radios': self.get_all_radios_json(),
            'checkboxes': self.get_all_checkboxes_json()
        }
        return state_data
Exemple #16
0
def mk_relative_osx(path):
    assert sys.platform == 'darwin' and is_obj(path)
    s = macho.install_name_change(path, osx_ch_link)

    names = macho.otool(path)
    if names:
        # Strictly speaking, not all object files have install names (e.g.,
        # bundles and executables do not). In that case, the first name here
        # will not be the install name (i.e., the id), but it isn't a problem,
        # because in that case it will be a no-op (with the exception of stub
        # files, which give an error, which is handled below).
        args = [
            'install_name_tool',
            '-id',
            join('@rpath', relpath(dirname(path),
                join(config.build_prefix, 'lib')), basename(names[0])),
            path,
        ]
        print(' '.join(args))
        p = Popen(args, stderr=PIPE)
        stdout, stderr = p.communicate()
        stderr = stderr.decode('utf-8')
        if "Mach-O dynamic shared library stub file" in stderr:
            print("Skipping Mach-O dynamic shared library stub file %s" % path)
            return
        else:
            print(stderr, file=sys.stderr)
            if p.returncode:
                raise RuntimeError("install_name_tool failed with exit status %d"
            % p.returncode)

        # Add an rpath to every executable to increase the chances of it
        # being found.
        args = [
            'install_name_tool',
            '-add_rpath',
            join('@loader_path', relpath(join(config.build_prefix, 'lib'),
                dirname(path)), '').replace('/./', '/'),
            path,
            ]
        print(' '.join(args))
        p = Popen(args, stderr=PIPE)
        stdout, stderr = p.communicate()
        stderr = stderr.decode('utf-8')
        if "Mach-O dynamic shared library stub file" in stderr:
            print("Skipping Mach-O dynamic shared library stub file %s\n" % path)
            return
        elif "would duplicate path, file already has LC_RPATH for:" in stderr:
            print("Skipping -add_rpath, file already has LC_RPATH set")
            return
        else:
            print(stderr, file=sys.stderr)
            if p.returncode:
                raise RuntimeError("install_name_tool failed with exit status %d"
            % p.returncode)

    if s:
        # Skip for stub files, which have to use binary_has_prefix_files to be
        # made relocatable.
        assert_relative_osx(path)
Exemple #17
0
def copy_template(src, dst, dvcs, values):
    for root, dirs, files in os.walk(src):
        for dir in dirs:
            src_path = join(root, dir)
            dst_path = join(dst, relpath(src_path, src))
            if not isdir(dst_path):
                os.makedirs(dst_path)

        for file in files:
            src_path = join(root, file)
            dst_path = join(dst, relpath(src_path, src))

            try:
                with io.open(src_path, 'r', encoding='utf-8') as fd:
                    content = fd.read()
            except UnicodeDecodeError:
                # File is some sort of binary file...  just copy it
                # directly with no template substitution
                with io.open(src_path, 'rb') as fd:
                    content = fd.read()
                with io.open(dst_path, 'wb') as fd:
                    fd.write(content)
            else:
                content = content.format(**values)
                with io.open(dst_path, 'w', encoding='utf-8') as fd:
                    fd.write(content)

            dvcs.add(dst_path)
Exemple #18
0
def configuration(parent_package="", top_path=None):
    from numpy.distutils.misc_util import Configuration

    config = Configuration("statsmodels", parent_package, top_path)

    # these are subpackages because they have Cython code
    config.add_subpackage("nonparametric")
    config.add_subpackage("tsa")

    # TODO: delegate the non-test stuff to subpackages
    config.add_data_files("sandbox/panel/test_data.txt")
    config.add_data_files("stats/libqsturng/tests/bootleg.dat")
    config.add_data_files("stats/libqsturng/CH.r")
    config.add_data_files("stats/libqsturng/LICENSE.txt")

    curdir = os.path.abspath(os.path.dirname(__file__))

    extradatafiles = [
        relpath(os.path.join(r, d), start=curdir)
        for r, ds, f in os.walk(os.path.join(curdir, "datasets"))
        for d in f
        if not os.path.splitext(d)[1] in [".py", ".pyc"]
    ]
    for f in extradatafiles:
        config.add_data_files(f)

    # add all the test and results directories for non *.py files
    for root, dirnames, filenames in os.walk(curdir):
        for dir_name in dirnames:
            if dir_name in ["tests", "results"] and root != "sandbox":
                config.add_data_dir(relpath(os.path.join(root, dir_name), start=curdir))

    return config
Exemple #19
0
  def __iter_alljobs__(self):
    """ Goes through all directories with an OUTVAR. """
    from os import walk
    from os.path import relpath, join
    from glob import iglob
    from .relax import RelaxExtract

    if self.glob is None:
      for dirpath, dirnames, filenames in walk( self.rootpath, topdown=True, 
                                                followlinks=True ):
        if 'crystal.out' not in filenames: continue
        if 'relax' in dirnames:
          dirnames[:] = [u for u in dirnames if u != 'relax']
          try: result = RelaxExtract(join(self.rootpath, dirpath))
          except:
            try: result = self.__class__.DefaultExtract(join(self.rootpath, dirpath))
            except: continue
        else: 
          try: result = self.__class__.DefaultExtract(join(self.rootpath, dirpath))
          except: continue
        
        yield join('/', relpath(dirpath, self.rootpath)), result
    else: 
      for dirpath, dirnames, filenames in walk( self.rootpath, topdown=True, 
                                                followlinks=True ):
        for filename in iglob(join(join(self.rootpath, dirpath), self.glob)):
          try: result = self.__class__.DefaultExtract(filename)
          except: continue
          else: yield join('/',relpath(filename, self.rootpath)), result
    def filterTrainUserPrefFile(self):
        file_path = path.relpath(self.orig_train_triplet_file_path)
        forig = open(file_path, 'r')

        goodTrainUsers = []
        for line in forig:
            if any(x in line for x in self.user_feed_tracks):
                goodTrainUsers.append(line.split('\t', 1)[0])
        forig.close()

        # TODO search in track database for additional echo song ids
        #if any(x in line for x in self.user_feed_artists):
        #    goodTrainUsers.append(line.split('\t', 1)[0])
        #    self.user_feed_artists_tracks.append(line.split('\t')[1])

        goodTrainUsers = set(goodTrainUsers) # make list unique
        file_path = path.relpath(self.train_triplet_file_path)
        ffiltered = open(file_path, 'w')

        file_path = path.relpath(self.orig_train_triplet_file_path)
        forig = open(file_path, 'r')
        for line in forig:
            if any(x in line for x in goodTrainUsers):
                userId, song_id_echo, playCount = line.strip().split('\t')
                if int(playCount) > 10: # using constant to limit size of train data
                    ffiltered.write(line)

        forig.close()

        ffiltered.close()
Exemple #21
0
 def get_subdatasets(self, pattern=None, fulfilled=None, absolute=False,
                     recursive=False, recursion_limit=None, edges=False):
     """DEPRECATED: use `subdatasets()`"""
     # TODO wipe this function out completely once we are comfortable
     # with it. Internally we don't need or use it anymore.
     import inspect
     lgr.warning('%s still uses Dataset.get_subdatasets(). RF to use `subdatasets` command', inspect.stack()[1][3])
     from datalad.coreapi import subdatasets
     if edges:
         return [(r['parentpath'] if absolute else relpath(r['parentpath'], start=self.path),
                  r['path'] if absolute else relpath(r['path'], start=self.path))
                 for r in subdatasets(
                     dataset=self,
                     fulfilled=fulfilled,
                     recursive=recursive,
                     recursion_limit=recursion_limit,
                     bottomup=True)]
     else:
         return subdatasets(
             dataset=self,
             fulfilled=fulfilled,
             recursive=recursive,
             recursion_limit=recursion_limit,
             bottomup=True,
             result_xfm='{}paths'.format('' if absolute else 'rel'))
def write_makefile_am_from_apps_dir( directory, moc_generated_files ):
    '''Makefile writer for an /apps directory'''
    makefile = open(P.join(directory,'Makefile.am'), 'w')

    # For some reason ISIS repeats a lot of headers and source
    # files. I think they actually compile some of their sources
    # multiple times.
    #
    # To save on errors from autogen, we must keep a dictionary of files
    sources_thus_far = []

    app_names = []
    moc_sources = []
    xml_files = []
    for sdirectory in glob(P.join(directory,'*')): # Loop through all folders in the directory
        if not P.isdir( sdirectory ):
            continue
        app_name = P.relpath( sdirectory, directory ).split(".")[0] # Each folder is an app

        # Identify XML files
        xml_files.extend( P.relpath(x, directory) for x in glob(P.join(sdirectory,'*.xml')) )

        # Write instructions to create an executable from the sources
        sourcefiles = glob(P.join(sdirectory,'*.cpp'))
        if sourcefiles:
            app_names.append( app_name )
            print('%s_SOURCES = ' % app_name, file=makefile, end='')
            ld_add = []
            for source in sourcefiles:
                relative_source = P.relpath( source, directory )
                filename = P.relpath( source, sdirectory )
                if filename in sources_thus_far:
                    ld_add.append("%s.$(OBJEXT)" % filename.split('.')[0])
                else:
                    sources_thus_far.append(filename)
                    print(' \\\n  %s' % relative_source, file=makefile, end='')
            # Identify headers that need to be moc generated
            for header in (glob(P.join(sdirectory,'*.h')) + glob(P.join(sdirectory,'*.hpp'))):
                for line in open( header ):
                    if "Q_OBJECT" in line:
                        moc_sources.append( "%s.dir/%s.moc.cc" % (app_name,P.basename(header).split('.')[0]) )
                        print(' \\\n  %s' % moc_sources[-1], file=makefile, end='')
                        break
            print('\n', file=makefile, end='')
            ld_add.append("../src/Core/libisis3.la") # They're referenced by directory path
            # Mission specific stuff is DLopened I believe.
            print('%s_LDADD = %s' % (app_name, " ".join(ld_add)), file=makefile)
            print('%s_CFLAGS = $(AM_CFLAGS)' % app_name, file=makefile)
            print('\n', file=makefile, end='')

    print('bin_PROGRAMS =', file=makefile, end='')
    for app in app_names:
        print(' %s' % app, file=makefile, end='')
    print('', file=makefile)

    # Write out where the XML files should be installed
    print('xmlhelpdir = $(bindir)/xml', file=makefile)
    print('xmlhelp_DATA = %s' % ' '.join(xml_files), file=makefile)

    write_makefile_am_closing( directory, makefile, [], moc_sources, moc_sources, xml_files )
def main():
    args = prog_options()

# Iterate through all directories and access their files
    for root, dirs, files in walk(args.sample_dir):
        if root != args.sample_dir:
            print root

# Map primer names to their 16S regions
            gene_region = {"519R": "V1-V3", "806R": "V4-V5"}

# Iterate through all files and perform quality filtering
            for file in files:
                if file.endswith("R-assigned-01.fastq"):
                    fname = file.split("_")[0] + "_" +\
                            gene_region[file.split("_")[1][:4]]
                    cmd = "fastq_quality_filter -v -q {} -p {} -i {} "\
                          "-o {}_qual_fil.fastq"\
                          .format(args.min_qual_score, args.min_base_pct,
                                  relpath(join(root, file)),
                                  relpath(join(root, fname)))
                    kwargs = shlex.split(cmd)
                    print kwargs, "\n"
                    out = sp.check_output(kwargs)
                    print out
                    with open(relpath(join(root, fname+"_qual_fil.txt")), "w")as fo:
                        fo.write("{}\n{}".format(fname, out))
    return
Exemple #24
0
def move(filename, basedir, newbase):
  ''' Given a filename and two directory names, this function generates
  a new filename which is constructed from the relative path of the
  filename and the first directory and the joint of this relative path
  with the second directory.

  This is useful to generate a new filename in a different directory
  based on another. Craftr uses this function to generate object
  filenames.

  Example:

      >>> move('src/main.c', 'src', 'build/obj')
      build/obj/main.c

  *path* may be an iterable other than a string in which case the
  function is applied recursively to all its items and a list is
  returned instead of a string. '''

  if isinstance(filename, str):
    rel = relpath(filename, basedir)
    if rel == os.curdir or rel.startswith(os.pardir):
      raise ValueError('pathname not a subpath of basedir', filename, basedir)
    return join(newbase, relpath(filename, basedir))
  elif isinstance(filename, collections.Iterable):
    result = []
    for item in filename:
      result.append(move(item, basedir, newbase))
    return result
  else:
    raise TypeError('move() expected string or iterable')
def page_list_to_dict(mem_cap, val, file_list):
    file_number = 0
    result_dict = {}
    cnt = 0
    ele = 0
    fil_max = len(file_list)
    # loop: if element in queue is 'DONE', no more files to read --> function saves current dict to file & terminates.
    # otherwise: converting list of tuples to dict.
    for file in file_list:
        # reading file & updating dict with tuples from file
        tmp_list = (pickle.load(open(file, "rb")))
        result_dict.update(dict(tmp_list))
        cnt += 1
        print("Process ", val, " || Processed files: ", cnt, " of ", fil_max )

        # check memory usage: if dict gets to big, save and clear
        if memory_usage() >= mem_cap:
            print("saving pages dict...")
            full_path = path.relpath("page_dict/dict_"+str(val)+str(file_number)+'.pickle')
            with open(full_path, 'wb') as pickle_file:
                pickle.dump(result_dict, pickle_file, pickle.HIGHEST_PROTOCOL)
                result_dict.clear()
                file_number += 1
    full_path = path.relpath("page_dict/dict_"+str(val)+str(file_number)+'.pickle')
    with open(full_path, 'wb') as pickle_file:
        print("Process ", val, " ||    finishing pages dict...")
        pickle.dump(result_dict, pickle_file, pickle.HIGHEST_PROTOCOL)
        result_dict.clear()
        file_number += 1
Exemple #26
0
def translate(old_commit, new_commit, filename, repo_path, inp):
    filename = relpath(filename, repo_path)
    repo_path = relpath(repo_path)

    try:
        git = Git(repo_path)
    except Exception as e:
        stderr.write(str(e))
        return

    # Load the line numbers and the times they have been looked at
    line_counts = loads(sub('\s', '', inp))
    # Put those line numbers in a dictionary so we can translate them to the new diff
    line_counts = dict((int(k), v) for k, v in line_counts.iteritems())

    diff = create_diff(
        old_content=git.get_file_content(filename, old_commit),
        new_content=git.get_file_content(filename, new_commit))
    new_lines = convert_line_numbers(diff, line_counts.keys())

    result = {}

    for i, v in enumerate(line_counts.values()):
        if new_lines[i] is not None:
            result[new_lines[i]] = v

    return dumps(result)
Exemple #27
0
def compile(conf, env):
    """Copy or compile assets to output directory.  If an asset is used as template, it
    won't be copied to the output directory. All assets are tracked by the event object
    and should not be removed during `acrylamid clean`."""

    global __writers, __default

    ext_map = {
        '.sass': SASSWriter,
        '.scss': SCSSWriter,
        '.less': LESSWriter,
        # '.haml': 'HAMLWriter',
        # ...
    }

    other = [(prefix, readers.filelist(prefix)) for prefix in conf['static']]
    other = [((relpath(path, prefix), prefix) for path in generator)
        for prefix, generator in other]

    files = ((path, conf['theme']) for path in readers.filelist(conf['theme']))
    files = ((relpath(path, prefix), prefix) for path, prefix in files)
    files = ((path, prefix) for path, prefix in files if path not in env.engine.templates)

    for path, directory in chain(files, chain(*other)):

        # initialize writer for extension if not already there
        _, ext = splitext(path)
        if ext in ext_map and ext not in __writers:
            __writers[ext] = ext_map[ext]()

        src, dest = join(directory, path), join(conf['output_dir'], path)
        writer = __writers.get(ext, __defaultwriter)
        writer.write(src, dest, force=env.options.force, dryrun=env.options.dryrun)
Exemple #28
0
def get_current_path():
    home = expanduser("~")
    view = sublime.active_window().active_view()
    data = sublime.active_window().project_data()
    here = None

    if USE_PROJECT_DIR and data and "folders" in data and len(data["folders"]) == 1:
        specified_path = data["folders"][0]["path"]
        if isabs(specified_path):
            here = specified_path+sep
        else:
            project_file_name = sublime.active_window().project_file_name()
            project_file_dir = split(project_file_name)[0]
            here = normpath(join(project_file_dir, specified_path))+sep

    elif view != None and view.file_name() != None:        
        here = split(view.file_name())[0]
        if here != sep: here += sep
    else:
        here = "~" + sep

    # Return path relative to home if applicable.
    if len(commonprefix([home, here])) > 1:
        relative_path = relpath(here,home)
        if len(relative_path) > 1:
            return join("~", relpath(here,home)) + sep
        else:
            return "~" + sep
    else:
        return here
Exemple #29
0
def zip_export(file_name, prefix, resources, project_files, inc_repos, notify):
    """Create a zip file from an exported project.

    Positional Parameters:
    file_name - the file name of the resulting zip file
    prefix - a directory name that will prefix the entire zip file's contents
    resources - a resources object with files that must be included in the zip
    project_files - a list of extra files to be added to the root of the prefix
      directory
    """
    to_zip_list = list(_inner_zip_export(resources, inc_repos))
    total_files = sum(len(to_zip) for _, to_zip in to_zip_list)
    total_files += len(project_files)
    zipped = 0
    with zipfile.ZipFile(file_name, "w") as zip_file:
        for prj_file in project_files:
            zip_file.write(prj_file, join(prefix, basename(prj_file)))
        for loc, to_zip in to_zip_list:
            res = resources[loc]
            for source in to_zip:
                if source:
                    zip_file.write(
                        source,
                        join(prefix, loc,
                             relpath(source, res.file_basepath[source])))
                    notify.progress("Zipping", source,
                                    100 * (zipped / total_files))
                    zipped += 1
        for lib, res in resources.items():
            for source in res.lib_builds:
                target_dir, _ = splitext(source)
                dest = join(prefix, loc,
                            relpath(target_dir, res.file_basepath[source]),
                            ".bld", "bldrc")
                zip_file.write(source, dest)
    def parseCurrentUserPref(self):
        file_path = path.relpath(self.echo_spotify_translation_file_path)
        fTrackTranslation = open(file_path, 'r')
        trackTranslation = dict()
        for line in fTrackTranslation:
            song_id_echo, song_id_spotify = line.strip().split(':')
            trackTranslation[song_id_spotify] = song_id_echo
        fTrackTranslation.close()

        file_path = path.relpath(self.user_triplet_file_path)
        fUserTrackPref = open(file_path, 'r')
        for line in fUserTrackPref:
            userId, songIdSpotify, playCount = line.strip().split('\t')
            if songIdSpotify in trackTranslation:
                echoSongId = trackTranslation[songIdSpotify]
                self.user_feed_tracks.append(echoSongId)
#                print("Translated %s" % echoSongId)
#                if echoSongId in self.echoSongIdToIdxMap:
#                    print("Found %s" % echoSongId)
#            else:
#                print("Not found %s in translations" % songIdSpotify)

        fUserTrackPref.close()

        # add artist for filtering
        fArtists = open(path.relpath("user_artist_preferences.txt"), 'r')
        for artist in fArtists:
            self.user_feed_artists.append(artist.replace("\n", ""))
        fArtists.close()
        print(self.user_feed_tracks)
def run(arguments, content, options, state_machine, state, lineno):
    # The user may provide a filename *or* Python code content, but not both
    if arguments and content:
        raise RuntimeError("plot:: directive can't have both args and content")

    document = state_machine.document
    config = document.settings.env.config
    nofigs = 'nofigs' in options

    options.setdefault('include-source', config.plot_include_source)
    keep_context = 'context' in options
    context_opt = None if not keep_context else options['context']

    rst_file = document.attributes['source']
    rst_dir = os.path.dirname(rst_file)

    if len(arguments):
        if not config.plot_basedir:
            source_file_name = os.path.join(setup.app.builder.srcdir,
                                            directives.uri(arguments[0]))
        else:
            source_file_name = os.path.join(setup.confdir, config.plot_basedir,
                                            directives.uri(arguments[0]))

        # If there is content, it will be passed as a caption.
        caption = '\n'.join(content)

        # If the optional function name is provided, use it
        if len(arguments) == 2:
            function_name = arguments[1]
        else:
            function_name = None

        with io.open(source_file_name, 'r', encoding='utf-8') as fd:
            code = fd.read()
        output_base = os.path.basename(source_file_name)
    else:
        source_file_name = rst_file
        code = textwrap.dedent("\n".join(map(str, content)))
        counter = document.attributes.get('_plot_counter', 0) + 1
        document.attributes['_plot_counter'] = counter
        base, ext = os.path.splitext(os.path.basename(source_file_name))
        output_base = '%s-%d.py' % (base, counter)
        function_name = None
        caption = ''

    base, source_ext = os.path.splitext(output_base)
    if source_ext in ('.py', '.rst', '.txt'):
        output_base = base
    else:
        source_ext = ''

    # ensure that LaTeX includegraphics doesn't choke in foo.bar.pdf filenames
    output_base = output_base.replace('.', '-')

    # is it in doctest format?
    is_doctest = contains_doctest(code)
    if 'format' in options:
        if options['format'] == 'python':
            is_doctest = False
        else:
            is_doctest = True

    # determine output directory name fragment
    source_rel_name = relpath(source_file_name, setup.confdir)
    source_rel_dir = os.path.dirname(source_rel_name)
    while source_rel_dir.startswith(os.path.sep):
        source_rel_dir = source_rel_dir[1:]

    # build_dir: where to place output files (temporarily)
    build_dir = os.path.join(os.path.dirname(setup.app.doctreedir),
                             'plot_directive', source_rel_dir)
    # get rid of .. in paths, also changes pathsep
    # see note in Python docs for warning about symbolic links on Windows.
    # need to compare source and dest paths at end
    build_dir = os.path.normpath(build_dir)

    if not os.path.exists(build_dir):
        os.makedirs(build_dir)

    # output_dir: final location in the builder's directory
    dest_dir = os.path.abspath(
        os.path.join(setup.app.builder.outdir, source_rel_dir))
    if not os.path.exists(dest_dir):
        os.makedirs(dest_dir)  # no problem here for me, but just use built-ins

    # how to link to files from the RST file
    dest_dir_link = os.path.join(relpath(setup.confdir, rst_dir),
                                 source_rel_dir).replace(os.path.sep, '/')
    build_dir_link = relpath(build_dir, rst_dir).replace(os.path.sep, '/')
    source_link = dest_dir_link + '/' + output_base + source_ext

    # make figures
    try:
        results = render_figures(code,
                                 source_file_name,
                                 build_dir,
                                 output_base,
                                 keep_context,
                                 function_name,
                                 config,
                                 context_reset=context_opt == 'reset',
                                 close_figs=context_opt == 'close-figs')
        errors = []
    except PlotError as err:
        reporter = state.memo.reporter
        sm = reporter.system_message(
            2,
            "Exception occurred in plotting %s\n from %s:\n%s" %
            (output_base, source_file_name, err),
            line=lineno)
        results = [(code, [])]
        errors = [sm]

    # Properly indent the caption
    caption = '\n'.join('      ' + line.strip()
                        for line in caption.split('\n'))

    # generate output restructuredtext
    total_lines = []
    for j, (code_piece, images) in enumerate(results):
        if options['include-source']:
            if is_doctest:
                lines = ['']
                lines += [row.rstrip() for row in code_piece.split('\n')]
            else:
                lines = ['.. code-block:: python', '']
                lines += [
                    '    %s' % row.rstrip() for row in code_piece.split('\n')
                ]
            source_code = "\n".join(lines)
        else:
            source_code = ""

        if nofigs:
            images = []

        opts = [
            ':%s: %s' % (key, val) for key, val in six.iteritems(options)
            if key in ('alt', 'height', 'width', 'scale', 'align', 'class')
        ]

        only_html = ".. only:: html"
        only_latex = ".. only:: latex"
        only_texinfo = ".. only:: texinfo"

        # Not-None src_link signals the need for a source link in the generated
        # html
        if j == 0 and config.plot_html_show_source_link:
            src_link = source_link
        else:
            src_link = None

        result = format_template(
            config.plot_template or TEMPLATE,
            dest_dir=dest_dir_link,
            build_dir=build_dir_link,
            source_link=src_link,
            multi_image=len(images) > 1,
            only_html=only_html,
            only_latex=only_latex,
            only_texinfo=only_texinfo,
            options=opts,
            images=images,
            source_code=source_code,
            html_show_formats=config.plot_html_show_formats and not nofigs,
            caption=caption)

        total_lines.extend(result.split("\n"))
        total_lines.extend("\n")

    if total_lines:
        state_machine.insert_input(total_lines, source=source_file_name)

    # copy image files to builder's output directory, if necessary
    if not os.path.exists(dest_dir):
        cbook.mkdirs(dest_dir)

    for code_piece, images in results:
        for img in images:
            for fn in img.filenames():
                destimg = os.path.join(dest_dir, os.path.basename(fn))
                if fn != destimg:
                    shutil.copyfile(fn, destimg)

    # copy script (if necessary)
    target_name = os.path.join(dest_dir, output_base + source_ext)
    with io.open(target_name, 'w', encoding="utf-8") as f:
        if source_file_name == rst_file:
            code_escaped = unescape_doctest(code)
        else:
            code_escaped = code
        f.write(code_escaped)

    return errors
    def run(self,
            params,
            release=False,
            dev=False,
            android=None,
            debug=False,
            debugger=None,
            browserhtml=False):
        env = self.build_env()
        env["RUST_BACKTRACE"] = "1"

        if android is None:
            android = self.config["build"]["android"]

        if android:
            if debug:
                print(
                    "Android on-device debugging is not supported by mach yet. See"
                )
                print(
                    "https://github.com/servo/servo/wiki/Building-for-Android#debugging-on-device"
                )
                return
            script = [
                "am force-stop com.mozilla.servo",
                "echo servo >/sdcard/servo/android_params"
            ]
            for param in params:
                script += [
                    "echo '%s' >>/sdcard/servo/android_params" %
                    param.replace("'", "\\'")
                ]
            script += [
                "am start com.mozilla.servo/com.mozilla.servo.MainActivity",
                "exit"
            ]
            shell = subprocess.Popen(["adb", "shell"], stdin=subprocess.PIPE)
            shell.communicate("\n".join(script) + "\n")
            return shell.wait()

        args = [self.get_binary_path(release, dev)]

        if browserhtml:
            browserhtml_path = find_dep_path_newest('browserhtml', args[0])
            if browserhtml_path is None:
                print(
                    "Could not find browserhtml package; perhaps you haven't built Servo."
                )
                return 1

            if is_macosx():
                # Enable borderless on OSX
                args = args + ['-b']
            elif is_windows():
                # Convert to a relative path to avoid mingw -> Windows path conversions
                browserhtml_path = path.relpath(browserhtml_path, os.getcwd())

            args = args + [
                '-w', '--pref', 'dom.mozbrowser.enabled', '--pref',
                'dom.forcetouch.enabled', '--pref',
                'shell.builtin-key-shortcuts.enabled=false',
                path.join(browserhtml_path, 'out', 'index.html')
            ]

        # Borrowed and modified from:
        # http://hg.mozilla.org/mozilla-central/file/c9cfa9b91dea/python/mozbuild/mozbuild/mach_commands.py#l883
        if debug:
            import mozdebug
            if not debugger:
                # No debugger name was provided. Look for the default ones on
                # current OS.
                debugger = mozdebug.get_default_debugger_name(
                    mozdebug.DebuggerSearch.KeepLooking)

            self.debuggerInfo = mozdebug.get_debugger_info(debugger)
            if not self.debuggerInfo:
                print("Could not find a suitable debugger in your PATH.")
                return 1

            command = self.debuggerInfo.path
            if debugger == 'gdb' or debugger == 'lldb':
                rustCommand = 'rust-' + debugger
                try:
                    subprocess.check_call([rustCommand, '--version'],
                                          env=env,
                                          stdout=open(os.devnull, 'w'))
                except (OSError, subprocess.CalledProcessError):
                    pass
                else:
                    command = rustCommand

            # Prepend the debugger args.
            args = ([command] + self.debuggerInfo.args + args + params)
        else:
            args = args + params

        try:
            check_call(args, env=env)
        except subprocess.CalledProcessError as e:
            print("Servo exited with return value %d" % e.returncode)
            return e.returncode
        except OSError as e:
            if e.errno == 2:
                print("Servo Binary can't be found! Run './mach build'"
                      " and try again!")
            else:
                raise e
Exemple #33
0
def handle(args):
    cmd.find_config()
    config = cmd.load_config()

    assert config.LOCAL_ARCHIVES_BACKUP_DIR, "No LOCAL_ARCHIVES_BACKUP_DIR!"

    db = config.archive_db
    old_dir = os.getcwd()
    os.chdir(config.LOCAL_ARCHIVES_BACKUP_DIR)

    missing = set(db.entries.keys())
    seen = set()

    changes = 0
    need_backup = False

    for root, dirs, files in os.walk('.'):
        for f in files:
            if f.startswith('.'): continue

            rel_path = relpath(join(root, f), '.')

            sha1 = archives.get_sha1(rel_path)
            new = archives.StoredArchive(url=config.ARCHIVES_BASE_URL +
                                         rel_path,
                                         sha1=sha1)

            existing = db.entries.get(f, None)

            if f in seen:
                raise SafeException(
                    "{}: DUPLICATE basename - not allowed!\nFirst:{}\nSecord:{}"
                    .format(f, existing, new))
            seen.add(f)

            if existing:
                missing.remove(f)

                if existing != new:
                    need_backup = True
                    changes += 1

                    print("{}:".format(rel_path))
                    if existing.sha1 != new.sha1:
                        print("  Old SHA1: {old}\n  New SHA1: {new}".format(
                            file=rel_path, old=existing.sha1, new=new.sha1))
                    if existing.url != new.url:
                        print("  Old URL: {old}\n  New URL: {new}".format(
                            file=rel_path, old=existing.url, new=new.url))
            else:
                changes += 1
                print("{}: added to database: {url}".format(rel_path,
                                                            url=new.url))

            db.entries[f] = new

    if missing:
        print(
            "These archives were missing (but were not removed from the database)"
        )
        for m in sorted(missing):
            print("  " + m)

    os.chdir(old_dir)

    if need_backup:
        backup_path = db.path + '.old'
        print("Old database saved as {}".format(backup_path))
        shutil.copyfile(db.path, backup_path)

    if changes:
        db.save_all()
        print("Updated {} (changes: {})".format(db.path, changes))

        if need_backup:
            print("Run '0repo update' to update public feeds.")
    else:
        print("No changes found")
Exemple #34
0
def main():
    import sys

    # Separate the nose params and the pydev params.
    pydev_params = []
    other_test_framework_params = []
    found_other_test_framework_param = None

    NOSE_PARAMS = '--nose-params'
    PY_TEST_PARAMS = '--py-test-params'

    for arg in sys.argv[1:]:
        if not found_other_test_framework_param and arg != NOSE_PARAMS and arg != PY_TEST_PARAMS:
            pydev_params.append(arg)

        else:
            if not found_other_test_framework_param:
                found_other_test_framework_param = arg
            else:
                other_test_framework_params.append(arg)

    try:
        # Convert to the case stored in the filesystem
        import win32api

        def get_with_filesystem_case(f):
            return win32api.GetLongPathName(win32api.GetShortPathName(f))
    except:

        def get_with_filesystem_case(f):
            return f

    # Here we'll run either with nose or with the pydev_runfiles.
    from _pydev_runfiles import pydev_runfiles
    from _pydev_runfiles import pydev_runfiles_xml_rpc
    from _pydevd_bundle import pydevd_constants
    from pydevd_file_utils import _NormFile

    DEBUG = 0
    if DEBUG:
        sys.stdout.write('Received parameters: %s\n' % (sys.argv, ))
        sys.stdout.write('Params for pydev: %s\n' % (pydev_params, ))
        if found_other_test_framework_param:
            sys.stdout.write('Params for test framework: %s, %s\n' %
                             (found_other_test_framework_param,
                              other_test_framework_params))

    try:
        configuration = pydev_runfiles.parse_cmdline([sys.argv[0]] +
                                                     pydev_params)
    except:
        sys.stderr.write('Command line received: %s\n' % (sys.argv, ))
        raise
    pydev_runfiles_xml_rpc.initialize_server(
        configuration.port
    )  # Note that if the port is None, a Null server will be initialized.

    NOSE_FRAMEWORK = "nose"
    PY_TEST_FRAMEWORK = "py.test"
    test_framework = None  # Default (pydev)
    try:
        if found_other_test_framework_param:
            if found_other_test_framework_param == NOSE_PARAMS:
                test_framework = NOSE_FRAMEWORK
                import nose

            elif found_other_test_framework_param == PY_TEST_PARAMS:
                test_framework = PY_TEST_FRAMEWORK
                import pytest

            else:
                raise ImportError('Test framework: %s not supported.' %
                                  (found_other_test_framework_param, ))

        else:
            raise ImportError()

    except ImportError:
        if found_other_test_framework_param:
            raise

        test_framework = None

    # Clear any exception that may be there so that clients don't see it.
    # See: https://sourceforge.net/tracker/?func=detail&aid=3408057&group_id=85796&atid=577329
    if hasattr(sys, 'exc_clear'):
        sys.exc_clear()

    if not test_framework:

        return pydev_runfiles.main(
            configuration)  # Note: still doesn't return a proper value.

    else:
        # We'll convert the parameters to what nose or py.test expects.
        # The supported parameters are:
        # runfiles.py  --config-file|-t|--tests <Test.test1,Test2>  dirs|files --nose-params xxx yyy zzz
        # (all after --nose-params should be passed directly to nose)

        # In java:
        # --tests = Constants.ATTR_UNITTEST_TESTS
        # --config-file = Constants.ATTR_UNITTEST_CONFIGURATION_FILE

        # The only thing actually handled here are the tests that we want to run, which we'll
        # handle and pass as what the test framework expects.

        py_test_accept_filter = {}
        files_to_tests = configuration.files_to_tests

        if files_to_tests:
            # Handling through the file contents (file where each line is a test)
            files_or_dirs = []
            for file, tests in files_to_tests.items():
                if test_framework == NOSE_FRAMEWORK:
                    for test in tests:
                        files_or_dirs.append(file + ':' + test)

                elif test_framework == PY_TEST_FRAMEWORK:
                    file = _NormFile(file)
                    py_test_accept_filter[file] = tests
                    files_or_dirs.append(file)

                else:
                    raise AssertionError(
                        'Cannot handle test framework: %s at this point.' %
                        (test_framework, ))

        else:
            if configuration.tests:
                # Tests passed (works together with the files_or_dirs)
                files_or_dirs = []
                for file in configuration.files_or_dirs:
                    if test_framework == NOSE_FRAMEWORK:
                        for t in configuration.tests:
                            files_or_dirs.append(file + ':' + t)

                    elif test_framework == PY_TEST_FRAMEWORK:
                        file = _NormFile(file)
                        py_test_accept_filter[file] = configuration.tests
                        files_or_dirs.append(file)

                    else:
                        raise AssertionError(
                            'Cannot handle test framework: %s at this point.' %
                            (test_framework, ))
            else:
                # Only files or dirs passed (let it do the test-loading based on those paths)
                files_or_dirs = configuration.files_or_dirs

        argv = other_test_framework_params + files_or_dirs

        if test_framework == NOSE_FRAMEWORK:
            # Nose usage: http://somethingaboutorange.com/mrl/projects/nose/0.11.2/usage.html
            # show_stdout_option = ['-s']
            # processes_option = ['--processes=2']
            argv.insert(0, sys.argv[0])
            if DEBUG:
                sys.stdout.write('Final test framework args: %s\n' %
                                 (argv[1:], ))

            from _pydev_runfiles import pydev_runfiles_nose
            PYDEV_NOSE_PLUGIN_SINGLETON = pydev_runfiles_nose.start_pydev_nose_plugin_singleton(
                configuration)
            argv.append('--with-pydevplugin')
            # Return 'not' because it will return 'success' (so, exit == 0 if success)
            return not nose.run(argv=argv,
                                addplugins=[PYDEV_NOSE_PLUGIN_SINGLETON])

        elif test_framework == PY_TEST_FRAMEWORK:

            if '--coverage_output_dir' in pydev_params and '--coverage_include' in pydev_params:
                coverage_output_dir = pydev_params[
                    pydev_params.index('--coverage_output_dir') + 1]
                coverage_include = pydev_params[
                    pydev_params.index('--coverage_include') + 1]
                try:
                    import pytest_cov
                except ImportError:
                    sys.stderr.write(
                        'To do a coverage run with pytest the pytest-cov library is needed (i.e.: pip install pytest-cov).\n\n'
                    )
                    raise

                argv.insert(0, '--cov-append')
                argv.insert(1, '--cov-report=')
                argv.insert(2, '--cov=%s' % (coverage_include, ))

                import time
                os.environ['COVERAGE_FILE'] = os.path.join(
                    coverage_output_dir, '.coverage.%s' % (time.time(), ))

            if DEBUG:
                sys.stdout.write('Final test framework args: %s\n' % (argv, ))
                sys.stdout.write('py_test_accept_filter: %s\n' %
                                 (py_test_accept_filter, ))

            def dotted(p):
                # Helper to convert path to have dots instead of slashes
                return os.path.normpath(p).replace(os.sep,
                                                   "/").replace('/', '.')

            curr_dir = os.path.realpath('.')
            curr_dotted = dotted(curr_dir) + '.'

            # Overcome limitation on py.test:
            # When searching conftest if we have a structure as:
            # /my_package
            # /my_package/conftest.py
            # /my_package/tests
            # /my_package/tests/test_my_package.py
            # The test_my_package won't have access to the conftest contents from the
            # test_my_package.py file unless the working dir is set to /my_package.
            #
            # See related issue (for which we work-around below):
            # https://bitbucket.org/hpk42/pytest/issue/639/conftest-being-loaded-twice-giving

            for path in sys.path:
                path_dotted = dotted(path)
                if curr_dotted.startswith(path_dotted):
                    os.chdir(path)
                    break

            remove = []
            for i in xrange(len(argv)):
                arg = argv[i]
                # Workaround bug in py.test: if we pass the full path it ends up importing conftest
                # more than once (so, always work with relative paths).
                if os.path.isfile(arg) or os.path.isdir(arg):

                    # Args must be passed with the proper case in the filesystem (otherwise
                    # python itself may not recognize it).
                    arg = get_with_filesystem_case(arg)
                    argv[i] = arg

                    from os.path import relpath
                    try:
                        # May fail if on different drives
                        arg = relpath(arg)
                    except ValueError:
                        pass
                    else:
                        argv[i] = arg
                elif '<unable to get>' in arg:
                    remove.append(i)

            for i in reversed(remove):
                del argv[i]

            # To find our runfile helpers (i.e.: plugin)...
            d = os.path.dirname(__file__)
            if d not in sys.path:
                sys.path.insert(0, d)

            import pickle, zlib, base64

            # Update environment PYTHONPATH so that it finds our plugin if using xdist.
            os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)

            # Set what should be skipped in the plugin through an environment variable
            s = base64.b64encode(
                zlib.compress(pickle.dumps(py_test_accept_filter)))
            if pydevd_constants.IS_PY3K:
                s = s.decode('ascii')  # Must be str in py3.
            os.environ['PYDEV_PYTEST_SKIP'] = s

            # Identifies the main pid (i.e.: if it's not the main pid it has to connect back to the
            # main pid to give xml-rpc notifications).
            os.environ['PYDEV_MAIN_PID'] = str(os.getpid())
            os.environ['PYDEV_PYTEST_SERVER'] = str(configuration.port)

            argv.append('-p')
            argv.append('_pydev_runfiles.pydev_runfiles_pytest2')
            if 'unittest' in sys.modules or 'unittest2' in sys.modules:
                sys.stderr.write(
                    'pydev test runner error: imported unittest before running pytest.main\n'
                )
            return pytest.main(argv)

        else:
            raise AssertionError(
                'Cannot handle test framework: %s at this point.' %
                (test_framework, ))
Exemple #35
0
    def __getitem__(self, index, tocache=False):
        if tocache and (self.data_format == 'data'):
            assert False, 'Caching only enabled for pano format'

        # Select the data
        rgb_img_fp, labels_color_fp = self.image_list[index]

        # Load the data
        basename = osp.splitext(osp.basename(rgb_img_fp))[0]
        cache_path = osp.join(
            self.cache_folder,
            osp.splitext(osp.relpath(rgb_img_fp, self.root_path))[0])
        if os.path.exists(cache_path):
            if tocache:
                # If we're trying to cache the data, and it's already there, don't do anything
                return
            tmp = torch.load(cache_path)
            c = tmp.shape[0]
            rgb, labels = torch.split(tmp, [c - 1, 1], dim=0)

            # Assemble the pano set
            pano_data = [rgb, labels, basename]

            # Return the set of pano data
            return pano_data

        # Load the label files using OpenCV
        if self.data_format == 'data':
            # uint16 image, labels are stored in the last channel
            labels = cv2.imread(labels_color_fp, cv2.IMREAD_UNCHANGED)[:, :, 2]
        else:
            # uint8 image, value=255 introduced for invalid region
            labels = cv2.imread(labels_color_fp, cv2.IMREAD_UNCHANGED)
            labels[labels == 255] = 0
        labels = labels.astype('float32')

        # Handle missing classes
        unq = np.unique(labels.tolist())
        assert not 13 in unq and not 14 in unq
        labels[labels == 15] = 13

        # Load the RGB image using PIL
        rgb = Image.open(rgb_img_fp)

        # Preprocess the loaded data for training
        rgb = self.preprocess(rgb)
        labels = numpy2torch(labels).unsqueeze(0)

        # Camera normalize the perspective images
        if self.data_format == 'data':
            K = torch.tensor(self.K).view(3, 3)

            # Compute a random shift (could use either camnorm instance as both have same intrinsic parameters)
            shift = self.camnorm_bilinear.compute_random_shift(
                rgb.shape[-2:], K)

            # Normalize with a random crop
            rgb = self.camnorm_bilinear(rgb, K, shift)
            labels = self.camnorm_nearest(labels, K, shift)

        # Resize the pano images as desired
        elif self.data_format == 'pano':
            rgb = F.interpolate(rgb.unsqueeze(0),
                                scale_factor=self.scale_factor,
                                mode='bilinear',
                                align_corners=False,
                                recompute_scale_factor=True).squeeze(0)
            labels = F.interpolate(labels.unsqueeze(0),
                                   scale_factor=self.scale_factor,
                                   mode='nearest',
                                   recompute_scale_factor=True).squeeze(0)

            # If caching data
            if tocache:
                cache_dir = osp.dirname(cache_path)
                if not osp.exists(cache_dir):
                    os.makedirs(cache_dir, exist_ok=True)
                if os.path.exists(cache_path):
                    raise IOError('{} already exists'.format(cache_path))
                tmp = torch.cat([rgb, labels], 0)
                torch.save(tmp, cache_path, pickle_protocol=4)
                return None

        # Assemble the pano set
        pano_data = [rgb, labels, basename]

        # Return the set of pano data
        return pano_data
Exemple #36
0
class DocumentCache(object):
    """
    A high-level document cache for caching the content of files.
    
    This is a read-only cache which uses the OS-reported modification timestamps
    for files (via `os.stat()`) to determine cache dirtiness, and then refreshes
    its cache behind the scenes when files are requested.
    
    You can access values via `.get()` (which supports several options) or via
    simple subscription syntax (i.e. `cache[path]`). The cache is configured
    with a 'root' on instantiation, by which all relative paths are resolved.
    """
    def __init__(self, base=None, cache=None, encoding='utf-8'):
        if cache is None:
            cache = {}
        self.cache = cache

        if base is None:
            base = os.getcwd()
        self.base = base

        self.encoding = encoding

    absolute = lambda self, relpath: p.join(self.base, relpath)
    relative = lambda self, abspath: p.relpath(abspath, start=self.base)

    def has_latest_version(self, path):
        """Determine whether the cache for a path is up to date."""

        # No-op for already-absolute paths.
        path = self.absolute(path)
        if path not in self.cache:
            return False
        cached_mtime = self.cache[path][0]
        return os.stat(path).st_mtime <= cached_mtime

    def refresh_cache(self, path, encoding=None):
        """Refresh the cache, no matter what, with an optional encoding."""

        path = self.absolute(path)
        encoding = encoding or self.encoding
        data = read_from(path, encoding=encoding)
        mtime = os.stat(path).st_mtime
        self.cache[path] = (mtime, data)

    def update_to_latest_version(self, path):
        """If necessary, refresh the cache's copy of a file."""

        if not self.has_latest_version(path):
            self.refresh_cache(path)

    def get(self, path, cache=True, encoding=None):
        """Retrieve the data for a given path, optionally using the cache."""

        path = self.absolute(path)

        if cache:
            self.update_to_latest_version(path)
            return self.cache[path][1]  # (mtime, data)[1]

        if not p.isfile(path):
            return None

        if encoding is None:
            encoding = self.encoding
        return read_from(path, encoding=encoding)

    def __getitem__(self, path):
        result = self.get(path)
        if result is None:
            raise KeyError(path)
        return result
Exemple #37
0
    def is_safe(self, directory):
        """Make sure the given absolute path does not point above the htroot."""

        return p.pardir not in p.relpath(
            directory, start=self.config.html_dir).split(p.sep)
Exemple #38
0
def _get_flexible_source_candidates_for_submodule(ds, sm):
    """Assemble candidate locations from where to clone a submodule

    The following locations candidates are considered. For each candidate a
    cost is given in parenthesis, lower values indicate higher cost:

    - URL of any configured superdataset remote that is known to have the
      desired submodule commit, with the submodule path appended to it.
      There can be more than one candidate (cost 500).

    - A URL or absolute path recorded in `.gitmodules` (cost 600).

    - In case `.gitmodules` contains a relative path instead of a URL,
      the URL of any configured superdataset remote that is known to have the
      desired submodule commit, with this relative path appended to it.
      There can be more than one candidate (cost 500).

    - In case `.gitmodules` contains a relative path as a URL, the absolute
      path of the superdataset, appended with this relative path (cost 900).

    Additional candidate URLs can be generated based on templates specified as
    configuration variables with the pattern

      `datalad.get.subdataset-source-candidate-<name>`

    where `name` is an arbitrary identifier. If name starts with three digits
    (e.g. '400myserver') these will be interpreted as a cost, and the
    respective candidate will be sorted into the generated candidate list
    according to this cost. If no cost is given, a default of 700
    is used.

    A template string assigned to such a variable can utilize the Python format
    mini language and may reference a number of properties that are inferred
    from the parent dataset's knowledge about the target subdataset. Properties
    include any submodule property specified in the respective `.gitmodules`
    record. For convenience, an existing `datalad-id` record is made available
    under the shortened name `id`.

    Additionally, the URL of any configured remote that contains the respective
    submodule commit is available as `remote-<name>` properties, where `name`
    is the configured remote name.

    Lastly, all candidates are sorted according to their cost (lower values
    first, and duplicate URLs are stripped, while preserving the first item in the
    candidate list.

    Parameters
    ----------
    ds : Dataset
      Parent dataset of to-be-installed subdataset.
    sm : dict
      Submodule record as produced by `subdatasets()`.

    Returns
    -------
    list of dict
      Where each dict has keys 'cost' (int), 'name' (str), 'url' (str).
      Names are not unique and either derived from the name of the respective
      remote, template configuration variable, or 'local'.
    """
    # short cuts
    ds_repo = ds.repo
    sm_url = sm.get('gitmodule_url', None)
    sm_path = op.relpath(sm['path'], start=sm['parentds'])

    clone_urls = []

    # CANDIDATE: tracking remote of the current branch
    tracking_remote, tracking_branch = ds_repo.get_tracking_branch()
    candidate_remotes = [tracking_remote] if tracking_remote else []

    # if we have a remote, let's check the location of that remote
    # for the presence of the desired submodule
    last_commit = ds_repo.get_last_commit_hexsha(sm_path)
    if last_commit:
        # CANDIDATE: any remote that has the commit when the submodule was
        # last modified

        # ideally should also give preference to the remotes which have
        # the same branch checked out I guess
        candidate_remotes += list(
            _get_remotes_having_commit(ds_repo, last_commit))

    # prepare a dict to generate URL candidates from templates
    sm_candidate_props = {
        k[10:].replace('datalad-id', 'id'): v
        for k, v in sm.items() if k.startswith('gitmodule_')
    }

    for remote in unique(candidate_remotes):
        remote_url = ds_repo.get_remote_url(remote, push=False)

        # Directly on parent's ds url
        if remote_url:
            # make remotes and their URLs available to template rendering
            sm_candidate_props['remoteurl-{}'.format(remote)] = remote_url
            # attempt: submodule checkout at parent remote URL
            # We might need to quote sm_path portion, e.g. for spaces etc
            if isinstance(RI(remote_url), URL):
                sm_path_url = urlquote(sm_path)
            else:
                sm_path_url = sm_path

            clone_urls.extend(
                dict(cost=500, name=remote, url=url)
                for url in _get_flexible_source_candidates(
                    # alternate suffixes are tested by `clone` anyways
                    sm_path_url,
                    remote_url,
                    alternate_suffix=False))

            # attempt: provided (configured?) submodule URL
            # TODO: consider supporting DataLadRI here?  or would confuse
            #  git and we wouldn't want that (i.e. not allow pure git clone
            #  --recursive)
            if sm_url:
                clone_urls.extend(
                    dict(cost=600, name=remote, url=url)
                    for url in _get_flexible_source_candidates(
                        sm_url, remote_url, alternate_suffix=False))
    cost_candidate_expr = re.compile('[0-9][0-9][0-9].*')
    candcfg_prefix = 'datalad.get.subdataset-source-candidate-'
    for name, tmpl in [(c[len(candcfg_prefix):], ds_repo.config[c])
                       for c in ds_repo.config.keys()
                       if c.startswith(candcfg_prefix)]:
        url = tmpl.format(**sm_candidate_props)
        # we don't want "flexible_source_candidates" here, this is
        # configuration that can be made arbitrarily precise from the
        # outside. Additional guesswork can only make it slower
        has_cost = cost_candidate_expr.match(name) is not None
        clone_urls.append(
            # assign a default cost, if a config doesn't have one
            dict(
                cost=int(name[:3]) if has_cost else 700,
                name=name[3:] if has_cost else name,
                url=url,
                from_config=True,
            ))

    # CANDIDATE: the actual configured gitmodule URL
    if sm_url:
        clone_urls.extend(
            dict(cost=900, name='local', url=url)
            for url in _get_flexible_source_candidates(
                sm_url, ds.path, alternate_suffix=False)
            # avoid inclusion of submodule location itself
            if url != sm['path'])

    # sort all candidates by their label, thereby allowing a
    # candidate provided by configuration to purposefully
    # sort before or after automatically generated configuration
    clone_urls = sorted(clone_urls, key=lambda x: x['cost'])
    # take out any duplicate source candidates
    # unique() takes out the duplicated at the tail end
    clone_urls = unique(clone_urls, lambda x: x['url'])

    return clone_urls
Exemple #39
0
        except Exception:
            print("\tFile was not GMA.".format(title))
            continue

    lua = join(ext, "lua")

    curs.execute("DELETE FROM files WHERE owner = ?", (wsid, ))
    curs.execute("DELETE FROM components WHERE owner = ?", (wsid, ))
    curs.execute("DELETE FROM cars WHERE owner = ?", (wsid, ))
    curs.execute("DELETE FROM errors WHERE owner = ?", (wsid, ))

    for rt, dirs, files in walk(lua):
        tld = basename(rt)
        for f in files:
            pf = join(rt, f)
            p = normpath(normcase(relpath(join(rt, f), ext)))

            curs.execute(
                "SELECT files.path, addons.name FROM files INNER JOIN addons ON files.owner = addons.wsid WHERE path = ? AND owner != ?",
                (
                    p,
                    wsid,
                ))
            for res in curs:
                print("\tADON-FILE: {}, '{}'!".format(*res))
            curs.execute("INSERT OR IGNORE INTO files VALUES (?, ?)", (
                p,
                wsid,
            ))

            if tld == "auto":
Exemple #40
0
def _install_subds_from_flexible_source(ds, sm, **kwargs):
    """Tries to obtain a given subdataset from several meaningful locations

    Parameters
    ----------
    ds : Dataset
      Parent dataset of to-be-installed subdataset.
    sm : dict
      Submodule record as produced by `subdatasets()`.
    **kwargs
      Passed onto clone()
    """
    sm_path = op.relpath(sm['path'], start=sm['parentds'])
    # compose a list of candidate clone URLs
    clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm)

    # prevent inevitable exception from `clone`
    dest_path = op.join(ds.path, sm_path)
    clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path]

    if not clone_urls:
        # yield error
        yield get_status_dict(
            action='install',
            ds=ds,
            status='error',
            message=("Have got no candidates to install subdataset %s from.",
                     sm_path),
            logger=lgr,
        )
        return

    for res in clone_dataset(clone_urls_,
                             Dataset(dest_path),
                             cfg=ds.config,
                             **kwargs):
        # make sure to fix a detached HEAD before yielding the install success
        # result. The resetting of the branch would undo any change done
        # to the repo by processing in response to the result
        if res.get('action', None) == 'install' and \
                res.get('status', None) == 'ok' and \
                res.get('type', None) == 'dataset' and \
                res.get('path', None) == dest_path:
            _fixup_submodule_dotgit_setup(ds, sm_path)

            target_commit = sm['gitshasum']
            lgr.debug(
                "Update cloned subdataset {0} in parent".format(dest_path))
            section_name = 'submodule.{}'.format(sm['gitmodule_name'])
            # do not use `git-submodule update --init`, it would make calls
            # to git-config which will not obey datalad inter-process locks for
            # modifying .git/config
            sub = GitRepo(res['path'])
            # record what branch we were on right after the clone
            # TODO instead of the active branch, this should first consider
            # a configured branch in the submodule record of the superdataset
            sub_orig_branch = sub.get_active_branch()
            # if we are on a branch this hexsha will be the tip of that branch
            sub_orig_hexsha = sub.get_hexsha()
            if sub_orig_hexsha != target_commit:
                # make sure we have the desired commit locally
                # expensive and possibly error-prone fetch conditional on cheap
                # local check
                if not sub.commit_exists(target_commit):
                    try:
                        sub.fetch(remote='origin', refspec=target_commit)
                    except CommandError:
                        pass
                    # instead of inspecting the fetch results for possible ways
                    # with which it could failed to produced the desired result
                    # let's verify the presence of the commit directly, we are in
                    # expensive-land already anyways
                    if not sub.commit_exists(target_commit):
                        res.update(
                            status='error',
                            message=
                            ('Target commit %s does not exist in the clone, and '
                             'a fetch that commit from origin failed',
                             target_commit[:8]),
                        )
                        yield res
                        # there is nothing we can do about this
                        # MIH thinks that removing the clone is not needed, as a likely
                        # next step will have to be a manual recovery intervention
                        # and not another blind attempt
                        continue
                # checkout the desired commit
                sub.call_git(['checkout', target_commit])
                # did we detach?
                # XXX: This is a less generic variant of a part of
                # GitRepo.update_submodule(). It makes use of already available
                # information and trusts the existence of the just cloned repo
                # and avoids (redoing) some safety checks
                if sub_orig_branch and not sub.get_active_branch():
                    # trace if current state is a predecessor of the branch_hexsha
                    lgr.debug(
                        "Detached HEAD after updating submodule %s "
                        "(original branch: %s)", sub, sub_orig_branch)
                    if sub.get_merge_base([sub_orig_hexsha,
                                           target_commit]) == target_commit:
                        # TODO: config option?
                        # MIH: There is no real need here. IMHO this should all not
                        # happen, unless the submodule record has a branch
                        # configured. And Datalad should leave such a record, when
                        # a submodule is registered.

                        # we assume the target_commit to be from the same branch,
                        # because it is an ancestor -- update that original branch
                        # to point to the target_commit, and update HEAD to point to
                        # that location -- this readies the subdataset for
                        # further modification
                        lgr.info(
                            "Reset subdataset branch '%s' to %s (from %s) to "
                            "avoid a detached HEAD", sub_orig_branch,
                            target_commit[:8], sub_orig_hexsha[:8])
                        branch_ref = 'refs/heads/%s' % sub_orig_branch
                        sub.update_ref(branch_ref, target_commit)
                        sub.update_ref('HEAD', branch_ref, symbolic=True)
                    else:
                        lgr.warning(
                            "%s has a detached HEAD, because the recorded "
                            "subdataset state %s has no unique ancestor with "
                            "branch '%s'", sub, target_commit[:8],
                            sub_orig_branch)

            # register the submodule as "active" in the superdataset
            ds.config.set(
                '{}.active'.format(section_name),
                'true',
                reload=False,
                force=True,
                where='local',
            )
            ds.config.set(
                '{}.url'.format(section_name),
                # record the actual source URL of the successful clone
                # and not a funky prediction based on the parent ds
                # like ds.repo.update_submodule() would do (does not
                # accept a URL)
                res['source']['giturl'],
                reload=True,
                force=True,
                where='local',
            )
        yield res

    subds = Dataset(dest_path)
    if not subds.is_installed():
        lgr.debug('Desired subdataset %s did not materialize, stopping', subds)
        return

    # check whether clone URL generators were involved
    cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)]
    if cand_cfg:
        # get a handle on the configuration that is specified in the
        # dataset itself (local and dataset)
        super_cfg = ConfigManager(dataset=ds, source='dataset-local')
        need_reload = False
        for rec in cand_cfg:
            # check whether any of this configuration originated from the
            # superdataset. if so, inherit the config in the new subdataset
            # clone. if not, keep things clean in order to be able to move with
            # any outside configuration change
            for c in ('datalad.get.subdataset-source-candidate-{}{}'.format(
                    rec['cost'], rec['name']),
                      'datalad.get.subdataset-source-candidate-{}'.format(
                          rec['name'])):
                if c in super_cfg.keys():
                    subds.config.set(c,
                                     super_cfg.get(c),
                                     where='local',
                                     reload=False)
                    need_reload = True
                    break
        if need_reload:
            subds.config.reload(force=True)
Exemple #41
0
 def setup(self):
     """Stata reg output from `sysuse auto; reg price mpg`"""
     test_path = path.split(path.relpath(__file__))[0]
     auto_path = path.join(test_path, 'data', 'auto.dta')
     autodata = pd.read_stata(auto_path)
     self.df = autodata
Exemple #42
0
def test_temp_directory_respects_dir_arg():
    with tempfile.TemporaryDirectory() as parent:
        with tempfile.TemporaryDirectory(dir=parent) as child:
            assert path.relpath(child, parent) == path.basename(child)
Exemple #43
0
 def append_expected_output_pattern(f):
     if f == '/':
         expected_patterns.append(r'{}{}{}'.format(size, " +", '.' + f))
     else:
         expected_patterns.append(r'{}{}{}'.format(size, " +",
             path_prefix + path.relpath(f, self.mount_a.mountpoint)))
    from datalad.config import anything2bool

    import hirni_heuristic as heuristic

    dataset = Dataset(sys.argv[1])
    rel_spec_path = sys.argv[2]
    anonymize = anything2bool(sys.argv[3])
    subject = sys.argv[4]
    location = sys.argv[5]

    from mock import patch

    from tempfile import mkdtemp

    # relative path to heuristic to be recorded by datalad-run
    heuristic_path = op.relpath(heuristic.__file__, dataset.path)

    # relative path to not-needed-heudiconv output:
    rel_trash_path = op.relpath(
        mkdtemp(prefix="hirni-tmp-", dir=op.join(dataset.path, ".git")),
        dataset.path)

    # heudiconv may create README, CHANGES, dataset_description.json if they
    # weren't there. So, if those aren't there before, we want to kill them
    # afterwards
    keep_readme = op.lexists(op.join(dataset.path, "README"))
    keep_changes = op.lexists(op.join(dataset.path, "CHANGES"))
    keep_description = op.lexists(
        op.join(dataset.path, "dataset_description.json"))

    # at least narrow down the output target:
def _copy_binaries(source, destination, toolchain, target):
    """
    Copy TF-M binaries from source to destination

    :param source: directory where TF-M binaries are available
    :param destination: directory to which TF-M binaries are copied to
    :param toolchain: build toolchain
    :param target: target name
    """
    if destination.endswith("/"):
        output_dir = destination
    else:
        output_dir = destination + "/"

    tfm_secure_axf = join(source, "tfm_s.axf")
    logging.info(
        "Copying %s to %s" %
        (relpath(tfm_secure_axf, mbed_path), relpath(output_dir, mbed_path)))
    shutil.copy2(tfm_secure_axf, output_dir)

    try:
        out_ext = TARGET_MAP[target].TFM_OUTPUT_EXT
    except AttributeError:
        tfm_secure_bin = join(source, "tfm_s.bin")
        logging.info("Copying %s to %s" % (
            relpath(tfm_secure_bin, mbed_path),
            relpath(output_dir, mbed_path),
        ))
        shutil.copy2(tfm_secure_bin, output_dir)
    else:
        if out_ext == "hex":
            tfm_secure_bin = join(source, "tfm_s.hex")
            global TC_DICT
            if toolchain == "ARMCLANG":
                elf2bin = join(TOOLCHAIN_PATHS[TC_DICT.get(toolchain)],
                               "fromelf")
                cmd = [
                    elf2bin,
                    "--i32",
                    "--output=" + tfm_secure_bin,
                    tfm_secure_axf,
                ]
            elif toolchain == "GNUARM":
                elf2bin = join(
                    TOOLCHAIN_PATHS[TC_DICT.get(toolchain)],
                    "arm-none-eabi-objcopy",
                )
                cmd = [elf2bin, "-O", "ihex", tfm_secure_axf, tfm_secure_bin]

            run_cmd_and_return(cmd)

            logging.info("Copying %s to %s" % (
                relpath(tfm_secure_bin, mbed_path),
                relpath(output_dir, mbed_path),
            ))
            shutil.copy2(tfm_secure_bin, output_dir)

    if TARGET_MAP[target].tfm_bootloader_supported:
        mcu_bin = join(source, "mcuboot.bin")
        shutil.copy2(mcu_bin, output_dir)

    if "TFM_V8M" in TARGET_MAP[target].extra_labels:
        install_dir = abspath(join(source, os.pardir, os.pardir))
        tfm_veneer = join(install_dir, "export", "tfm", "veneers",
                          "s_veneers.o")
        shutil.copy2(tfm_veneer, output_dir)
    logging.info("Start script")

    proj_image_dir = "/share/mikro/IMX/MDC Polina Georgiev/exp-WIDE/"
    last_mod_date = getLastModificationInDir(proj_image_dir, '*')

    # Get connection to server
    conn = getOmeroConn()

    # Get all subdirs (these are the top plate dir)
    plate_dirs = get_subdirs(proj_image_dir)
    logging.debug("plate_dirs" + str(plate_dirs))
    for plate_dir in plate_dirs:
        plate_subdirs = get_subdirs(plate_dir)
        for plate_date_dir in plate_subdirs:
            logging.debug("plate_subdir: " + str(plate_date_dir))
            rel_plate_date_dir = relpath(plate_date_dir, proj_image_dir)

            # Parse filename for metadata (e.g. platename well, site, channet etc.)
            metadata = parse_path_plate_date(rel_plate_date_dir)
            logging.debug("metadata" + str(metadata))

            # Check if plate exists in database (if no - then import folder)
            #
            # TODO (and aquisition-date?)
            #
            # TODO create screen?
            #
            plate_ID = getPlateID(conn, metadata['plate'])
            if plate_ID is None:
                # import images and create database entries for plate, well, site etc.
                import_plate_images_and_meta(plate_date_dir, conn)
def _train_async(model: Model,
                 data: TrainingData,
                 checkpoint: Union[str, None],
                 parameter_checkpoint: Union[str, None],
                 save_start: bool,
                 train_params: trainer.TrainParams,
                 evaluators: List[Evaluator],
                 out: ModelDir,
                 notes=None,
                 dry_run=False,
                 start_eval=False):
    """ Train while encoding batches on a seperate thread and storing them in a tensorflow Queue, can
    be much faster then using the feed_dict approach """

    train = data.get_train()

    eval_datasets = data.get_eval()
    loader = data.get_resource_loader()

    print("Training on %d batches" % len(train))
    print("Evaluation datasets: " + " ".join("%s (%d)" % (name, len(data)) for name, data in eval_datasets.items()))

    # spec the model for the given datasets
    model.set_inputs([train] + list(eval_datasets.values()), loader)
    placeholders = model.get_placeholders()

    train_queue = tf.FIFOQueue(train_params.async_encoding, [x.dtype for x in placeholders], name="train_queue")
    evaluator_runner = AysncEvaluatorRunner(evaluators, model, train_params.async_encoding)
    train_enqeue = train_queue.enqueue(placeholders)
    train_close = train_queue.close(True)

    is_train = tf.placeholder(tf.bool, ())
    input_tensors = tf.cond(is_train, lambda: train_queue.dequeue(),
                            lambda: evaluator_runner.eval_queue.dequeue())

    # tensorfow can't infer the shape for an unsized queue, so set it manually
    for input_tensor, pl in zip(input_tensors, placeholders):
        input_tensor.set_shape(pl.shape)

    bcast = hvd.broadcast_global_variables(0)
    print("Init model...")
    config = tf.ConfigProto(allow_soft_placement=True)
    config.gpu_options.allow_growth = True
    config.gpu_options.visible_device_list = str(hvd.local_rank())
    sess = tf.Session(config=config)
    with sess.as_default():
        pred = model.get_predictions_for(dict(zip(placeholders, input_tensors)))

    evaluator_runner.set_input(pred)

    if parameter_checkpoint is not None:
        print("Restoring parameters from %s" % parameter_checkpoint)
        saver = tf.train.Saver()
        saver.restore(sess, checkpoint)
        saver = None

    print("Setting up model prediction / tf...")
    all_vars = tf.global_variables()

    loss, summary_tensor, train_opt, global_step, weight_ema = _build_train_ops(train_params)

    # Pre-compute tensors we need at evaluations time
    eval_tensors = []
    for ev in evaluators:
        eval_tensors.append(ev.tensors_needed(pred))

    if train_params.best_weights is not None:
        lst = all_vars
        if weight_ema is not None:
            for x in lst:
                v = weight_ema.average(x)
                if v is not None:
                    lst.append(v)
        best_weight_saver = tf.train.Saver(var_list=lst, max_to_keep=1)
        cur_best = None
    else:
        best_weight_saver = None
        cur_best = None

    saver = tf.train.Saver(max_to_keep=train_params.max_checkpoints_to_keep)
    summary_writer = tf.summary.FileWriter(out.log_dir)

    # Load or initialize the model parameters
    if checkpoint is not None:
        print("Restoring from checkpoint...")
        saver.restore(sess, checkpoint)
        print("Loaded checkpoint: " + str(sess.run(global_step)))
    else:
        print("Initializing parameters...")
        sess.run(tf.global_variables_initializer())
    sess.run(bcast)

    # Make sure no bugs occur that add to the graph in the train loop, that can cause (eventuall) OOMs
    tf.get_default_graph().finalize()

    if dry_run:
        return

    on_step = sess.run(global_step)

    if save_start:
        # summary_writer.add_graph(sess.graph, global_step=on_step)
        if hvd.rank() == 0:
            trainer.save_train_start(out.dir, data, sess.run(global_step), evaluators, train_params, notes)

    def enqueue_train():
        try:
            # feed data from the dataset iterator -> encoder -> queue
            for epoch in range(train_params.num_epochs):
                for batch in train.get_epoch():
                    feed_dict = model.encode(batch, True)
                    sess.run(train_enqeue, feed_dict)
        except tf.errors.CancelledError:
            # The queue_close operator has been called, exit gracefully
            return
        except Exception as e:
            # Crashes the main thread with a queue exception
            sess.run(train_close)
            raise e

    train_enqueue_thread = Thread(target=enqueue_train)
    train_enqueue_thread.daemon = True  # Ensure we exit the program on an excpetion

    print("Start training!")

    batch_time = 0

    train_dict = {is_train: True}
    eval_dict = {is_train: False}
    try:
        train_enqueue_thread.start()

        for epoch in range(train_params.num_epochs):
            for batch_ix in range(len(train)):
                t0 = time.perf_counter()
                on_step = sess.run(global_step) + 1
                get_summary = on_step % train_params.log_period == 0

                if get_summary:
                    summary, _, batch_loss = sess.run([summary_tensor, train_opt, loss], feed_dict=train_dict)
                else:
                    summary = None
                    _, batch_loss = sess.run([train_opt, loss], feed_dict=train_dict)

                if np.isnan(batch_loss):
                    raise RuntimeError("NaN loss!")

                batch_time += time.perf_counter() - t0
                if hvd.rank() == 0:
                    if summary is not None:
                        print("on epoch=%d batch=%d step=%d, time=%.3f" %
                              (epoch, batch_ix + 1, on_step, batch_time))
                        summary_writer.add_summary(
                            tf.Summary(value=[tf.Summary.Value(tag="time", simple_value=batch_time)]), on_step)
                        summary_writer.add_summary(summary, on_step)
                        batch_time = 0

                # occasional saving
                if hvd.rank() == 0:
                    if on_step % train_params.save_period == 0:
                        print("Checkpointing")
                        saver.save(sess, join(out.save_dir, "checkpoint-" + str(on_step)), global_step=global_step)
                # Occasional evaluation
                if (on_step % train_params.eval_period == 0) or start_eval:
                    print("Running evaluation...")
                    start_eval = False
                    t0 = time.perf_counter()
                    for name, data in eval_datasets.items():
                        n_samples = train_params.eval_samples.get(name)
                        evaluation = evaluator_runner.run_evaluators(sess, data, name, n_samples, eval_dict)
                        if hvd.rank() == 0:
                            for s in evaluation.to_summaries(name + "-"):
                                summary_writer.add_summary(s, on_step)

                            # Maybe save as the best weights
                            if train_params.best_weights is not None and name == train_params.best_weights[0]:
                                val = evaluation.scalars[train_params.best_weights[1]]
                                if cur_best is None or val > cur_best:
                                    print("Save weights with current best weights (%s vs %.5f)" % (
                                        "None" if cur_best is None else ("%.5f" % cur_best), val))
                                    best_weight_saver.save(sess, join(out.best_weight_dir, "best"), global_step=global_step)
                                    cur_best = val

                            print("Evaluation took: %.3f seconds" % (time.perf_counter() - t0))
    finally:
        sess.run(train_close)  # terminates the enqueue thread with an exception

    train_enqueue_thread.join()

    saver.save(sess, relpath(join(out.save_dir, "checkpoint-" + str(on_step))), global_step=global_step)
    sess.close()
Exemple #48
0
    def _setup_files(self, return_path_to_files=False, path_prefix='./'):
        dirname = 'dir1'
        regfilename = 'regfile'
        hlinkname = 'hlink'
        slinkname = 'slink1'
        slink2name = 'slink2'

        dir_abspath = path.join(self.mount_a.mountpoint, dirname)
        regfile_abspath = path.join(self.mount_a.mountpoint, regfilename)
        hlink_abspath = path.join(self.mount_a.mountpoint, hlinkname)
        slink_abspath = path.join(self.mount_a.mountpoint, slinkname)
        slink2_abspath = path.join(self.mount_a.mountpoint, slink2name)

        self.mount_a.run_shell('mkdir ' + dir_abspath)
        self.mount_a.run_shell('touch ' + regfile_abspath)
        self.mount_a.run_shell(['ln', regfile_abspath, hlink_abspath])
        self.mount_a.run_shell(['ln', '-s', regfile_abspath, slink_abspath])
        self.mount_a.run_shell(['ln', '-s', dir_abspath, slink2_abspath])

        dir2_name = 'dir2'
        dir21_name = 'dir21'
        regfile121_name = 'regfile121'
        dir2_abspath = path.join(self.mount_a.mountpoint, dir2_name)
        dir21_abspath = path.join(dir2_abspath, dir21_name)
        regfile121_abspath = path.join(dir21_abspath, regfile121_name)
        self.mount_a.run_shell('mkdir -p ' + dir21_abspath)
        self.mount_a.run_shell('touch ' + regfile121_abspath)

        sudo_write_file(self.mount_a.client_remote, regfile_abspath,
            'somedata')
        sudo_write_file(self.mount_a.client_remote, regfile121_abspath,
            'somemoredata')

        # TODO: is there a way to trigger/force update ceph.dir.rbytes?
        # wait so that attr ceph.dir.rbytes gets a chance to be updated.
        sleep(20)

        expected_patterns = []
        path_to_files = []

        def append_expected_output_pattern(f):
            if f == '/':
                expected_patterns.append(r'{}{}{}'.format(size, " +", '.' + f))
            else:
                expected_patterns.append(r'{}{}{}'.format(size, " +",
                    path_prefix + path.relpath(f, self.mount_a.mountpoint)))

        for f in [dir_abspath, regfile_abspath, regfile121_abspath,
                  hlink_abspath, slink_abspath, slink2_abspath]:
            size = humansize(self.mount_a.stat(f, follow_symlinks=
                                               False)['st_size'])
            append_expected_output_pattern(f)

        # get size for directories containig regfiles within
        for f in [dir2_abspath, dir21_abspath]:
            size = humansize(self.mount_a.stat(regfile121_abspath,
                             follow_symlinks=False)['st_size'])
            append_expected_output_pattern(f)

        # get size for CephFS root
        size = 0
        for f in [regfile_abspath, regfile121_abspath, slink_abspath,
                  slink2_abspath]:
            size += self.mount_a.stat(f, follow_symlinks=False)['st_size']
        size = humansize(size)
        append_expected_output_pattern('/')

        if return_path_to_files:
            for p in [dir_abspath, regfile_abspath, dir2_abspath,
                      dir21_abspath, regfile121_abspath, hlink_abspath,
                      slink_abspath, slink2_abspath]:
                 path_to_files.append(path.relpath(p, self.mount_a.mountpoint))

            return (expected_patterns, path_to_files)
        else:
            return expected_patterns
Exemple #49
0
app_dir = str(os.environ['APP_DIR'])

host="193.2.205.19"  #IP address of the server
port=50000

print("Connect to server")
try:
	s.connect((host,port))

except socket.gaierror as err:
	print("Address-related error connecting to server: ", err)
	sys.exit(1)

except socket.error as err:
	print("Connection error: ", err)
	sys.exit(1)

filename= path.relpath("/root/LOG-a-TEC-testbed/applications/" + app_dir + "/node_results.txt")

f=open(filename, "rb")  #with open (filename, "rb") as f:
line=f.read(1024)
while(line):
	s.send(line)
	#print('sent' , repr(line))
	line=f.read(1024)
f.close()

print("Done sending file to server.")

s.close()
Exemple #50
0
def generate_depfile(outpath, out_dir):
    reloutpath = path.relpath(outpath, out_dir)
    # Just lie and say we depend on build.ninja so we get re-run every gen.
    # Despite the lie, this is more or less correct since we want to observe
    # every build graph change.
    return "%s: build.ninja" % reloutpath
Exemple #51
0
    def run(self):
        if "skip-manim" in self.state.document.settings.env.app.builder.tags.tags:
            node = skip_manim_node()
            self.state.nested_parse(StringList(self.content[0]),
                                    self.content_offset, node)
            return [node]

        from manim import config

        global classnamedict

        clsname = self.arguments[0]
        if clsname not in classnamedict:
            classnamedict[clsname] = 1
        else:
            classnamedict[clsname] += 1

        hide_source = "hide_source" in self.options
        save_as_gif = "save_as_gif" in self.options
        save_last_frame = "save_last_frame" in self.options
        assert not (save_as_gif and save_last_frame)

        ref_content = (self.options.get("ref_modules", []) +
                       self.options.get("ref_classes", []) +
                       self.options.get("ref_functions", []))
        if ref_content:
            ref_block = f"""
.. admonition:: Example References
    :class: example-reference

    {' '.join(ref_content)}"""
        else:
            ref_block = ""

        if "quality" in self.options:
            quality = f'{self.options["quality"]}_quality'
        else:
            quality = "example_quality"
        frame_rate = QUALITIES[quality]["frame_rate"]
        pixel_height = QUALITIES[quality]["pixel_height"]
        pixel_width = QUALITIES[quality]["pixel_width"]
        qualitydir = f"{pixel_height}p{frame_rate}"

        state_machine = self.state_machine
        document = state_machine.document

        source_file_name = document.attributes["source"]
        source_rel_name = relpath(source_file_name, setup.confdir)
        source_rel_dir = os.path.dirname(source_rel_name)
        while source_rel_dir.startswith(os.path.sep):
            source_rel_dir = source_rel_dir[1:]

        dest_dir = os.path.abspath(
            os.path.join(setup.app.builder.outdir, source_rel_dir))
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)

        source_block = [
            ".. code-block:: python",
            "",
            *["    " + line for line in self.content],
        ]
        source_block = "\n".join(source_block)

        config.media_dir = Path(setup.confdir) / "media"
        config.images_dir = "{media_dir}/images"
        config.video_dir = "{media_dir}/videos/{quality}"
        output_file = f"{clsname}-{classnamedict[clsname]}"
        config.assets_dir = Path("_static")

        config_code = [
            f'config["frame_rate"] = {frame_rate}',
            f'config["pixel_height"] = {pixel_height}',
            f'config["pixel_width"] = {pixel_width}',
            f'config["save_last_frame"] = {save_last_frame}',
            f'config["save_as_gif"] = {save_as_gif}',
            f'config["output_file"] = r"{output_file}"',
        ]

        user_code = self.content
        if user_code[0].startswith(
                ">>> "):  # check whether block comes from doctest
            user_code = [
                line[4:] for line in user_code
                if line.startswith((">>> ", "... "))
            ]

        code = [
            "from manim import *",
            *config_code,
            *user_code,
            f"{clsname}().render()",
        ]
        exec("\n".join(code), globals())

        # copy video file to output directory
        if not (save_as_gif or save_last_frame):
            filename = f"{output_file}.mp4"
            filesrc = config.get_dir("video_dir") / filename
            destfile = os.path.join(dest_dir, filename)
            shutil.copyfile(filesrc, destfile)
        elif save_as_gif:
            filename = f"{output_file}.gif"
            filesrc = config.get_dir("video_dir") / filename
        elif save_last_frame:
            filename = f"{output_file}.png"
            filesrc = config.get_dir("images_dir") / filename
        else:
            raise ValueError("Invalid combination of render flags received.")

        rendered_template = jinja2.Template(TEMPLATE).render(
            clsname=clsname,
            clsname_lowercase=clsname.lower(),
            hide_source=hide_source,
            filesrc_rel=os.path.relpath(filesrc, setup.confdir),
            output_file=output_file,
            save_last_frame=save_last_frame,
            save_as_gif=save_as_gif,
            source_block=source_block,
            ref_block=ref_block,
        )
        state_machine.insert_input(rendered_template.split("\n"),
                                   source=document.attributes["source"])

        return []
 def get_relative_path_from_output_directory(file_to_resolve):
     return path.relpath(path.join(os.getcwd(), file_to_resolve), tsconfig_output_directory)
Exemple #53
0
from bokeh.application import Application
from bokeh.application.handlers.function import FunctionHandler
from bokeh.plotting import figure

from bokeh.io.state import curstate
from bokeh.resources import Resources

from functools import partial  # by https://stackoverflow.com/questions/41926478/python-bokeh-send-additional-parameters-to-widget-event-handler

import step_1_action

DATA_PATH_NAME = 'static'
USER_FILE_NAME = 'user_definitions.json'

DATA_DIR = join(dirname(__file__), DATA_PATH_NAME)  # full abs path
rel_DATA_DIR = relpath(DATA_DIR)

rel_User_fileName = join(rel_DATA_DIR, USER_FILE_NAME)

case_test = True
case_test = False

default_text = """
                    this is an initialized text widget
                """
default_abs_script_path = join(dirname(__file__))
"""**************************************************"""


def load_JSON_file_to_Dict(fileName):
    """
def rela_path(path_a, path_b):
    return relpath(path_a, path_b)
Exemple #55
0
                  href="mailto:[email protected]">[email protected]</a></b>
            </p>
          </td>
          <td style="vertical-align: top;">
            <p class="copyright"><a href="https://creativecommons.org/licenses/by/4.0/"
            target="_blank"><img
            src="{BASE_URL}/static/images/CC-BY-icon-80x15.png" alt="CC-BY" /></a></p>
          </td>
        </tr>
      </table>
    </div>
  </body>
</html>
"""

DKIP_KEY_PATH = path.join(path.dirname(path.relpath(path.relpath(__file__))),
                          ".keys", "dkim.key")

# RQ
RQ_REDIS_URL = "redis://redis:6379/0"
RQ_QUEUE_CLASS = "orcid_hub.queuing.ThrottledQueue"
# rq-dashboard config:
RQ_POLL_INTERVAL = 5000  #: Web interface poll period for updates in ms
WEB_BACKGROUND = "gray"

# Celery
CELERY_BROKER_URL = "redis://localhost:6379/0"
CELERY_RESULT_BACKEND = "redis://localhost:6379/0"

# CELERY_ACCEPT_CONTENT = ["application/x-python-serialize"]
# CELERY_TASK_SERIALIZER = "pickle"
Exemple #56
0
def write_state(source, layer, data_source_name, skipped, destination,
                log_handler, tests_passed, cache_result, conform_result,
                preview_path, slippymap_path, temp_dir):
    '''
    '''
    source_id, _ = splitext(basename(source))
    statedir = join(destination, source_id)

    if not exists(statedir):
        mkdir(statedir)

    statedir = join(statedir, layer)
    if not exists(statedir):
        mkdir(statedir)

    statedir = join(statedir, data_source_name)
    if not exists(statedir):
        mkdir(statedir)

    if cache_result.cache:
        scheme, _, cache_path1, _, _, _ = urlparse(cache_result.cache)
        if scheme in ('file', ''):
            cache_path2 = join(statedir,
                               'cache{1}'.format(*splitext(cache_path1)))
            copy(cache_path1, cache_path2)
            state_cache = relpath(cache_path2, statedir)
        else:
            state_cache = cache_result.cache
    else:
        state_cache = None

    if conform_result.path:
        _, _, processed_path1, _, _, _ = urlparse(conform_result.path)
        processed_path2 = join(statedir,
                               'out{1}'.format(*splitext(processed_path1)))
        copy(processed_path1, processed_path2)

    # Write the sample data to a sample.json file
    if conform_result.sample:
        sample_path = join(statedir, 'sample.json')
        with open(sample_path, 'w') as sample_file:
            json.dump(conform_result.sample, sample_file, indent=2)

    if preview_path:
        preview_path2 = join(statedir, 'preview.png')
        copy(preview_path, preview_path2)

    if slippymap_path:
        slippymap_path2 = join(statedir, 'slippymap.mbtiles')
        copy(slippymap_path, slippymap_path2)

    log_handler.flush()
    output_path = join(statedir, 'output.txt')
    copy(log_handler.stream.name, output_path)

    if skipped:
        source_problem = SourceProblem.skip_source
    else:
        with open(output_path) as file:
            log_content = file.read()
        if exists(source):
            with open(source) as file:
                source_data = json.load(file)
        else:
            source_data = {}

        source_problem = find_source_problem(log_content, source_data)

    state = [
        ('source', basename(source)),
        ('skipped', bool(skipped)),
        ('cache', state_cache),
        ('sample', conform_result.sample and relpath(sample_path, statedir)),
        ('website', conform_result.website),
        ('license', conform_result.license),
        ('geometry type', conform_result.geometry_type),
        ('address count', conform_result.address_count),
        ('version', cache_result.version),
        ('fingerprint', cache_result.fingerprint),
        ('cache time', cache_result.elapsed and str(cache_result.elapsed)),
        ('processed', conform_result.path
         and relpath(processed_path2, statedir)),
        ('process time', conform_result.elapsed
         and str(conform_result.elapsed)),
        ('output', relpath(output_path, statedir)),
        ('preview', preview_path and relpath(preview_path2, statedir)),
        ('slippymap', slippymap_path and relpath(slippymap_path2, statedir)),
        ('attribution required', boolstr(conform_result.attribution_flag)),
        ('attribution name', conform_result.attribution_name),
        ('share-alike', boolstr(conform_result.sharealike_flag)),
        ('source problem', getattr(source_problem, 'value', None)),
        ('code version', __version__),
        ('tests passed', tests_passed),
    ]

    with open(join(statedir, 'index.txt'), 'w', encoding='utf8') as file:
        out = csv.writer(file, dialect='excel-tab')
        for row in zip(*state):
            out.writerow(row)

    with open(join(statedir, 'index.json'), 'w') as file:
        json.dump(list(zip(*state)), file, indent=2)

        _L.info(u'Wrote to state: {}'.format(file.name))
        return file.name
Exemple #57
0
def get_include_files(file_tree, base_dir):
    include_files = {}
    for f in file_tree:
        include_files[path.relpath(f, base_dir)] = f
    return include_files
Exemple #58
0
def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    parser.add_argument('input_dir', help='input annotated directory')
    parser.add_argument('output_dir', help='output dataset directory')
    parser.add_argument('--labels', help='labels file', required=True)
    parser.add_argument(
        '--label-encoding',
        default='utf-8',
        help='file encoding of label files',
    )
    args = parser.parse_args()

    if osp.exists(args.output_dir):
        print('Output directory already exists:', args.output_dir)
        sys.exit(1)
    os.makedirs(args.output_dir)
    os.makedirs(osp.join(args.output_dir, 'JPEGImages'))
    print('Creating dataset:', args.output_dir)

    now = datetime.datetime.now()

    data = dict(
        info=dict(
            description=None,
            url=None,
            version=None,
            year=now.year,
            contributor=None,
            date_created=now.strftime('%Y-%m-%d %H:%M:%S.%f'),
        ),
        licenses=[dict(
            url=None,
            id=0,
            name=None,
        )],
        images=[
            # license, url, file_name, height, width, date_captured, id
        ],
        type='instances',
        annotations=[
            # segmentation, area, iscrowd, image_id, bbox, category_id, id
        ],
        categories=[
            # supercategory, id, name
        ],
    )

    class_name_to_id = {}
    for i, line in enumerate(open(args.labels).readlines()):
        class_id = i - 1  # starts with -1
        class_name = line.strip()
        if class_id == -1:
            assert class_name == '__ignore__'
            continue
        class_name_to_id[class_name] = class_id
        data['categories'].append(dict(
            supercategory=None,
            id=class_id,
            name=class_name,
        ))

    out_ann_file = osp.join(args.output_dir, 'annotations.json')
    label_files = glob.glob(osp.join(args.input_dir, '*.json'))
    for image_id, filename in enumerate(label_files):
        print('Generating dataset from:', filename)

        label_file = labelme.LabelFile(filename=filename,
                                       encoding=args.label_encoding)

        base = osp.splitext(osp.basename(filename))[0]
        out_img_file = osp.join(
            args.output_dir, 'JPEGImages', base + '.jpg'
        )

        img = labelme.utils.img_data_to_arr(label_file.imageData)
        PIL.Image.fromarray(img).convert("RGB").save(out_img_file)
        data['images'].append(dict(
            license=0,
            url=None,
            file_name=osp.relpath(out_img_file, osp.dirname(out_ann_file)),
            height=img.shape[0],
            width=img.shape[1],
            date_captured=None,
            id=image_id,
        ))

        masks = {}                                     # for area
        segmentations = collections.defaultdict(list)  # for segmentation
        for shape in label_file.shapes:
            points = shape['points']
            label = shape['label']
            group_id = shape.get('group_id')
            shape_type = shape.get('shape_type', 'polygon')
            mask = labelme.utils.shape_to_mask(
                img.shape[:2], points, shape_type
            )

            if group_id is None:
                group_id = uuid.uuid1()

            instance = (label, group_id)

            if instance in masks:
                masks[instance] = masks[instance] | mask
            else:
                masks[instance] = mask

            if shape_type == 'rectangle':
                (x1, y1), (x2, y2) = points
                x1, x2 = sorted([x1, x2])
                y1, y2 = sorted([y1, y2])
                points = [x1, y1, x2, y1, x2, y2, x1, y2]
            else:
                points = np.asarray(points).flatten().tolist()

            segmentations[instance].append(points)
        segmentations = dict(segmentations)

        for instance, mask in masks.items():
            cls_name, group_id = instance
            if cls_name not in class_name_to_id:
                continue
            cls_id = class_name_to_id[cls_name]

            mask = np.asfortranarray(mask.astype(np.uint8))
            mask = pycocotools.mask.encode(mask)
            area = float(pycocotools.mask.area(mask))
            bbox = pycocotools.mask.toBbox(mask).flatten().tolist()

            data['annotations'].append(dict(
                id=len(data['annotations']),
                image_id=image_id,
                category_id=cls_id,
                segmentation=segmentations[instance],
                area=area,
                bbox=bbox,
                iscrowd=0,
            ))

    with open(out_ann_file, 'w') as f:
        json.dump(data, f)
Exemple #59
0
def is_subdir(suspect_child, suspect_parent):
    suspect_child = path.realpath(suspect_child)
    suspect_parent = path.realpath(suspect_parent)
    relative = path.relpath(suspect_child, start=suspect_parent)
    return not relative.startswith(os.pardir)
try:
    word = client.DispatchEx("Word.Application")
    for files in listdir(getcwd()):
        match = 0
        if files.endswith(".doc"):
            s, match = "doc", 1
        elif files.endswith(".docx"):
            s, match = "docx", 1
        if match:
            new_name = files.replace("." + s, r".pdf")
            in_file = path.abspath(folder + "\\" + files)
            new_file = path.abspath(folder + "\\" + new_name)
            doc = word.Documents.Open(in_file)
            print(strftime("%H:%M:%S"), " " + s + " -> pdf ",
                  path.relpath(new_file))
            doc.SaveAs(new_file, FileFormat=17)
            doc.Close()

except (Exception, e):
    print(e)
finally:
    word.Quit()

print("\n", strftime("%H:%M:%S"), "Finished converting files.")

# Count the number of pdf files.

num_pdf = count_files(".pdf")

print("\nNumber of pdf files: ", num_pdf)