Esempio n. 1
0
    def __call__(self, oeb, opts):
        self.log = oeb.logger
        self.opts = opts
        self.oeb = oeb

        self.seen_filenames = set([])
        self.rename_map = {}

        for item in list(oeb.manifest.items):
            fname = posixpath.basename(item.href)
            if fname in self.seen_filenames:
                suffix = self.unique_suffix(fname)
                data = item.data
                base, ext = posixpath.splitext(item.href)
                nhref = base + suffix + ext
                nhref = oeb.manifest.generate(href=nhref)[1]
                spine_pos = item.spine_position
                oeb.manifest.remove(item)
                nitem = oeb.manifest.add(item.id, nhref, item.media_type, data=data,
                        fallback=item.fallback)
                self.seen_filenames.add(posixpath.basename(nhref))
                self.rename_map[item.href] = nhref
                if spine_pos is not None:
                    oeb.spine.insert(spine_pos, nitem, item.linear)
            else:
                self.seen_filenames.add(fname)

        if self.rename_map:
            self.log('Found non-unique filenames, renaming to support broken'
                    ' EPUB readers like FBReader, Aldiko and Stanza...')
            from pprint import pformat
            self.log.debug(pformat(self.rename_map))

            renamer = RenameFiles(self.rename_map)
            renamer(oeb, opts)
  def RemoteHostCopy(self, file_path, remote_path='', copy_to=True):
    """Copies a file to or from the VM.

    Args:
      file_path: Local path to file.
      remote_path: Optional path of where to copy file on remote host.
      copy_to: True to copy to vm, False to copy from vm.

    Raises:
      RemoteCommandError: If there was a problem copying the file.
    """
    if copy_to:
      file_name = posixpath.basename(file_path)
      src_spec, dest_spec = file_path, '%s:%s' % (self.name, file_name)
    else:
      remote_path, _ = self.RemoteCommand('readlink -f %s' % remote_path)
      remote_path = remote_path.strip()
      src_spec, dest_spec = '%s:%s' % (self.name, remote_path), file_path
    cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig,
           'cp', src_spec, dest_spec]
    stdout, stderr, retcode = vm_util.IssueCommand(cmd)
    if retcode:
      error_text = ('Got non-zero return code (%s) executing %s\n'
                    'STDOUT: %sSTDERR: %s' %
                    (retcode, ' '.join(cmd), stdout, stderr))
      raise errors.VirtualMachine.RemoteCommandError(error_text)
    if copy_to:
      file_name = posixpath.basename(file_path)
      remote_path = remote_path or file_name
      self.RemoteCommand('mv %s %s; chmod 777 %s' %
                         (file_name, remote_path, remote_path))
Esempio n. 3
0
    def run(self):
        """Run substitutions on files."""
        super(checkbox_install_data, self).run()

        examplesfiles = [o for o in self.outfiles if "examples" in o]
        if not examplesfiles:
            return

        # Create etc directory
        if self.install_dir == "/usr":
            basedir = posixpath.sep
        else:
            basedir = self.install_dir
        etcdir = posixpath.join(basedir, "etc", "checkbox.d")
        self.mkpath(etcdir)

        # Create configs symbolic link
        dstdir = posixpath.dirname(examplesfiles[0]).replace("examples",
            "configs")
        if not os.path.exists(dstdir):
            os.symlink(etcdir, dstdir)

        # Substitute version in examplesfiles and etcfiles
        version = changelog_version()
        for examplesfile in examplesfiles:
            etcfile = posixpath.join(etcdir,
                posixpath.basename(examplesfile))
            infile = posixpath.join("examples",
                posixpath.basename(examplesfile))
            for outfile in examplesfile, etcfile:
                substitute_variables(infile, outfile, {
                    "version = dev": "version = %s" % version})
Esempio n. 4
0
    def findNodeIndex(self, filename, nodename):
        nexus_widget = self.neXusWidget()
        file_model = nexus_widget.model()

        for node in file_model.rootItem.children:
            if node.file.filename == filename:
                file_node = node
                break
        else:
            raise Exception("Could not find file %s" % filename)

        index = file_model.index(file_node.row, 0, Qt.QModelIndex())

        node_parts = nodename.split(posixpath.sep)
        while node_parts:
            name = posixpath.basename(node_parts.pop(0))
            for child in node.children:
                child_name = posixpath.basename(child.name)
                if child_name == name:
                    node = child
                    index = file_model.index(node.row, 0, index)
                    break
            else:
                raise Exception("Could not find node %s in %s" % (name, filename))
        return index
Esempio n. 5
0
 def cmd_list(self):
     if self.args.podcast_id is None:
         print 'Podcast feeds available:'
         print
         for pid in self.config.podcast:
             print '    %s - %s' % (pid, self.config.podcast[pid])
         print
         return os.EX_OK
     else:
         print 'Fetched files available for "%s" (sorted by name):' \
               % self.args.podcast_id
         print
         count = 1
         for filepath in self.podcast.list_fetched_chapters():
             print '    %i: %s' % (count, os.path.basename(filepath))
             count += 1
         if count == 1:
             print '    **** No fetched files.'
         print
         print 'Remote files available for "%s" (reverse sorted by date):' \
               % self.args.podcast_id
         print
         count = 1
         for url in self.podcast.list_chapters():
             if isinstance(url, basestring):
                 print '    %i: %s' % (count, posixpath.basename(url))
             else:
                 print '    %i: %s (%s)' % (count,
                                            posixpath.basename(url[0]),
                                            url[1])
             count += 1
         if count == 1:
             print '    **** No remote files. Try running this script ' \
                   'with `--sync\''
         print
Esempio n. 6
0
    def process_presence_update(msg):
        message_type = msg.delivery_info["routing_key"]
        payload = json.loads(msg.body)

        namespace = payload["namespace"]
        if not namespace.startswith("/robin/"):
            return

        user_id36 = posixpath.basename(namespace)
        room_namespace = posixpath.dirname(namespace)
        room_id = posixpath.basename(room_namespace)

        account = Account._byID36(user_id36, data=True, stale=True)
        try:
            room = RobinRoom._byID(room_id)
        except tdb_cassandra.NotFoundException:
            return

        if not room.is_participant(account):
            return

        presence_type = "join" if message_type == "websocket.connect" else "part"

        websockets.send_broadcast(
            namespace=room_namespace,
            type=presence_type,
            payload={
                "user": account.name,
            },
        )

        if presence_type == "join":
            ParticipantPresenceByRoom.mark_joined(room, account)
        else:
            ParticipantPresenceByRoom.mark_exited(room, account)
Esempio n. 7
0
 def data(self, index, role):
     if role != qt.Qt.DisplayRole:
         return MyQVariant()
     item = self.getProxyFromIndex(index)
     column = index.column()
     if column == 0:
         if isinstance(item, H5FileProxy):
             return MyQVariant(os.path.basename(item.file.filename))
         else:
             return MyQVariant(posixpath.basename(item.name))
     if column == 1:
         showtitle = True
         if showtitle:
             if hasattr(item, 'type'):
                 if item.type in ["Entry", "NXentry"]:
                     children = item.children
                     names = [posixpath.basename(o.name) for o in children]
                     if "title" in names:
                         idx = names.index("title")
                         if len(children[idx].getNode().shape):
                             #stored as an array of strings!!!
                             #return just the first item
                             return MyQVariant("%s" % children[idx].getNode().value[0])
                         else:
                             #stored as a string
                             return MyQVariant("%s" % children[idx].getNode().value)
         return MyQVariant(item.type)
     if column == 2:
         return MyQVariant(item.shape)
     if column == 3:
         return MyQVariant(item.dtype)
     return MyQVariant()
    def test_basename(self):
        self.assertEqual(posixpath.basename("/foo/bar"), "bar")
        self.assertEqual(posixpath.basename("/"), "")
        self.assertEqual(posixpath.basename("foo"), "foo")
        self.assertEqual(posixpath.basename("////foo"), "foo")
        self.assertEqual(posixpath.basename("//foo//bar"), "bar")

        self.assertRaises(TypeError, posixpath.basename)
Esempio n. 9
0
def parseAll(path):
    for root, dirs, files in os.walk(path):
        print("processing: " + basename(root.replace("\\", "/")))
        if len(files) and basename(root.replace("\\", "/")) != "ARM_SAMPLE":
            for file in files:
                filename, file_extension = posixpath.splitext(file)
                if file_extension == ".svd":
                    parseFile(basename(root.replace("\\", "/")), filename)
Esempio n. 10
0
    def makeHDF5(self, rewrite=False):
        """
        Create the HDF5 structure if needed ...
        """
        print("Initialization of HDF5 file")
        if os.path.exists(self.hdf5) and rewrite:
            os.unlink(self.hdf5)

        spath = self.hdf5path.split("/")
        assert len(spath) > 2
        nxs = Nexus(self.hdf5, mode="w")
        entry = nxs.new_entry(entry=spath[0], program_name="pyFAI", title="diffmap")
        grp = entry
        for subgrp in spath[1:-2]:
            grp = nxs.new_class(grp, name=subgrp, class_type="NXcollection")

        processgrp = nxs.new_class(grp, "pyFAI", class_type="NXprocess")
        processgrp["program"] = numpy.array([numpy.str_(i) for i in sys.argv])
        processgrp["version"] = numpy.str_(PyFAI_VERSION)
        processgrp["date"] = numpy.str_(get_isotime())
        if self.mask:
            processgrp["maskfile"] = numpy.str_(self.mask)
        if self.flat:
            processgrp["flatfiles"] = numpy.array([numpy.str_(i) for i in self.flat])
        if self.dark:
            processgrp["darkfiles"] = numpy.array([numpy.str_(i) for i in self.dark])
        processgrp["inputfiles"] = numpy.array([numpy.str_(i) for i in self.inputfiles])
        processgrp["PONIfile"] = numpy.str_(self.poni)

        processgrp["dim0"] = self.npt_slow
        processgrp["dim0"].attrs["axis"] = "Rotation"
        processgrp["dim1"] = self.npt_fast
        processgrp["dim1"].attrs["axis"] = "Translation"
        processgrp["dim2"] = self.npt_rad
        processgrp["dim2"].attrs["axis"] = "Diffraction"
        for k, v in self.ai.getPyFAI().items():
            if "__len__" in dir(v):
                processgrp[k] = numpy.str_(v)
            elif v:
                processgrp[k] = v

        self.group = nxs.new_class(grp, name=spath[-2], class_type="NXdata")

        if posixpath.basename(self.hdf5path) in self.group:
            self.dataset = self.group[posixpath.basename(self.hdf5path)]
        else:
            self.dataset = self.group.create_dataset(
                name=posixpath.basename(self.hdf5path),
                shape=(self.npt_slow, self.npt_fast, self.npt_rad),
                dtype="float32",
                chunks=(1, self.npt_fast, self.npt_rad),
                maxshape=(None, None, self.npt_rad))
            self.dataset.attrs["signal"] = "1"
            self.dataset.attrs["interpretation"] = "spectrum"
            self.dataset.attrs["axes"] = str(self.unit).split("_")[0]
            self.dataset.attrs["creator"] = "pyFAI"
            self.dataset.attrs["long_name"] = "Diffraction imaging experiment"
        self.nxs = nxs
Esempio n. 11
0
def get_trace(filename, fileno):
    data = get_data(filename)
    group_label = ""
    if fileno % 2 == 0:
        group_label = posixpath.basename(filename).split(".")[0] + ":after stimulation"
    else:
        group_label = posixpath.basename(filename).split(".")[0] + ":before stimulation"
    trace = go.Histogram(x=data, opacity=0.75, name=group_label)
    return trace
Esempio n. 12
0
def print_tree(photos, top):
    #TODO:  Make this work for both posix and windows.  Hardwired for posix (looking for / and using posixpath now)
    offset = top.count('/')
    indent = 4
    print "Walking tree from: {}".format(top)
    for dirpath, dirs, files in walk_db_tree(top):
        print '{}{}'.format(' ' * indent * (dirpath.count('/') - offset), posixpath.basename(dirpath))
        for f in files:
            print '{}{}'.format(' ' * indent * (f.count('/') - offset), posixpath.basename(f))
Esempio n. 13
0
def Search(term):
	resultText = ""
	with ix.searcher() as searcher:
		query = QueryParser("content", ix.schema).parse(term)
		corrector = searcher.corrector("content")
		# for mistyped_word in mistyped_words:
		Suggestions = corrector.suggest(term, limit=3)
		if(Suggestions):
			resultText += "<li><h2 class='suggestion_header'>Suggestions based on content</h2></li>"
			for Suggestion in Suggestions:
				resultText += "<li><h3><a href='#' class='suggestions'> " + Suggestion + "</a></h3></li>"
		resultText += "<li>"
		corrected = searcher.correct_query(query, term)
		if corrected.query != query:
			resultText += "<li><h3 class='did_you_mean'>Did You Mean: </h3><p><a class='suggestions'>" + corrected.string + "</a></p>"

		results = searcher.search(query, terms=True,limit=None)
		
		# results.fragmenter = highlight.PinpointFragmenter(autotrim=True)
		# results.formatter = highlight.UppercaseFormatter(between = "...")
		terms=term.split(" ")
		# resultText += " estimated_min_length " + str(results.estimated_min_length()) 
		if results.estimated_min_length() < 11:
			urls = open("uniqueUrltest.txt",'r')
			i = 0
			for url in urls:
				if i < 6:
					if any(word in url for word in terms):
						parse_object = urlparse.urlparse(url)
						if "." not in url and "/" == url[-1]:
							url = url[0:-2]
							title_head = basename(url).decode("utf-8", 'ignore')
						else:
							title_head = basename(parse_object.path).split(".")[0].decode("utf-8", 'ignore')
						
						resultText += "<h3><a href='"+url+"'>" + title_head + "</a></h3><p class='green'>" + url+ "</p>"
						i += 1
			urls.close()
		for result in results:
			# i=0
			# if ".pdf" in str(result["path"]) and i<3:
			# 	resultText += "<h1>" + "<a href='" + result["path"] + "'>" + result["path"] + "</a><br/></h1>" + result["path"] + "<br/><br/>" + result.highlights("content") + "<br/>"
			# 	i += 1
			# 	del results[result]
			if "title" in result:
				if "error" in result["title"].lower():
					resultText += "<h3>" + "<a href='" + result["path"] + "'>" + result["path"] + "</a></h3><p class='url'>" + result["path"] + "</p><p>" + result.highlights("content") + "</p>"
				else:
					if(result["title"] == "" ) or result["title"].isspace():
						resultText += "<h3 class='title_head'>" + "<a href='" + result["path"] + "'>" + "No Title Present" + "</a></h3><p class='url'>" + result["path"] + "</p><p>"  + result.highlights("content") + "</p>"
					else:
						resultText += "<h3 class='title_head'>" + "<a href='" + result["path"] + "'>" + result["title"] + "</a></h3><p class='url'>" + result["path"] + "</p><p>"  + result.highlights("content") + "</p>"
	if(resultText == "<li>"):
		resultText = ""
	else:	
		resultText += "</li>"
	return resultText
Esempio n. 14
0
    def plot_sensor_data_from_file(self, file_path):
        if not self.to_save and not self.to_show:
            return

        data = np.genfromtxt(file_path, delimiter=',', skip_header=1, names=['timestamp', 'x', 'y', 'z'])

        file_name = basename(file_path)
        file_name = file_name[:file_name.find(".")]

        View.plot_sensor_data(basename(file_name), data['timestamp'], data['x'], data['y'], data['z'])
Esempio n. 15
0
def download_path(request, filename):
    parts = posixpath.normpath(filename).split("/")
    if len(parts) < 3:
        return HttpResponseBadRequest()
    
    requested_file = parts[-1]

    try: version_num = str(int(parts[-2]))
    except ValueError: version_num = None
    
    if version_num is None:
        base_path = "/".join(parts[:-2])
        type_id = parts[-2]
        versions = get_versions('/' + base_path)
        if versions is None:
            return HttpResponseNotFound()
        version_num = str(max(map(int, versions)))
    else:
        base_path = "/".join(parts[:-3])
        type_id = parts[-3]

    try: 
        file_metadata = get_file_metadata("/%s/%s" % (base_path, version_num))
    except NotFoundError:
        return HttpResponseNotFound()

    if requested_file == posixpath.basename(base_path):
        is_mesh = True
        hash = file_metadata['types'][type_id]['hash']
    else:
        is_mesh = False
        subfile_map = {}
        for subfile in file_metadata['types'][type_id]['subfiles']:
            (subfile_base, vers) = posixpath.split(subfile)
            subfile_basename = posixpath.basename(subfile_base)
            subfile_map[subfile_basename] = subfile

        if requested_file not in subfile_map:
            return HttpResponseNotFound()

        subfile_metadata = get_file_metadata(subfile_map[requested_file])
        hash = subfile_metadata['hash']

    try:
        rec = get_hash(hash)
    except NotFoundError:
        return HttpResponseNotFound()

    data = rec['data']
    mime = rec['mimetype']

    response = HttpResponse(data, mimetype=mime)
    response['Content-Length'] = str(len(data))
    response['Access-Control-Allow-Origin'] = '*'
    return response
Esempio n. 16
0
 def hdf5Slot(self, ddict):
     if ddict['event'] == 'itemClicked':
         if ddict['mouse'] == "right":
             return self.itemRightClickedSlot(ddict)
     if ddict['event'] == "itemDoubleClicked":
         if ddict['type'] in ['Dataset']:
             if ddict['dtype'].startswith('|S'):
                 print("string")
             else:
                 root = ddict['name'].split('/')
                 root = "/" + root[1]
                 if len(ddict['name']) == len(root):
                     cnt = ddict['name']
                 else:
                     cnt  = ddict['name'][len(root):]
                 if cnt not in self._cntList:
                     self._cntList.append(cnt)
                     basename = posixpath.basename(cnt)
                     if basename not in self._aliasList:
                         self._aliasList.append(basename)
                     else:
                         self._aliasList.append(cnt)
                     self.cntTable.build(self._cntList, self._aliasList)
         elif ddict['type'] in ['NXentry', 'Entry']:
             if self._lastAction is None:
                 return
             action, selectionType = self._lastAction.split()
             if action == 'REMOVE':
                 action = 'ADD'
             ddict['action'] = "%s %s" % (action, selectionType)
             self.buttonsSlot(ddict)
         else:
             if self.data is not None:
                 name = ddict['name']
                 filename = ddict['file']
                 fileIndex = self.data.sourceName.index(filename)
                 phynxFile  = self.data._sourceObjectList[fileIndex]
                 dataset = phynxFile[name]
                 if isinstance(dataset, h5py.Dataset):
                     root = ddict['name'].split('/')
                     root = "/" + root[1]
                     cnt  = ddict['name'].split(root)[-1]
                     if cnt not in self._cntList:
                         if DEBUG:
                             print("USING SECOND WAY")
                         self._cntList.append(cnt)
                         basename = posixpath.basename(cnt)
                         if basename not in self._aliasList:
                             self._aliasList.append(basename)
                         else:
                             self._aliasList.append(cnt)
                         self.cntTable.build(self._cntList, self._aliasList)
                     return
             if DEBUG:
                 print("Unhandled item type: %s" % ddict['dtype'])
Esempio n. 17
0
    def compare_links(self):
        ret = False
        if isinstance(self.devpi_body, Exception):
            tw.line("%s: devpi request failed: %s" %
                    (self.name, self.devpi_body), red=True)
            ret = True
        if isinstance(self.pypi_body, Exception):
            tw.line("%s: pypi request failed: %s" %
                    (self.name, self.pypi_body), red=True)
            ret = True
        if ret:
            return

        pypi_links = BeautifulSoup(self.pypi_body).findAll("a")
        devpi_links = BeautifulSoup(self.devpi_body).findAll("a")
        misses = []
        count = 0
        pypi_basenames = set()
        for link in pypi_links:
            href = link.get("href")
            if href.startswith("../../packages"):
                count += 1
                # stored at pypi, should appear directly with devpi
                package_prefix = href[6:]
                pypi_basenames.add(posixpath.basename(href))
                #print package_prefix
                for link2 in devpi_links:
                    #print "   ", link2.get("href")
                    if link2.get("href").endswith(package_prefix):
                        break
                else:
                    misses.append((href))
        b = logbase.join("res", self.name).ensure(dir=1)
        misses_path = b.join("devpi_misses")
        if misses:
            tw.line("%s: devpi misses %d out of %d links" %
                    (self.name, len(misses), count), red=True)
            misses_path.write("\n".join(misses))
        else:
            if misses_path.check():
                misses_path.remove()
            extralinks = []
            for link2 in devpi_links:
                devpi_basename = posixpath.basename(link2.get("href"))
                if devpi_basename not in pypi_basenames:
                    extralinks.append(devpi_basename)
            if extralinks:
                extra = ", externally scraped: %s" % (extralinks)
            else:
                extra = ""

            tw.line("%s: devpi has %d pypi internal links%s" %
                    (self.name, count, extra), green=True)
        b.join("pypi_body").write(self.pypi_body)
        b.join("devpi_body").write(self.devpi_body)
Esempio n. 18
0
    def insert(self, path, entry):
        if path == '/':
            assert entry.ftype == DIR
            self.index = entry
        else:
            assert ppath.basename(path) == entry.name

            parent = self.lookup(ppath.dirname(path))
            assert parent.ftype == DIR

            parent.items[ppath.basename(path)] = entry
Esempio n. 19
0
    def test_basename(self):
        self.assertEqual(posixpath.basename("/foo/bar"), "bar")
        self.assertEqual(posixpath.basename("/"), "")
        self.assertEqual(posixpath.basename("foo"), "foo")
        self.assertEqual(posixpath.basename("////foo"), "foo")
        self.assertEqual(posixpath.basename("//foo//bar"), "bar")

        self.assertEqual(posixpath.basename(b"/foo/bar"), b"bar")
        self.assertEqual(posixpath.basename(b"/"), b"")
        self.assertEqual(posixpath.basename(b"foo"), b"foo")
        self.assertEqual(posixpath.basename(b"////foo"), b"foo")
        self.assertEqual(posixpath.basename(b"//foo//bar"), b"bar")
Esempio n. 20
0
def find_in_zip(filepath, zip):
    if filepath in zip.namelist():
        return zip.read(filepath)
    basename = posixpath.basename(filepath)
    matching = []
    for zipname in zip.namelist():
        if posixpath.basename(zipname) == basename:
            matching.append(zipname)
    if len(matching) == 1:
        return zip.read(matching[0])
    else:
        return None
Esempio n. 21
0
File: main.py Progetto: mazhe/cimtp
 def _walk(self, path):
     dirs, nondirs = [], []
     for e in self.ls(path):
         if self.isdir(e):
             dirs.append(posixpath.basename(re.sub("/$", "", e)))
         else:
             nondirs.append(posixpath.basename(e))
     yield path, dirs, nondirs
     for d in dirs:
         path2 = posixpath.join(path, d)
         #yield from self.walk(path2)
         for t in self.walk(path2):
             yield t
Esempio n. 22
0
def HTML(path):
	file_=open(use.join(path,use.basename(path)+'.htm'),"w")
	Str="<html><title>"+use.basename(path)+"</title><body>"
	Dict={}
	for files in os.listdir(path):
		if use.splitext(files)[1]=='.txt':
			Dict[int(use.splitext(files)[0])]=files
	list_=Dict.keys()
	list_.sort()
	for files in list_:
		Str+="<img src="+Dict[files]+">"
	file_.write(Str+"<br><br><font color='red'>&copy AUTOMATIC file generator</font></body></center></html>")
	file_.close()
Esempio n. 23
0
def flash(target, firmware_file, firmware_md5, args):
    # Connect to controller...
    if target == 'controller':
        errprinter('connecting to Controller...')
        client = soloutils.connect_controller(await=True)
    else:
        errprinter('connecting to Solo...')
        client = soloutils.connect_solo(await=True)

    # Prepare the update.
    # older versions don't have sololink_config and ssh returns 127, so do it manually
    code = soloutils.command_stream(client, 'sololink_config --update-prepare sololink')
    if code != 0:
        soloutils.command_stream(client, 'rm -rf /log/updates && mkdir -p /log/updates')

    # Upload the files.
    errprinter('uploading files...')
    scp = SCPClient(client.get_transport())
    scp.put(firmware_file, posixpath.join('/log/updates/', posixpath.basename(firmware_file)))
    scp.put(firmware_md5, posixpath.join('/log/updates/', posixpath.basename(firmware_md5)))
    scp.close()

    if target == 'controller':
        errprinter("starting update on the Controller...")
    else:
        errprinter("starting update on Solo...")

    if args['--clean']:
        errprinter('marking all user-local changes to be reset...')
        code = soloutils.command_stream(client, 'sololink_config --update-apply sololink --reset')
    else:
        code = soloutils.command_stream(client, 'sololink_config --update-apply sololink')
    # Fallback to earlier versions.
    if code != 0:
        if args['--clean']:
            code = soloutils.command_stream(client, 'touch /log/updates/UPDATE && touch /log/updates/RESETSETTINGS && shutdown -r now')
        else:
            code = soloutils.command_stream(client, 'touch /log/updates/UPDATE && shutdown -r now')
        
    if target == 'controller':
        errprinter('the Controller will update once it reboots!')
    else:
        errprinter('Solo will update once it reboots!')

    dt = datetime.today() + timedelta(minutes=4)
    errprinter('please wait up to four minutes longer for the installation to complete (by {}).'.format(dt.strftime('%-I:%M')))

    # Complete!
    client.close()

    return code
Esempio n. 24
0
def get_trace(filename, fileno):
    data = get_data(filename)
    label = ""
    if fileno % 2 == 1:
        label = posixpath.basename(filename).split(".")[0] + ":before stimulation"
    else:
        label = posixpath.basename(filename).split(".")[0] + ":after stimulation"
    trace = go.Histogram(
        x=data,
        name=label,
        opacity=0.75,
        histnorm='density'
    )
    return trace
Esempio n. 25
0
        def test_realpath_resolve_before_normalizing(self):
            # Bug #990669: Symbolic links should be resolved before we
            # normalize the path. E.g.: if we have directories 'a', 'k' and 'y'
            # in the following hierarchy:
            # a/k/y
            #
            # and a symbolic link 'link-y' pointing to 'y' in directory 'a',
            # then realpath("link-y/..") should return 'k', not 'a'.
            try:
                os.mkdir(ABSTFN)
                os.mkdir(ABSTFN + "/k")
                os.mkdir(ABSTFN + "/k/y")
                os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y")

                # Absolute path.
                self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k")
                # Relative path.
                with support.change_cwd(dirname(ABSTFN)):
                    self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."),
                                     ABSTFN + "/k")
            finally:
                test_support.unlink(ABSTFN + "/link-y")
                safe_rmdir(ABSTFN + "/k/y")
                safe_rmdir(ABSTFN + "/k")
                safe_rmdir(ABSTFN)
Esempio n. 26
0
 def get_poster(self, only_cached=False):
     """Downloads this poster to a local cache and returns the path"""
     from flexget.manager import manager
     base_dir = os.path.join(manager.config_base, 'userstatic')
     if os.path.isfile(os.path.join(base_dir, self.poster_file or '')):
         return self.poster_file
     elif only_cached:
         return
     # If we don't already have a local copy, download one.
     url = get_mirror('banner') + self.poster
     log.debug('Downloading poster %s', url)
     dirname = os.path.join('tvdb', 'posters')
     # Create folders if the don't exist
     fullpath = os.path.join(base_dir, dirname)
     if not os.path.isdir(fullpath):
         os.makedirs(fullpath)
     filename = os.path.join(dirname, posixpath.basename(self.poster))
     thefile = file(os.path.join(base_dir, filename), 'wb')
     thefile.write(requests.get(url).content)
     self.poster_file = filename
     # If we are detached from a session, update the db
     if not Session.object_session(self):
         with Session() as session:
             session.query(TVDBSeries).filter(TVDBSeries.id == self.id).update(values={'poster_file': filename})
     return filename
Esempio n. 27
0
def download(url,output_dir,retry=False):
	global tried_urls, failed_urls
	url_hash=hashlib.sha224(url.encode('utf-8')).digest()
	if url_hash in tried_urls:
		return
	pool_sema.acquire() 
	path = urllib.parse.urlsplit(url).path
	filename = posixpath.basename(path)
	if len(filename)>40:
		filename=filename[:36]+filename[-4:]
	while os.path.exists(output_dir + '/' + filename):
		filename = str(random.randint(0,100)) + filename
	in_progress.append(filename)
	try:
		request=urllib.request.Request(url,None,urlopenheader)
		image=urllib.request.urlopen(request).read()
		if len(image)==0:
			print('no image')
		imagefile=open(output_dir + '/' + filename,'wb')
		imagefile.write(image)
		imagefile.close()
		in_progress.remove(filename)
		if retry:
			print('Retry OK '+ filename)
		else:
			print("OK " + filename)
		tried_urls.append(url_hash)
	except Exception as e:
		if retry:
			print('Retry Fail ' + filename)
		else:
			print("FAIL " + filename)
			failed_urls.append((url, output_dir))
	pool_sema.release()
Esempio n. 28
0
    def __call__(self, args={'--quiet': False, '--verbose': False},
                 fmt=None, name=None, stream=sys.stderr):

        if not name:
            name = posixpath.basename(sys.argv[0])

        if name in self.loggers:
            return self.loggers[name]

        log = logging.getLogger(name)

        if not log.handlers:
            if fmt == 'brief':
                fmt = '[%(levelname)-4s] %(message)s'
            elif fmt == 'msg':
                fmt = '%(message)s'
            else:
                fmt = '%(asctime)s %(levelname)-6s %(name)s %(message)s'
            logging.basicConfig(stream=stream, format=fmt)

        if not log.level:  # pragma: no cover
            if args.get('--quiet'):
                level = logging.ERROR
            elif args.get('--debug'):
                level = logging.DEBUG
            else:
                level = logging.INFO

            log.setLevel(level)
        self.loggers[name] = log
        return log
def download_and_cache(url, session=None):
    """This function downloads a file into a folder called 
    original_data and returns the local filepath."""
    path = urllib.parse.urlsplit(url).path
    filename = posixpath.basename(path)
    filepath = "input/original_data/" + filename
    print(url)

    # check if file exists, if not download it
    filepath = "input/original_data/" + filename
    print(filepath)
    if not os.path.exists(filepath):
        if not session:
            print('No session')
            session = requests.session()
        
        print("Downloading file: ", filename)
        r = session.get(url, stream=True)

        chuncksize = 1024
        with open(filepath, 'wb') as file:
            for chunck in r.iter_content(chuncksize):
                file.write(chunck)
    else:
        print("Using local file from", filepath)
    filepath = '' + filepath
    return filepath
def main(args, loglevel):
    logging.basicConfig(format="%(levelname)s: %(message)s", level=loglevel)

    socket.setdefaulttimeout(args.timeout)
    
    g = Github()
    with open(args.repo_file, 'r') as f:
        file_counter = 0
        for line in f.readlines():
            logging.info('Fetching repository: %s' % line)
            try:
                repo_str = line.rstrip().split('github.com/')[-1]
                repo = g.get_repo(repo_str)
                tree = repo.get_git_tree('master', recursive=True)
                files_to_download = []
                for file in tree.tree:
                    if fnmatch.fnmatch(file.path, args.wildcard):
                        files_to_download.append('https://github.com/%s/raw/master/%s' % (repo_str, file.path))
                for file in files_to_download:
                    logging.info('Downloading %s' % file)
                    file_counter += 1
                    filename = posixpath.basename(urlparse.urlsplit(file).path)
                    output_path = os.path.join(args.output_dir, filename)
                    if os.path.exists(output_path):
                        output_path += "-" + str(file_counter)
                    try:
                        urllib.urlretrieve(file, output_path)
                    except:
                        logging.error('Error downloading %s' % file)
            except:
                 logging.error('Error fetching repository %s' % line)
Esempio n. 31
0
 def basename(self, path):
     if self.sys.platform.startswith('win'):
         return ntpath.basename(path)
     else:
         return posixpath.basename(path)
Esempio n. 32
0
    def buttonsSlot(self, ddict, emit=True):
        _logger.debug("buttonsSlot(self, %s,emit=%s)", ddict, emit)
        if self.data is None:
            return
        action, selectionType = ddict['action'].split()
        entryList = self.getSelectedEntries()
        if not len(entryList):
            return
        text = qt.safe_str(self.tableTab.tabText(self.tableTab.currentIndex()))
        mcaSelection = {'mcalist': [], 'selectionindex': []}
        cntSelection = {'cntlist': [], 'y': []}
        if text.upper() == "AUTO":
            cntSelection = self.autoTable.getCounterSelection()
            # self._aliasList = cntSelection['aliaslist']
        elif text.upper() == "MCA":
            mcaSelection = self.mcaTable.getMcaSelection()
        else:
            cntSelection = self.cntTable.getCounterSelection()
            self._aliasList = cntSelection['aliaslist']
        selectionList = []
        for entry, filename in entryList:
            if not len(cntSelection['cntlist']) and \
               not len(mcaSelection['mcalist']):
                continue
            if not len(cntSelection['y']) and \
               not len(mcaSelection['selectionindex']):
                #nothing to plot
                continue
            mcaIdx = 0
            for yMca in mcaSelection['selectionindex']:
                sel = {}
                sel['SourceName'] = self.data.sourceName * 1
                sel['SourceType'] = "HDF5"
                fileIndex = self.data.sourceName.index(filename)
                phynxFile = self.data._sourceObjectList[fileIndex]
                entryIndex = list(phynxFile["/"].keys()).index(entry[1:])
                sel['Key'] = "%d.%d" % (fileIndex + 1, entryIndex + 1)
                sel['legend']     = os.path.basename(sel['SourceName'][0])+\
                                    " " + posixpath.basename(entry) #it was sel['Key']
                sel['selection'] = {}
                sel['selection']['sourcename'] = filename
                #deal with the case the "entry" is a dataset hunging at root level
                if isinstance(phynxFile[entry], h5py.Dataset):
                    entry = "/"
                sel['selection']['entry'] = entry
                sel['selection']['key'] = "%d.%d" % (fileIndex + 1,
                                                     entryIndex + 1)
                sel['selection']['mca'] = [yMca]
                sel['selection']['mcaselectiontype'] = mcaSelection[
                    'selectiontype'][mcaIdx]
                mcaIdx += 1
                sel['selection']['mcalist'] = mcaSelection['mcalist']
                sel['selection']['LabelNames'] = mcaSelection['aliaslist']
                #sel['selection']['aliaslist'] = cntSelection['aliaslist']
                sel['selection']['selectiontype'] = "MCA"
                sel['mcaselection'] = True
                aliases = mcaSelection['aliaslist']
                selectionList.append(sel)

            for yCnt in cntSelection['y']:
                sel = {}
                sel['SourceName'] = self.data.sourceName * 1
                sel['SourceType'] = "HDF5"
                fileIndex = self.data.sourceName.index(filename)
                phynxFile = self.data._sourceObjectList[fileIndex]
                if entry == "/":
                    entryIndex = 1
                else:
                    entryIndex = list(phynxFile["/"].keys()).index(entry[1:])
                sel['Key'] = "%d.%d" % (fileIndex + 1, entryIndex + 1)
                sel['legend']     = os.path.basename(sel['SourceName'][0])+\
                                    " " + posixpath.basename(entry) #it was sel['Key']
                sel['selection'] = {}
                sel['selection']['sourcename'] = filename
                #deal with the case the "entry" is a dataset hunging at root level
                if isinstance(phynxFile[entry], h5py.Dataset):
                    entry = "/"
                sel['selection']['entry'] = entry
                sel['selection']['key'] = "%d.%d" % (fileIndex + 1,
                                                     entryIndex + 1)
                sel['selection']['x'] = cntSelection['x']
                sel['selection']['y'] = [yCnt]
                sel['selection']['m'] = cntSelection['m']
                sel['selection']['cntlist'] = cntSelection['cntlist']
                sel['selection']['LabelNames'] = cntSelection['aliaslist']
                #sel['selection']['aliaslist'] = cntSelection['aliaslist']
                sel['selection']['selectiontype'] = selectionType
                if selectionType.upper() == "SCAN":
                    if cntSelection['cntlist'][yCnt].startswith("/"):
                        actualDatasetPath = posixpath.join(
                            entry, cntSelection['cntlist'][yCnt][1:])
                    else:
                        actualDatasetPath = posixpath.join(
                            entry, cntSelection['cntlist'][yCnt])
                    actualDataset = phynxFile[actualDatasetPath]
                    sel['scanselection'] = True
                    if hasattr(actualDataset, "shape"):
                        if len(actualDataset.shape) > 1:
                            if 1 in actualDataset.shape[-2:]:
                                #shape (1, n) or (n, 1)
                                pass
                            else:
                                # at least twoD dataset
                                selectionType = "2D"
                                sel['scanselection'] = False
                    sel['mcaselection'] = False
                elif selectionType.upper() == "MCA":
                    sel['scanselection'] = False
                    sel['mcaselection'] = True
                    if cntSelection['cntlist'][yCnt].startswith("/"):
                        actualDatasetPath = posixpath.join(
                            entry, cntSelection['cntlist'][yCnt][1:])
                    else:
                        actualDatasetPath = posixpath.join(
                            entry, cntSelection['cntlist'][yCnt])
                    actualDataset = phynxFile[actualDatasetPath]
                    if hasattr(actualDataset, "shape"):
                        actualDatasetLen = len(actualDataset.shape)
                        if (actualDatasetLen
                                == 2) and (1 in actualDataset.shape):
                            # still can be used
                            pass
                        elif (actualDatasetLen > 1) and (not hasattr(
                                self, "_messageShown")):
                            # at least twoD dataset
                            msg = qt.QMessageBox(self)
                            msg.setIcon(qt.QMessageBox.Information)
                            msg.setText(
                                "Multidimensional data set as MCA. Using Average. You should use ROI Imaging"
                            )
                            msg.exec_()
                            self._messageShown = True
                        sel['selection']['mcaselectiontype'] = "avg"
                else:
                    sel['scanselection'] = False
                    sel['mcaselection'] = False
                sel['selection']['selectiontype'] = selectionType
                aliases = cntSelection['aliaslist']
                if len(cntSelection['x']) and len(cntSelection['m']):
                    addLegend = " (%s/%s) vs %s" % (aliases[yCnt], aliases[
                        cntSelection['m'][0]], aliases[cntSelection['x'][0]])
                elif len(cntSelection['x']):
                    addLegend = " %s vs %s" % (aliases[yCnt],
                                               aliases[cntSelection['x'][0]])
                elif len(cntSelection['m']):
                    addLegend = " (%s/%s)" % (aliases[yCnt],
                                              aliases[cntSelection['m'][0]])
                else:
                    addLegend = " %s" % aliases[yCnt]
                sel['legend'] += addLegend
                selectionList.append(sel)

        if not emit:
            return selectionList
        self._lastAction = "%s" % ddict['action']
        if len(selectionList):
            if selectionType.upper() in ["SCAN", "MCA"]:
                ddict = {}
                ddict['event'] = "SelectionTypeChanged"
                ddict['SelectionType'] = selectionType.upper()
                self.sigOtherSignals.emit(ddict)
            if action.upper() == "ADD":
                self.sigAddSelection.emit(selectionList)
            if action.upper() == "REMOVE":
                self.sigRemoveSelection.emit(selectionList)
            if action.upper() == "REPLACE":
                self.sigReplaceSelection.emit(selectionList)
Esempio n. 33
0
def is_special_file(name):
    """Returns true if a file is some special Git metadata and not content."""
    return posixpath.basename(name) in [
        '.gitignore', '.gitattributes', '.gitmodules'
    ]
Esempio n. 34
0
    def show_url(self):

        # type: () -> str

        return posixpath.basename(self._url.split('#', 1)[0].split('?', 1)[0])
Esempio n. 35
0
def url_matches_criterion(purl: 'ParseResult', url: str, unquoted_path: str, mc: MatchCriteria) -> bool:
    if mc.type == 'url':
        import re
        try:
            pat = re.compile(mc.value)
        except re.error:
            return False
        return pat.search(unquote(url)) is not None

    if mc.type == 'mime':
        import fnmatch
        mt = guess_type(unquoted_path)
        if not mt:
            return False
        mt = mt.lower()
        for mpat in mc.value.split(','):
            mpat = mpat.strip()
            with suppress(Exception):
                if fnmatch.fnmatchcase(mt, mpat):
                    return True
        return False

    if mc.type == 'ext':
        if not purl.path:
            return False
        path = unquoted_path.lower()
        for ext in mc.value.split(','):
            ext = ext.strip()
            if path.endswith('.' + ext):
                return True
        return False

    if mc.type == 'protocol':
        protocol = (purl.scheme or 'file').lower()
        for key in mc.value.split(','):
            if key.strip() == protocol:
                return True
        return False

    if mc.type == 'fragment_matches':
        import re
        try:
            pat = re.compile(mc.value)
        except re.error:
            return False

        return pat.search(unquote(purl.fragment)) is not None

    if mc.type == 'path':
        import fnmatch
        try:
            return fnmatch.fnmatchcase(unquoted_path.lower(), mc.value)
        except Exception:
            return False

    if mc.type == 'file':
        import fnmatch
        import posixpath
        try:
            fname = posixpath.basename(unquoted_path)
        except Exception:
            return False
        try:
            return fnmatch.fnmatchcase(fname.lower(), mc.value)
        except Exception:
            return False
Esempio n. 36
0
 def hdf5Slot(self, ddict):
     entryName = NexusTools.getEntryName(ddict['name'])
     currentEntry = "%s::%s" % (ddict['file'], entryName)
     if (currentEntry != self._lastEntry) and not self._BUTTONS:
         self._lastEntry = None
         cntList = []
         mcaList = []
         aliasList = []
         measurement = None
         scanned = []
         mcaList = []
         if posixpath.dirname(entryName) != entryName:
             with h5py.File(ddict['file'], "r") as h5file:
                 measurement = NexusTools.getMeasurementGroup(
                     h5file, ddict['name'])
                 scanned = NexusTools.getScannedPositioners(
                     h5file, ddict['name'])
                 if measurement is not None:
                     measurement = [item.name for key,item in measurement.items() \
                                    if self._isNumeric(item)]
                 if self._mca:
                     mcaList = NexusTools.getMcaList(h5file, entryName)
         for i in range(len(scanned)):
             key = scanned[i]
             cntList.append(key)
             aliasList.append(posixpath.basename(key))
         if measurement is not None:
             for key in measurement:
                 if key not in cntList:
                     cntList.append(key)
                     aliasList.append(posixpath.basename(key))
         cleanedCntList = []
         for key in cntList:
             root = key.split('/')
             root = "/" + root[1]
             if len(key) == len(root):
                 cleanedCntList.append(key)
             else:
                 cleanedCntList.append(key[len(root):])
         self._autoAliasList = aliasList
         self._autoCntList = cleanedCntList
         self.autoTable.build(self._autoCntList, self._autoAliasList)
         currentTab = qt.safe_str(self.tableTab.tabText( \
                                 self.tableTab.currentIndex()))
         if self._mca:
             mcaAliasList = []
             cleanedMcaList = []
             for key in mcaList:
                 root = key.split('/')
                 root = "/" + root[1]
                 if len(key) == len(root):
                     cleanedMcaList.append(key)
                 else:
                     cleanedMcaList.append(key[len(root):])
                 mcaAliasList.append(posixpath.basename(key))
             self.mcaTable.build(cleanedMcaList, mcaAliasList)
             nTabs = self.tableTab.count()
             if (len(mcaList) > 0) and (nTabs < 3):
                 self.tableTab.insertTab(2, self.mcaTable, "MCA")
             elif (len(mcaList) == 0) and (nTabs > 2):
                 self.tableTab.removeTab(2)
         _logger.debug("currentTab = %s", currentTab)
         if currentTab != "USER":
             if (len(mcaList) > 0) and (len(cntList) == 0):
                 idx = self.tableTabOrder.index("MCA")
                 self.tableTab.setCurrentIndex(idx)
                 _logger.debug("setting tab = %s MCA", idx)
             elif (len(mcaList) == 0) and (len(cntList) > 0):
                 idx = self.tableTabOrder.index("AUTO")
                 self.tableTab.setCurrentIndex(idx)
                 _logger.debug("setting tab = %s AUTO", idx)
         self._lastEntry = currentEntry
     if ddict['event'] == 'itemClicked':
         if ddict['mouse'] == "right":
             return self.itemRightClickedSlot(ddict)
         if ddict['mouse'] == "left":
             # If parent is root do it even if not NXentry??
             if ddict['type'] in ['NXentry', 'Entry']:
                 if not self._BUTTONS:
                     auto = self.actions.getConfiguration()["auto"]
                     if auto == "ADD":
                         self._addAction()
                     elif auto == "REPLACE":
                         self._replaceAction()
     if ddict['event'] == "itemDoubleClicked":
         if ddict['type'] in ['Dataset']:
             currentIndex = self.tableTab.currentIndex()
             text = safe_str(self.tableTab.tabText(currentIndex))
             if text.upper() != "USER":
                 if currentIndex == 0:
                     self.tableTab.setCurrentIndex(1)
                 else:
                     self.tableTab.setCurrentIndex(0)
             if not self._isNumericType(ddict['dtype']):
                 _logger.debug("string like %s", ddict['dtype'])
             else:
                 root = ddict['name'].split('/')
                 root = "/" + root[1]
                 if len(ddict['name']) == len(root):
                     cnt = ddict['name']
                 else:
                     cnt = ddict['name'][len(root):]
                 if cnt not in self._cntList:
                     self._cntList.append(cnt)
                     basename = posixpath.basename(cnt)
                     if basename not in self._aliasList:
                         self._aliasList.append(basename)
                     else:
                         self._aliasList.append(cnt)
                     self.cntTable.build(self._cntList, self._aliasList)
         elif (ddict['color'] == qt.Qt.blue) and ("silx" in sys.modules):
             # there is an action to be applied
             self.showInfoWidget(ddict["file"], ddict["name"], dset=False)
         elif ddict['type'] in ['NXentry', 'Entry']:
             if self._lastAction is None:
                 return
             action, selectionType = self._lastAction.split()
             if action == 'REMOVE':
                 action = 'ADD'
             ddict['action'] = "%s %s" % (action, selectionType)
             self.buttonsSlot(ddict)
         else:
             if self.data is not None:
                 name = ddict['name']
                 filename = ddict['file']
                 fileIndex = self.data.sourceName.index(filename)
                 phynxFile = self.data._sourceObjectList[fileIndex]
                 dataset = phynxFile[name]
                 if isinstance(dataset, h5py.Dataset):
                     root = ddict['name'].split('/')
                     root = "/" + root[1]
                     cnt = ddict['name'].split(root)[-1]
                     if cnt not in self._cntList:
                         _logger.debug("USING SECOND WAY")
                         self._cntList.append(cnt)
                         basename = posixpath.basename(cnt)
                         if basename not in self._aliasList:
                             self._aliasList.append(basename)
                         else:
                             self._aliasList.append(cnt)
                         self.cntTable.build(self._cntList, self._aliasList)
                     return
             _logger.debug("Unhandled item type: %s", ddict['dtype'])
Esempio n. 37
0
 def show_url(self):
     # type: () -> Optional[str]
     return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
Esempio n. 38
0
    def __str__(self):
        """String representation of the primitive"""

        # Build a list of files to make the logic a bit simpler below.
        # The items in the files list are dictionaries with keys 'src'
        # and 'dest'.
        files = []
        if self.__files:
            # Sort to make it deterministic
            files.extend([{
                'dest': dest,
                'src': src
            } for src, dest in sorted(self.__files.items())])
        elif self.__dest and self.__src:
            files.append({'dest': self.__dest, 'src': self.__src})
        else:
            # No files!
            return ''

        if hpccm.config.g_ctype == container_type.DOCKER:
            if self._app:
                logging.warning('The Singularity specific SCI-F syntax '
                                'was requested. Docker does not have an '
                                'equivalent: using regular COPY!')

            # Format:
            # COPY src1 \
            #     src2 \
            #     src3 \
            #     dest/
            # COPY src1 dest1
            # COPY src2 dest2
            # COPY src3 dest3
            base_inst = 'COPY '

            if self.__chown:
                base_inst = base_inst + '--chown={} '.format(self.__chown)

            if self.__from:
                base_inst = base_inst + '--from={} '.format(self.__from)

            # Docker does not have the notion of copying a set of
            # files to different locations inside the container in a
            # single instruction.  So generate multiple COPY
            # instructions in that case.
            instructions = []
            for pair in files:
                dest = pair['dest']
                src = pair['src']
                c = [base_inst]

                if isinstance(src, list):
                    c[0] = c[0] + src[0]
                    c.extend(['    {}'.format(x) for x in src[1:]])
                    # Docker requires a trailing slash.  Add one if missing.
                    c.append('    {}'.format(posixpath.join(dest, '')))
                else:
                    c[0] = c[0] + '{0} {1}'.format(src, dest)

                instructions.append(' \\\n'.join(c))

            return '\n'.join(instructions)

        elif hpccm.config.g_ctype == container_type.SINGULARITY:
            # Format:
            # %files
            #     src1 dest
            #     src2 dest
            #     src3 dest
            # %files
            #     src1 dest1
            #     src2 dest2
            #     src3 dest3
            section = '%files'
            if (self.__from and hpccm.config.g_singularity_version >=
                    StrictVersion('3.2')):
                section = section + ' from {}'.format(self.__from)

            if self._app:
                # SCIF appfiles does not support "from"
                section = '%appfiles {0}'.format(self._app)

            # Singularity will error if the destination does not
            # already exist in the container.  The workarounds are to
            # either 1) prior to copying the files, create the
            # destination directories with %setup or 2) copy the files
            # to a path guaranteed to exist, "/", and then move them
            # later with %post.  Option 1 is the "pre" approach,
            # option 2 is the "post" approach.
            flat_files = []
            post = []  # post actions if _post is enabled
            pre = []  # pre actions if _mkdir is enabled
            for pair in files:
                dest = pair['dest']
                src = pair['src']

                if self._post:
                    dest = '/'

                if isinstance(src, list):
                    for s in src:
                        flat_files.append('    {0} {1}'.format(s, dest))

                        if self._post:
                            post.append('    mv /{0} {1}'.format(
                                posixpath.basename(s),
                                posixpath.join(pair['dest'], s)))
                    if (self._mkdir and posixpath.dirname(dest) != '/'
                            and posixpath.basename(dest) != dest):
                        # When multiple files are to be copied to the
                        # same destination, assume the destination is
                        # a directory
                        pre.append(
                            '    mkdir -p ${{SINGULARITY_ROOTFS}}{0}'.format(
                                dest))
                else:
                    flat_files.append('    {0} {1}'.format(src, dest))
                    if (self._mkdir and posixpath.dirname(dest) != '/'
                            and posixpath.basename(dest) != dest):
                        # When a single file is to be copied to a
                        # destination, assume the destination is a
                        # file.
                        pre.append(
                            '    mkdir -p ${{SINGULARITY_ROOTFS}}{0}'.format(
                                posixpath.dirname(dest)))
                    elif self._post:
                        post.append('    mv /{0} {1}'.format(
                            posixpath.basename(src), pair['dest']))

            s = ''
            if pre:
                s += '%setup\n' + '\n'.join(pre) + '\n'
            s += section + '\n' + '\n'.join(flat_files)
            if post:
                s += '\n%post\n' + '\n'.join(post)

            return s

        elif hpccm.config.g_ctype == container_type.BASH:
            logging.warning('copy primitive does not map into bash')
            return ''
        else:
            raise RuntimeError('Unknown container type')
Esempio n. 39
0
 async def translate_cell(self,
                          cell: JupyterCell,
                          metadata: Optional[MutableMapping[Text, Any]],
                          autoawait: bool = False) -> Step:
     # Build execution target
     target = metadata.get('target')
     if target is not None:
         if isinstance(target['model'], Text):
             target = {**target, **{'model': metadata['models'][target['model']]}}
         model_name = hashlib.md5(json.dumps(
             obj=target['model'],
             sort_keys=True,
             ensure_ascii=True,
             default=str).encode('ascii')).hexdigest()
         target = _build_target(model_name, target)
     # Extract Python interpreter from metadata
     interpreter = metadata.get('interpreter', 'ipython')
     # Create a step structure
     step = BaseStep(
         name=metadata['cell_id'],
         context=self.context,
         target=target)
     step.workdir = metadata['step'].get('workdir')
     # Process cell inputs
     cell_inputs = metadata['step'].get('in', [])
     input_tokens = {}
     scatter_inputs = _get_scatter_inputs(metadata['step'].get('scatter'))
     for element in cell_inputs:
         # If is a string, it refers to the name of a variable
         if isinstance(element, Text):
             element = {
                 'type': 'name',
                 'name': element
             }
         # Otherwise it must be a dictionary
         if isinstance(element, MutableMapping):
             # Create input port
             name = element.get('name') or element.get('valueFrom')
             if name in scatter_inputs:
                 port = ScatterInputPort(name=name, step=step)
             else:
                 port = DefaultInputPort(name=name, step=step)
             # Get serializer if present
             serializer = (metadata['serializers'][element['serializer']]
                           if isinstance(element['serializer'], Text)
                           else element['serializer']) if 'serializer' in element else None
             # Process port type
             element_type = element['type']
             # If type is equal to `file`, it refers to a file path in the local resource
             if element_type == 'file':
                 port.token_processor = FileTokenProcessor(
                     port=port,
                     value=element.get('value'),
                     value_from=element.get('valueFrom'))
             # If type is equal to `name` or `env`, it refers to a variable
             elif element_type in ['name', 'env']:
                 port.token_processor = NameTokenProcessor(
                     port=port,
                     serializer=serializer,
                     compiler=cell.compiler,
                     value=element.get('value'),
                     value_from=element.get('valueFrom', name))
             # If type is equal to `control`, simply add an empty dependency
             elif element_type == 'control':
                 port.token_processor = ControlTokenProcessor(port=port)
             # Add command token
             input_tokens[name] = JupyterCommandToken(
                 name=name,
                 token_type=element_type,
                 serializer=serializer)
             # Register step port
             step.input_ports[port.name] = port
     # Retrieve inputs automatically if necessary
     if metadata['step'].get('autoin', True):
         input_names = _extract_dependencies(cell.name, cell.compiler, cell.code)
         for name in input_names:
             if name not in step.input_ports:
                 if name in scatter_inputs:
                     port = ScatterInputPort(name=name, step=step)
                 else:
                     port = DefaultInputPort(name=name, step=step)
                 port.token_processor = NameTokenProcessor(
                     port=port,
                     compiler=cell.compiler,
                     value_from=name)
                 input_tokens[name] = JupyterCommandToken(name=name, token_type='name')
                 step.input_ports[port.name] = port
     # If outputs are defined for the current cell
     output_tokens = {}
     if 'out' in metadata['step']:
         for element in metadata['step']['out']:
             # If is a string, it refers to the name of a variable
             if isinstance(element, Text):
                 element = {
                     'type': 'name',
                     'name': element
                 }
             # Otherwise it must be a dictionary
             if isinstance(element, MutableMapping):
                 # Create port
                 name = element.get('name') or element.get('valueFrom')
                 if scatter_inputs:
                     output_port = GatherOutputPort(
                         name=name,
                         step=step,
                         merge_strategy=lambda values: sorted(values, key=lambda t: int(posixpath.basename(t.tag))))
                 else:
                     output_port = DefaultOutputPort(name=name, step=step)
                 # Get serializer if present
                 serializer = (metadata['serializers'][element['serializer']]
                               if isinstance(element['serializer'], Text)
                               else element['serializer']) if 'serializer' in element else None
                 # Process port type
                 element_type = element['type']
                 # If type is equal to `file`, it refers to a file path in the remote resource
                 if element_type == 'file':
                     output_port.token_processor = FileTokenProcessor(
                         port=output_port,
                         value=element.get('value'),
                         value_from=element.get('valueFrom'))
                 # If type is equal to `name` or `env`, it refers to a variable
                 elif element_type in ['name', 'env']:
                     output_port.token_processor = NameTokenProcessor(
                         port=output_port,
                         compiler=cell.compiler,
                         serializer=serializer,
                         value_from=element.get('valueFrom', name))
                 # If type is equal to `control`, simply add an empty dependency
                 elif element_type == 'control':
                     output_port.token_processor = ControlTokenProcessor(port=output_port)
                 # Add command token
                 output_tokens[name] = JupyterCommandToken(
                     name=name,
                     token_type=element_type,
                     serializer=serializer)
                 # Register step port
                 step.output_ports[name] = output_port
     # Add output log port
     if scatter_inputs:
         output_log_port = GatherOutputPort(
             name=executor.CELL_OUTPUT,
             step=step,
             merge_strategy=lambda values: sorted(values, key=lambda t: int(posixpath.basename(t.tag))))
     else:
         output_log_port = DefaultOutputPort(name=executor.CELL_OUTPUT, step=step)
     output_log_port.token_processor = OutputLogTokenProcessor(port=output_log_port)
     step.output_ports[executor.CELL_OUTPUT] = output_log_port
     # Set input combinator for the step
     step.input_combinator = _get_input_combinator(
         step=step,
         scatter=metadata['step'].get('scatter'))
     # Create the command to be executed remotely
     step.command = JupyterCommand(
         step=step,
         ast_nodes=cell.code,
         compiler=cell.compiler,
         interpreter=interpreter,
         input_tokens=input_tokens,
         output_tokens=output_tokens,
         autoawait=autoawait)
     return step
Esempio n. 40
0
 def axes(self):
     from .dataset import Axis
     return OrderedDict((posixpath.basename(j.name), j) for j in sorted(
         i for i in self.values() if isinstance(i, Axis)))
Esempio n. 41
0
 def signals(self):
     from .dataset import Signal
     return OrderedDict((posixpath.basename(j.name), j) for j in sorted(
         i for i in self.values() if isinstance(i, Signal)))
Esempio n. 42
0
def id_from_url(url):
    from urlparse import urlsplit
    from posixpath import dirname, basename
    path = urlsplit(url).path
    b = basename(path)
    return int(b if b else dirname(path))
Esempio n. 43
0
 def GetBaseName(self):
     if self.base_name is None:
         # We must use posixpath explicitly so this will work on Windows.
         self.base_name = posixpath.basename(
             six.moves.urllib.parse.urlparse(self.full_path).path)
     return self.base_name
Esempio n. 44
0
def GenerateApkAnalysis(apk_filename,
                        tool_prefix,
                        out_dir,
                        unknown_handler=None):
    """Analyse APK to determine size contributions of different file classes."""
    file_groups = []

    def make_group(name):
        group = _FileGroup(name)
        file_groups.append(group)
        return group

    native_code = make_group('Native code')
    java_code = make_group('Java code')
    native_resources_no_translations = make_group('Native resources (no l10n)')
    translations = make_group('Native resources (l10n)')
    stored_translations = make_group('Native resources stored (l10n)')
    icu_data = make_group('ICU (i18n library) data')
    v8_snapshots = make_group('V8 Snapshots')
    png_drawables = make_group('PNG drawables')
    res_directory = make_group('Non-compiled Android resources')
    arsc = make_group('Compiled Android resources')
    metadata = make_group('Package metadata')
    unknown = make_group('Unknown files')
    notices = make_group('licenses.notice file')
    unwind_cfi = make_group('unwind_cfi (dev and canary only)')

    apk = zipfile.ZipFile(apk_filename, 'r')
    try:
        apk_contents = apk.infolist()
    finally:
        apk.close()

    dex_multiplier, skip_extract_lib = _ParseManifestAttributes(apk_filename)
    total_apk_size = os.path.getsize(apk_filename)
    for member in apk_contents:
        filename = member.filename
        if filename.endswith('/'):
            continue
        if filename.endswith('.so'):
            basename = posixpath.basename(filename)
            should_extract_lib = not skip_extract_lib and basename.startswith(
                'lib')
            native_code.AddZipInfo(
                member, extracted_multiplier=int(should_extract_lib))
        elif filename.endswith('.dex'):
            java_code.AddZipInfo(member, extracted_multiplier=dex_multiplier)
        elif re.search(_RE_NON_LANGUAGE_PAK, filename):
            native_resources_no_translations.AddZipInfo(member)
        elif re.search(_RE_COMPRESSED_LANGUAGE_PAK, filename):
            translations.AddZipInfo(
                member,
                extracted_multiplier=int('en_' in filename
                                         or 'en-' in filename))
        elif re.search(_RE_STORED_LANGUAGE_PAK, filename):
            stored_translations.AddZipInfo(member)
        elif filename == 'assets/icudtl.dat':
            icu_data.AddZipInfo(member)
        elif filename.endswith('.bin'):
            v8_snapshots.AddZipInfo(member)
        elif filename.endswith('.png') or filename.endswith('.webp'):
            png_drawables.AddZipInfo(member)
        elif filename.startswith('res/'):
            res_directory.AddZipInfo(member)
        elif filename.endswith('.arsc'):
            arsc.AddZipInfo(member)
        elif filename.startswith(
                'META-INF') or filename == 'AndroidManifest.xml':
            metadata.AddZipInfo(member)
        elif filename.endswith('.notice'):
            notices.AddZipInfo(member)
        elif filename.startswith('assets/unwind_cfi'):
            unwind_cfi.AddZipInfo(member)
        else:
            unknown.AddZipInfo(member)

    total_install_size = total_apk_size
    zip_overhead = total_apk_size

    for group in file_groups:
        actual_size = group.ComputeZippedSize()
        install_size = group.ComputeInstallSize()
        uncompressed_size = group.ComputeUncompressedSize()

        total_install_size += group.ComputeExtractedSize()
        zip_overhead -= actual_size

        yield ('Breakdown', group.name + ' size', actual_size, 'bytes')
        yield ('InstallBreakdown', group.name + ' size', install_size, 'bytes')
        # Only a few metrics are compressed in the first place.
        # To avoid over-reporting, track uncompressed size only for compressed
        # entries.
        if uncompressed_size != actual_size:
            yield ('Uncompressed', group.name + ' size', uncompressed_size,
                   'bytes')

    # Per-file zip overhead is caused by:
    # * 30 byte entry header + len(file name)
    # * 46 byte central directory entry + len(file name)
    # * 0-3 bytes for zipalign.
    yield ('Breakdown', 'Zip Overhead', zip_overhead, 'bytes')
    yield ('InstallSize', 'APK size', total_apk_size, 'bytes')
    yield ('InstallSize', 'Estimated installed size', total_install_size,
           'bytes')
    transfer_size = _CalculateCompressedSize(apk_filename)
    yield ('TransferSize', 'Transfer size (deflate)', transfer_size, 'bytes')

    # Size of main dex vs remaining.
    main_dex_info = java_code.FindByPattern('classes.dex')
    if main_dex_info:
        main_dex_size = main_dex_info.file_size
        yield ('Specifics', 'main dex size', main_dex_size, 'bytes')
        secondary_size = java_code.ComputeUncompressedSize() - main_dex_size
        yield ('Specifics', 'secondary dex size', secondary_size, 'bytes')

    # Size of main .so vs remaining.
    main_lib_info = native_code.FindLargest()
    if main_lib_info:
        main_lib_size = main_lib_info.file_size
        yield ('Specifics', 'main lib size', main_lib_size, 'bytes')
        secondary_size = native_code.ComputeUncompressedSize() - main_lib_size
        yield ('Specifics', 'other lib size', secondary_size, 'bytes')

        main_lib_section_sizes = _ExtractMainLibSectionSizesFromApk(
            apk_filename, main_lib_info.filename, tool_prefix)
        for metric_name, size in main_lib_section_sizes.iteritems():
            yield ('MainLibInfo', metric_name, size, 'bytes')

    # Main metric that we want to monitor for jumps.
    normalized_apk_size = total_apk_size
    # unwind_cfi exists only in dev, canary, and non-channel builds.
    normalized_apk_size -= unwind_cfi.ComputeZippedSize()
    # Always look at uncompressed .so.
    normalized_apk_size -= native_code.ComputeZippedSize()
    normalized_apk_size += native_code.ComputeUncompressedSize()
    # Normalized dex size: size within the zip + size on disk for Android Go
    # devices (which ~= uncompressed dex size).
    normalized_apk_size += java_code.ComputeUncompressedSize()
    # Avoid noise caused when strings change and translations haven't yet been
    # updated.
    num_translations = translations.GetNumEntries()
    num_stored_translations = stored_translations.GetNumEntries()

    if num_translations > 1:
        # Multipliers found by looking at MonochromePublic.apk and seeing how much
        # smaller en-US.pak is relative to the average locale.pak.
        normalized_apk_size = _NormalizeLanguagePaks(translations,
                                                     normalized_apk_size, 1.17)
    if num_stored_translations > 1:
        normalized_apk_size = _NormalizeLanguagePaks(stored_translations,
                                                     normalized_apk_size, 1.43)
    if num_translations + num_stored_translations > 1:
        if num_translations == 0:
            # WebView stores all locale paks uncompressed.
            num_arsc_translations = num_stored_translations
        else:
            # Monochrome has more configurations than Chrome since it includes
            # WebView (which supports more locales), but these should mostly be empty
            # so ignore them here.
            num_arsc_translations = num_translations
        normalized_apk_size += int(
            _NormalizeResourcesArsc(apk_filename, arsc.GetNumEntries(),
                                    num_arsc_translations, out_dir))

    yield ('Specifics', 'normalized apk size', normalized_apk_size, 'bytes')
    yield ('Specifics', 'file count', len(apk_contents), 'zip entries')

    if unknown_handler is not None:
        for info in unknown.AllEntries():
            unknown_handler(info)
Esempio n. 45
0
 def _clone_repository(metadata_repo_url, branch):
     directory_name = posixpath.splitext(posixpath.basename(metadata_repo_url))[0]
     LOGGER.info("Cloning the repository %s into the directory %s",
                 metadata_repo_url, directory_name)
     return git.Repository.clone(metadata_repo_url, directory_name, branch)
Esempio n. 46
0
def url2filename(url):
    urlpath = urlsplit(url).path
    return posixpath.basename(unquote(urlpath))
Esempio n. 47
0
 def sort_key(p):
     return (len(p.split('/')), len(posixpath.basename(p)), p)
    def scrapePageCont(self, code_syntax, url, classname, filepath):

        """
        CLIFF NOTE: ADD FUNCTION COMMENT HERE
        """
        logging.debug('Scraping page')
        soup = BeautifulSoup(code_syntax)
        now = datetime.datetime.now()
        testMethods = []

        #Class name cannot start with digit.  
        m = classname[0].isdigit()
        if m:
            logging.debug('Class name cannot start with a digit')
            classname = "c" + classname
            filepath = "c" + filepath

        #Define variable for page object file name
        filetext = classname
        classname = classname.replace ("_", "")
        filename = dest_dir + "/pages/auto/" + classname.lower() + ".py"
        
        #Create new page object file using file name Class Name
        fh = open(filename, "w")
        logging.debug('URL: ' + filepath)
        logging.debug('FILENAME: ' + classname)        
        fh.write("#!/usr/bin/env python\n")

        #Write comment for creation date and author
        fh.write("\"\"\"")
        fh.write("\nCreated on " + now.strftime("%Y-%m-%d %H:%M"))
        fh.write("\n@author: Selenium Webdriver Code Generator\n")
        fh.write("\"\"\"\n")

        #Create page object header code by manipulating library code
        ph = open("lib/page_header.py", "r")
        fh.write("\n\n" + ph.read())
        ph.close()
        fh.write("\n\nclass " + classname + "(Base):\n") 
        
        #Write main comment for class
        fh.write("    \"\"\"\n")
        fh.write("    Variable definitions of  " + classname + " page object based on site HTML elements.\n") 
        fh.write("    \"\"\"")

        """
        Parse Title:
           (a) HTML Syntax: <title>CSCE 499 - Design Document</title>
           (b) BeautifulSoup Syntax: print soup.head.title
           (c) Variable Definition: _page_title = "CSCE 499 - Design Document"
           (d) Generated Selenium: The generated code should verify that the title string matches the ex-
               pected text as defined by the title variable.

        """
        logging.debug('Parse page title')
        fh.write("\n\n    # Page Title Variable")
        title = soup.head.title.string 
        gen = Generate()
        gen.generateVariable(fh, "_page_title", title)
        logging.debug('Title: ' + title)

        """
        All DIV with ID tag
           (a) HTML Syntax: <div id="banner-top">
           (b) BeautifulSoup Syntax: divs = soup.findAll('div','id':True)
           (c) Variable Definition: _div_banner_top_locator = (By.ID, "banner-top")
           (d) Generated Selenium: The generated code should verify that the DIV ID exists on the page.
        """
        
        logging.debug('Parse all DIVS with ids')
        fh.write("\n\n    # Variables defined by ID element")
        divs = soup.findAll('div',{'id':True})    
        for div in divs:
	    d = div['id']
            logging.debug(d)

            varName = "_id_" + d.replace("-", "_") + "_locator"
            byIdSyntax = "(By.ID, \""+ d + "\")"
            gen.generateVariable(fh, varName, byIdSyntax)
        
        """
        H1 Text
           (a) HTML Syntax: <h1>CSCE 499 Capstone - 2012-2013</h1>
           (b) BeautifulSoup Syntax: h1s = soup.findAll('h1')
           (c) Variable Definition: _h1_CSCE_499_Capstone = "CSCE 499 Capstone - 2012-2013"
           (d) Generated Selenium: The generated code should verify that the H1 text exists and matches
               the expected text as defined by the specific H1 variable.
        """
        #CLIFF NOTE: Possibly delete or add debug condition for printing.
        logging.debug('Parse H1')
        h1s = soup.findAll('h1')    
        for h1 in h1s:
	    logging.debug(h1)

        """
        Image Files
           (a) HTML Syntax: <img src="/home-assets/images/banner-images/honors.jpg" alt="" border="0">
           (b) BeautifulSoup Syntax: imgs = soup.findAll('img')
           (c) Variable Definition: _img_honors_locator = (By.SRC, "/home-assets/images/banner-images/honors
           (d) Generated Selenium: The generat

        """
        #CLIFF NOTE: Possibly delete or add debug condition for printing.
        logging.debug('Parse Image')
        imgs = soup.findAll('img')    
        for img in imgs:
            logging.debug(img)
        """
        CLIFF NOTE: Re-investigate this one to see if it is really needed for testing
        """
        logging.debug('Parse iFrames')
        iframes = soup.findAll('iframe')    
        for iframe in iframes:
            logging.debug(iframe)

        """
        Radio Buttons
           (a) HTML Syntax: <input type="radio" name="group1" value="Cheese">
           (b) BeautifulSoup Syntax: radios = soup.findAll('input','type':'radio')
           (c) Variable Definition: _radio_cheese_locator = (By.Value, "Cheese")
           (d) Generated Selenium: The generated code should verify that the radio button exists, is selectable
               and simulates the "click" action in a basic sanity test.
        """
        #CLIFF NOTE: Possibly delete or add debug condition for printing.
        logging.debug('Parse Radio Buttons')
        radios = soup.findAll('input',{'type':'radio'})
        for radio in radios:
            logging.debug(radio)

        """
        Checkboxes
           (a) HTML Syntax: <input type="checkbox" name="sports" value="football">
           (b) BeautifulSoup Syntax: checkboxes = soup.findAll('input','type':'checkbox')
           (c) Variable Definition: _checkbox_sports_locator = (By.Value, "football")
           (d) Generated Selenium: The generated code should verify that the check box exists, is selectable
               and simulates the "click" action in a basic sanity test.
        """
        #CLIFF NOTE: Possibly delete or add debug condition for printing.
        logging.debug('Parse Checkbox Buttons')
        checkboxes = soup.findAll('input',{'type':'checkbox'})
        for checkbox in checkboxes:
            logging.debug(checkbox)

        """
        Text Fields
           (a) HTML Syntax: <textarea name="comments" cols="25" rows="5">Enter your comments here...
               </textarea>
           (b) BeautifulSoup Syntax: textboxes = soup.findAll('input','type':'text')
           (c) Variable Definition: _text_comments_locator = (By.Name, "comments")
           (d) Generated Selenium: The generated code should verify that the text area exists, is accessible
               by typing sample text and does not fail any SQL Injection negative test cases.
        """

        logging.debug('Parse Text Boxes')
        textboxes = soup.findAll('input',{'type':'text'})
        for textbox in textboxes:

            logging.debug(textbox)
            text_id = textbox.get('id')
            text_class = textbox.get('class')

            if text_id != "" and text_class != "" :
                out_str = ""
                for tc in text_class:
                    out_str += tc + "."
                
                if out_str[-1:] == '.':
                    out_str = out_str[:-1]

                #Print Class
                logging.debug(out_str)
                out_str2 = out_str.replace("-", "_")
                out_str2 = out_str2.replace(".", "_")
                varTextboxClass = "_textbox_class_" + out_str2 + "_locator"
                byCssSelectorSyntax = "(By.CSS_Selector, \""+ out_str + "\")"
                gen.generateVariable(fh, varTextboxClass, byCssSelectorSyntax)
                fh.write("\n")
                gen.generateFindElement(fh, varTextboxClass)

            if text_id  != "" and text_class == "":
                logging.debug(text_id)
                varTextboxId = "_textbox_id_" + out_str.replace("-", "_") + "_locator"
                byIdSyntax = "(By.ID, \""+ d + "\")"
                gen.generateVariable(fh, varTextboxId, byIdSyntax)
                fh.write("\n")
                gen.generateFindElement(fh, varTextboxId)


            if text_id == "" and text_class  != "":
                out_str = ""
                for tc in text_class:
                    out_str += tc + "."
                
                if out_str[-1:] == '.':
                    out_str = out_str[:-1]

                #Print Class
                logging.debug(out_str)
        
                out_str2 = out_str.replace("-", "_")
                out_str2 = out_str2.replace(".", "_")
                varTextboxClass = "_textbox_class_" + out_str2 + "_locator"
                byCssSelectorSyntax = "(By.CSS_Selector, \""+ out_str + "\")"
                gen.generateVariable(fh, varTextboxClass, byCssSelectorSyntax)
                fh.write("\n")                
                gen.generateFindElement(fh, varTextboxClass)
   
        """
        Password Text Fields
           (a) HTML Syntax: <input type="password" size="25">
           (b) BeautifulSoup Syntax: passwords = soup.findAll('input','type':'password')
           (c) Variable Definition: _text_password_locator = (By.Type, "password")
           (d) Generated Selenium: The generated code should verify that the password text area exists, is
               accessible by typing hidden characters and does not fail any negative test case scenarios (e.g., SQL
               Injections and Security Tests).
        """
        #CLIFF NOTE: Possibly delete or add debug condition for printing.
        logging.debug('Parse Text Box')
        passwords = soup.findAll('input',{'type':'password'})
        for password in passwords:
            logging.debug(password)
    
        """
        Buttons
           (a) HTML Syntax: <input type="submit" value="Submit">
           (b) BeautifulSoup Syntax: button = soup.findAll('input','type':'submit')
           (c) Variable Definition: _button_submit_locator = (By.Type, "submit")
           (d) Generated Selenium: The generated code should verify that the button exists, is able to be
               clicked and functions correctly when selected.
        """
        #CLIFF NOTE: Possibly delete or add debug condition for printing.
        logging.debug('Parse Buttons')
        buttons = soup.findAll('input',{'type':'submit'})
        for button in buttons:
            logging.debug(button)

        #Extract base name for url
        o = urlparse.urlparse(url)
        bname = basename(o.path)
        logging.debug('Base Name: ' + bname)

        #Creates a new instance of the class and gets the page ready for testing
        wc4 = "{{WILDCARD4}}"
        page_init = "lib/page_init.py"
        ph = open(page_init).read()
        initialize = ph.replace(wc4, bname)
        fh.write("\n\n" + initialize)

        """
        Generate DIV Find Element Methods
        """
        fh.write("\n")

        for div in divs:
	    d = div['id']
            logging.debug(d)
            varName = "_id_" + d.replace("-", "_") + "_locator"
            methodName = "is" + varName + "_available"
            testMethods.append(methodName)
            gen.generateFindElement(fh, varName)

        fh.close()

        if classname != "":
            gen.generateTestElement(classname, filetext, testMethods)    
Esempio n. 49
0
 def done(name):
     if not name or name == '' or name == posixpath.basename(path):
         return cancel()
     target = posixpath.join(posixpath.dirname(path), name)
     debug('renaming %s to %s' % (path, target))
     this.connection.rename(path, target)
Esempio n. 50
0
 def Basename(self):
     return posixpath.basename(self.Path())
Esempio n. 51
0
def GetRemotePackageName(remote_package_file):
    """Returns the package name given a remote posix based package file."""
    package_basename = posixpath.basename(remote_package_file)
    return GetFileBaseName(package_basename)
Esempio n. 52
0
 def name(self):
     return posixpath.basename(self.path)
Esempio n. 53
0
    def __setup(self):
        """Construct the series of shell commands, i.e., fill in
           self.__commands"""

        if self.__url:
            # Set the name of the tarball, untarred package directory, and
            # source directory inside the extracted directory
            tarball = posixpath.basename(self.__url)
            match = re.search(r'(.*)(?:(?:\.tar)|(?:\.tar\.gz)|(?:\.tgz)|(?:\.tar\.bz2)|(?:\.tar\.xz))$',
                              tarball)
            if match:
                pkgdir = match.group(1)
            else:
                raise RuntimeError('unrecognized package format')

            # directory containing the unarchived package
            if self.__directory:
                if posixpath.isabs(self.__directory):
                    directory = self.__directory
                else:
                    directory = posixpath.join(self.__wd, self.__directory)
            else:
                directory = posixpath.join(self.__wd, pkgdir)

            # Download source from web
            self.__commands.append(self.download_step(url=self.__url,
                                                      directory=self.__wd))

            # Untar source package
            self.__commands.append(self.untar_step(
                tarball=posixpath.join(self.__wd, tarball),
                directory=self.__wd))

        if self.__repository:
            # Clone git repository
            self.__commands.append(self.clone_step(
                branch=self.__branch, path=self.__wd,
                repository=self.__repository))

            directory = posixpath.join(self.__wd, posixpath.splitext(
                posixpath.basename(self.__repository))[0])

        # Preconfigure setup
        if self.__preconfigure:
            # Assume the preconfigure commands should be run from the
            # source directory
            self.__commands.append('cd {}'.format(directory))
            self.__commands.extend(self.__preconfigure)

        # Configure
        environment = []
        if self.__environment:
            for key, val in sorted(self.__environment.items()):
                environment.append('{0}={1}'.format(key, val))
        self.__commands.append(self.configure_step(
            build_directory=self.__build_directory,
            directory=directory, environment=environment,
            toolchain=self.__toolchain))

        # Build
        if self.__make:
            self.__commands.append(self.build_step())

        # Check the build
        if self.__check:
            self.__commands.append(self.build_step(target='check'))

        # Install
        if self.__install:
            self.__commands.append(self.build_step(target='install'))

        if self.__postinstall:
            # Assume the postinstall commands should be run from the
            # install directory
            self.__commands.append('cd {}'.format(self.prefix))
            self.__commands.extend(self.__postinstall)

        # Cleanup
        remove = [directory]
        if self.__url:
            remove.append(posixpath.join(self.__wd, tarball))
        if self.__build_directory:
            if posixpath.isabs(self.__build_directory):
                remove.append(self.__build_directory)
        self.__commands.append(self.cleanup_step(items=remove))
Esempio n. 54
0
 def _path_to_checksum(self, path):
     relpath = posixpath.relpath(path, self.url)
     return posixpath.dirname(relpath) + posixpath.basename(relpath)
Esempio n. 55
0
 def splitext(self):
     return splitext(posixpath.basename(self.path.rstrip('/')))
Esempio n. 56
0
    def splitext(self):

        # type: () -> Tuple[str, str]

        return splitext(posixpath.basename(self.path.rstrip('/')))
Esempio n. 57
0
 def filename(self):
     _, netloc, path, _, _ = urllib_parse.urlsplit(self.url)
     name = posixpath.basename(path.rstrip('/')) or netloc
     name = urllib_parse.unquote(name)
     assert name, ('URL %r produced no filename' % self.url)
     return name
Esempio n. 58
0
 def show_url(self):
     return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
Esempio n. 59
0
 def __unicode__(self):
     return posixpath.basename(self.url_path)
Esempio n. 60
0
def make_editor_session(pad,
                        path,
                        is_attachment=None,
                        alt=PRIMARY_ALT,
                        datamodel=None):
    """Creates an editor session for the given path object."""
    if alt != PRIMARY_ALT and not pad.db.config.is_valid_alternative(alt):
        raise BadEdit('Attempted to edit an invalid alternative (%s)' % alt)

    raw_data = pad.db.load_raw_data(path,
                                    cls=OrderedDict,
                                    alt=alt,
                                    fallback=False)
    raw_data_fallback = None
    if alt != PRIMARY_ALT:
        raw_data_fallback = pad.db.load_raw_data(path, cls=OrderedDict)
        all_data = OrderedDict()
        all_data.update(raw_data_fallback or ())
        all_data.update(raw_data or ())
    else:
        all_data = raw_data

    id = posixpath.basename(path)
    if not is_valid_id(id):
        raise BadEdit('Invalid ID')

    record = None
    exists = all_data is not None
    if raw_data is None:
        raw_data = OrderedDict()

    if is_attachment is None:
        if not exists:
            is_attachment = False
        else:
            is_attachment = bool(all_data.get('_attachment_for'))
    elif bool(all_data.get('_attachment_for')) != is_attachment:
        raise BadEdit('The attachment flag passed is conflicting with the '
                      'record\'s attachment flag.')

    if exists:
        # XXX: what about changing the datamodel after the fact?
        if datamodel is not None:
            raise BadEdit('When editing an existing record, a datamodel '
                          'must not be provided.')
        datamodel = pad.db.get_datamodel_for_raw_data(all_data, pad)
    else:
        if datamodel is None:
            datamodel = pad.db.get_implied_datamodel(path, is_attachment, pad)
        elif isinstance(datamodel, string_types):
            datamodel = pad.db.datamodels[datamodel]

    if exists:
        record = pad.instance_from_data(dict(all_data), datamodel)

    for key in implied_keys:
        raw_data.pop(key, None)
        if raw_data_fallback:
            raw_data_fallback.pop(key, None)

    return EditorSession(pad, id, text_type(path), raw_data, raw_data_fallback,
                         datamodel, record, exists, is_attachment, alt)