def __call__(self, oeb, opts):
self.log = oeb.logger
self.opts = opts
self.oeb = oeb
self.seen_filenames = set([])
self.rename_map = {}
for item in list(oeb.manifest.items):
fname = posixpath.basename(item.href)
if fname in self.seen_filenames:
suffix = self.unique_suffix(fname)
data = item.data
base, ext = posixpath.splitext(item.href)
nhref = base + suffix + ext
nhref = oeb.manifest.generate(href=nhref)[1]
spine_pos = item.spine_position
oeb.manifest.remove(item)
nitem = oeb.manifest.add(item.id, nhref, item.media_type, data=data,
fallback=item.fallback)
self.seen_filenames.add(posixpath.basename(nhref))
self.rename_map[item.href] = nhref
if spine_pos is not None:
oeb.spine.insert(spine_pos, nitem, item.linear)
else:
self.seen_filenames.add(fname)
if self.rename_map:
self.log('Found non-unique filenames, renaming to support broken'
' EPUB readers like FBReader, Aldiko and Stanza...')
from pprint import pformat
self.log.debug(pformat(self.rename_map))
renamer = RenameFiles(self.rename_map)
renamer(oeb, opts)
def RemoteHostCopy(self, file_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
file_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
if copy_to:
file_name = posixpath.basename(file_path)
src_spec, dest_spec = file_path, '%s:%s' % (self.name, file_name)
else:
remote_path, _ = self.RemoteCommand('readlink -f %s' % remote_path)
remote_path = remote_path.strip()
src_spec, dest_spec = '%s:%s' % (self.name, remote_path), file_path
cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig,
'cp', src_spec, dest_spec]
stdout, stderr, retcode = vm_util.IssueCommand(cmd)
if retcode:
error_text = ('Got non-zero return code (%s) executing %s\n'
'STDOUT: %sSTDERR: %s' %
(retcode, ' '.join(cmd), stdout, stderr))
raise errors.VirtualMachine.RemoteCommandError(error_text)
if copy_to:
file_name = posixpath.basename(file_path)
remote_path = remote_path or file_name
self.RemoteCommand('mv %s %s; chmod 777 %s' %
(file_name, remote_path, remote_path))
def run(self):
"""Run substitutions on files."""
super(checkbox_install_data, self).run()
examplesfiles = [o for o in self.outfiles if "examples" in o]
if not examplesfiles:
return
# Create etc directory
if self.install_dir == "/usr":
basedir = posixpath.sep
else:
basedir = self.install_dir
etcdir = posixpath.join(basedir, "etc", "checkbox.d")
self.mkpath(etcdir)
# Create configs symbolic link
dstdir = posixpath.dirname(examplesfiles[0]).replace("examples",
"configs")
if not os.path.exists(dstdir):
os.symlink(etcdir, dstdir)
# Substitute version in examplesfiles and etcfiles
version = changelog_version()
for examplesfile in examplesfiles:
etcfile = posixpath.join(etcdir,
posixpath.basename(examplesfile))
infile = posixpath.join("examples",
posixpath.basename(examplesfile))
for outfile in examplesfile, etcfile:
substitute_variables(infile, outfile, {
"version = dev": "version = %s" % version})
def findNodeIndex(self, filename, nodename):
nexus_widget = self.neXusWidget()
file_model = nexus_widget.model()
for node in file_model.rootItem.children:
if node.file.filename == filename:
file_node = node
break
else:
raise Exception("Could not find file %s" % filename)
index = file_model.index(file_node.row, 0, Qt.QModelIndex())
node_parts = nodename.split(posixpath.sep)
while node_parts:
name = posixpath.basename(node_parts.pop(0))
for child in node.children:
child_name = posixpath.basename(child.name)
if child_name == name:
node = child
index = file_model.index(node.row, 0, index)
break
else:
raise Exception("Could not find node %s in %s" % (name, filename))
return index
def cmd_list(self):
if self.args.podcast_id is None:
print 'Podcast feeds available:'
print
for pid in self.config.podcast:
print ' %s - %s' % (pid, self.config.podcast[pid])
print
return os.EX_OK
else:
print 'Fetched files available for "%s" (sorted by name):' \
% self.args.podcast_id
print
count = 1
for filepath in self.podcast.list_fetched_chapters():
print ' %i: %s' % (count, os.path.basename(filepath))
count += 1
if count == 1:
print ' **** No fetched files.'
print
print 'Remote files available for "%s" (reverse sorted by date):' \
% self.args.podcast_id
print
count = 1
for url in self.podcast.list_chapters():
if isinstance(url, basestring):
print ' %i: %s' % (count, posixpath.basename(url))
else:
print ' %i: %s (%s)' % (count,
posixpath.basename(url[0]),
url[1])
count += 1
if count == 1:
print ' **** No remote files. Try running this script ' \
'with `--sync\''
print
def process_presence_update(msg):
message_type = msg.delivery_info["routing_key"]
payload = json.loads(msg.body)
namespace = payload["namespace"]
if not namespace.startswith("/robin/"):
return
user_id36 = posixpath.basename(namespace)
room_namespace = posixpath.dirname(namespace)
room_id = posixpath.basename(room_namespace)
account = Account._byID36(user_id36, data=True, stale=True)
try:
room = RobinRoom._byID(room_id)
except tdb_cassandra.NotFoundException:
return
if not room.is_participant(account):
return
presence_type = "join" if message_type == "websocket.connect" else "part"
websockets.send_broadcast(
namespace=room_namespace,
type=presence_type,
payload={
"user": account.name,
},
)
if presence_type == "join":
ParticipantPresenceByRoom.mark_joined(room, account)
else:
ParticipantPresenceByRoom.mark_exited(room, account)
def data(self, index, role):
if role != qt.Qt.DisplayRole:
return MyQVariant()
item = self.getProxyFromIndex(index)
column = index.column()
if column == 0:
if isinstance(item, H5FileProxy):
return MyQVariant(os.path.basename(item.file.filename))
else:
return MyQVariant(posixpath.basename(item.name))
if column == 1:
showtitle = True
if showtitle:
if hasattr(item, 'type'):
if item.type in ["Entry", "NXentry"]:
children = item.children
names = [posixpath.basename(o.name) for o in children]
if "title" in names:
idx = names.index("title")
if len(children[idx].getNode().shape):
#stored as an array of strings!!!
#return just the first item
return MyQVariant("%s" % children[idx].getNode().value[0])
else:
#stored as a string
return MyQVariant("%s" % children[idx].getNode().value)
return MyQVariant(item.type)
if column == 2:
return MyQVariant(item.shape)
if column == 3:
return MyQVariant(item.dtype)
return MyQVariant()
def test_basename(self):
self.assertEqual(posixpath.basename("/foo/bar"), "bar")
self.assertEqual(posixpath.basename("/"), "")
self.assertEqual(posixpath.basename("foo"), "foo")
self.assertEqual(posixpath.basename("////foo"), "foo")
self.assertEqual(posixpath.basename("//foo//bar"), "bar")
self.assertRaises(TypeError, posixpath.basename)
def parseAll(path):
for root, dirs, files in os.walk(path):
print("processing: " + basename(root.replace("\\", "/")))
if len(files) and basename(root.replace("\\", "/")) != "ARM_SAMPLE":
for file in files:
filename, file_extension = posixpath.splitext(file)
if file_extension == ".svd":
parseFile(basename(root.replace("\\", "/")), filename)
def makeHDF5(self, rewrite=False):
"""
Create the HDF5 structure if needed ...
"""
print("Initialization of HDF5 file")
if os.path.exists(self.hdf5) and rewrite:
os.unlink(self.hdf5)
spath = self.hdf5path.split("/")
assert len(spath) > 2
nxs = Nexus(self.hdf5, mode="w")
entry = nxs.new_entry(entry=spath[0], program_name="pyFAI", title="diffmap")
grp = entry
for subgrp in spath[1:-2]:
grp = nxs.new_class(grp, name=subgrp, class_type="NXcollection")
processgrp = nxs.new_class(grp, "pyFAI", class_type="NXprocess")
processgrp["program"] = numpy.array([numpy.str_(i) for i in sys.argv])
processgrp["version"] = numpy.str_(PyFAI_VERSION)
processgrp["date"] = numpy.str_(get_isotime())
if self.mask:
processgrp["maskfile"] = numpy.str_(self.mask)
if self.flat:
processgrp["flatfiles"] = numpy.array([numpy.str_(i) for i in self.flat])
if self.dark:
processgrp["darkfiles"] = numpy.array([numpy.str_(i) for i in self.dark])
processgrp["inputfiles"] = numpy.array([numpy.str_(i) for i in self.inputfiles])
processgrp["PONIfile"] = numpy.str_(self.poni)
processgrp["dim0"] = self.npt_slow
processgrp["dim0"].attrs["axis"] = "Rotation"
processgrp["dim1"] = self.npt_fast
processgrp["dim1"].attrs["axis"] = "Translation"
processgrp["dim2"] = self.npt_rad
processgrp["dim2"].attrs["axis"] = "Diffraction"
for k, v in self.ai.getPyFAI().items():
if "__len__" in dir(v):
processgrp[k] = numpy.str_(v)
elif v:
processgrp[k] = v
self.group = nxs.new_class(grp, name=spath[-2], class_type="NXdata")
if posixpath.basename(self.hdf5path) in self.group:
self.dataset = self.group[posixpath.basename(self.hdf5path)]
else:
self.dataset = self.group.create_dataset(
name=posixpath.basename(self.hdf5path),
shape=(self.npt_slow, self.npt_fast, self.npt_rad),
dtype="float32",
chunks=(1, self.npt_fast, self.npt_rad),
maxshape=(None, None, self.npt_rad))
self.dataset.attrs["signal"] = "1"
self.dataset.attrs["interpretation"] = "spectrum"
self.dataset.attrs["axes"] = str(self.unit).split("_")[0]
self.dataset.attrs["creator"] = "pyFAI"
self.dataset.attrs["long_name"] = "Diffraction imaging experiment"
self.nxs = nxs
def get_trace(filename, fileno):
data = get_data(filename)
group_label = ""
if fileno % 2 == 0:
group_label = posixpath.basename(filename).split(".")[0] + ":after stimulation"
else:
group_label = posixpath.basename(filename).split(".")[0] + ":before stimulation"
trace = go.Histogram(x=data, opacity=0.75, name=group_label)
return trace
def print_tree(photos, top):
#TODO: Make this work for both posix and windows. Hardwired for posix (looking for / and using posixpath now)
offset = top.count('/')
indent = 4
print "Walking tree from: {}".format(top)
for dirpath, dirs, files in walk_db_tree(top):
print '{}{}'.format(' ' * indent * (dirpath.count('/') - offset), posixpath.basename(dirpath))
for f in files:
print '{}{}'.format(' ' * indent * (f.count('/') - offset), posixpath.basename(f))
def Search(term):
resultText = ""
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(term)
corrector = searcher.corrector("content")
# for mistyped_word in mistyped_words:
Suggestions = corrector.suggest(term, limit=3)
if(Suggestions):
resultText += "<li><h2 class='suggestion_header'>Suggestions based on content</h2></li>"
for Suggestion in Suggestions:
resultText += "<li><h3><a href='#' class='suggestions'> " + Suggestion + "</a></h3></li>"
resultText += "<li>"
corrected = searcher.correct_query(query, term)
if corrected.query != query:
resultText += "<li><h3 class='did_you_mean'>Did You Mean: </h3><p><a class='suggestions'>" + corrected.string + "</a></p>"
results = searcher.search(query, terms=True,limit=None)
# results.fragmenter = highlight.PinpointFragmenter(autotrim=True)
# results.formatter = highlight.UppercaseFormatter(between = "...")
terms=term.split(" ")
# resultText += " estimated_min_length " + str(results.estimated_min_length())
if results.estimated_min_length() < 11:
urls = open("uniqueUrltest.txt",'r')
i = 0
for url in urls:
if i < 6:
if any(word in url for word in terms):
parse_object = urlparse.urlparse(url)
if "." not in url and "/" == url[-1]:
url = url[0:-2]
title_head = basename(url).decode("utf-8", 'ignore')
else:
title_head = basename(parse_object.path).split(".")[0].decode("utf-8", 'ignore')
resultText += "<h3><a href='"+url+"'>" + title_head + "</a></h3><p class='green'>" + url+ "</p>"
i += 1
urls.close()
for result in results:
# i=0
# if ".pdf" in str(result["path"]) and i<3:
# resultText += "<h1>" + "<a href='" + result["path"] + "'>" + result["path"] + "</a><br/></h1>" + result["path"] + "<br/><br/>" + result.highlights("content") + "<br/>"
# i += 1
# del results[result]
if "title" in result:
if "error" in result["title"].lower():
resultText += "<h3>" + "<a href='" + result["path"] + "'>" + result["path"] + "</a></h3><p class='url'>" + result["path"] + "</p><p>" + result.highlights("content") + "</p>"
else:
if(result["title"] == "" ) or result["title"].isspace():
resultText += "<h3 class='title_head'>" + "<a href='" + result["path"] + "'>" + "No Title Present" + "</a></h3><p class='url'>" + result["path"] + "</p><p>" + result.highlights("content") + "</p>"
else:
resultText += "<h3 class='title_head'>" + "<a href='" + result["path"] + "'>" + result["title"] + "</a></h3><p class='url'>" + result["path"] + "</p><p>" + result.highlights("content") + "</p>"
if(resultText == "<li>"):
resultText = ""
else:
resultText += "</li>"
return resultText
def plot_sensor_data_from_file(self, file_path):
if not self.to_save and not self.to_show:
return
data = np.genfromtxt(file_path, delimiter=',', skip_header=1, names=['timestamp', 'x', 'y', 'z'])
file_name = basename(file_path)
file_name = file_name[:file_name.find(".")]
View.plot_sensor_data(basename(file_name), data['timestamp'], data['x'], data['y'], data['z'])
def download_path(request, filename):
parts = posixpath.normpath(filename).split("/")
if len(parts) < 3:
return HttpResponseBadRequest()
requested_file = parts[-1]
try: version_num = str(int(parts[-2]))
except ValueError: version_num = None
if version_num is None:
base_path = "/".join(parts[:-2])
type_id = parts[-2]
versions = get_versions('/' + base_path)
if versions is None:
return HttpResponseNotFound()
version_num = str(max(map(int, versions)))
else:
base_path = "/".join(parts[:-3])
type_id = parts[-3]
try:
file_metadata = get_file_metadata("/%s/%s" % (base_path, version_num))
except NotFoundError:
return HttpResponseNotFound()
if requested_file == posixpath.basename(base_path):
is_mesh = True
hash = file_metadata['types'][type_id]['hash']
else:
is_mesh = False
subfile_map = {}
for subfile in file_metadata['types'][type_id]['subfiles']:
(subfile_base, vers) = posixpath.split(subfile)
subfile_basename = posixpath.basename(subfile_base)
subfile_map[subfile_basename] = subfile
if requested_file not in subfile_map:
return HttpResponseNotFound()
subfile_metadata = get_file_metadata(subfile_map[requested_file])
hash = subfile_metadata['hash']
try:
rec = get_hash(hash)
except NotFoundError:
return HttpResponseNotFound()
data = rec['data']
mime = rec['mimetype']
response = HttpResponse(data, mimetype=mime)
response['Content-Length'] = str(len(data))
response['Access-Control-Allow-Origin'] = '*'
return response
def hdf5Slot(self, ddict):
if ddict['event'] == 'itemClicked':
if ddict['mouse'] == "right":
return self.itemRightClickedSlot(ddict)
if ddict['event'] == "itemDoubleClicked":
if ddict['type'] in ['Dataset']:
if ddict['dtype'].startswith('|S'):
print("string")
else:
root = ddict['name'].split('/')
root = "/" + root[1]
if len(ddict['name']) == len(root):
cnt = ddict['name']
else:
cnt = ddict['name'][len(root):]
if cnt not in self._cntList:
self._cntList.append(cnt)
basename = posixpath.basename(cnt)
if basename not in self._aliasList:
self._aliasList.append(basename)
else:
self._aliasList.append(cnt)
self.cntTable.build(self._cntList, self._aliasList)
elif ddict['type'] in ['NXentry', 'Entry']:
if self._lastAction is None:
return
action, selectionType = self._lastAction.split()
if action == 'REMOVE':
action = 'ADD'
ddict['action'] = "%s %s" % (action, selectionType)
self.buttonsSlot(ddict)
else:
if self.data is not None:
name = ddict['name']
filename = ddict['file']
fileIndex = self.data.sourceName.index(filename)
phynxFile = self.data._sourceObjectList[fileIndex]
dataset = phynxFile[name]
if isinstance(dataset, h5py.Dataset):
root = ddict['name'].split('/')
root = "/" + root[1]
cnt = ddict['name'].split(root)[-1]
if cnt not in self._cntList:
if DEBUG:
print("USING SECOND WAY")
self._cntList.append(cnt)
basename = posixpath.basename(cnt)
if basename not in self._aliasList:
self._aliasList.append(basename)
else:
self._aliasList.append(cnt)
self.cntTable.build(self._cntList, self._aliasList)
return
if DEBUG:
print("Unhandled item type: %s" % ddict['dtype'])
def compare_links(self):
ret = False
if isinstance(self.devpi_body, Exception):
tw.line("%s: devpi request failed: %s" %
(self.name, self.devpi_body), red=True)
ret = True
if isinstance(self.pypi_body, Exception):
tw.line("%s: pypi request failed: %s" %
(self.name, self.pypi_body), red=True)
ret = True
if ret:
return
pypi_links = BeautifulSoup(self.pypi_body).findAll("a")
devpi_links = BeautifulSoup(self.devpi_body).findAll("a")
misses = []
count = 0
pypi_basenames = set()
for link in pypi_links:
href = link.get("href")
if href.startswith("../../packages"):
count += 1
# stored at pypi, should appear directly with devpi
package_prefix = href[6:]
pypi_basenames.add(posixpath.basename(href))
#print package_prefix
for link2 in devpi_links:
#print " ", link2.get("href")
if link2.get("href").endswith(package_prefix):
break
else:
misses.append((href))
b = logbase.join("res", self.name).ensure(dir=1)
misses_path = b.join("devpi_misses")
if misses:
tw.line("%s: devpi misses %d out of %d links" %
(self.name, len(misses), count), red=True)
misses_path.write("\n".join(misses))
else:
if misses_path.check():
misses_path.remove()
extralinks = []
for link2 in devpi_links:
devpi_basename = posixpath.basename(link2.get("href"))
if devpi_basename not in pypi_basenames:
extralinks.append(devpi_basename)
if extralinks:
extra = ", externally scraped: %s" % (extralinks)
else:
extra = ""
tw.line("%s: devpi has %d pypi internal links%s" %
(self.name, count, extra), green=True)
b.join("pypi_body").write(self.pypi_body)
b.join("devpi_body").write(self.devpi_body)
def insert(self, path, entry):
if path == '/':
assert entry.ftype == DIR
self.index = entry
else:
assert ppath.basename(path) == entry.name
parent = self.lookup(ppath.dirname(path))
assert parent.ftype == DIR
parent.items[ppath.basename(path)] = entry
def test_basename(self):
self.assertEqual(posixpath.basename("/foo/bar"), "bar")
self.assertEqual(posixpath.basename("/"), "")
self.assertEqual(posixpath.basename("foo"), "foo")
self.assertEqual(posixpath.basename("////foo"), "foo")
self.assertEqual(posixpath.basename("//foo//bar"), "bar")
self.assertEqual(posixpath.basename(b"/foo/bar"), b"bar")
self.assertEqual(posixpath.basename(b"/"), b"")
self.assertEqual(posixpath.basename(b"foo"), b"foo")
self.assertEqual(posixpath.basename(b"////foo"), b"foo")
self.assertEqual(posixpath.basename(b"//foo//bar"), b"bar")
def find_in_zip(filepath, zip):
if filepath in zip.namelist():
return zip.read(filepath)
basename = posixpath.basename(filepath)
matching = []
for zipname in zip.namelist():
if posixpath.basename(zipname) == basename:
matching.append(zipname)
if len(matching) == 1:
return zip.read(matching[0])
else:
return None
def _walk(self, path):
dirs, nondirs = [], []
for e in self.ls(path):
if self.isdir(e):
dirs.append(posixpath.basename(re.sub("/$", "", e)))
else:
nondirs.append(posixpath.basename(e))
yield path, dirs, nondirs
for d in dirs:
path2 = posixpath.join(path, d)
#yield from self.walk(path2)
for t in self.walk(path2):
yield t
def HTML(path):
file_=open(use.join(path,use.basename(path)+'.htm'),"w")
Str="<html><title>"+use.basename(path)+"</title><body>"
Dict={}
for files in os.listdir(path):
if use.splitext(files)[1]=='.txt':
Dict[int(use.splitext(files)[0])]=files
list_=Dict.keys()
list_.sort()
for files in list_:
Str+="<img src="+Dict[files]+">"
file_.write(Str+"<br><br><font color='red'>© AUTOMATIC file generator</font></body></center></html>")
file_.close()
def flash(target, firmware_file, firmware_md5, args):
# Connect to controller...
if target == 'controller':
errprinter('connecting to Controller...')
client = soloutils.connect_controller(await=True)
else:
errprinter('connecting to Solo...')
client = soloutils.connect_solo(await=True)
# Prepare the update.
# older versions don't have sololink_config and ssh returns 127, so do it manually
code = soloutils.command_stream(client, 'sololink_config --update-prepare sololink')
if code != 0:
soloutils.command_stream(client, 'rm -rf /log/updates && mkdir -p /log/updates')
# Upload the files.
errprinter('uploading files...')
scp = SCPClient(client.get_transport())
scp.put(firmware_file, posixpath.join('/log/updates/', posixpath.basename(firmware_file)))
scp.put(firmware_md5, posixpath.join('/log/updates/', posixpath.basename(firmware_md5)))
scp.close()
if target == 'controller':
errprinter("starting update on the Controller...")
else:
errprinter("starting update on Solo...")
if args['--clean']:
errprinter('marking all user-local changes to be reset...')
code = soloutils.command_stream(client, 'sololink_config --update-apply sololink --reset')
else:
code = soloutils.command_stream(client, 'sololink_config --update-apply sololink')
# Fallback to earlier versions.
if code != 0:
if args['--clean']:
code = soloutils.command_stream(client, 'touch /log/updates/UPDATE && touch /log/updates/RESETSETTINGS && shutdown -r now')
else:
code = soloutils.command_stream(client, 'touch /log/updates/UPDATE && shutdown -r now')
if target == 'controller':
errprinter('the Controller will update once it reboots!')
else:
errprinter('Solo will update once it reboots!')
dt = datetime.today() + timedelta(minutes=4)
errprinter('please wait up to four minutes longer for the installation to complete (by {}).'.format(dt.strftime('%-I:%M')))
# Complete!
client.close()
return code
def get_trace(filename, fileno):
data = get_data(filename)
label = ""
if fileno % 2 == 1:
label = posixpath.basename(filename).split(".")[0] + ":before stimulation"
else:
label = posixpath.basename(filename).split(".")[0] + ":after stimulation"
trace = go.Histogram(
x=data,
name=label,
opacity=0.75,
histnorm='density'
)
return trace
def test_realpath_resolve_before_normalizing(self):
# Bug #990669: Symbolic links should be resolved before we
# normalize the path. E.g.: if we have directories 'a', 'k' and 'y'
# in the following hierarchy:
# a/k/y
#
# and a symbolic link 'link-y' pointing to 'y' in directory 'a',
# then realpath("link-y/..") should return 'k', not 'a'.
try:
os.mkdir(ABSTFN)
os.mkdir(ABSTFN + "/k")
os.mkdir(ABSTFN + "/k/y")
os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y")
# Absolute path.
self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k")
# Relative path.
with support.change_cwd(dirname(ABSTFN)):
self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."),
ABSTFN + "/k")
finally:
test_support.unlink(ABSTFN + "/link-y")
safe_rmdir(ABSTFN + "/k/y")
safe_rmdir(ABSTFN + "/k")
safe_rmdir(ABSTFN)
def get_poster(self, only_cached=False):
"""Downloads this poster to a local cache and returns the path"""
from flexget.manager import manager
base_dir = os.path.join(manager.config_base, 'userstatic')
if os.path.isfile(os.path.join(base_dir, self.poster_file or '')):
return self.poster_file
elif only_cached:
return
# If we don't already have a local copy, download one.
url = get_mirror('banner') + self.poster
log.debug('Downloading poster %s', url)
dirname = os.path.join('tvdb', 'posters')
# Create folders if the don't exist
fullpath = os.path.join(base_dir, dirname)
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
filename = os.path.join(dirname, posixpath.basename(self.poster))
thefile = file(os.path.join(base_dir, filename), 'wb')
thefile.write(requests.get(url).content)
self.poster_file = filename
# If we are detached from a session, update the db
if not Session.object_session(self):
with Session() as session:
session.query(TVDBSeries).filter(TVDBSeries.id == self.id).update(values={'poster_file': filename})
return filename
def download(url,output_dir,retry=False):
global tried_urls, failed_urls
url_hash=hashlib.sha224(url.encode('utf-8')).digest()
if url_hash in tried_urls:
return
pool_sema.acquire()
path = urllib.parse.urlsplit(url).path
filename = posixpath.basename(path)
if len(filename)>40:
filename=filename[:36]+filename[-4:]
while os.path.exists(output_dir + '/' + filename):
filename = str(random.randint(0,100)) + filename
in_progress.append(filename)
try:
request=urllib.request.Request(url,None,urlopenheader)
image=urllib.request.urlopen(request).read()
if len(image)==0:
print('no image')
imagefile=open(output_dir + '/' + filename,'wb')
imagefile.write(image)
imagefile.close()
in_progress.remove(filename)
if retry:
print('Retry OK '+ filename)
else:
print("OK " + filename)
tried_urls.append(url_hash)
except Exception as e:
if retry:
print('Retry Fail ' + filename)
else:
print("FAIL " + filename)
failed_urls.append((url, output_dir))
pool_sema.release()
def __call__(self, args={'--quiet': False, '--verbose': False},
fmt=None, name=None, stream=sys.stderr):
if not name:
name = posixpath.basename(sys.argv[0])
if name in self.loggers:
return self.loggers[name]
log = logging.getLogger(name)
if not log.handlers:
if fmt == 'brief':
fmt = '[%(levelname)-4s] %(message)s'
elif fmt == 'msg':
fmt = '%(message)s'
else:
fmt = '%(asctime)s %(levelname)-6s %(name)s %(message)s'
logging.basicConfig(stream=stream, format=fmt)
if not log.level: # pragma: no cover
if args.get('--quiet'):
level = logging.ERROR
elif args.get('--debug'):
level = logging.DEBUG
else:
level = logging.INFO
log.setLevel(level)
self.loggers[name] = log
return log
def download_and_cache(url, session=None):
"""This function downloads a file into a folder called
original_data and returns the local filepath."""
path = urllib.parse.urlsplit(url).path
filename = posixpath.basename(path)
filepath = "input/original_data/" + filename
print(url)
# check if file exists, if not download it
filepath = "input/original_data/" + filename
print(filepath)
if not os.path.exists(filepath):
if not session:
print('No session')
session = requests.session()
print("Downloading file: ", filename)
r = session.get(url, stream=True)
chuncksize = 1024
with open(filepath, 'wb') as file:
for chunck in r.iter_content(chuncksize):
file.write(chunck)
else:
print("Using local file from", filepath)
filepath = '' + filepath
return filepath
def main(args, loglevel):
logging.basicConfig(format="%(levelname)s: %(message)s", level=loglevel)
socket.setdefaulttimeout(args.timeout)
g = Github()
with open(args.repo_file, 'r') as f:
file_counter = 0
for line in f.readlines():
logging.info('Fetching repository: %s' % line)
try:
repo_str = line.rstrip().split('github.com/')[-1]
repo = g.get_repo(repo_str)
tree = repo.get_git_tree('master', recursive=True)
files_to_download = []
for file in tree.tree:
if fnmatch.fnmatch(file.path, args.wildcard):
files_to_download.append('https://github.com/%s/raw/master/%s' % (repo_str, file.path))
for file in files_to_download:
logging.info('Downloading %s' % file)
file_counter += 1
filename = posixpath.basename(urlparse.urlsplit(file).path)
output_path = os.path.join(args.output_dir, filename)
if os.path.exists(output_path):
output_path += "-" + str(file_counter)
try:
urllib.urlretrieve(file, output_path)
except:
logging.error('Error downloading %s' % file)
except:
logging.error('Error fetching repository %s' % line)
def basename(self, path):
if self.sys.platform.startswith('win'):
return ntpath.basename(path)
else:
return posixpath.basename(path)
def buttonsSlot(self, ddict, emit=True):
_logger.debug("buttonsSlot(self, %s,emit=%s)", ddict, emit)
if self.data is None:
return
action, selectionType = ddict['action'].split()
entryList = self.getSelectedEntries()
if not len(entryList):
return
text = qt.safe_str(self.tableTab.tabText(self.tableTab.currentIndex()))
mcaSelection = {'mcalist': [], 'selectionindex': []}
cntSelection = {'cntlist': [], 'y': []}
if text.upper() == "AUTO":
cntSelection = self.autoTable.getCounterSelection()
# self._aliasList = cntSelection['aliaslist']
elif text.upper() == "MCA":
mcaSelection = self.mcaTable.getMcaSelection()
else:
cntSelection = self.cntTable.getCounterSelection()
self._aliasList = cntSelection['aliaslist']
selectionList = []
for entry, filename in entryList:
if not len(cntSelection['cntlist']) and \
not len(mcaSelection['mcalist']):
continue
if not len(cntSelection['y']) and \
not len(mcaSelection['selectionindex']):
#nothing to plot
continue
mcaIdx = 0
for yMca in mcaSelection['selectionindex']:
sel = {}
sel['SourceName'] = self.data.sourceName * 1
sel['SourceType'] = "HDF5"
fileIndex = self.data.sourceName.index(filename)
phynxFile = self.data._sourceObjectList[fileIndex]
entryIndex = list(phynxFile["/"].keys()).index(entry[1:])
sel['Key'] = "%d.%d" % (fileIndex + 1, entryIndex + 1)
sel['legend'] = os.path.basename(sel['SourceName'][0])+\
" " + posixpath.basename(entry) #it was sel['Key']
sel['selection'] = {}
sel['selection']['sourcename'] = filename
#deal with the case the "entry" is a dataset hunging at root level
if isinstance(phynxFile[entry], h5py.Dataset):
entry = "/"
sel['selection']['entry'] = entry
sel['selection']['key'] = "%d.%d" % (fileIndex + 1,
entryIndex + 1)
sel['selection']['mca'] = [yMca]
sel['selection']['mcaselectiontype'] = mcaSelection[
'selectiontype'][mcaIdx]
mcaIdx += 1
sel['selection']['mcalist'] = mcaSelection['mcalist']
sel['selection']['LabelNames'] = mcaSelection['aliaslist']
#sel['selection']['aliaslist'] = cntSelection['aliaslist']
sel['selection']['selectiontype'] = "MCA"
sel['mcaselection'] = True
aliases = mcaSelection['aliaslist']
selectionList.append(sel)
for yCnt in cntSelection['y']:
sel = {}
sel['SourceName'] = self.data.sourceName * 1
sel['SourceType'] = "HDF5"
fileIndex = self.data.sourceName.index(filename)
phynxFile = self.data._sourceObjectList[fileIndex]
if entry == "/":
entryIndex = 1
else:
entryIndex = list(phynxFile["/"].keys()).index(entry[1:])
sel['Key'] = "%d.%d" % (fileIndex + 1, entryIndex + 1)
sel['legend'] = os.path.basename(sel['SourceName'][0])+\
" " + posixpath.basename(entry) #it was sel['Key']
sel['selection'] = {}
sel['selection']['sourcename'] = filename
#deal with the case the "entry" is a dataset hunging at root level
if isinstance(phynxFile[entry], h5py.Dataset):
entry = "/"
sel['selection']['entry'] = entry
sel['selection']['key'] = "%d.%d" % (fileIndex + 1,
entryIndex + 1)
sel['selection']['x'] = cntSelection['x']
sel['selection']['y'] = [yCnt]
sel['selection']['m'] = cntSelection['m']
sel['selection']['cntlist'] = cntSelection['cntlist']
sel['selection']['LabelNames'] = cntSelection['aliaslist']
#sel['selection']['aliaslist'] = cntSelection['aliaslist']
sel['selection']['selectiontype'] = selectionType
if selectionType.upper() == "SCAN":
if cntSelection['cntlist'][yCnt].startswith("/"):
actualDatasetPath = posixpath.join(
entry, cntSelection['cntlist'][yCnt][1:])
else:
actualDatasetPath = posixpath.join(
entry, cntSelection['cntlist'][yCnt])
actualDataset = phynxFile[actualDatasetPath]
sel['scanselection'] = True
if hasattr(actualDataset, "shape"):
if len(actualDataset.shape) > 1:
if 1 in actualDataset.shape[-2:]:
#shape (1, n) or (n, 1)
pass
else:
# at least twoD dataset
selectionType = "2D"
sel['scanselection'] = False
sel['mcaselection'] = False
elif selectionType.upper() == "MCA":
sel['scanselection'] = False
sel['mcaselection'] = True
if cntSelection['cntlist'][yCnt].startswith("/"):
actualDatasetPath = posixpath.join(
entry, cntSelection['cntlist'][yCnt][1:])
else:
actualDatasetPath = posixpath.join(
entry, cntSelection['cntlist'][yCnt])
actualDataset = phynxFile[actualDatasetPath]
if hasattr(actualDataset, "shape"):
actualDatasetLen = len(actualDataset.shape)
if (actualDatasetLen
== 2) and (1 in actualDataset.shape):
# still can be used
pass
elif (actualDatasetLen > 1) and (not hasattr(
self, "_messageShown")):
# at least twoD dataset
msg = qt.QMessageBox(self)
msg.setIcon(qt.QMessageBox.Information)
msg.setText(
"Multidimensional data set as MCA. Using Average. You should use ROI Imaging"
)
msg.exec_()
self._messageShown = True
sel['selection']['mcaselectiontype'] = "avg"
else:
sel['scanselection'] = False
sel['mcaselection'] = False
sel['selection']['selectiontype'] = selectionType
aliases = cntSelection['aliaslist']
if len(cntSelection['x']) and len(cntSelection['m']):
addLegend = " (%s/%s) vs %s" % (aliases[yCnt], aliases[
cntSelection['m'][0]], aliases[cntSelection['x'][0]])
elif len(cntSelection['x']):
addLegend = " %s vs %s" % (aliases[yCnt],
aliases[cntSelection['x'][0]])
elif len(cntSelection['m']):
addLegend = " (%s/%s)" % (aliases[yCnt],
aliases[cntSelection['m'][0]])
else:
addLegend = " %s" % aliases[yCnt]
sel['legend'] += addLegend
selectionList.append(sel)
if not emit:
return selectionList
self._lastAction = "%s" % ddict['action']
if len(selectionList):
if selectionType.upper() in ["SCAN", "MCA"]:
ddict = {}
ddict['event'] = "SelectionTypeChanged"
ddict['SelectionType'] = selectionType.upper()
self.sigOtherSignals.emit(ddict)
if action.upper() == "ADD":
self.sigAddSelection.emit(selectionList)
if action.upper() == "REMOVE":
self.sigRemoveSelection.emit(selectionList)
if action.upper() == "REPLACE":
self.sigReplaceSelection.emit(selectionList)
def is_special_file(name):
"""Returns true if a file is some special Git metadata and not content."""
return posixpath.basename(name) in [
'.gitignore', '.gitattributes', '.gitmodules'
]
def show_url(self):
# type: () -> str
return posixpath.basename(self._url.split('#', 1)[0].split('?', 1)[0])
def url_matches_criterion(purl: 'ParseResult', url: str, unquoted_path: str, mc: MatchCriteria) -> bool:
if mc.type == 'url':
import re
try:
pat = re.compile(mc.value)
except re.error:
return False
return pat.search(unquote(url)) is not None
if mc.type == 'mime':
import fnmatch
mt = guess_type(unquoted_path)
if not mt:
return False
mt = mt.lower()
for mpat in mc.value.split(','):
mpat = mpat.strip()
with suppress(Exception):
if fnmatch.fnmatchcase(mt, mpat):
return True
return False
if mc.type == 'ext':
if not purl.path:
return False
path = unquoted_path.lower()
for ext in mc.value.split(','):
ext = ext.strip()
if path.endswith('.' + ext):
return True
return False
if mc.type == 'protocol':
protocol = (purl.scheme or 'file').lower()
for key in mc.value.split(','):
if key.strip() == protocol:
return True
return False
if mc.type == 'fragment_matches':
import re
try:
pat = re.compile(mc.value)
except re.error:
return False
return pat.search(unquote(purl.fragment)) is not None
if mc.type == 'path':
import fnmatch
try:
return fnmatch.fnmatchcase(unquoted_path.lower(), mc.value)
except Exception:
return False
if mc.type == 'file':
import fnmatch
import posixpath
try:
fname = posixpath.basename(unquoted_path)
except Exception:
return False
try:
return fnmatch.fnmatchcase(fname.lower(), mc.value)
except Exception:
return False
def hdf5Slot(self, ddict):
entryName = NexusTools.getEntryName(ddict['name'])
currentEntry = "%s::%s" % (ddict['file'], entryName)
if (currentEntry != self._lastEntry) and not self._BUTTONS:
self._lastEntry = None
cntList = []
mcaList = []
aliasList = []
measurement = None
scanned = []
mcaList = []
if posixpath.dirname(entryName) != entryName:
with h5py.File(ddict['file'], "r") as h5file:
measurement = NexusTools.getMeasurementGroup(
h5file, ddict['name'])
scanned = NexusTools.getScannedPositioners(
h5file, ddict['name'])
if measurement is not None:
measurement = [item.name for key,item in measurement.items() \
if self._isNumeric(item)]
if self._mca:
mcaList = NexusTools.getMcaList(h5file, entryName)
for i in range(len(scanned)):
key = scanned[i]
cntList.append(key)
aliasList.append(posixpath.basename(key))
if measurement is not None:
for key in measurement:
if key not in cntList:
cntList.append(key)
aliasList.append(posixpath.basename(key))
cleanedCntList = []
for key in cntList:
root = key.split('/')
root = "/" + root[1]
if len(key) == len(root):
cleanedCntList.append(key)
else:
cleanedCntList.append(key[len(root):])
self._autoAliasList = aliasList
self._autoCntList = cleanedCntList
self.autoTable.build(self._autoCntList, self._autoAliasList)
currentTab = qt.safe_str(self.tableTab.tabText( \
self.tableTab.currentIndex()))
if self._mca:
mcaAliasList = []
cleanedMcaList = []
for key in mcaList:
root = key.split('/')
root = "/" + root[1]
if len(key) == len(root):
cleanedMcaList.append(key)
else:
cleanedMcaList.append(key[len(root):])
mcaAliasList.append(posixpath.basename(key))
self.mcaTable.build(cleanedMcaList, mcaAliasList)
nTabs = self.tableTab.count()
if (len(mcaList) > 0) and (nTabs < 3):
self.tableTab.insertTab(2, self.mcaTable, "MCA")
elif (len(mcaList) == 0) and (nTabs > 2):
self.tableTab.removeTab(2)
_logger.debug("currentTab = %s", currentTab)
if currentTab != "USER":
if (len(mcaList) > 0) and (len(cntList) == 0):
idx = self.tableTabOrder.index("MCA")
self.tableTab.setCurrentIndex(idx)
_logger.debug("setting tab = %s MCA", idx)
elif (len(mcaList) == 0) and (len(cntList) > 0):
idx = self.tableTabOrder.index("AUTO")
self.tableTab.setCurrentIndex(idx)
_logger.debug("setting tab = %s AUTO", idx)
self._lastEntry = currentEntry
if ddict['event'] == 'itemClicked':
if ddict['mouse'] == "right":
return self.itemRightClickedSlot(ddict)
if ddict['mouse'] == "left":
# If parent is root do it even if not NXentry??
if ddict['type'] in ['NXentry', 'Entry']:
if not self._BUTTONS:
auto = self.actions.getConfiguration()["auto"]
if auto == "ADD":
self._addAction()
elif auto == "REPLACE":
self._replaceAction()
if ddict['event'] == "itemDoubleClicked":
if ddict['type'] in ['Dataset']:
currentIndex = self.tableTab.currentIndex()
text = safe_str(self.tableTab.tabText(currentIndex))
if text.upper() != "USER":
if currentIndex == 0:
self.tableTab.setCurrentIndex(1)
else:
self.tableTab.setCurrentIndex(0)
if not self._isNumericType(ddict['dtype']):
_logger.debug("string like %s", ddict['dtype'])
else:
root = ddict['name'].split('/')
root = "/" + root[1]
if len(ddict['name']) == len(root):
cnt = ddict['name']
else:
cnt = ddict['name'][len(root):]
if cnt not in self._cntList:
self._cntList.append(cnt)
basename = posixpath.basename(cnt)
if basename not in self._aliasList:
self._aliasList.append(basename)
else:
self._aliasList.append(cnt)
self.cntTable.build(self._cntList, self._aliasList)
elif (ddict['color'] == qt.Qt.blue) and ("silx" in sys.modules):
# there is an action to be applied
self.showInfoWidget(ddict["file"], ddict["name"], dset=False)
elif ddict['type'] in ['NXentry', 'Entry']:
if self._lastAction is None:
return
action, selectionType = self._lastAction.split()
if action == 'REMOVE':
action = 'ADD'
ddict['action'] = "%s %s" % (action, selectionType)
self.buttonsSlot(ddict)
else:
if self.data is not None:
name = ddict['name']
filename = ddict['file']
fileIndex = self.data.sourceName.index(filename)
phynxFile = self.data._sourceObjectList[fileIndex]
dataset = phynxFile[name]
if isinstance(dataset, h5py.Dataset):
root = ddict['name'].split('/')
root = "/" + root[1]
cnt = ddict['name'].split(root)[-1]
if cnt not in self._cntList:
_logger.debug("USING SECOND WAY")
self._cntList.append(cnt)
basename = posixpath.basename(cnt)
if basename not in self._aliasList:
self._aliasList.append(basename)
else:
self._aliasList.append(cnt)
self.cntTable.build(self._cntList, self._aliasList)
return
_logger.debug("Unhandled item type: %s", ddict['dtype'])
def show_url(self):
# type: () -> Optional[str]
return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
def __str__(self):
"""String representation of the primitive"""
# Build a list of files to make the logic a bit simpler below.
# The items in the files list are dictionaries with keys 'src'
# and 'dest'.
files = []
if self.__files:
# Sort to make it deterministic
files.extend([{
'dest': dest,
'src': src
} for src, dest in sorted(self.__files.items())])
elif self.__dest and self.__src:
files.append({'dest': self.__dest, 'src': self.__src})
else:
# No files!
return ''
if hpccm.config.g_ctype == container_type.DOCKER:
if self._app:
logging.warning('The Singularity specific SCI-F syntax '
'was requested. Docker does not have an '
'equivalent: using regular COPY!')
# Format:
# COPY src1 \
# src2 \
# src3 \
# dest/
# COPY src1 dest1
# COPY src2 dest2
# COPY src3 dest3
base_inst = 'COPY '
if self.__chown:
base_inst = base_inst + '--chown={} '.format(self.__chown)
if self.__from:
base_inst = base_inst + '--from={} '.format(self.__from)
# Docker does not have the notion of copying a set of
# files to different locations inside the container in a
# single instruction. So generate multiple COPY
# instructions in that case.
instructions = []
for pair in files:
dest = pair['dest']
src = pair['src']
c = [base_inst]
if isinstance(src, list):
c[0] = c[0] + src[0]
c.extend([' {}'.format(x) for x in src[1:]])
# Docker requires a trailing slash. Add one if missing.
c.append(' {}'.format(posixpath.join(dest, '')))
else:
c[0] = c[0] + '{0} {1}'.format(src, dest)
instructions.append(' \\\n'.join(c))
return '\n'.join(instructions)
elif hpccm.config.g_ctype == container_type.SINGULARITY:
# Format:
# %files
# src1 dest
# src2 dest
# src3 dest
# %files
# src1 dest1
# src2 dest2
# src3 dest3
section = '%files'
if (self.__from and hpccm.config.g_singularity_version >=
StrictVersion('3.2')):
section = section + ' from {}'.format(self.__from)
if self._app:
# SCIF appfiles does not support "from"
section = '%appfiles {0}'.format(self._app)
# Singularity will error if the destination does not
# already exist in the container. The workarounds are to
# either 1) prior to copying the files, create the
# destination directories with %setup or 2) copy the files
# to a path guaranteed to exist, "/", and then move them
# later with %post. Option 1 is the "pre" approach,
# option 2 is the "post" approach.
flat_files = []
post = [] # post actions if _post is enabled
pre = [] # pre actions if _mkdir is enabled
for pair in files:
dest = pair['dest']
src = pair['src']
if self._post:
dest = '/'
if isinstance(src, list):
for s in src:
flat_files.append(' {0} {1}'.format(s, dest))
if self._post:
post.append(' mv /{0} {1}'.format(
posixpath.basename(s),
posixpath.join(pair['dest'], s)))
if (self._mkdir and posixpath.dirname(dest) != '/'
and posixpath.basename(dest) != dest):
# When multiple files are to be copied to the
# same destination, assume the destination is
# a directory
pre.append(
' mkdir -p ${{SINGULARITY_ROOTFS}}{0}'.format(
dest))
else:
flat_files.append(' {0} {1}'.format(src, dest))
if (self._mkdir and posixpath.dirname(dest) != '/'
and posixpath.basename(dest) != dest):
# When a single file is to be copied to a
# destination, assume the destination is a
# file.
pre.append(
' mkdir -p ${{SINGULARITY_ROOTFS}}{0}'.format(
posixpath.dirname(dest)))
elif self._post:
post.append(' mv /{0} {1}'.format(
posixpath.basename(src), pair['dest']))
s = ''
if pre:
s += '%setup\n' + '\n'.join(pre) + '\n'
s += section + '\n' + '\n'.join(flat_files)
if post:
s += '\n%post\n' + '\n'.join(post)
return s
elif hpccm.config.g_ctype == container_type.BASH:
logging.warning('copy primitive does not map into bash')
return ''
else:
raise RuntimeError('Unknown container type')
async def translate_cell(self,
cell: JupyterCell,
metadata: Optional[MutableMapping[Text, Any]],
autoawait: bool = False) -> Step:
# Build execution target
target = metadata.get('target')
if target is not None:
if isinstance(target['model'], Text):
target = {**target, **{'model': metadata['models'][target['model']]}}
model_name = hashlib.md5(json.dumps(
obj=target['model'],
sort_keys=True,
ensure_ascii=True,
default=str).encode('ascii')).hexdigest()
target = _build_target(model_name, target)
# Extract Python interpreter from metadata
interpreter = metadata.get('interpreter', 'ipython')
# Create a step structure
step = BaseStep(
name=metadata['cell_id'],
context=self.context,
target=target)
step.workdir = metadata['step'].get('workdir')
# Process cell inputs
cell_inputs = metadata['step'].get('in', [])
input_tokens = {}
scatter_inputs = _get_scatter_inputs(metadata['step'].get('scatter'))
for element in cell_inputs:
# If is a string, it refers to the name of a variable
if isinstance(element, Text):
element = {
'type': 'name',
'name': element
}
# Otherwise it must be a dictionary
if isinstance(element, MutableMapping):
# Create input port
name = element.get('name') or element.get('valueFrom')
if name in scatter_inputs:
port = ScatterInputPort(name=name, step=step)
else:
port = DefaultInputPort(name=name, step=step)
# Get serializer if present
serializer = (metadata['serializers'][element['serializer']]
if isinstance(element['serializer'], Text)
else element['serializer']) if 'serializer' in element else None
# Process port type
element_type = element['type']
# If type is equal to `file`, it refers to a file path in the local resource
if element_type == 'file':
port.token_processor = FileTokenProcessor(
port=port,
value=element.get('value'),
value_from=element.get('valueFrom'))
# If type is equal to `name` or `env`, it refers to a variable
elif element_type in ['name', 'env']:
port.token_processor = NameTokenProcessor(
port=port,
serializer=serializer,
compiler=cell.compiler,
value=element.get('value'),
value_from=element.get('valueFrom', name))
# If type is equal to `control`, simply add an empty dependency
elif element_type == 'control':
port.token_processor = ControlTokenProcessor(port=port)
# Add command token
input_tokens[name] = JupyterCommandToken(
name=name,
token_type=element_type,
serializer=serializer)
# Register step port
step.input_ports[port.name] = port
# Retrieve inputs automatically if necessary
if metadata['step'].get('autoin', True):
input_names = _extract_dependencies(cell.name, cell.compiler, cell.code)
for name in input_names:
if name not in step.input_ports:
if name in scatter_inputs:
port = ScatterInputPort(name=name, step=step)
else:
port = DefaultInputPort(name=name, step=step)
port.token_processor = NameTokenProcessor(
port=port,
compiler=cell.compiler,
value_from=name)
input_tokens[name] = JupyterCommandToken(name=name, token_type='name')
step.input_ports[port.name] = port
# If outputs are defined for the current cell
output_tokens = {}
if 'out' in metadata['step']:
for element in metadata['step']['out']:
# If is a string, it refers to the name of a variable
if isinstance(element, Text):
element = {
'type': 'name',
'name': element
}
# Otherwise it must be a dictionary
if isinstance(element, MutableMapping):
# Create port
name = element.get('name') or element.get('valueFrom')
if scatter_inputs:
output_port = GatherOutputPort(
name=name,
step=step,
merge_strategy=lambda values: sorted(values, key=lambda t: int(posixpath.basename(t.tag))))
else:
output_port = DefaultOutputPort(name=name, step=step)
# Get serializer if present
serializer = (metadata['serializers'][element['serializer']]
if isinstance(element['serializer'], Text)
else element['serializer']) if 'serializer' in element else None
# Process port type
element_type = element['type']
# If type is equal to `file`, it refers to a file path in the remote resource
if element_type == 'file':
output_port.token_processor = FileTokenProcessor(
port=output_port,
value=element.get('value'),
value_from=element.get('valueFrom'))
# If type is equal to `name` or `env`, it refers to a variable
elif element_type in ['name', 'env']:
output_port.token_processor = NameTokenProcessor(
port=output_port,
compiler=cell.compiler,
serializer=serializer,
value_from=element.get('valueFrom', name))
# If type is equal to `control`, simply add an empty dependency
elif element_type == 'control':
output_port.token_processor = ControlTokenProcessor(port=output_port)
# Add command token
output_tokens[name] = JupyterCommandToken(
name=name,
token_type=element_type,
serializer=serializer)
# Register step port
step.output_ports[name] = output_port
# Add output log port
if scatter_inputs:
output_log_port = GatherOutputPort(
name=executor.CELL_OUTPUT,
step=step,
merge_strategy=lambda values: sorted(values, key=lambda t: int(posixpath.basename(t.tag))))
else:
output_log_port = DefaultOutputPort(name=executor.CELL_OUTPUT, step=step)
output_log_port.token_processor = OutputLogTokenProcessor(port=output_log_port)
step.output_ports[executor.CELL_OUTPUT] = output_log_port
# Set input combinator for the step
step.input_combinator = _get_input_combinator(
step=step,
scatter=metadata['step'].get('scatter'))
# Create the command to be executed remotely
step.command = JupyterCommand(
step=step,
ast_nodes=cell.code,
compiler=cell.compiler,
interpreter=interpreter,
input_tokens=input_tokens,
output_tokens=output_tokens,
autoawait=autoawait)
return step
def axes(self):
from .dataset import Axis
return OrderedDict((posixpath.basename(j.name), j) for j in sorted(
i for i in self.values() if isinstance(i, Axis)))
def signals(self):
from .dataset import Signal
return OrderedDict((posixpath.basename(j.name), j) for j in sorted(
i for i in self.values() if isinstance(i, Signal)))
def id_from_url(url):
from urlparse import urlsplit
from posixpath import dirname, basename
path = urlsplit(url).path
b = basename(path)
return int(b if b else dirname(path))
def GetBaseName(self):
if self.base_name is None:
# We must use posixpath explicitly so this will work on Windows.
self.base_name = posixpath.basename(
six.moves.urllib.parse.urlparse(self.full_path).path)
return self.base_name
def GenerateApkAnalysis(apk_filename,
tool_prefix,
out_dir,
unknown_handler=None):
"""Analyse APK to determine size contributions of different file classes."""
file_groups = []
def make_group(name):
group = _FileGroup(name)
file_groups.append(group)
return group
native_code = make_group('Native code')
java_code = make_group('Java code')
native_resources_no_translations = make_group('Native resources (no l10n)')
translations = make_group('Native resources (l10n)')
stored_translations = make_group('Native resources stored (l10n)')
icu_data = make_group('ICU (i18n library) data')
v8_snapshots = make_group('V8 Snapshots')
png_drawables = make_group('PNG drawables')
res_directory = make_group('Non-compiled Android resources')
arsc = make_group('Compiled Android resources')
metadata = make_group('Package metadata')
unknown = make_group('Unknown files')
notices = make_group('licenses.notice file')
unwind_cfi = make_group('unwind_cfi (dev and canary only)')
apk = zipfile.ZipFile(apk_filename, 'r')
try:
apk_contents = apk.infolist()
finally:
apk.close()
dex_multiplier, skip_extract_lib = _ParseManifestAttributes(apk_filename)
total_apk_size = os.path.getsize(apk_filename)
for member in apk_contents:
filename = member.filename
if filename.endswith('/'):
continue
if filename.endswith('.so'):
basename = posixpath.basename(filename)
should_extract_lib = not skip_extract_lib and basename.startswith(
'lib')
native_code.AddZipInfo(
member, extracted_multiplier=int(should_extract_lib))
elif filename.endswith('.dex'):
java_code.AddZipInfo(member, extracted_multiplier=dex_multiplier)
elif re.search(_RE_NON_LANGUAGE_PAK, filename):
native_resources_no_translations.AddZipInfo(member)
elif re.search(_RE_COMPRESSED_LANGUAGE_PAK, filename):
translations.AddZipInfo(
member,
extracted_multiplier=int('en_' in filename
or 'en-' in filename))
elif re.search(_RE_STORED_LANGUAGE_PAK, filename):
stored_translations.AddZipInfo(member)
elif filename == 'assets/icudtl.dat':
icu_data.AddZipInfo(member)
elif filename.endswith('.bin'):
v8_snapshots.AddZipInfo(member)
elif filename.endswith('.png') or filename.endswith('.webp'):
png_drawables.AddZipInfo(member)
elif filename.startswith('res/'):
res_directory.AddZipInfo(member)
elif filename.endswith('.arsc'):
arsc.AddZipInfo(member)
elif filename.startswith(
'META-INF') or filename == 'AndroidManifest.xml':
metadata.AddZipInfo(member)
elif filename.endswith('.notice'):
notices.AddZipInfo(member)
elif filename.startswith('assets/unwind_cfi'):
unwind_cfi.AddZipInfo(member)
else:
unknown.AddZipInfo(member)
total_install_size = total_apk_size
zip_overhead = total_apk_size
for group in file_groups:
actual_size = group.ComputeZippedSize()
install_size = group.ComputeInstallSize()
uncompressed_size = group.ComputeUncompressedSize()
total_install_size += group.ComputeExtractedSize()
zip_overhead -= actual_size
yield ('Breakdown', group.name + ' size', actual_size, 'bytes')
yield ('InstallBreakdown', group.name + ' size', install_size, 'bytes')
# Only a few metrics are compressed in the first place.
# To avoid over-reporting, track uncompressed size only for compressed
# entries.
if uncompressed_size != actual_size:
yield ('Uncompressed', group.name + ' size', uncompressed_size,
'bytes')
# Per-file zip overhead is caused by:
# * 30 byte entry header + len(file name)
# * 46 byte central directory entry + len(file name)
# * 0-3 bytes for zipalign.
yield ('Breakdown', 'Zip Overhead', zip_overhead, 'bytes')
yield ('InstallSize', 'APK size', total_apk_size, 'bytes')
yield ('InstallSize', 'Estimated installed size', total_install_size,
'bytes')
transfer_size = _CalculateCompressedSize(apk_filename)
yield ('TransferSize', 'Transfer size (deflate)', transfer_size, 'bytes')
# Size of main dex vs remaining.
main_dex_info = java_code.FindByPattern('classes.dex')
if main_dex_info:
main_dex_size = main_dex_info.file_size
yield ('Specifics', 'main dex size', main_dex_size, 'bytes')
secondary_size = java_code.ComputeUncompressedSize() - main_dex_size
yield ('Specifics', 'secondary dex size', secondary_size, 'bytes')
# Size of main .so vs remaining.
main_lib_info = native_code.FindLargest()
if main_lib_info:
main_lib_size = main_lib_info.file_size
yield ('Specifics', 'main lib size', main_lib_size, 'bytes')
secondary_size = native_code.ComputeUncompressedSize() - main_lib_size
yield ('Specifics', 'other lib size', secondary_size, 'bytes')
main_lib_section_sizes = _ExtractMainLibSectionSizesFromApk(
apk_filename, main_lib_info.filename, tool_prefix)
for metric_name, size in main_lib_section_sizes.iteritems():
yield ('MainLibInfo', metric_name, size, 'bytes')
# Main metric that we want to monitor for jumps.
normalized_apk_size = total_apk_size
# unwind_cfi exists only in dev, canary, and non-channel builds.
normalized_apk_size -= unwind_cfi.ComputeZippedSize()
# Always look at uncompressed .so.
normalized_apk_size -= native_code.ComputeZippedSize()
normalized_apk_size += native_code.ComputeUncompressedSize()
# Normalized dex size: size within the zip + size on disk for Android Go
# devices (which ~= uncompressed dex size).
normalized_apk_size += java_code.ComputeUncompressedSize()
# Avoid noise caused when strings change and translations haven't yet been
# updated.
num_translations = translations.GetNumEntries()
num_stored_translations = stored_translations.GetNumEntries()
if num_translations > 1:
# Multipliers found by looking at MonochromePublic.apk and seeing how much
# smaller en-US.pak is relative to the average locale.pak.
normalized_apk_size = _NormalizeLanguagePaks(translations,
normalized_apk_size, 1.17)
if num_stored_translations > 1:
normalized_apk_size = _NormalizeLanguagePaks(stored_translations,
normalized_apk_size, 1.43)
if num_translations + num_stored_translations > 1:
if num_translations == 0:
# WebView stores all locale paks uncompressed.
num_arsc_translations = num_stored_translations
else:
# Monochrome has more configurations than Chrome since it includes
# WebView (which supports more locales), but these should mostly be empty
# so ignore them here.
num_arsc_translations = num_translations
normalized_apk_size += int(
_NormalizeResourcesArsc(apk_filename, arsc.GetNumEntries(),
num_arsc_translations, out_dir))
yield ('Specifics', 'normalized apk size', normalized_apk_size, 'bytes')
yield ('Specifics', 'file count', len(apk_contents), 'zip entries')
if unknown_handler is not None:
for info in unknown.AllEntries():
unknown_handler(info)
def _clone_repository(metadata_repo_url, branch):
directory_name = posixpath.splitext(posixpath.basename(metadata_repo_url))[0]
LOGGER.info("Cloning the repository %s into the directory %s",
metadata_repo_url, directory_name)
return git.Repository.clone(metadata_repo_url, directory_name, branch)
def url2filename(url):
urlpath = urlsplit(url).path
return posixpath.basename(unquote(urlpath))
def sort_key(p):
return (len(p.split('/')), len(posixpath.basename(p)), p)
def scrapePageCont(self, code_syntax, url, classname, filepath):
"""
CLIFF NOTE: ADD FUNCTION COMMENT HERE
"""
logging.debug('Scraping page')
soup = BeautifulSoup(code_syntax)
now = datetime.datetime.now()
testMethods = []
#Class name cannot start with digit.
m = classname[0].isdigit()
if m:
logging.debug('Class name cannot start with a digit')
classname = "c" + classname
filepath = "c" + filepath
#Define variable for page object file name
filetext = classname
classname = classname.replace ("_", "")
filename = dest_dir + "/pages/auto/" + classname.lower() + ".py"
#Create new page object file using file name Class Name
fh = open(filename, "w")
logging.debug('URL: ' + filepath)
logging.debug('FILENAME: ' + classname)
fh.write("#!/usr/bin/env python\n")
#Write comment for creation date and author
fh.write("\"\"\"")
fh.write("\nCreated on " + now.strftime("%Y-%m-%d %H:%M"))
fh.write("\n@author: Selenium Webdriver Code Generator\n")
fh.write("\"\"\"\n")
#Create page object header code by manipulating library code
ph = open("lib/page_header.py", "r")
fh.write("\n\n" + ph.read())
ph.close()
fh.write("\n\nclass " + classname + "(Base):\n")
#Write main comment for class
fh.write(" \"\"\"\n")
fh.write(" Variable definitions of " + classname + " page object based on site HTML elements.\n")
fh.write(" \"\"\"")
"""
Parse Title:
(a) HTML Syntax: <title>CSCE 499 - Design Document</title>
(b) BeautifulSoup Syntax: print soup.head.title
(c) Variable Definition: _page_title = "CSCE 499 - Design Document"
(d) Generated Selenium: The generated code should verify that the title string matches the ex-
pected text as defined by the title variable.
"""
logging.debug('Parse page title')
fh.write("\n\n # Page Title Variable")
title = soup.head.title.string
gen = Generate()
gen.generateVariable(fh, "_page_title", title)
logging.debug('Title: ' + title)
"""
All DIV with ID tag
(a) HTML Syntax: <div id="banner-top">
(b) BeautifulSoup Syntax: divs = soup.findAll('div','id':True)
(c) Variable Definition: _div_banner_top_locator = (By.ID, "banner-top")
(d) Generated Selenium: The generated code should verify that the DIV ID exists on the page.
"""
logging.debug('Parse all DIVS with ids')
fh.write("\n\n # Variables defined by ID element")
divs = soup.findAll('div',{'id':True})
for div in divs:
d = div['id']
logging.debug(d)
varName = "_id_" + d.replace("-", "_") + "_locator"
byIdSyntax = "(By.ID, \""+ d + "\")"
gen.generateVariable(fh, varName, byIdSyntax)
"""
H1 Text
(a) HTML Syntax: <h1>CSCE 499 Capstone - 2012-2013</h1>
(b) BeautifulSoup Syntax: h1s = soup.findAll('h1')
(c) Variable Definition: _h1_CSCE_499_Capstone = "CSCE 499 Capstone - 2012-2013"
(d) Generated Selenium: The generated code should verify that the H1 text exists and matches
the expected text as defined by the specific H1 variable.
"""
#CLIFF NOTE: Possibly delete or add debug condition for printing.
logging.debug('Parse H1')
h1s = soup.findAll('h1')
for h1 in h1s:
logging.debug(h1)
"""
Image Files
(a) HTML Syntax: <img src="/home-assets/images/banner-images/honors.jpg" alt="" border="0">
(b) BeautifulSoup Syntax: imgs = soup.findAll('img')
(c) Variable Definition: _img_honors_locator = (By.SRC, "/home-assets/images/banner-images/honors
(d) Generated Selenium: The generat
"""
#CLIFF NOTE: Possibly delete or add debug condition for printing.
logging.debug('Parse Image')
imgs = soup.findAll('img')
for img in imgs:
logging.debug(img)
"""
CLIFF NOTE: Re-investigate this one to see if it is really needed for testing
"""
logging.debug('Parse iFrames')
iframes = soup.findAll('iframe')
for iframe in iframes:
logging.debug(iframe)
"""
Radio Buttons
(a) HTML Syntax: <input type="radio" name="group1" value="Cheese">
(b) BeautifulSoup Syntax: radios = soup.findAll('input','type':'radio')
(c) Variable Definition: _radio_cheese_locator = (By.Value, "Cheese")
(d) Generated Selenium: The generated code should verify that the radio button exists, is selectable
and simulates the "click" action in a basic sanity test.
"""
#CLIFF NOTE: Possibly delete or add debug condition for printing.
logging.debug('Parse Radio Buttons')
radios = soup.findAll('input',{'type':'radio'})
for radio in radios:
logging.debug(radio)
"""
Checkboxes
(a) HTML Syntax: <input type="checkbox" name="sports" value="football">
(b) BeautifulSoup Syntax: checkboxes = soup.findAll('input','type':'checkbox')
(c) Variable Definition: _checkbox_sports_locator = (By.Value, "football")
(d) Generated Selenium: The generated code should verify that the check box exists, is selectable
and simulates the "click" action in a basic sanity test.
"""
#CLIFF NOTE: Possibly delete or add debug condition for printing.
logging.debug('Parse Checkbox Buttons')
checkboxes = soup.findAll('input',{'type':'checkbox'})
for checkbox in checkboxes:
logging.debug(checkbox)
"""
Text Fields
(a) HTML Syntax: <textarea name="comments" cols="25" rows="5">Enter your comments here...
</textarea>
(b) BeautifulSoup Syntax: textboxes = soup.findAll('input','type':'text')
(c) Variable Definition: _text_comments_locator = (By.Name, "comments")
(d) Generated Selenium: The generated code should verify that the text area exists, is accessible
by typing sample text and does not fail any SQL Injection negative test cases.
"""
logging.debug('Parse Text Boxes')
textboxes = soup.findAll('input',{'type':'text'})
for textbox in textboxes:
logging.debug(textbox)
text_id = textbox.get('id')
text_class = textbox.get('class')
if text_id != "" and text_class != "" :
out_str = ""
for tc in text_class:
out_str += tc + "."
if out_str[-1:] == '.':
out_str = out_str[:-1]
#Print Class
logging.debug(out_str)
out_str2 = out_str.replace("-", "_")
out_str2 = out_str2.replace(".", "_")
varTextboxClass = "_textbox_class_" + out_str2 + "_locator"
byCssSelectorSyntax = "(By.CSS_Selector, \""+ out_str + "\")"
gen.generateVariable(fh, varTextboxClass, byCssSelectorSyntax)
fh.write("\n")
gen.generateFindElement(fh, varTextboxClass)
if text_id != "" and text_class == "":
logging.debug(text_id)
varTextboxId = "_textbox_id_" + out_str.replace("-", "_") + "_locator"
byIdSyntax = "(By.ID, \""+ d + "\")"
gen.generateVariable(fh, varTextboxId, byIdSyntax)
fh.write("\n")
gen.generateFindElement(fh, varTextboxId)
if text_id == "" and text_class != "":
out_str = ""
for tc in text_class:
out_str += tc + "."
if out_str[-1:] == '.':
out_str = out_str[:-1]
#Print Class
logging.debug(out_str)
out_str2 = out_str.replace("-", "_")
out_str2 = out_str2.replace(".", "_")
varTextboxClass = "_textbox_class_" + out_str2 + "_locator"
byCssSelectorSyntax = "(By.CSS_Selector, \""+ out_str + "\")"
gen.generateVariable(fh, varTextboxClass, byCssSelectorSyntax)
fh.write("\n")
gen.generateFindElement(fh, varTextboxClass)
"""
Password Text Fields
(a) HTML Syntax: <input type="password" size="25">
(b) BeautifulSoup Syntax: passwords = soup.findAll('input','type':'password')
(c) Variable Definition: _text_password_locator = (By.Type, "password")
(d) Generated Selenium: The generated code should verify that the password text area exists, is
accessible by typing hidden characters and does not fail any negative test case scenarios (e.g., SQL
Injections and Security Tests).
"""
#CLIFF NOTE: Possibly delete or add debug condition for printing.
logging.debug('Parse Text Box')
passwords = soup.findAll('input',{'type':'password'})
for password in passwords:
logging.debug(password)
"""
Buttons
(a) HTML Syntax: <input type="submit" value="Submit">
(b) BeautifulSoup Syntax: button = soup.findAll('input','type':'submit')
(c) Variable Definition: _button_submit_locator = (By.Type, "submit")
(d) Generated Selenium: The generated code should verify that the button exists, is able to be
clicked and functions correctly when selected.
"""
#CLIFF NOTE: Possibly delete or add debug condition for printing.
logging.debug('Parse Buttons')
buttons = soup.findAll('input',{'type':'submit'})
for button in buttons:
logging.debug(button)
#Extract base name for url
o = urlparse.urlparse(url)
bname = basename(o.path)
logging.debug('Base Name: ' + bname)
#Creates a new instance of the class and gets the page ready for testing
wc4 = "{{WILDCARD4}}"
page_init = "lib/page_init.py"
ph = open(page_init).read()
initialize = ph.replace(wc4, bname)
fh.write("\n\n" + initialize)
"""
Generate DIV Find Element Methods
"""
fh.write("\n")
for div in divs:
d = div['id']
logging.debug(d)
varName = "_id_" + d.replace("-", "_") + "_locator"
methodName = "is" + varName + "_available"
testMethods.append(methodName)
gen.generateFindElement(fh, varName)
fh.close()
if classname != "":
gen.generateTestElement(classname, filetext, testMethods)
def done(name):
if not name or name == '' or name == posixpath.basename(path):
return cancel()
target = posixpath.join(posixpath.dirname(path), name)
debug('renaming %s to %s' % (path, target))
this.connection.rename(path, target)
def Basename(self):
return posixpath.basename(self.Path())
def GetRemotePackageName(remote_package_file):
"""Returns the package name given a remote posix based package file."""
package_basename = posixpath.basename(remote_package_file)
return GetFileBaseName(package_basename)
def name(self):
return posixpath.basename(self.path)
def __setup(self):
"""Construct the series of shell commands, i.e., fill in
self.__commands"""
if self.__url:
# Set the name of the tarball, untarred package directory, and
# source directory inside the extracted directory
tarball = posixpath.basename(self.__url)
match = re.search(r'(.*)(?:(?:\.tar)|(?:\.tar\.gz)|(?:\.tgz)|(?:\.tar\.bz2)|(?:\.tar\.xz))$',
tarball)
if match:
pkgdir = match.group(1)
else:
raise RuntimeError('unrecognized package format')
# directory containing the unarchived package
if self.__directory:
if posixpath.isabs(self.__directory):
directory = self.__directory
else:
directory = posixpath.join(self.__wd, self.__directory)
else:
directory = posixpath.join(self.__wd, pkgdir)
# Download source from web
self.__commands.append(self.download_step(url=self.__url,
directory=self.__wd))
# Untar source package
self.__commands.append(self.untar_step(
tarball=posixpath.join(self.__wd, tarball),
directory=self.__wd))
if self.__repository:
# Clone git repository
self.__commands.append(self.clone_step(
branch=self.__branch, path=self.__wd,
repository=self.__repository))
directory = posixpath.join(self.__wd, posixpath.splitext(
posixpath.basename(self.__repository))[0])
# Preconfigure setup
if self.__preconfigure:
# Assume the preconfigure commands should be run from the
# source directory
self.__commands.append('cd {}'.format(directory))
self.__commands.extend(self.__preconfigure)
# Configure
environment = []
if self.__environment:
for key, val in sorted(self.__environment.items()):
environment.append('{0}={1}'.format(key, val))
self.__commands.append(self.configure_step(
build_directory=self.__build_directory,
directory=directory, environment=environment,
toolchain=self.__toolchain))
# Build
if self.__make:
self.__commands.append(self.build_step())
# Check the build
if self.__check:
self.__commands.append(self.build_step(target='check'))
# Install
if self.__install:
self.__commands.append(self.build_step(target='install'))
if self.__postinstall:
# Assume the postinstall commands should be run from the
# install directory
self.__commands.append('cd {}'.format(self.prefix))
self.__commands.extend(self.__postinstall)
# Cleanup
remove = [directory]
if self.__url:
remove.append(posixpath.join(self.__wd, tarball))
if self.__build_directory:
if posixpath.isabs(self.__build_directory):
remove.append(self.__build_directory)
self.__commands.append(self.cleanup_step(items=remove))
def _path_to_checksum(self, path):
relpath = posixpath.relpath(path, self.url)
return posixpath.dirname(relpath) + posixpath.basename(relpath)
def splitext(self):
return splitext(posixpath.basename(self.path.rstrip('/')))
def splitext(self):
# type: () -> Tuple[str, str]
return splitext(posixpath.basename(self.path.rstrip('/')))
def filename(self):
_, netloc, path, _, _ = urllib_parse.urlsplit(self.url)
name = posixpath.basename(path.rstrip('/')) or netloc
name = urllib_parse.unquote(name)
assert name, ('URL %r produced no filename' % self.url)
return name
def show_url(self):
return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
def __unicode__(self):
return posixpath.basename(self.url_path)
def make_editor_session(pad,
path,
is_attachment=None,
alt=PRIMARY_ALT,
datamodel=None):
"""Creates an editor session for the given path object."""
if alt != PRIMARY_ALT and not pad.db.config.is_valid_alternative(alt):
raise BadEdit('Attempted to edit an invalid alternative (%s)' % alt)
raw_data = pad.db.load_raw_data(path,
cls=OrderedDict,
alt=alt,
fallback=False)
raw_data_fallback = None
if alt != PRIMARY_ALT:
raw_data_fallback = pad.db.load_raw_data(path, cls=OrderedDict)
all_data = OrderedDict()
all_data.update(raw_data_fallback or ())
all_data.update(raw_data or ())
else:
all_data = raw_data
id = posixpath.basename(path)
if not is_valid_id(id):
raise BadEdit('Invalid ID')
record = None
exists = all_data is not None
if raw_data is None:
raw_data = OrderedDict()
if is_attachment is None:
if not exists:
is_attachment = False
else:
is_attachment = bool(all_data.get('_attachment_for'))
elif bool(all_data.get('_attachment_for')) != is_attachment:
raise BadEdit('The attachment flag passed is conflicting with the '
'record\'s attachment flag.')
if exists:
# XXX: what about changing the datamodel after the fact?
if datamodel is not None:
raise BadEdit('When editing an existing record, a datamodel '
'must not be provided.')
datamodel = pad.db.get_datamodel_for_raw_data(all_data, pad)
else:
if datamodel is None:
datamodel = pad.db.get_implied_datamodel(path, is_attachment, pad)
elif isinstance(datamodel, string_types):
datamodel = pad.db.datamodels[datamodel]
if exists:
record = pad.instance_from_data(dict(all_data), datamodel)
for key in implied_keys:
raw_data.pop(key, None)
if raw_data_fallback:
raw_data_fallback.pop(key, None)
return EditorSession(pad, id, text_type(path), raw_data, raw_data_fallback,
datamodel, record, exists, is_attachment, alt)
