def process_requirements_txt(self, repo: Addon, data: QtCore.QByteArray):
"""Process the requirements.txt metadata file"""
self.status_message.emit(
translate(
"AddonsInstaller",
"Downloaded requirements.txt for {}",
).format(repo.display_name)
)
f = io.StringIO(data.data().decode("utf8"))
lines = f.readlines()
for line in lines:
break_chars = " <>=~!+#"
package = line
for n, c in enumerate(line):
if c in break_chars:
package = line[:n].strip()
break
if package:
repo.python_requires.add(package)
# For review and debugging purposes, store the file locally
package_cache_directory = os.path.join(self.store, repo.name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "requirements.txt")
with open(new_xml_file, "wb") as f:
f.write(data.data())
def process_metadata_txt(self, repo: Addon, data: QtCore.QByteArray):
"""Process the metadata.txt metadata file"""
self.status_message.emit(
translate("AddonsInstaller", "Downloaded metadata.txt for {}").format(
repo.display_name
)
)
f = io.StringIO(data.data().decode("utf8"))
while True:
line = f.readline()
if not line:
break
if line.startswith("workbenches="):
depswb = line.split("=")[1].split(",")
for wb in depswb:
wb_name = wb.strip()
if wb_name:
repo.requires.add(wb_name)
FreeCAD.Console.PrintLog(
f"{repo.display_name} requires FreeCAD Addon '{wb_name}'\n"
)
elif line.startswith("pylibs="):
depspy = line.split("=")[1].split(",")
for pl in depspy:
dep = pl.strip()
if dep:
repo.python_requires.add(dep)
FreeCAD.Console.PrintLog(
f"{repo.display_name} requires python package '{dep}'\n"
)
elif line.startswith("optionalpylibs="):
opspy = line.split("=")[1].split(",")
for pl in opspy:
dep = pl.strip()
if dep:
repo.python_optional.add(dep)
FreeCAD.Console.PrintLog(
f"{repo.display_name} optionally imports python package"
+ f" '{pl.strip()}'\n"
)
# For review and debugging purposes, store the file locally
package_cache_directory = os.path.join(self.store, repo.name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "metadata.txt")
with open(new_xml_file, "wb") as f:
f.write(data.data())
def testIt(self):
w = QGLWidget()
w.makeCurrent()
b = QGLBuffer()
b.setUsagePattern(QGLBuffer.DynamicDraw)
self.assertTrue(b.create())
self.assertTrue(b.bufferId() != 0)
self.assertTrue(b.bind())
data = QByteArray(py3k.b("12345"))
b.allocate(data)
self.assertEqual(b.size(), data.size())
m = b.map(QGLBuffer.ReadOnly)
if m:
self.assertEqual(m, py3k.buffer(py3k.b(data.data())))
b.unmap()
m = b.map(QGLBuffer.ReadWrite)
m[3] = py3k.b('A')[0]
b.unmap()
result, rdata = b.read(3, 1)
self.assertTrue(result)
self.assertEqual(py3k.b('A'), rdata.data())
else:
print(" memory mapping is not possible in this OpenGL implementation.")
b.release()
def testIt(self):
w = QGLWidget()
w.makeCurrent()
b = QGLBuffer()
b.setUsagePattern(QGLBuffer.DynamicDraw)
self.assertTrue(b.create())
self.assertTrue(b.bufferId() != 0)
self.assertTrue(b.bind())
data = QByteArray("12345")
b.allocate(data)
self.assertEqual(b.size(), data.size())
m = b.map(QGLBuffer.ReadOnly)
if m:
self.assertEqual(m, py3k.buffer(py3k.b(data.data())))
b.unmap()
m = b.map(QGLBuffer.ReadWrite)
m[3] = py3k.b('A')
b.unmap()
result, rdata = b.read(3, 1)
self.assertTrue(result)
self.assertEqual(py3k.b('A'), rdata.data())
else:
print(" memory mapping is not possible in this OpenGL implementation.")
b.release()
def _handle_bin(self, raw_data: QByteArray):
data = raw_data.data()
self._buffer.extend(data)
if data.endswith(ZLIB_SUFFIX):
data = self._inflator.decompress(data)
self._buffer.clear()
self._handle_text(data.decode("utf-8"))
def testRawData(self):
data = QDataStream()
self.assertEqual(data.readRawData(4), None)
ba = QByteArray()
data = QDataStream(ba, QIODevice.WriteOnly)
data.writeRawData('AB\x00C')
self.assertEqual(ba.data(), py3k.b('AB\x00C'))
data = QDataStream(ba)
self.assertEqual(data.readRawData(4), py3k.b('AB\x00C'))
def testRawData(self):
data = QDataStream()
self.assertEqual(data.readRawData(4), None)
ba = QByteArray()
data = QDataStream(ba, QIODevice.WriteOnly)
data.writeRawData('AB\x00C')
self.assertEqual(ba.data(), py3k.b('AB\x00C'))
data = QDataStream(ba)
self.assertEqual(data.readRawData(4), py3k.b('AB\x00C'))
def readData(self, maxlen):
data = QByteArray()
total = 0
while maxlen > total:
chunk = min(self.m_buffer.size() - self.m_pos, maxlen - total)
data.append(self.m_buffer.mid(self.m_pos, chunk))
self.m_pos = (self.m_pos + chunk) % self.m_buffer.size()
total += chunk
return data.data()
def readData(self, maxlen):
data = QByteArray()
total = 0
while maxlen > total:
chunk = min(self.m_buffer.size() - self.m_pos, maxlen - total)
data.append(self.m_buffer.mid(self.m_pos, chunk))
self.m_pos = (self.m_pos + chunk) % self.m_buffer.size()
total += chunk
return data.data()
def process_icon(self, repo: Addon, data: QtCore.QByteArray):
"""Convert icon data into a valid icon file and store it"""
self.status_message.emit(
translate("AddonsInstaller", "Downloaded icon for {}").format(
repo.display_name
)
)
cache_file = repo.get_cached_icon_filename()
with open(cache_file, "wb") as icon_file:
icon_file.write(data.data())
repo.cached_icon_filename = cache_file
class AudioDataHandler(QIODevice):
"""Class for storing incoming audio data in a buffer for later analysis.
_buffer_size determines how much audio data is used when trying to find
beats per minute (BPM) value. The value set has been found experimentally
to give accurate BPM values without too much delay when BPM changes.
With current settings (44100Hz, 8bit uint), the buffe fills with 44100B/s.
Therefore, buffer of 350000 holds a bit more than 7 seconds of audio data.
"""
data_ready = Signal(np.ndarray)
_buffer_size = 350000
def __init__(self, format_):
super().__init__()
self._buffer = QByteArray()
self._format = format_
self.data = None
def start(self):
self.open(QIODevice.WriteOnly)
def stop(self):
self.close()
@Slot(QByteArray, int)
def writeData(self, new_data, len_):
"""Write new data to buffer.
Keeps buffer at constant size.
Emits:
data_ready(numpy.ndarray): When new data is written, emit all data
for later analysis.
"""
self._buffer.append(new_data.data())
data = self._buffer.data()
self._buffer = self._buffer.right(self._buffer_size)
data_array = np.frombuffer(data, dtype="uint8")
data_array = data_array.astype("int")
self.data_ready.emit(data_array)
return len_
class WebEngineScreenshot(QApplication):
def __init__(self, url, width, height, *args, **kwargs):
self.display = Xvfb(int(width * 1.2), int(height * 1.2))
self.display.start()
super().__init__(*args, **kwargs)
self.engine = QWebEngineView()
size = QSize(width, height)
self.engine.setFixedSize(size)
self.engine.setPage(QWebEnginePage())
settings = self.engine.page().settings()
settings.setAttribute(QWebEngineSettings.ShowScrollBars, False)
settings.setAttribute(QWebEngineSettings.JavascriptCanOpenWindows,
False)
settings.setAttribute(QWebEngineSettings.LocalStorageEnabled, False)
settings.setAttribute(QWebEngineSettings.HyperlinkAuditingEnabled,
False)
settings.setAttribute(QWebEngineSettings.PluginsEnabled, False)
settings.setAttribute(QWebEngineSettings.FullScreenSupportEnabled,
False)
settings.setAttribute(QWebEngineSettings.ScreenCaptureEnabled, False)
self.engine.loadFinished.connect(self.load_finished)
self.engine.load(QUrl(url))
self.engine.show()
@Slot(bool)
def load_finished(self, state):
pixmap = self.engine.grab()
self.image = QByteArray()
buf = QBuffer(self.image)
buf.open(QIODevice.WriteOnly)
pixmap.save(buf, "PNG")
buf.close()
self.quit()
def run(self):
try:
self.exec_()
except Exception:
return None
finally:
self.display.stop()
return self.image.data()
def process_package_xml(self, repo: Addon, data: QtCore.QByteArray):
"""Process the package.xml metadata file"""
repo.repo_type = Addon.Kind.PACKAGE # By definition
package_cache_directory = os.path.join(self.store, repo.name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "package.xml")
with open(new_xml_file, "wb") as f:
f.write(data.data())
metadata = FreeCAD.Metadata(new_xml_file)
repo.metadata = metadata
self.status_message.emit(
translate("AddonsInstaller", "Downloaded package.xml for {}").format(
repo.name
)
)
# Grab a new copy of the icon as well: we couldn't enqueue this earlier because
# we didn't know the path to it, which is stored in the package.xml file.
icon = metadata.Icon
if not icon:
# If there is no icon set for the entire package, see if there are
# any workbenches, which are required to have icons, and grab the first
# one we find:
content = repo.metadata.Content
if "workbench" in content:
wb = content["workbench"][0]
if wb.Icon:
if wb.Subdirectory:
subdir = wb.Subdirectory
else:
subdir = wb.Name
repo.Icon = subdir + wb.Icon
icon = repo.Icon
icon_url = utils.construct_git_url(repo, icon)
index = NetworkManager.AM_NETWORK_MANAGER.submit_unmonitored_get(icon_url)
self.requests[index] = (repo, UpdateMetadataCacheWorker.RequestType.ICON)
self.total_requests += 1
def testDataWithZeros(self):
s1 = "123\000321"
ba = QByteArray(s1)
s2 = ba.data()
self.assertEqual(py3k.b(s1), s2)
self.assertEqual(s1, ba)
def changeItem(self,
item,
newName=None,
newDescription=None,
newContent=None,
newAccess=None,
metadataChanges=None):
assert isinstance(item,
CollectionItem), 'item must be a collection item'
#newName=str(newName)
#newDescription=str(newDescription)
#newContent=str(newContent)
#just in case we have random unicode coming in
#TODO: check that item belongs to this collection. just in case
if item.readonly(): raise CollectionReadonlyError()
# Upgrade version if needed
self.updateItemIfNeeded(item)
if newAccess is not None and newAccess != item.access():
newitem = self.addItem(
item.name() if newName is None else newName,
item.description()
if newDescription is None else newDescription,
item.content() if newContent is None else newContent,
newAccess, item.metadata())
# all auxiliary files MUST BE COPIED before deleting original item
# icon:
id, filename = newitem.id().split('@', 1)
if 'iconpixmap' in item.metadata(
) and 'iconfullname' in item.metadata(
) and 'icondata' in item.metadata():
data = {
'files': {
item.metadata()['iconfullname']: {
'content': item.metadata()['icondata']
}
}
}
req = urllib2.Request('https://api.github.com/gists/%s' % id,
json.dumps(data),
headers=self.__headers)
req.get_method = lambda: 'PATCH'
code, rep = urlopen_nt(req)
if (code != 200):
if (code == 403): raise InvalidToken('github auth failed')
raise CollectionSyncError(
"unexpected server return level %d" % code)
self.removeItem(
copy.copy(item)
) # remove the copy cuz item gets invalidated and we dont want that for original item
item._name = newitem._name
item._desc = newitem._desc
item._meta['raw_url'] = newitem._meta['raw_url']
item._meta['nettype'] = newitem._meta['nettype']
item._id = newitem._id
item._access = newitem._access
item._readonly = newitem._readonly
# if access is changed - we have to destroy this gist and create a new one with proper 'public' key
# Butt Beware - we need to modify item's contents and return it WITHOUT reassigning the item itself
# That's what we are doing here currently
return
if 'nettype' not in item.metadata():
item.invalidate()
raise CollectionItemInvalidError(
'required metadata was not found in the item')
id, filename = item.id().split('@', 1)
data = {}
proceed = False
if (newName is not None):
if filename.startswith(
'item:'
): # new format. can cause trouble if someone actually called item starting with 'item:'
data['filename'] = 'item:' + newName
else: # old format
data['filename'] = newName
proceed = True
if (newContent is not None):
data['content'] = newContent
proceed = True
if (data != {}):
data = {'files': {filename: data}}
if (newDescription is not None):
data['description'] = ':'.join(
(item.metadata()['nettype'], newDescription))
proceed = True
if proceed:
req = urllib2.Request('https://api.github.com/gists/%s' % id,
json.dumps(data),
headers=self.__headers)
req.get_method = lambda: 'PATCH'
code, rep = urlopen_nt(req)
if (code != 200):
if (code == 403): raise InvalidToken('github auth failed')
raise CollectionSyncError("unexpected server return level %d" %
code)
gist = json.loads(rep.read())
newfilenames = gist['files'].keys()
newfilenames.remove('00_HPASTE_SNIPPET')
if (len(newfilenames) == 1):
newfilename = newfilenames[0]
newname = newfilename
else:
itemfileanmes = [
x for x in newfilenames if x.startswith("item:")
]
if len(itemfileanmes) != 1:
raise CollectionInconsistentError(
'something went wrong during item creation: could not find unique item data in gist'
)
newfilename = itemfileanmes[0]
newname = newfilename.split(':', 1)[1]
desc = gist['description']
nettype = ''
if (':' in desc):
nettype, desc = desc.split(':', 1)
item._desc = desc
item._name = newname
item._meta['raw_url'] = gist['files'][newfilename]['raw_url']
item._meta['nettype'] = nettype
item._id = '%s@%s' % (gist['id'], newfilename)
item._access = CollectionItem.AccessType.public if gist[
'public'] else CollectionItem.AccessType.private
item._readonly = False
# metadata changes processing
if metadataChanges:
metaspecialkeys = [
'raw_url', 'nettype', 'icondata', 'iconfullname', 'iconpixmap',
'iconfullname', 'icondata'
]
#if 'icondata' in metadataChanges and ('iconfullname' in item.metadata() or 'iconfullname' in metadataChanges):
# Shall i implement this case? for when qt is not loaded
if 'iconpixmap' in metadataChanges and qtAvailable:
pix = metadataChanges['iconpixmap']
if pix is None: # removing pixmap
if 'iconpixmap' in item.metadata():
data = {
'files': {
item.metadata()['iconfullname']: None
}
}
req = urllib2.Request(
'https://api.github.com/gists/%s' %
item.id().split('@', 1)[0],
json.dumps(data),
headers=self.__headers)
req.get_method = lambda: 'PATCH'
code, rep = urlopen_nt(req)
if (code != 200):
if (code == 403):
raise InvalidToken('github auth failed')
raise CollectionSyncError(
"unexpected server return level %d" % code)
rep.close()
del item._meta['iconpixmap']
del item._meta['iconfullname']
del item._meta['icondata']
else:
barr = QByteArray()
buff = QBuffer(barr)
pix.save(buff, "PNG")
imagedata = base64.b64encode(barr.data())
buff.deleteLater()
oldiconname = item.metadata().get('iconfullname', None)
newiconname = 'icon:PNG-base64:autoicon'
data = {'files': {}}
if oldiconname is not None and oldiconname != newiconname:
data['files'][oldiconname] = None
data['files'][newiconname] = {'content': imagedata}
req = urllib2.Request('https://api.github.com/gists/%s' %
item.id().split('@', 1)[0],
json.dumps(data),
headers=self.__headers)
req.get_method = lambda: 'PATCH'
code, rep = urlopen_nt(req)
if (code != 200):
if (code == 403):
raise InvalidToken('github auth failed')
raise CollectionSyncError(
"unexpected server return level %d" % code)
replydict = json.loads(rep.read())
if newiconname not in replydict['files']:
raise CollectionSyncError(
"icon file was not uploaded properly")
globalIconCacher[replydict['files'][newiconname]
['raw_url']] = imagedata
item._meta['iconfullname'] = newiconname
item._meta['icondata'] = imagedata
item._meta['iconpixmap'] = pix
for metakey in metadataChanges.keys():
# All special cases are taken care of, so now just blind copy all remaining changes
if metakey in metaspecialkeys: continue
item._meta[metakey] = metadataChanges[metakey]
def testData(self):
url = QByteArray("http://web.openbossa.org/")
self.assertEqual(url.data(), py3k.b("http://web.openbossa.org/"))
def testData(self):
url = QByteArray(py3k.b("http://pyside.org"))
self.assertEqual(url.data(), py3k.b("http://pyside.org"))
def testData(self):
url = QByteArray("http://web.openbossa.org/")
self.assertEqual(url.data(), py3k.b("http://web.openbossa.org/"))
def testDataWithZeros(self):
s1 = "123\000321"
ba = QByteArray(s1)
s2 = ba.data()
self.assertEqual(py3k.b(s1), s2)
self.assertEqual(s1, ba)
