def get(self, key, default):
fs = None
try:
fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key))
dirName = os.path.dirname(fullFileName)
pkg = os.path.basename(dirName)
fileName = os.path.basename(fullFileName)
isZip = pkg.lower().endswith('.zip')
if os.path.exists(dirName):
if isZip:
fs = ZipFS(dirName, mode='r', compression='stored')
else:
fs = OSFS(dirName, create=True)
if fs.exists(fileName):
try:
#log(fileName)
#log(cPickle.loads(fs.getcontents(fileName)))
return cPickle.loads(fs.getcontents(fileName))
except Exception:
if isZip:
log('[WARNING] Broken file: %s' % fullFileName)
else:
log('[WARNING] Remove broken file: %s' %
fullFileName)
fs.remove(fileName)
raise
return default
except Exception:
err(traceback.format_exc())
return default
finally:
if fs is not None:
fs.close()
def set(self, key, value):
fs = None
try:
if not key:
return
key = key.format(accountDBID=utils.getAccountDBID())
fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key))
dirName = os.path.dirname(fullFileName)
pkg = os.path.basename(dirName)
fileName = os.path.basename(fullFileName)
isZip = pkg.lower().endswith('.zip')
save = True
if isZip:
fs = ZipFS(dirName, mode='a', compression='stored')
if fs.exists(fileName):
log('[WARNING] archive "{}" already contains file "{}". Do not save the new data.'
.format(pkg, fileName))
save = False
else:
fs = OSFS(dirName, create=True)
if save:
fs.setcontents(fileName, cPickle.dumps(value))
except Exception:
err(traceback.format_exc())
finally:
if fs is not None:
fs.close()
def get(self, key, default):
fs = None
try:
fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key))
dirName = os.path.dirname(fullFileName)
pkg = os.path.basename(dirName)
fileName = os.path.basename(fullFileName)
isZip = pkg.lower().endswith('.zip')
if os.path.exists(dirName):
if isZip:
fs = ZipFS(dirName, mode='r', compression='stored')
else:
fs = OSFS(dirName, create=True)
if fs.exists(fileName):
try:
#log(fileName)
#log(cPickle.loads(fs.getcontents(fileName)))
return cPickle.loads(fs.getcontents(fileName))
except Exception:
if isZip:
log('[WARNING] Broken file: %s' % fullFileName)
else:
log('[WARNING] Remove broken file: %s' % fullFileName)
fs.remove(fileName)
raise
return default
except Exception:
err(traceback.format_exc())
return default
finally:
if fs is not None:
fs.close()
def test_binary_write_read(self):
# GIVEN zipfs
z = self.z
# WHEN binary data is written to a test file in zipfs
f = z.open('test.data', 'wb')
f.write(self.test_content)
f.close()
z.close()
# THEN the same binary data is retrieved when opened again
z = ZipFS('test.zip', 'r')
f = z.open('test.data', 'rb')
content = f.read()
f.close()
z.close()
self.assertEqual(content, self.test_content)
def test_url_on_sys_path(self):
t = TempFS()
zpath = t.getsyspath("modules.zip")
z = ZipFS(zpath, "w")
self._init_modules(z)
z.close()
z = ZipFS(zpath, "r")
assert z.isfile("fsih_hello.py")
z.close()
sys.path.append("zip://" + zpath)
FSImportHook.install()
try:
self._check_imports_are_working()
finally:
sys.path_hooks.remove(FSImportHook)
sys.path.pop()
t.close()
def test_url_on_sys_path(self):
t = TempFS()
zpath = t.getsyspath("modules.zip")
z = ZipFS(zpath,"w")
self._init_modules(z)
z.close()
z = ZipFS(zpath,"r")
assert z.isfile("fsih_hello.py")
z.close()
sys.path.append("zip://" + zpath)
FSImportHook.install()
try:
self._check_imports_are_working()
finally:
sys.path_hooks.remove(FSImportHook)
sys.path.pop()
t.close()
def do_build(dataplicity_path):
"""Build firmware in project directory"""
with fsopendir(dataplicity_path) as src_fs:
version = firmware.get_version(src_fs)
print("Building version {:010}...".format(version))
filename = "firmware-{}.zip".format(version)
firmware_path = join('__firmware__', filename)
src_fs.makedir('__firmware__', allow_recreate=True)
with src_fs.open(firmware_path, 'wb') as zip_file:
dst_fs = ZipFS(zip_file, 'w')
firmware.build(src_fs, dst_fs)
dst_fs.close()
size = src_fs.getsize(firmware_path)
print("Wrote {} ({:,} bytes)".format(firmware_path, size))
def test_binary_write_read(self):
# GIVEN zipfs
z = self.z
# WHEN binary data is written to a test file in zipfs
f = z.open('test.data', 'wb')
f.write(self.test_content)
f.close()
z.close()
# THEN the same binary data is retrieved when opened again
z = ZipFS('test.zip', 'r')
f = z.open('test.data', 'rb')
content = f.read()
f.close()
z.close()
self.assertEqual(content, self.test_content)
def do_build(dataplicity_path):
"""Build firmware in project directory"""
with fsopendir(dataplicity_path) as src_fs:
version = firmware.get_version(src_fs)
print "Building version {:010}...".format(version)
filename = "firmware-{}.zip".format(version)
firmware_path = join('__firmware__', filename)
src_fs.makedir('__firmware__', allow_recreate=True)
with src_fs.open(firmware_path, 'wb') as zip_file:
dst_fs = ZipFS(zip_file, 'w')
firmware.build(src_fs, dst_fs)
dst_fs.close()
size = src_fs.getsize(firmware_path)
print "Wrote {} ({:,} bytes)".format(firmware_path, size)
def set(self, key, value):
fs = None
try:
fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key))
dirName = os.path.dirname(fullFileName)
pkg = os.path.basename(dirName)
fileName = os.path.basename(fullFileName)
isZip = pkg.lower().endswith('.zip')
save = True
if isZip:
fs = ZipFS(dirName, mode='a', compression='stored')
if fs.exists(fileName):
log('[WARNING] archive "{}" already contains file "{}". Do not save the new data.'.format(pkg, fileName))
save = False
else:
fs = OSFS(dirName, create=True)
if save:
fs.setcontents(fileName, cPickle.dumps(value))
except Exception:
err(traceback.format_exc())
finally:
if fs is not None:
fs.close()
def install_encoded(device_class, version, firmware_b64, activate_firmware=True, firmware_path=None):
"""Install firmware from a b64 encoded zip file"""
# TODO: implement this in a less memory hungry way
# decode from b64
firmware_bin = base64.b64decode(firmware_b64)
# Make a file-like object
firmware_file = BytesIO(firmware_bin)
# Open zip
firmware_fs = ZipFS(firmware_file)
# Open firmware dir
dst_fs = OSFS(firmware_path or constants.FIRMWARE_PATH, create=True, dir_mode=0o755)
# Install
install_path = install(device_class, version, firmware_fs, dst_fs)
# Move symlink to active firmware
if activate_firmware:
activate(device_class, version, dst_fs, fw_path=firmware_path)
# Clean up any temporary files
firmware_fs.close()
dst_fs.close()
# Return install_path
return install_path
def install_encoded(device_class, version, firmware_b64, activate_firmware=True):
"""Install firmware from a b64 encoded zip file"""
# TODO: implement this in a less memory hungry way
# decode from b64
firmware_bin = base64.b64decode(firmware_b64)
# Make a file-like object
firmware_file = StringIO(firmware_bin)
# Open zip
firmware_fs = ZipFS(firmware_file)
# Open firmware dir
dst_fs = OSFS(constants.FIRMWARE_PATH, create=True)
# Install
install_path = install(device_class, version, firmware_fs, dst_fs)
# Move symlink to active firmware
if activate_firmware:
activate(device_class, version, dst_fs)
# Clean up any temporary files
firmware_fs.close()
dst_fs.close()
# Return install_path
return install_path
