def post_process_3mf(filename):
print('post processing 3MF file (extracting XML data from ZIP): ',
filename)
from zipfile import ZipFile
xml_content = ZipFile(filename).read("3D/3dmodel.model")
xml_content = re.sub('UUID="[^"]*"',
'UUID="XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXX"',
xml_content.decode('utf-8'))
with open(filename, 'wb') as xml_file:
xml_file.write(xml_content.encode('utf-8'))
def fetch_application(app_url, directory=None):
domain, path = split_url(app_url)
url = urlparse(app_url)
metadata = {'origin': domain}
manifest_filename = 'manifest.webapp'
if url.scheme:
print 'manifest: ' + app_url
print 'fetching manifest...'
manifest_url = urllib2.urlopen(app_url)
manifest = json.loads(manifest_url.read().decode('utf-8-sig'))
metadata['installOrigin'] = domain
if 'etag' in manifest_url.headers:
metadata['etag'] = manifest_url.headers['etag']
else:
print 'extract manifest from zip...'
appzip = ZipFile(app_url, 'r').read('manifest.webapp')
manifest = json.loads(appzip.decode('utf-8-sig'))
appname = get_directory_name(manifest['name'])
manifest["shortname"] = appname
apppath = appname
if directory is not None:
apppath = os.path.join(directory, appname)
if not os.path.exists(apppath):
os.mkdir(apppath)
if 'package_path' in manifest or not url.scheme:
manifest_filename = 'update.webapp'
filename = 'application.zip'
metadata['origin'] = ''.join(['app://', appname])
metadata['type'] = 'web'
if url.scheme:
print 'downloading app...'
path = manifest['package_path']
#urllib.urlretrieve(
# manifest['package_path'],
# filename=os.path.join(apppath, filename))
filename='%s%s%s' % (appname, os.sep, filename)
#filename=os.path.join(apppath, filename)
f = open(filename, "wb")
f.write(urllib2.urlopen(path).read())
f.close()
#
metadata['manifestURL'] = url.geturl()
metadata['packageEtag'] = urllib2.urlopen(path).headers['etag']
else:
print 'copying app...'
shutil.copyfile(app_url, '%s%s%s' % (appname, os.sep, filename))
metadata['manifestURL'] = ''.join([domain, path, 'manifest.webapp'])
manifest['package_path'] = ''.join(['/', filename])
def get_element_tree_from_file(self, file_path):
'''
Returns the root of an ElementTree object loaded with the content
of the given file at file_path.
The file can be an .ods file or its constituent content.xml file.
'''
if not os.path.exists(file_path):
return None
if file_path.endswith('.ods'):
from zipfile import ZipFile
content = ZipFile(file_path).read(name='content.xml')
content = content.decode('utf-8')
elif file_path.endswith('.xml'):
content = self._read_file(file_path, encoding='utf-8')
else:
self.warn(
'Unsupported file extension, only accept .xml or .ods.',
status='ERROR'
)
content = re.sub(r'(table|office|style|text|draw|fo):', '', content)
return ET.fromstring(content)
def fetch_webapp(app_url, directory=None, removable=False):
"""
get webapp file and parse for preinstalled webapp
output:
[appname]/manifest.webapp
[appname]/metadata.json
[appname]/update.webapp (if package_path is defined)
[appname]/cache/ (if appcache_path is defined)
"""
domain, path = split_url(app_url)
url = urlparse(app_url)
metadata = {'origin': domain}
manifest_filename = 'manifest.webapp'
if url.scheme:
logger.info('manifest: ' + app_url)
logger.info('fetching manifest...')
manifest_url = open_from_url(app_url)
manifest = json.loads(manifest_url.read().decode(MANIFEST_CODE_PAGE))
metadata['installOrigin'] = 'app://kaios-plus.kaiostech.com'
if 'etag' in manifest_url.headers:
metadata['etag'] = manifest_url.headers['etag']
else:
logger.info('extract manifest from zip...')
appzip = ZipFile(app_url, 'r').read('manifest.webapp')
manifest = json.loads(appzip.decode(MANIFEST_CODE_PAGE))
appname = get_directory_name(manifest['name'])
app_dir = appname
if directory is not None:
app_dir = os.path.join(directory, appname)
if not os.path.exists(app_dir):
os.mkdir(app_dir)
if 'package_path' in manifest or not url.scheme:
manifest_filename = 'update.webapp'
filename = 'application.zip'
metadata.pop('origin', None)
if url.scheme:
logger.info('downloading app...')
path = manifest['package_path']
retrieve_from_url(manifest['package_path'],
os.path.join(app_dir, filename))
metadata['manifestURL'] = url.geturl()
package_url = open_from_url(path)
if 'etag' in package_url.headers:
metadata['packageEtag'] = package_url.headers['etag']
else:
logger.info('copying app...')
shutil.copyfile(app_url, '%s%s%s' % (appname, os.sep, filename))
metadata['manifestURL'] = ''.join(
[domain, path, 'manifest.webapp'])
manifest['package_path'] = ''.join(['/', filename])
logger.info('fetching icons...')
for key in manifest['icons']:
manifest['icons'][key] = fetch_icon(key, manifest['icons'], domain,
path, app_dir)
if 'appcache_path' in manifest:
metadata_info = []
logger.info('fetching appcache...', )
appcache_manifest = get_appcache_manifest(domain, path, app_dir,
manifest['appcache_path'])
if not os.path.exists(appcache_manifest['local_dir']):
os.makedirs(appcache_manifest['local_dir'])
logger.info(' from ' + appcache_manifest['url'])
logger.info(' save to ' + appcache_manifest['local_path'], )
(_filename,
headerAC) = retrieve_from_url(appcache_manifest['url'],
appcache_manifest['local_path'])
lines = []
with open(appcache_manifest['local_path']) as fd:
lines = fd.readlines()
(lines,
headers) = fetch_appcache(domain, appcache_manifest['remote_dir'],
appcache_manifest['local_dir'], lines)
with open(appcache_manifest['local_path'], 'w') as fd:
logger.info('overwrite new appcache')
lines.append('')
fd.write('\n'.join(lines))
with open(
appcache_manifest['local_dir'] + '../resources_metadata.json',
'w') as resources:
resources.write('{\n')
headers.append(
format_resource_metadata(appcache_manifest['local_path'],
headerAC))
resources.write(',\n'.join(headers))
resources.write('\n}\n')
print app_url
# add manifestURL for update
metadata['manifestURL'] = app_url
metadata['external'] = True
if removable == 'true':
metadata['removable'] = True
else:
metadata['removable'] = False
if 'type' in manifest:
metadata['type'] = manifest['type']
f = file(os.path.join(app_dir, 'metadata.json'), 'w')
f.write(json.dumps(metadata))
f.close()
f = codecs.open(os.path.join(app_dir, manifest_filename), 'w', 'utf-8')
f.write(json.dumps(manifest, ensure_ascii=False))
return manifest
def fetch_webapp(app_url, directory=None):
"""
get webapp file and parse for preinstalled webapp
output:
[appname]/manifest.webapp
[appname]/metadata.json
[appname]/update.webapp (if package_path is defined)
[appname]/cache/ (if appcache_path is defined)
"""
domain, path = split_url(app_url)
url = urlparse(app_url)
metadata = {'origin': domain}
manifest_filename = 'manifest.webapp'
if url.scheme:
logger.info('manifest: ' + app_url)
logger.info('fetching manifest...')
manifest_url = open_from_url(app_url)
manifest = json.loads(manifest_url.read().decode('utf-8-sig'))
metadata['installOrigin'] = 'https://marketplace.firefox.com'
if 'etag' in manifest_url.headers:
metadata['etag'] = manifest_url.headers['etag']
else:
logger.info('extract manifest from zip...')
appzip = ZipFile(app_url, 'r').read('manifest.webapp')
manifest = json.loads(appzip.decode('utf-8-sig'))
appname = get_directory_name(manifest['name'])
app_dir = appname
if directory is not None:
app_dir = os.path.join(directory, appname)
if not os.path.exists(app_dir):
os.mkdir(app_dir)
if 'package_path' in manifest or not url.scheme:
manifest_filename = 'update.webapp'
filename = 'application.zip'
metadata.pop('origin', None)
if url.scheme:
logger.info('downloading app...')
path = manifest['package_path']
retrieve_from_url(
manifest['package_path'],
os.path.join(app_dir, filename))
metadata['manifestURL'] = url.geturl()
package_url = open_from_url(path)
if 'etag' in package_url.headers:
metadata['packageEtag'] = package_url.headers['etag']
else:
logger.info('copying app...')
shutil.copyfile(app_url, '%s%s%s' % (appname, os.sep, filename))
metadata['manifestURL'] = ''.join(
[domain, path, 'manifest.webapp'])
manifest['package_path'] = ''.join(['/', filename])
logger.info('fetching icons...')
for key in manifest['icons']:
manifest['icons'][key] = fetch_icon(
key, manifest['icons'], domain, path, app_dir)
if 'appcache_path' in manifest:
metadata_info = [];
logger.info('fetching appcache...',)
appcache_manifest = get_appcache_manifest(
domain, path, app_dir, manifest['appcache_path'])
if not os.path.exists(appcache_manifest['local_dir']):
os.makedirs(appcache_manifest['local_dir'])
logger.info(' from ' + appcache_manifest['url'])
logger.info(' save to ' + appcache_manifest['local_path'],)
( _filename, headerAC ) = retrieve_from_url(appcache_manifest['url'],
appcache_manifest['local_path'])
lines = []
with open(appcache_manifest['local_path']) as fd:
lines = fd.readlines()
(lines, headers) = fetch_appcache(domain, appcache_manifest['remote_dir'],
appcache_manifest['local_dir'], lines)
with open(appcache_manifest['local_path'], 'w') as fd:
logger.info('overwrite new appcache')
lines.append('')
fd.write('\n'.join(lines))
with open(appcache_manifest['local_dir'] + '../resources_metadata.json', 'w') as resources:
resources.write('{\n');
headers.append(format_resource_metadata(appcache_manifest['local_path'], headerAC))
resources.write(',\n'.join(headers))
resources.write('\n}\n');
# add manifestURL for update
metadata['manifestURL'] = app_url
metadata['external'] = True
f = file(os.path.join(app_dir, 'metadata.json'), 'w')
f.write(json.dumps(metadata))
f.close()
f = codecs.open(os.path.join(app_dir, manifest_filename), 'w', 'utf-8')
f.write(json.dumps(manifest, ensure_ascii=False))
return manifest
def fetch_webapp(app_url, directory=None):
"""
get webapp file and parse for preinstalled webapp
output:
[appname]/manifest.webapp
[appname]/metadata.json
[appname]/update.webapp (if package_path is defined)
[appname]/cache/ (if appcache_path is defined)
"""
domain, path = split_url(app_url)
url = urlparse(app_url)
metadata = {'origin': domain}
manifest_filename = 'manifest.webapp'
if url.scheme:
print 'manifest: ' + app_url
print 'fetching manifest...'
manifest_url = open_from_url(app_url)
manifest = json.loads(manifest_url.read().decode('utf-8-sig'))
metadata['installOrigin'] = domain
if 'etag' in manifest_url.headers:
metadata['etag'] = manifest_url.headers['etag']
else:
print 'extract manifest from zip...'
appzip = ZipFile(app_url, 'r').read('manifest.webapp')
manifest = json.loads(appzip.decode('utf-8-sig'))
appname = get_directory_name(manifest['name'])
apppath = appname
if directory is not None:
apppath = os.path.join(directory, appname)
if not os.path.exists(apppath):
os.mkdir(apppath)
if 'package_path' in manifest or not url.scheme:
manifest_filename = 'update.webapp'
filename = 'application.zip'
metadata['origin'] = ''.join(['app://', appname])
if url.scheme:
print 'downloading app...'
path = manifest['package_path']
retrieve_from_url(manifest['package_path'],
os.path.join(apppath, filename))
metadata['manifestURL'] = url.geturl()
metadata['packageEtag'] = open_from_url(path).headers['etag']
else:
print 'copying app...'
shutil.copyfile(app_url, '%s%s%s' % (appname, os.sep, filename))
metadata['manifestURL'] = ''.join(
[domain, path, 'manifest.webapp'])
manifest['package_path'] = ''.join(['/', filename])
print 'fetching icons...'
for key in manifest['icons']:
manifest['icons'][key] = fetch_icon(key, manifest['icons'], domain,
path, apppath)
if 'appcache_path' in manifest:
print 'fetching appcache...',
fetch_appcache(domain, manifest['appcache_path'], apppath)
# add manifestURL for update
metadata['manifestURL'] = app_url
f = file(os.path.join(apppath, 'metadata.json'), 'w')
f.write(json.dumps(metadata))
f.close()
f = codecs.open(os.path.join(apppath, manifest_filename), 'w', 'utf-8')
f.write(json.dumps(manifest, ensure_ascii=False))
return manifest
def fetch_application(app_url, directory=None):
origin = get_origin(app_url)
url = urlparse(app_url)
metadata = {'origin': origin}
manifest_filename = 'manifest.webapp'
if url.scheme:
print 'manifest: ' + app_url
print 'fetching manifest...'
manifest_url = urllib.urlopen(app_url)
manifest = json.loads(manifest_url.read().decode('utf-8-sig'))
else:
print 'extract manifest from zip...'
appzip = ZipFile(app_url, 'r').read('manifest.webapp')
manifest = json.loads(appzip.decode('utf-8-sig'))
appname = get_directory_name(manifest['name'])
manifest["shortname"] = appname
apppath = appname
if directory is not None:
apppath = os.path.join(directory, appname)
if not os.path.exists(apppath):
os.mkdir(apppath)
if 'package_path' in manifest or not url.scheme:
manifest_filename = 'update.webapp'
filename = 'application.zip'
metadata['origin'] = 'app://%s/' % appname
metadata['type'] = 'web'
if url.scheme:
print 'downloading app...'
path = manifest['package_path']
urllib.urlretrieve(
manifest['package_path'],
filename=os.path.join(apppath, filename))
metadata['installOrigin'] = ''.join([url.scheme, '://', url.netloc])
metadata['manifestURL'] = url.geturl()
metadata['etag'] = manifest_url.headers['etag'][1:-1]
metadata['packageEtag'] = urllib.urlopen(path).headers['etag'][1:-1]
else:
print 'copying app...'
shutil.copyfile(app_url, '%s%s%s' % (appname, os.sep, filename))
metadata['manifestURL'] = ''.join([metadata['origin'],
'manifest.webapp'])
manifest['package_path'] = ''.join(['/', filename])
else:
print 'fetching icons...'
for key in manifest['icons']:
iconurl = get_absolute_url(urlparse(origin),
urlparse(manifest['icons'][key]))
image = urllib.urlopen(iconurl).read()
manifest['icons'][key] = convert(image,
mimetypes.guess_type(iconurl)[0])
f = file(os.path.join(apppath, 'metadata.json'), 'w')
f.write(json.dumps(metadata))
f.close()
f = codecs.open(os.path.join(apppath, manifest_filename), 'w', 'utf-8')
f.write(json.dumps(manifest, ensure_ascii=False))
return manifest
def fetch_webapp(app_url, directory=None):
"""
get webapp file and parse for preinstalled webapp
output:
[appname]/manifest.webapp
[appname]/metadata.json
[appname]/update.webapp (if package_path is defined)
[appname]/cache/ (if appcache_path is defined)
"""
domain, path = split_url(app_url)
url = urlparse(app_url)
metadata = {'origin': domain}
manifest_filename = 'manifest.webapp'
if url.scheme:
logger.info('manifest: ' + app_url)
logger.info('fetching manifest...')
manifest_url = open_from_url(app_url)
manifest = json.loads(manifest_url.read().decode('utf-8-sig'))
metadata['installOrigin'] = 'https://marketplace.firefox.com'
if 'etag' in manifest_url.headers:
metadata['etag'] = manifest_url.headers['etag']
else:
logger.info('extract manifest from zip...')
appzip = ZipFile(app_url, 'r').read('manifest.webapp')
manifest = json.loads(appzip.decode('utf-8-sig'))
appname = get_directory_name(manifest['name'])
app_dir = appname
if directory is not None:
app_dir = os.path.join(directory, appname)
if not os.path.exists(app_dir):
os.mkdir(app_dir)
if 'package_path' in manifest or not url.scheme:
manifest_filename = 'update.webapp'
filename = 'application.zip'
metadata['origin'] = ''.join(['app://', appname])
if url.scheme:
logger.info('downloading app...')
path = manifest['package_path']
retrieve_from_url(manifest['package_path'],
os.path.join(app_dir, filename))
metadata['manifestURL'] = url.geturl()
metadata['packageEtag'] = open_from_url(path).headers['etag']
else:
logger.info('copying app...')
shutil.copyfile(app_url, '%s%s%s' % (appname, os.sep, filename))
metadata['manifestURL'] = ''.join(
[domain, path, 'manifest.webapp'])
manifest['package_path'] = ''.join(['/', filename])
logger.info('fetching icons...')
for key in manifest['icons']:
manifest['icons'][key] = fetch_icon(key, manifest['icons'], domain,
path, app_dir)
if 'appcache_path' in manifest:
logger.info('fetching appcache...', )
appcache_manifest = get_appcache_manifest(domain, path, app_dir,
manifest['appcache_path'])
if not os.path.exists(appcache_manifest['local_dir']):
os.makedirs(appcache_manifest['local_dir'])
logger.info(' from ' + appcache_manifest['url'])
logger.info(' save to ' + appcache_manifest['local_path'], )
retrieve_from_url(appcache_manifest['url'],
appcache_manifest['local_path'])
lines = []
with open(appcache_manifest['local_path']) as fd:
lines = fd.readlines()
lines = fetch_appcache(domain, appcache_manifest['remote_dir'],
appcache_manifest['local_dir'], lines)
with open(appcache_manifest['local_path'], 'w') as fd:
logger.info('overwrite new appcache')
lines.append('')
fd.write('\n'.join(lines))
# add manifestURL for update
metadata['manifestURL'] = app_url
f = file(os.path.join(app_dir, 'metadata.json'), 'w')
f.write(json.dumps(metadata))
f.close()
f = codecs.open(os.path.join(app_dir, manifest_filename), 'w', 'utf-8')
f.write(json.dumps(manifest, ensure_ascii=False))
return manifest
