def iCalendarZipArchiveData(self):
calendarComponents = yield self.calendarComponents()
fileHandle = StringIO()
try:
zipFile = ZipFile(fileHandle, "w", allowZip64=True)
try:
zipFile.comment = ("Calendars for UID: {}".format(
self._record.uid))
names = set()
for name, component in calendarComponents:
if name in names:
i = 0
while True:
i += 1
nextName = "{} {:d}".format(name, i)
if nextName not in names:
name = nextName
break
assert i < len(calendarComponents)
text = component.getText().encode("utf-8")
zipFile.writestr(name.encode("utf-8"), text)
finally:
zipFile.close()
data = fileHandle.getvalue()
finally:
fileHandle.close()
returnValue(data)
def createZip(targetpath, contents, comment=''):
zfile = ZipFile(str(targetpath), mode='w', compression=compression)
zfile.comment = bytes(comment, 'utf-8')
try:
for name, file, in contents:
zfile.writestr(name, file, compression)
finally:
zfile.close()
def snapshot(snapshot_path, *paths):
"""Create an arcive, consisting of specified paths. Recursive.
Appends comment for zip_file, as package data
Args:
snapshot_path (str): The path to the snapshot.zip.
*paths (str): The paths to files to include
in the said snapshot.zip.
Returns:
None
"""
zip_file = ZipFile(snapshot_path, 'a')
zip_file.comment = 'unimplemented' # will be implemented in the future
for path in paths:
if os.path.isfile(path):
zip_file.write(path, path, ZIP_DEFLATED)
continue
# it's a directory, so recursively add files from a directory.
for directory, files in lib.index(path).items():
zip_file.write(directory, directory, ZIP_DEFLATED)
for file_name in files:
file_path = os.path.join(directory, file_name)
try:
zip_file.write(file_path, file_path, ZIP_DEFLATED)
except IOError: # I forgot what this weird exception actually is
# the timestamp preceeded 1980 for some reason
os.utime(file_path, None)
zip_file.write(file_path, file_path, ZIP_DEFLATED)
zip_file.close()
return None
def _update_noversion(self, filepath):
logging.debug('Updating from "no version"')
oldzip = ZipFile(filepath, 'r')
newzip = ZipFile(filepath + ".new", 'w')
# Update keys.ini
config = ConfigParser()
fp = oldzip.open(KEYS_INI_FILENAME, 'r')
config.read(StringIO(fp.read().decode('ascii')))
for section, option, value in config:
value = value.replace('pymontecarlo.result.base.result.', '')
setattr(getattr(config, section), option, value)
fp = StringIO()
config.write(fp)
newzip.writestr(KEYS_INI_FILENAME, fp.getvalue())
# Add other files to new zip
for zipinfo in oldzip.infolist():
if zipinfo.filename == KEYS_INI_FILENAME:
continue
data = oldzip.read(zipinfo)
newzip.writestr(zipinfo, data)
# Add version
newzip.comment = b'version=2'
oldzip.close()
newzip.close()
# Remove old zip and replace with new one
os.remove(filepath)
os.rename(filepath + ".new", filepath)
return self._update_version2(filepath)
def _update_version1(self, filepath):
logging.debug('Updating from "version 1"')
oldzip = ZipFile(filepath, 'r')
newzip = ZipFile(filepath + ".new", 'w')
# Update stats.cfg
config = ConfigParser()
config.read(oldzip.open('stats.cfg', 'r'))
limit = config.stats.convergence_limit
del config.stats.convergence_limit
config.stats.convergor = '<CompositionConvergor(limit=%s)>' % limit
fp = StringIO()
config.write(fp)
newzip.writestr('stats.cfg', fp.getvalue())
# Add other files to new zip
for zipinfo in oldzip.infolist():
if zipinfo.filename == 'stats.cfg':
continue
data = oldzip.read(zipinfo)
newzip.writestr(zipinfo, data)
# Add version
newzip.comment = 'version=%s' % VERSION
oldzip.close()
newzip.close()
# Remove old zip and replace with new one
os.remove(filepath)
os.rename(filepath + ".new", filepath)
return filepath
def _update_version1(self, filepath):
logging.debug('Updating from "version 1"')
oldzip = ZipFile(filepath, 'r')
newzip = ZipFile(filepath + ".new", 'w')
# Update stats.cfg
config = ConfigParser()
config.read(oldzip.open('stats.cfg', 'r'))
limit = config.stats.convergence_limit
del config.stats.convergence_limit
config.stats.convergor = '<CompositionConvergor(limit=%s)>' % limit
fp = StringIO()
config.write(fp)
newzip.writestr('stats.cfg', fp.getvalue())
# Add other files to new zip
for zipinfo in oldzip.infolist():
if zipinfo.filename == 'stats.cfg':
continue
data = oldzip.read(zipinfo)
newzip.writestr(zipinfo, data)
# Add version
newzip.comment = 'version=%s' % VERSION
oldzip.close()
newzip.close()
# Remove old zip and replace with new one
os.remove(filepath)
os.rename(filepath + ".new", filepath)
return filepath
def iCalendarZipArchiveData(self):
calendarComponents = yield self.calendarComponents()
fileHandle = StringIO()
try:
zipFile = ZipFile(fileHandle, "w", allowZip64=True)
try:
zipFile.comment = (
"Calendars for UID: {}".format(self._record.uid)
)
names = set()
for name, component in calendarComponents:
if name in names:
i = 0
while True:
i += 1
nextName = "{} {:d}".format(name, i)
if nextName not in names:
name = nextName
break
assert i < len(calendarComponents)
text = component.getText().encode("utf-8")
zipFile.writestr(name.encode("utf-8"), text)
finally:
zipFile.close()
data = fileHandle.getvalue()
finally:
fileHandle.close()
returnValue(data)
def save(self, source):
"""
Saves results in a results ZIP.
:arg source: filepath or file-object
"""
zipfile = ZipFile(source, 'w', compression=ZIP_DEFLATED)
zipfile.comment = 'version=%s' % VERSION
# Save compositions
fp = StringIO()
writer = csv.writer(fp)
zs = self._compositions[0].keys()
writer.writerow(['iteration'] + zs)
for i, composition in enumerate(self._compositions):
writer.writerow([i + 1] + [composition.get(z, 0.0) for z in zs])
zipfile.writestr('compositions.csv', fp.getvalue())
# Save stats
config = ConfigParser()
section = config.add_section('stats')
section.elapsed_time_s = self.elapsed_time_s
section.iterations = self.iterations
section.max_iterations = self.max_iterations
section.iterator = self.iterator
section.convergor = self.convergor
fp = StringIO()
config.write(fp)
zipfile.writestr('stats.cfg', fp.getvalue())
zipfile.close()
def _update_version2(self, filepath):
logging.debug('Updating from "version 2"')
# Find options
xmlfilepath = os.path.splitext(filepath)[0] + '.xml'
if not os.path.exists(xmlfilepath):
raise ValueError('Update requires an options file saved at %s' % xmlfilepath)
with open(xmlfilepath, 'rb') as fp:
source = fp.read()
source = _update_options(source)
options = Options.read(BytesIO(source))
oldzip = ZipFile(filepath, 'r')
newzip = ZipFile(filepath + ".new", 'w')
# Add options file
fp = BytesIO()
options.write(fp)
newzip.writestr(OPTIONS_FILENAME, fp.getvalue())
# Add other files to new zip
for zipinfo in oldzip.infolist():
data = oldzip.read(zipinfo)
newzip.writestr(zipinfo, data)
# Add version
newzip.comment = b'version=3'
oldzip.close()
newzip.close()
# Remove old zip and replace with new one
os.remove(filepath)
os.rename(filepath + ".new", filepath)
return self._update_version3(filepath)
def save(self, source):
"""
Saves results in a results ZIP.
:arg source: filepath or file-object
"""
zipfile = ZipFile(source, 'w', compression=ZIP_DEFLATED)
zipfile.comment = 'version=%s' % VERSION
# Save compositions
fp = StringIO()
writer = csv.writer(fp)
zs = self._compositions[0].keys()
writer.writerow(['iteration'] + zs)
for i, composition in enumerate(self._compositions):
writer.writerow([i + 1] + [composition.get(z, 0.0) for z in zs])
zipfile.writestr('compositions.csv', fp.getvalue())
# Save stats
config = ConfigParser()
section = config.add_section('stats')
section.elapsed_time_s = self.elapsed_time_s
section.iterations = self.iterations
section.max_iterations = self.max_iterations
section.iterator = self.iterator
section.convergor = self.convergor
fp = StringIO()
config.write(fp)
zipfile.writestr('stats.cfg', fp.getvalue())
zipfile.close()
def add_comment(zip, comment):
zipfile = ZipFile(zip, "a")
zipfile.comment = comment.encode()
zipfile.close()
return zip
def _process_archive(args):
path, root, urls, comment = args
archive = None
timestamp = 0
try:
zip = ZipFile(path)
try:
if zip.comment == comment:
timestamp = os.path.getmtime(path)
finally:
zip.close()
except (EnvironmentError, BadZipfile):
pass
for url in urls:
url = utils.make_url(url)
try:
response = urllib.request.urlopen(url, timeout=30)
current = parsedate(response.info().get('last-modified'))
if current is not None:
current = time.mktime(current)
try:
if current is None or current > timestamp:
archive = BytesIO(response.read())
else:
print(':: already up to date: [%s.files]' % root)
return True
finally:
response.close()
except IOError:
pass
else:
print(':: download succeeded: [%s.files] (%s)' % (root, url))
break
if archive is not None:
try:
with ZipFile(path, 'w', ZIP_DEFLATED) as zip:
print(':: converting archive: [%s.files] ...' % root)
with TarFile.open(fileobj=archive) as tar:
for info in tar:
if info.isfile() and info.name.endswith('/files'):
state = 0
lines = [b'']
stream = tar.extractfile(info)
for line in stream:
if line.isspace():
state = 0
elif state == 1:
lines.append(line)
elif state == 2:
continue
elif line.startswith(b'%FILES%'):
state = 1
elif line.startswith(b'%BACKUP%'):
state = 2
lines = b'/'.join(sorted(lines))
if len(lines) > 1:
lines = b'\n' + lines
zip.writestr(info.name.split('/')[0], lines)
stream.close()
zip.comment = comment
archive.close()
return True
except (EnvironmentError, TarError, UnicodeError) as exception:
print(':: ERROR: failed to convert archive: '
'[%s.files]' % root)
print(':: ', exception)
try:
os.remove(path)
except OSError:
pass
else:
print(':: ERROR: could not find a valid mirror: '
'[%s.files]' % root)
return False
