def write(cls, fn, pa):
fp = ZipFile(fn, 'w')
try:
keys = sorted(ifilter(lambda p: not p.untracked, pa.getJobKeys()))
fp.write('# %s\n' % json.dumps(keys))
maxN = pa.getMaxJobs()
if maxN:
activity = utils.ActivityLog('Writing parameter dump')
for jobNum in irange(maxN):
activity.finish()
activity = utils.ActivityLog(
'Writing parameter dump [%d/%d]' % (jobNum + 1, maxN))
meta = pa.getJobInfo(jobNum)
if meta.get(ParameterInfo.ACTIVE, True):
fp.write(
'%d\t%s\n' %
(jobNum,
str.join(
'\t',
imap(lambda k: json.dumps(meta.get(k, '')),
keys))))
else:
fp.write(
'%d!\t%s\n' %
(jobNum,
str.join(
'\t',
imap(lambda k: json.dumps(meta.get(k, '')),
keys))))
activity.finish()
finally:
fp.close()
def __init__(self, fn):
ParameterSource.__init__(self)
fp = ZipFile(fn, 'r')
try:
keyline = fp.readline().lstrip('#').strip()
self._keys = []
if keyline:
self._keys = parseJSON(keyline)
def parseLine(line):
if not line.startswith('#'):
pNumStr, stored = lmap(str.strip, line.split('\t', 1))
return ('!' in pNumStr, int(pNumStr.rstrip('!')), lmap(parseJSON, stored.split('\t')))
self._values = lmap(parseLine, fp.readlines())
finally:
fp.close()
def _writeJob2PID(self, fn):
fp = ZipFile(fn, 'w')
try:
fp.write('%d\n' % (self._rawSource.getMaxParameters() or 0))
data = ifilter(lambda jobNum_pNum: jobNum_pNum[0] != jobNum_pNum[1], self._mapJob2PID.items())
datastr = lmap(lambda jobNum_pNum: '%d:%d' % jobNum_pNum, data)
fp.write('%s\n' % str.join(',', datastr))
finally:
fp.close()
def write(cls, fn, pa):
fp = ZipFile(fn, 'w')
try:
keys = sorted(ifilter(lambda p: not p.untracked, pa.getJobKeys()))
fp.write('# %s\n' % json.dumps(keys))
maxN = pa.getMaxJobs()
if maxN:
activity = utils.ActivityLog('Writing parameter dump')
for jobNum in irange(maxN):
activity.finish()
activity = utils.ActivityLog('Writing parameter dump [%d/%d]' % (jobNum + 1, maxN))
meta = pa.getJobInfo(jobNum)
if meta.get(ParameterInfo.ACTIVE, True):
fp.write('%d\t%s\n' % (jobNum, str.join('\t', imap(lambda k: json.dumps(meta.get(k, '')), keys))))
else:
fp.write('%d!\t%s\n' % (jobNum, str.join('\t', imap(lambda k: json.dumps(meta.get(k, '')), keys))))
activity.finish()
finally:
fp.close()
def _readJob2PID(self):
fp = ZipFile(self._pathJob2PID, 'r')
try:
self.maxN = int(fp.readline())
if not self.maxN:
self.maxN = None
mapInfo = ifilter(identity, imap(str.strip, fp.readline().split(',')))
self._mapJob2PID = dict(imap(lambda x: tuple(imap(lambda y: int(y.lstrip('!')), x.split(':'))), mapInfo))
self._activeMap = {}
finally:
fp.close()
def __init__(self, fn):
ParameterSource.__init__(self)
fp = ZipFile(fn, 'r')
try:
keyline = fp.readline().lstrip('#').strip()
self._keys = []
if keyline:
self._keys = parseJSON(keyline)
def parseLine(line):
if not line.startswith('#'):
pNumStr, stored = lmap(str.strip, line.split('\t', 1))
return ('!' in pNumStr, int(pNumStr.rstrip('!')),
lmap(parseJSON, stored.split('\t')))
self._values = lmap(parseLine, fp.readlines())
finally:
fp.close()
if opts.dataset_show_removed:
if len(args) < 2:
utils.exitWithUsage('%s <dataset source 1> <dataset source 2> ... <dataset source N> ' % sys.argv[0])
removed = []
oldDP = DataProvider.createInstance('ListProvider', config, args[0], None)
for new in args[1:]:
newDP = DataProvider.createInstance('ListProvider', config, new, None)
(blocksAdded, blocksMissing, blocksChanged) = DataProvider.resyncSources(oldDP.getBlocks(show_stats = False), newDP.getBlocks(show_stats = False))
for block in blocksMissing:
tmp = dict(block)
tmp[-1] = new
removed.append(tmp)
oldDP = newDP
utils.printTabular([(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'), (-1, 'Removed in file')], removed)
if opts.logfile_decode:
import base64, gzip
from grid_control.utils.file_objects import ZipFile
if opts.logfile_decode.endswith('.gz'):
fp = ZipFile(opts.logfile_decode, 'r')
else:
fp = open(opts.logfile_decode, 'r')
for line in fp.readlines():
if line.startswith('(B64) '):
buffer = BytesBuffer(base64.b64decode(line.replace('(B64) ', '')))
line = gzip.GzipFile(fileobj = buffer).read().decode('ascii')
sys.stdout.write(line.rstrip() + '\n')
fp.close()
if opts.dataset_show_removed:
if len(args) < 2:
utils.exitWithUsage('%s <dataset source 1> <dataset source 2> ... <dataset source N> ' % sys.argv[0])
removed = []
utils.eprint = lambda *x: {}
oldDP = DataProvider.createInstance('ListProvider', config, args[0], None)
for new in args[1:]:
newDP = DataProvider.createInstance('ListProvider', config, new, None)
(blocksAdded, blocksMissing, blocksChanged) = DataProvider.resyncSources(oldDP.getBlocks(), newDP.getBlocks())
for block in blocksMissing:
tmp = dict(block)
tmp[-1] = new
removed.append(tmp)
oldDP = newDP
utils.printTabular([(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'), (-1, 'Removed in file')], removed)
if opts.logfile_decode:
import base64, gzip
from grid_control.utils.file_objects import ZipFile
if opts.logfile_decode.endswith('.gz'):
fp = ZipFile(opts.logfile_decode, 'r')
else:
fp = open(opts.logfile_decode, 'r')
for line in fp.readlines():
if line.startswith('(B64) '):
buffer = BytesBuffer(base64.b64decode(line.replace('(B64) ', '')))
line = gzip.GzipFile(fileobj = buffer).read().decode('ascii')
sys.stdout.write(line.rstrip() + '\n')