def setConversion(self, data, mime=None, date=None, **kw):
"""
"""
cache_id = self._getCacheKey(**kw)
if data is None:
cached_value = None
conversion_md5 = None
size = 0
elif isinstance(data, Pdata):
cached_value = aq_base(data)
conversion_md5 = hashPdataObject(cached_value)
size = len(cached_value)
elif isinstance(data, OFSImage):
cached_value = data
conversion_md5 = md5_new(str(data.data)).hexdigest()
size = len(data.data)
elif isinstance(data, (str, unicode,)):
cached_value = data
conversion_md5 = md5_new(cached_value).hexdigest()
size = len(cached_value)
elif isinstance(data, dict):
# Dict instance are used to store computed metadata
# from actual content.
# So this value is intimely related to cache of conversion.
# As it should be cleared each time the document is edited.
# Also may be a proper API should be used
cached_value = data
conversion_md5 = None
size = len(cached_value)
else:
raise NotImplementedError, 'Not able to store type:%r' % type(data)
if date is None:
date = DateTime()
stored_data_dict = {'content_md5': self.getContentMd5(),
'conversion_md5': conversion_md5,
'mime': mime,
'data': cached_value,
'date': date,
'size': size}
cache_factory = self._getCacheFactory()
if cache_factory is None:
if getattr(aq_base(self), 'temp_conversion_data', None) is None:
self.temp_conversion_data = {}
self.temp_conversion_data[cache_id] = stored_data_dict
return
cache_duration = cache_factory.cache_duration
# The purpose of this transaction cache is to help calls
# to the same cache value in the same transaction.
tv = getTransactionalVariable()
tv[cache_id] = stored_data_dict
for cache_plugin in cache_factory.getCachePluginList():
cache_plugin.set(cache_id, DEFAULT_CACHE_SCOPE,
stored_data_dict, cache_duration=cache_duration)
def get_hash(self, str1, str2):
"""Just return the hash for given strings"""
md5 = md5_new()
md5.update(str1.encode('utf-8'))
md5.update(str2.encode('utf-8'))
return str(md5.hexdigest())
def inner_func(*args, **kwargs):
from django.core.cache import cache
raw = [func.__name__, func.__module__, args, kwargs]
pickled = pickle.dumps(raw, protocol=pickle.HIGHEST_PROTOCOL)
key = md5_new(pickled).hexdigest()
value = cache.get(key)
if cache.has_key(key):
return value
else:
# This will set a temporary value while ``func`` is being
# processed. When using threads, this is vital, as otherwise
# the function can be called several times before it finishes
# and is put into the cache.
class MethodNotFinishedError(Exception):
pass
cache.set(
key,
MethodNotFinishedError(
'The function %s has not finished processing yet. This \
value will be replaced when it finishes.' % (func.__name__)), length)
result = func(*args, **kwargs)
cache.set(key, result, length)
return result
def _md5_for_file(self, filename):
hash = md5_new()
input = open(filename, "rb")
read = input.read(4096)
while read:
hash.update(read)
read = input.read(4096)
input.close()
return hash.hexdigest()
def _md5_for_file(self, filename):
hash = md5_new()
input = open(filename, "rb")
read = input.read(4096)
while read:
hash.update(read)
read = input.read(4096)
input.close()
return hash.hexdigest()
def _digest_md5_hash_from_pdata(self, pdata):
"""Compute hash part by part
"""
md5_hash = md5_new()
next = pdata
while next is not None:
md5_hash.update(next.data)
next = next.next
return md5_hash.hexdigest()
def hashPdataObject(pdata_object):
"""Pdata objects are iterable, use this feature strongly
to minimize memory footprint.
"""
md5_hash = md5_new()
next = pdata_object
while next is not None:
md5_hash.update(next.data)
next = next.next
return md5_hash.hexdigest()
def md5_hash(x, modulus=0x100000000, iterations=1):
assert (md5_new != None), "wasn't able to import md5"
for i in xrange(iterations):
hashVal = md5_new()
hashVal.update(str(x))
hashVal = int(hashVal.hexdigest()[:25], 16)
if (modulus != None):
hashVal = hashVal % modulus
x = hashVal
return hashVal
def updateContentMd5(self):
"""Update md5 checksum from the original file
"""
data = self.getData()
if data is not None:
if isinstance(data, Pdata):
self._setContentMd5(hashPdataObject(aq_base(data)))
else:
self._setContentMd5(md5_new(data).hexdigest()) # Reindex is useless
else:
self._setContentMd5(None)
def _createLDAPPassword(password, encoding='SHA'):
""" Create a password string suitable for the userPassword attribute
"""
encoding = encoding.upper()
if encoding in ('SSHA', 'SHA', 'CRYPT'):
pwd_str = AuthEncoding.pw_encrypt(password, encoding)
elif encoding == 'MD5':
m = md5_new(password)
pwd_str = '{MD5}' + base64.encodestring(m.digest())
elif encoding == 'CLEAR':
pwd_str = password
else:
pwd_str = AuthEncoding.pw_encrypt(password, 'SSHA')
return pwd_str.strip()
def _createLDAPPassword(password, encoding='SHA'):
""" Create a password string suitable for the userPassword attribute
"""
encoding = encoding.upper()
if encoding in ('SSHA', 'SHA', 'CRYPT'):
pwd_str = AuthEncoding.pw_encrypt(password, encoding)
elif encoding == 'MD5':
m = md5_new(password)
pwd_str = '{MD5}' + base64.encodestring(m.digest())
elif encoding == 'CLEAR':
pwd_str = password
else:
pwd_str = AuthEncoding.pw_encrypt(password, 'SSHA')
return pwd_str.strip()
def _generateUUID(self, args=""):
"""
Generate a unique id that will be used as url for password
"""
# this code is based on
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/213761
# by Carl Free Jr
# as uuid module is only available in pyhton 2.5
t = long( time.time() * 1000 )
r = long( random.random()*100000000000000000L )
try:
a = socket.gethostbyname( socket.gethostname() )
except:
# if we can't get a network address, just imagine one
a = random.random()*100000000000000000L
data = ' '.join((str(t), str(r), str(a), str(args)))
data = md5_new(data).hexdigest()
return data
def getERP5FormCSS(self):
"""
Returns a CSS file containing all layout instructions
"""
if self.getDefaultScribusFormValue() is None:
return
def generateERP5FormCSS():
parsed_scribus = self._getParsedScribusFile()
import_pdf_file = StringIO.StringIO(self.getDefaultPdfFormValue().getData())
pdf_parser = PDFParser(import_pdf_file)
import_scribus_file = StringIO.StringIO(self.getDefaultScribusFormValue().getData())
scribus_parser = ScribusParser(import_scribus_file)
page_gap = scribus_parser.getPageGap()
scribus_width = scribus_parser.getPageWidth()
scribus_height = scribus_parser.getPageHeight()
space_between_pages = 20 # XXX - hardcoded
image0 = self.getERP5FormImage(0)
properties_css_dict = getPropertiesCSSDict(
parsed_scribus,
page_gap,
image0.getWidth(),
image0.getHeight(),
scribus_width,
scribus_height,
space_between_pages,
self.getPortalObject().portal_preferences,
)
# declaring object that holds the CSS data
css_file_name = "%s_css.css" % self.getId().replace(" ", "")
css_file_content = generateCSSOutputContent(properties_css_dict)
css_file = DTMLDocument(css_file_content, __name__=css_file_name)
import_scribus_file.close()
import_pdf_file.close()
return css_file
generateERP5FormCSS = CachingMethod(
generateERP5FormCSS,
("PDFTypeInformation_generateERP5FormCSS", md5_new(self.getDefaultScribusFormValue().getData()).digest()),
cache_factory="dms_cache_factory",
)
self.REQUEST.RESPONSE.setHeader("Content-Type", "text/css")
return generateERP5FormCSS()
def inner_func(*args, **kwargs):
from django.core.cache import cache
raw = [func.__name__, func.__module__, args, kwargs]
pickled = pickle.dumps(raw, protocol=pickle.HIGHEST_PROTOCOL)
key = md5_new(pickled).hexdigest()
value = cache.get(key)
if cache.has_key(key):
return value
else:
# This will set a temporary value while ``func`` is being
# processed. When using threads, this is vital, as otherwise
# the function can be called several times before it finishes
# and is put into the cache.
class MethodNotFinishedError(Exception): pass
cache.set(key, MethodNotFinishedError(
'The function %s has not finished processing yet. This \
value will be replaced when it finishes.' % (func.__name__)
), length)
result = func(*args, **kwargs)
cache.set(key, result, length)
return result
def _getParsedScribusFile(self):
"""
Returns a reusable data structure which can
be used to generate ERP5 Form, ERP5 Form CSS,
PDF Form, etc.
"""
scribus_form = self.getDefaultScribusFormValue()
if scribus_form is None:
return
def generateParsedScribus():
import_scribus_file = StringIO.StringIO(scribus_form.getData())
scribus_parser = ScribusParser(import_scribus_file)
import_scribus_file.close()
return scribus_parser.getERP5PropertyDict()
generateParsedScribus = CachingMethod(
generateParsedScribus,
("PDFTypeInformation_generateParsedScribus", md5_new(scribus_form.getData()).digest()),
cache_factory="dms_cache_factory",
)
return generateParsedScribus()
# or implied, of Alexey V Michurin.
from random import randrange
try:
from hashlib import md5 as md5_new
except ImportError:
# for Python < 2.5
from md5 import new as md5_new
__all__ = 'gen_salt', 'passphrase_to_salted_key_and_iv'
__doc__ = '''Tools related to key derivation service
See RFC2898 PKCS #5: Password-Based Cryptography Specification Version 2.0'''
md5 = lambda x: md5_new(x).digest()
def gen_salt(saltlen=8):
return ''.join(map(lambda x: chr(randrange(256)), xrange(saltlen)))
def passphrase_to_salted_key_and_iv(passphrase, salt='', klen=16, ivlen=8):
dklen = klen + ivlen
dk = ''
d = ''
while len(dk) < dklen:
d = md5(d + passphrase + salt)
dk += d
return (dk[:klen], dk[klen:dklen])
if __name__ == '__main__':
from testutil import ok, qrepr
def hash_of_kmer(kmer,modulus): h = md5_new() h.update(kmer) return 1 + (int(h.hexdigest()[:25],16) % modulus)
def read_sam_simple(f):
lineNumber = recordNumber = 0
for line in stdin:
lineNumber += 1
line = line.strip()
if (line.startswith("@")):
if (outputWhat == ["sam record"]): # (nothing but sam is being output)
if (line.startswith("@SQ")): print line
continue
recordNumber += 1
if (reportProgress != None) and (recordNumber % reportProgress == 0):
progressCount = commatize(recordNumber)
if (progressId == None):
print >>stderr, "progress: %s sam records read" % progressCount
else:
print >>stderr, "progress: %s / %s sam records read" % (progressId,progressCount)
if (headLimit != None) and (recordNumber > headLimit):
print >>stderr, "limit of %d sam records reached" % headLimit
break
fields = line.split()
numFields = len(fields)
assert (numFields >= SAM_MIN_COLUMNS), \
"not enough columns at line %d (%d, expected %d)" \
% (lineNumber,numFields,SAM_MIN_COLUMNS)
if ("evaluation" in debug):
print >>stderr
print >>stderr, "line %d: \"%s\"" % (lineNumber," ".join(fields[:4]))
# if we are only to process a named-based subset, filter out any reads
# not in that subset
if (subsetN != None):
hashVal = md5_new()
hashVal.update(fields[SAM_QNAME_COLUMN])
hashK = 1 + (int(hashVal.hexdigest()[:25],16) % subsetN)
if (hashK != subsetK): continue
# create the context in which we will evaluate the criteria
context = dict(safeDict)
context["LINENUMBER"] = lineNumber
for name in samFieldToColumn:
col = samFieldToColumn[name]
context[name] = fields[col]
try:
context["FLAG" ] = int_or_string(context["FLAG" ])
context["POS" ] = int_or_string(context["POS" ])
context["MAPQ" ] = int_or_string(context["MAPQ" ])
context["PNEXT"] = int_or_string(context["PNEXT"])
context["TLEN" ] = int_or_string(context["TLEN" ])
except ValueError:
assert (False), "bad SAM record at line %d\n%s" % (lineNumber,line)
context["FLAGS"] = context["FLAG"]
if (type(context["POS"]) != str): context["POS"] -= 1
for field in fields[SAM_MIN_COLUMNS:]:
(tag,typeCode,val) = field.split(":",2)
if (tag not in tagsNeeded):
continue
variable = tagToVariable[tag]
if (typeCode == "i"): context[variable] = int(val)
elif (typeCode == "f"): context[variable] = float(val)
else: context[variable] = val
for tag in tagsNeeded:
variable = tagToVariable[tag]
if (variable not in context): context[variable] = None
for variable in computedVariables:
if (variable not in variablesNeeded): continue
(func,_) = computedVariables[variable]
context[variable] = func(context)
if ("context" in debug) or ("flags" in debug):
values = [(name,context[name]) for name in context if (name not in safeDict)]
if ("flags" in debug):
values = [(name,context[name]) for name in context if (name == "FLAGS")]
values.sort()
print >>stderr, "=== context for line %d ===" % lineNumber
nameWidth = max([len(name) for (name,_) in values])
for (name,val) in values:
if (name in ["SEQ","QUAL"]) and (len(val) > 20):
val = "\"%s...\"" % val[:20]
elif (name in ["FLAG","FLAGS"]):
val = flags_to_string(val)
elif (type(val) == str): val = "\"%s\"" % val
print >>stderr, " %-*s = %s" % (nameWidth,name,val)
# filter, by evaluating the requirements and prohibitions
reject = False
for criterionStr in requirements:
(criterion,_) = requirements[criterionStr]
if ("evaluation" in debug):
print >>stderr, "evaluating requirement \"%s\"" % criterion
try:
val = eval(criterion,{"__builtins__":None},context)
except NameError:
print >>stderr, "failed to evaluate requirement \"%s\"" % criterion
raise
#if ("evaluation" in debug):
# print >>stderr, " (evaluates to %s)" % val
if (val == False):
reject = True
if ("evaluation" in debug): print >>stderr, " (rejected)"
break
if (val != True):
assert (False), "requirement \"%s\" evaluates to \"%s\" on SAM record at line %d\n%s\nevaluated as \"%s\"" \
% (criterionStr,val,lineNumber,line,criterion)
if (reject): continue
for criterionStr in prohibitions:
(criterion,_) = prohibitions[criterionStr]
if ("evaluation" in debug):
print >>stderr, "evaluating prohibition \"%s\"" % criterion
try:
val = eval(criterion,{"__builtins__":None},context)
except NameError:
print >>stderr, "failed to evaluate prohibition \"%s\"" % criterion
raise
#if ("evaluation" in debug):
# print >>stderr, " (evaluates to %s)" % val
if (val == True):
reject = True
if ("evaluation" in debug): print >>stderr, " (rejected)"
break
if (val != False):
assert (False), "prohibition \"%s\" evaluates to \"%s\" on SAM record at line %d\n%s\n(evaluated as \"%s\")" \
% (criterionStr,lineNumber,line,criterion)
if (reject): continue
# if it made it through all that, keep it
yield (lineNumber,line,context)
def link_to_interval(chrom, start, end, number, comment, expansions=None):
if (expansions == None): expansions = []
intervalString = "%s:%s-%s" % (chrom, commatize(start + 1), commatize(end))
url = browserUrlBase \
+ "/hgTracks?" \
+ "db=" + refGenome \
+ "&position=%s%%3A%d-%d" % (chrom,start+1,end)
fields = []
# add number
if (number != None):
fields += ["<a href=\"%s\" target=browser>%s</a>" % (url, number)]
# add name
if (nameLength != None):
hashVal = md5_new()
hashVal.update(intervalString)
hashVal = md5_to_value(hashVal.hexdigest()[:25], 26**nameLength)
fakeName = value_to_name(hashVal, nameLength)
if ("md5" in debug):
print >> stderr, "%s -> %d -> %s" % (intervalString, hashVal,
fakeName)
fields += ["<a href=\"%s\" target=browser>%s</a>" % (url, fakeName)]
# add primary interval link
fields += ["<a href=\"%s\" target=browser>%s</a>" % (url, intervalString)]
# add expanded interval link(s)
for expansion in expansions:
if (expansion[0] == "ratio"):
(_, ratio, ratioStr) = expansion
mid = (start + end) / 2
exStart = int(mid + ((start - mid) * ratio))
exEnd = int(mid + ((end - mid) * ratio))
if (exStart < 0): exStart = 0
exUrl = browserUrlBase \
+ "/hgTracks?" \
+ "db=" + refGenome \
+ "&position=%s%%3A%d-%d" % (chrom,exStart+1,exEnd)
fields += [
"<a href=\"%s\" target=browser>zoom out %sx</a>" %
(exUrl, ratioStr)
]
if (expansion[0] == "center"):
(_, width, widthStr) = expansion
mid = (start + end) / 2
length = end - start
excess = length - width
exEnd = end - excess / 2
exStart = max(0, exEnd - width)
if (exStart < 0): exStart = 0
exUrl = browserUrlBase \
+ "/hgTracks?" \
+ "db=" + refGenome \
+ "&position=%s%%3A%d-%d" % (chrom,exStart+1,exEnd)
fields += ["<a href=\"%s\" target=browser>%s @ %s:%s</a>" \
% (exUrl,widthStr,chrom,mid)]
# add comment
if (comment != None):
if (commentsSeparator != None):
comment = comment.split(commentsSeparator)
comment = "\n".join([c.strip() for c in comment])
fields += [comment]
return "<tr>" + "".join(["<td>%s</td>" % s for s in fields]) + "</tr>"
def passcrypt_md5(passwd, salt=None, magic='$1$'):
if not salt:
pass
elif salt[:len(magic)] == magic:
# remove magic from salt if present
salt = salt[len(magic):]
# salt only goes up to first '$'
salt = salt.split('$')[0]
# limit length of salt to 8
salt = salt[:8]
ctx = md5_new(passwd)
ctx.update(magic)
ctx.update(salt)
ctx1 = md5_new(passwd)
ctx1.update(salt)
ctx1.update(passwd)
final = ctx1.digest()
for i in range(len(passwd), 0, -16):
if i > 16:
ctx.update(final)
else:
ctx.update(final[:i])
i = len(passwd)
while i:
if i & 1:
ctx.update('\0')
else:
ctx.update(passwd[:1])
i = i >> 1
final = ctx.digest()
for i in range(1000):
ctx1 = md5_new()
if i & 1:
ctx1.update(passwd)
else:
ctx1.update(final)
if i % 3: ctx1.update(salt)
if i % 7: ctx1.update(passwd)
if i & 1:
ctx1.update(final)
else:
ctx1.update(passwd)
final = ctx1.digest()
rv = magic + salt + '$'
final = map(ord, final)
l = (final[0] << 16) + (final[6] << 8) + final[12]
rv = rv + _to64(l, 4)
l = (final[1] << 16) + (final[7] << 8) + final[13]
rv = rv + _to64(l, 4)
l = (final[2] << 16) + (final[8] << 8) + final[14]
rv = rv + _to64(l, 4)
l = (final[3] << 16) + (final[9] << 8) + final[15]
rv = rv + _to64(l, 4)
l = (final[4] << 16) + (final[10] << 8) + final[5]
rv = rv + _to64(l, 4)
l = final[11]
rv = rv + _to64(l, 2)
return rv
def passcrypt_md5(passwd, salt=None, magic='$1$'):
if not salt:
pass
elif salt[:len(magic)] == magic:
# remove magic from salt if present
salt = salt[len(magic):]
# salt only goes up to first '$'
salt = salt.split('$')[0]
# limit length of salt to 8
salt = salt[:8]
ctx = md5_new(passwd)
ctx.update(magic)
ctx.update(salt)
ctx1 = md5_new(passwd)
ctx1.update(salt)
ctx1.update(passwd)
final = ctx1.digest()
for i in range(len(passwd), 0 , -16):
if i > 16:
ctx.update(final)
else:
ctx.update(final[:i])
i = len(passwd)
while i:
if i & 1:
ctx.update('\0')
else:
ctx.update(passwd[:1])
i = i >> 1
final = ctx.digest()
for i in range(1000):
ctx1 = md5_new()
if i & 1:
ctx1.update(passwd)
else:
ctx1.update(final)
if i % 3: ctx1.update(salt)
if i % 7: ctx1.update(passwd)
if i & 1:
ctx1.update(final)
else:
ctx1.update(passwd)
final = ctx1.digest()
rv = magic + salt + '$'
final = map(ord, final)
l = (final[0] << 16) + (final[6] << 8) + final[12]
rv = rv + _to64(l, 4)
l = (final[1] << 16) + (final[7] << 8) + final[13]
rv = rv + _to64(l, 4)
l = (final[2] << 16) + (final[8] << 8) + final[14]
rv = rv + _to64(l, 4)
l = (final[3] << 16) + (final[9] << 8) + final[15]
rv = rv + _to64(l, 4)
l = (final[4] << 16) + (final[10] << 8) + final[5]
rv = rv + _to64(l, 4)
l = final[11]
rv = rv + _to64(l, 2)
return rv
def get_checksum(self):
return md5_new(self.data).digest()
def link_to_interval(chrom,start,end,number,comment,expansions=None):
if (expansions == None): expansions = []
intervalString = "%s:%s-%s" % (chrom,commatize(start+1),commatize(end))
url = browserUrlBase \
+ "/hgTracks?" \
+ "db=" + refGenome \
+ "&position=%s%%3A%d-%d" % (chrom,start+1,end)
fields = []
# add number
if (number != None):
fields += ["<a href=\"%s\" target=browser>%s</a>" % (url,number)]
# add name
if (nameLength != None):
hashVal = md5_new()
hashVal.update(intervalString)
hashVal = md5_to_value(hashVal.hexdigest()[:25],26**nameLength)
fakeName = value_to_name(hashVal,nameLength)
if ("md5" in debug):
print >>stderr, "%s -> %d -> %s" % (intervalString,hashVal,fakeName)
fields += ["<a href=\"%s\" target=browser>%s</a>" % (url,fakeName)]
# add primary interval link
fields += ["<a href=\"%s\" target=browser>%s</a>" % (url,intervalString)]
# add expanded interval link(s)
for expansion in expansions:
if (expansion[0] == "ratio"):
(_,ratio,ratioStr) = expansion
mid = (start+end) / 2
exStart = int(mid + ((start - mid) * ratio))
exEnd = int(mid + ((end - mid) * ratio))
if (exStart < 0): exStart = 0
exUrl = browserUrlBase \
+ "/hgTracks?" \
+ "db=" + refGenome \
+ "&position=%s%%3A%d-%d" % (chrom,exStart+1,exEnd)
fields += ["<a href=\"%s\" target=browser>zoom out %sx</a>" % (exUrl,ratioStr)]
if (expansion[0] == "center"):
(_,width,widthStr) = expansion
mid = (start+end) / 2
length = end - start
excess = length - width
exEnd = end - excess/2
exStart = max(0,exEnd-width)
if (exStart < 0): exStart = 0
exUrl = browserUrlBase \
+ "/hgTracks?" \
+ "db=" + refGenome \
+ "&position=%s%%3A%d-%d" % (chrom,exStart+1,exEnd)
fields += ["<a href=\"%s\" target=browser>%s @ %s:%s</a>" \
% (exUrl,widthStr,chrom,mid)]
# add comment
if (comment != None):
if (commentsSeparator != None):
comment = comment.split(commentsSeparator)
comment = "\n".join([c.strip() for c in comment])
fields += [comment]
return "<tr>" + "".join(["<td>%s</td>" % s for s in fields]) + "</tr>"
def read_sam_simple(f):
lineNumber = recordNumber = 0
for line in stdin:
lineNumber += 1
line = line.strip()
if (line.startswith("@")):
if (outputWhat == ["sam record"
]): # (nothing but sam is being output)
if (line.startswith("@SQ")): print line
continue
recordNumber += 1
if (reportProgress != None) and (recordNumber % reportProgress == 0):
progressCount = commatize(recordNumber)
if (progressId == None):
print >> stderr, "progress: %s sam records read" % progressCount
else:
print >> stderr, "progress: %s / %s sam records read" % (
progressId, progressCount)
if (headLimit != None) and (recordNumber > headLimit):
print >> stderr, "limit of %d sam records reached" % headLimit
break
fields = line.split()
numFields = len(fields)
assert (numFields >= SAM_MIN_COLUMNS), \
"not enough columns at line %d (%d, expected %d)" \
% (lineNumber,numFields,SAM_MIN_COLUMNS)
if ("evaluation" in debug):
print >> stderr
print >> stderr, "line %d: \"%s\"" % (lineNumber, " ".join(
fields[:4]))
# if we are only to process a named-based subset, filter out any reads
# not in that subset
if (subsetN != None):
hashVal = md5_new()
hashVal.update(fields[SAM_QNAME_COLUMN])
hashK = 1 + (int(hashVal.hexdigest()[:25], 16) % subsetN)
if (hashK != subsetK): continue
# create the context in which we will evaluate the criteria
context = dict(safeDict)
context["LINENUMBER"] = lineNumber
for name in samFieldToColumn:
col = samFieldToColumn[name]
context[name] = fields[col]
try:
context["FLAG"] = int_or_string(context["FLAG"])
context["POS"] = int_or_string(context["POS"])
context["MAPQ"] = int_or_string(context["MAPQ"])
context["PNEXT"] = int_or_string(context["PNEXT"])
context["TLEN"] = int_or_string(context["TLEN"])
except ValueError:
assert (
False), "bad SAM record at line %d\n%s" % (lineNumber, line)
context["FLAGS"] = context["FLAG"]
if (type(context["POS"]) != str): context["POS"] -= 1
for field in fields[SAM_MIN_COLUMNS:]:
(tag, typeCode, val) = field.split(":", 2)
if (tag not in tagsNeeded):
continue
variable = tagToVariable[tag]
if (typeCode == "i"): context[variable] = int(val)
elif (typeCode == "f"): context[variable] = float(val)
else: context[variable] = val
for tag in tagsNeeded:
variable = tagToVariable[tag]
if (variable not in context): context[variable] = None
for variable in computedVariables:
if (variable not in variablesNeeded): continue
(func, _) = computedVariables[variable]
context[variable] = func(context)
if ("context" in debug) or ("flags" in debug):
values = [(name, context[name]) for name in context
if (name not in safeDict)]
if ("flags" in debug):
values = [(name, context[name]) for name in context
if (name == "FLAGS")]
values.sort()
print >> stderr, "=== context for line %d ===" % lineNumber
nameWidth = max([len(name) for (name, _) in values])
for (name, val) in values:
if (name in ["SEQ", "QUAL"]) and (len(val) > 20):
val = "\"%s...\"" % val[:20]
elif (name in ["FLAG", "FLAGS"]):
val = flags_to_string(val)
elif (type(val) == str):
val = "\"%s\"" % val
print >> stderr, " %-*s = %s" % (nameWidth, name, val)
# filter, by evaluating the requirements and prohibitions
reject = False
for criterionStr in requirements:
(criterion, _) = requirements[criterionStr]
if ("evaluation" in debug):
print >> stderr, "evaluating requirement \"%s\"" % criterion
try:
val = eval(criterion, {"__builtins__": None}, context)
except NameError:
print >> stderr, "failed to evaluate requirement \"%s\"" % criterion
raise
#if ("evaluation" in debug):
# print >>stderr, " (evaluates to %s)" % val
if (val == False):
reject = True
if ("evaluation" in debug): print >> stderr, " (rejected)"
break
if (val != True):
assert (False), "requirement \"%s\" evaluates to \"%s\" on SAM record at line %d\n%s\nevaluated as \"%s\"" \
% (criterionStr,val,lineNumber,line,criterion)
if (reject): continue
for criterionStr in prohibitions:
(criterion, _) = prohibitions[criterionStr]
if ("evaluation" in debug):
print >> stderr, "evaluating prohibition \"%s\"" % criterion
try:
val = eval(criterion, {"__builtins__": None}, context)
except NameError:
print >> stderr, "failed to evaluate prohibition \"%s\"" % criterion
raise
#if ("evaluation" in debug):
# print >>stderr, " (evaluates to %s)" % val
if (val == True):
reject = True
if ("evaluation" in debug): print >> stderr, " (rejected)"
break
if (val != False):
assert (False), "prohibition \"%s\" evaluates to \"%s\" on SAM record at line %d\n%s\n(evaluated as \"%s\")" \
% (criterionStr,lineNumber,line,criterion)
if (reject): continue
# if it made it through all that, keep it
yield (lineNumber, line, context)