def import_data(hmf_filename):
hmff = file(hmf_filename)
ferrors = file('/home/jvoight/lmfdb/backups/import_data.err', 'a')
# Parse field data
v = hmff.readline()
assert v[:9] == 'COEFFS :='
coeffs = eval(v[10:][:-2])
v = hmff.readline()
assert v[:4] == 'n :='
n = int(v[4:][:-2])
v = hmff.readline()
assert v[:4] == 'd :='
d = int(v[4:][:-2])
magma.eval('F<w> := NumberField(Polynomial(' + str(coeffs) + '));')
magma.eval('ZF := Integers(F);')
# Find the corresponding field in the database of number fields
print "Finding field..."
fields_matching = fields.find({
"signature": [n, int(0)],
"discriminant": d
})
cnt = fields_matching.count()
assert cnt >= 1
field_label = None
for i in range(cnt):
nf = fields_matching.next()
if nf['coefficients'] == coeffs:
field_label = nf['label']
assert field_label is not None
print "...found!"
# Find the field in the HMF database
print "Finding field in HMF..."
F_hmf = hmf_fields.find_one({"label": field_label})
if F_hmf is None:
# Create list of ideals
print "...adding!"
ideals = eval(preparse(magma.eval('nice_idealstr(F);')))
ideals_str = [str(c) for c in ideals]
hmf_fields.insert({
"label": field_label,
"degree": n,
"discriminant": d,
"ideals": ideals_str
})
F_hmf = hmf_fields.find_one({"label": field_label})
else:
print "...found!"
print "Computing ideals..."
ideals_str = F_hmf['ideals']
ideals = [eval(preparse(c)) for c in ideals_str]
ideals_norms = [c[0] for c in ideals]
magma.eval('ideals_str := [' +
''.join(F_hmf['ideals']).replace('][', '], [') + ']')
magma.eval('ideals := [ideal<ZF | {F!x : x in I}> : I in ideals_str];')
# Add the list of primes
print "Computing primes..."
v = hmff.readline() # Skip line
v = hmff.readline()
assert v[:9] == 'PRIMES :='
primes_str = v[10:][:-2]
primes_array = [str(t) for t in eval(preparse(primes_str))]
magma.eval('primes_array := ' + primes_str)
magma.eval('primes := [ideal<ZF | {F!x : x in I}> : I in primes_array];')
magma.eval('primes_indices := [Index(ideals, pp) : pp in primes];')
try:
assert magma(
'&and[primes_indices[j] gt primes_indices[j-1] : j in [2..#primes_indices]]'
)
resort = False
except AssertionError:
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Primes reordered!"
resort = True
magma.eval('_, sigma := Sort(primes_indices, func<x,y | x-y>);')
magma.eval(
'perm := [[xx : xx in x] : x in CycleDecomposition(sigma) | #x gt 1]'
)
# Check at least they have the same norm
magma.eval(
'for tau in perm do assert #{Norm(ideals[primes_indices[t]]) : t in tau} eq 1; end for;'
)
primes_resort = eval(magma.eval('Eltseq(sigma)'))
primes_resort = [c - 1 for c in primes_resort]
primes_indices = eval(magma.eval('primes_indices'))
primes_str = [ideals_str[j - 1] for j in primes_indices]
assert len(primes_array) == len(primes_str)
print "...comparing..."
for i in range(len(primes_array)):
assert magma('ideal<ZF | {F!x : x in ' + primes_array[i] + '}> eq ' +
'ideal<ZF | {F!x : x in ' + primes_str[i] + '}>;')
if 'primes' in F_hmf: # Nothing smart: make sure it is the same
assert F_hmf['primes'] == primes_str
else:
F_hmf['primes'] = primes_str
hmf_fields.save(F_hmf)
print "...saved!"
# Collect levels
v = hmff.readline()
if v[:9] == 'LEVELS :=':
# skip this line, we do not use the list of levels
v = hmff.readline()
for i in range(3):
if v[:11] != 'NEWFORMS :=':
v = hmff.readline()
else:
break
# Finally, newforms!
print "Starting newforms!"
while v != '':
v = hmff.readline()[1:-3]
if len(v) == 0:
break
data = eval(preparse(v))
level_ideal = data[0]
level_norm = data[0][0]
label_suffix = data[1]
weight = [2 for i in range(n)]
level_ind = int(
magma('Index(ideals, ideal<ZF | {F!x : x in ' + str(level_ideal) +
'}>)')) - 1 # Magma counts starting at 1
level_ideal = ideals_str[level_ind]
assert magma('ideal<ZF | {F!x : x in ' + str(level_ideal) + '}> eq ' +
'ideal<ZF | {F!x : x in ' + str(data[0]) + '}>;')
level_dotlabel = level_ind - ideals_norms.index(
level_norm) + 1 # Start counting at 1
assert level_dotlabel > 0
level_label = str(level_norm) + '.' + str(level_dotlabel)
label = parse_label(field_label, weight, level_label, label_suffix)
short_label = level_label + '-' + label_suffix
if len(data) == 3:
hecke_polynomial = x
hecke_eigenvalues = data[2]
else:
hecke_polynomial = data[2]
hecke_eigenvalues = data[3]
if resort:
hecke_eigenvalues = [hecke_eigenvalues[i] for i in primes_resort]
# Sort out Atkin-Lehner involutions
magma.eval('NN := ideal<ZF | {F!x : x in ' + str(level_ideal) + '}>;')
magma.eval('NNfact := Factorization(NN);')
ALind = eval(
preparse(
magma.eval(
'[Index(primes, pp[1])-1 : pp in NNfact | Index(primes,pp[1]) gt 0];'
)))
AL_eigsin = [hecke_eigenvalues[c] for c in ALind]
try:
assert all([abs(int(c)) <= 1 for c in AL_eigsin])
except:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
AL_eigenvalues = []
ees = eval(preparse(magma.eval('[pp[2] : pp in NNfact]')))
for i in range(len(AL_eigsin)):
if ees[i] >= 2:
if hecke_eigenvalues[ALind[i]] != 0:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
else:
try:
if abs(int(AL_eigsin[i])) != 1:
ferrors.write(
str(n) + '/' + str(d) + ' ' + label + '\n')
else:
AL_eigenvalues.append(
[primes_str[ALind[i]], -AL_eigsin[i]])
except TypeError:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
assert magma(
'[Valuation(NN, ideal<ZF | {F!x : x in a}>) gt 0 : a in [' +
str(''.join([a[0]
for a in AL_eigenvalues])).replace('][', '], [') +
']];')
# Constrain eigenvalues to size 2MB
hecke_eigenvalues = [str(c) for c in hecke_eigenvalues]
leftout = 0
while sum([len(s) for s in hecke_eigenvalues]) > 2000000:
leftout += 1
hecke_eigenvalues = hecke_eigenvalues[:-1]
if leftout > 0:
print "Left out", leftout
info = {
"label": label,
"short_label": short_label,
"field_label": field_label,
"level_norm": int(level_norm),
"level_ideal": str(level_ideal),
"level_label": level_label,
"weight": str(weight),
"label_suffix": label_suffix,
"dimension": hecke_polynomial.degree(),
"hecke_polynomial": str(hecke_polynomial),
"hecke_eigenvalues": hecke_eigenvalues,
"AL_eigenvalues": [[str(a[0]), str(a[1])] for a in AL_eigenvalues]
}
print info['label']
existing_forms = hmf_forms.find({"label": label})
assert existing_forms.count() <= 1
if existing_forms.count() == 0:
hmf_forms.insert(info)
else:
existing_form = existing_forms.next()
assert info['hecke_eigenvalues'] == existing_form[
'hecke_eigenvalues']
print "...duplicate"
def import_data_fix_perm_primes(hmf_filename,
fileprefix=None,
ferrors=None,
test=True):
if fileprefix == None:
fileprefix = "."
hmff = file(os.path.join(fileprefix, hmf_filename))
if ferrors == None:
ferrors = file('/home/jvoight/lmfdb/backups/import_data.err', 'a')
# Parse field data
v = hmff.readline()
assert v[:9] == 'COEFFS :='
coeffs = eval(v[10:].split(';')[0])
v = hmff.readline()
assert v[:4] == 'n :='
n = int(v[4:][:-2])
v = hmff.readline()
assert v[:4] == 'd :='
d = int(v[4:][:-2])
magma.eval('F<w> := NumberField(Polynomial(' + str(coeffs) + '));')
magma.eval('ZF := Integers(F);')
# Find the corresponding field in the database of number fields
dkey = make_disc_key(ZZ(d))[1]
sig = "%s,%s" % (n, 0)
print("Finding field with signature %s and disc_key %s ..." % (sig, dkey))
fields_matching = fields.find({"disc_abs_key": dkey, "signature": sig})
cnt = fields_matching.count()
print("Found %s fields" % cnt)
assert cnt >= 1
field_label = None
co = str(coeffs)[1:-1].replace(" ", "")
for i in range(cnt):
nf = fields_matching.next()
print("Comparing coeffs %s with %s" % (nf['coeffs'], co))
if nf['coeffs'] == co:
field_label = nf['label']
assert field_label is not None
print "...found!"
# Find the field in the HMF database
print("Finding field %s in HMF..." % field_label)
F_hmf = hmf_fields.find_one({"label": field_label})
assert F_hmf is not None # only proceed if field already in database
print "...found!"
print "Computing ideals..."
ideals_str = F_hmf['ideals']
# ideals = [eval(preparse(c)) for c in ideals_str] # doesn't appear to be used
# ideals_norms = [c[0] for c in ideals] # doesn't appear to be used
magma.eval('ideals_str := [' +
''.join(F_hmf['ideals']).replace('][', '], [') + ']')
magma.eval('ideals := [ideal<ZF | {F!x : x in I}> : I in ideals_str];')
# Add the list of primes
print "Computing primes..."
v = hmff.readline() # Skip line
v = hmff.readline()
assert v[:9] == 'PRIMES :='
primes_str = v[10:][:-2]
primes_array = [str(t) for t in eval(preparse(primes_str))]
magma.eval('primes_array := ' + primes_str)
magma.eval('primes := [ideal<ZF | {F!x : x in I}> : I in primes_array];')
magma.eval('primes_indices := [Index(ideals, pp) : pp in primes];')
try:
assert magma(
'&and[primes_indices[j] gt primes_indices[j-1] : j in [2..#primes_indices]]'
)
resort = False
except AssertionError:
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Primes reordered!"
resort = True
magma.eval('_, sigma := Sort(primes_indices, func<x,y | x-y>);')
magma.eval(
'perm := [[xx : xx in x] : x in CycleDecomposition(sigma) | #x gt 1]'
)
# Check at least they have the same norm
magma.eval(
'for tau in perm do assert #{Norm(ideals[primes_indices[t]]) : t in tau} eq 1; end for;'
)
primes_resort = eval(magma.eval('Eltseq(sigma)'))
primes_resort = [c - 1 for c in primes_resort]
if resort:
primes_indices = eval(magma.eval('primes_indices'))
primes_str = [ideals_str[j - 1] for j in primes_indices]
assert len(primes_array) == len(primes_str)
print "...comparing..."
for i in range(len(primes_array)):
assert magma('ideal<ZF | {F!x : x in ' + primes_array[i] +
'}> eq ' + 'ideal<ZF | {F!x : x in ' + primes_str[i] +
'}>;')
# Important also to resort the list of primes themselves!
# not just the a_pp's
primes_str = [primes_str[i] for i in primes_resort]
print("Compare primes in hmf.fields\n %s\n with NEW primes\n %s" %
(F_hmf['primes'], primes_str))
if test:
print("...Didn't do anything! Just a test")
else:
F_hmf['primes'] = primes_str
hmf_fields.save(F_hmf)
print "...saved!"
def import_data(hmf_filename):
hmff = file(hmf_filename)
ferrors = file('/home/jvoight/lmfdb/backups/import_data.err', 'a')
# Parse field data
v = hmff.readline()
assert v[:9] == 'COEFFS :='
coeffs = eval(v[10:][:-2])
v = hmff.readline()
assert v[:4] == 'n :='
n = int(v[4:][:-2])
v = hmff.readline()
assert v[:4] == 'd :='
d = int(v[4:][:-2])
magma.eval('F<w> := NumberField(Polynomial(' + str(coeffs) + '));')
magma.eval('ZF := Integers(F);')
# Find the corresponding field in the database of number fields
print "Finding field..."
fields_matching = fields.find({"signature": [n, int(0)],
"discriminant": d})
cnt = fields_matching.count()
assert cnt >= 1
field_label = None
for i in range(cnt):
nf = fields_matching.next()
if nf['coefficients'] == coeffs:
field_label = nf['label']
assert field_label is not None
print "...found!"
# Find the field in the HMF database
print "Finding field in HMF..."
F_hmf = hmf_fields.find_one({"label": field_label})
if F_hmf is None:
# Create list of ideals
print "...adding!"
ideals = eval(preparse(magma.eval('nice_idealstr(F);')))
ideals_str = [str(c) for c in ideals]
hmf_fields.insert({"label": field_label,
"degree": n,
"discriminant": d,
"ideals": ideals_str})
F_hmf = hmf_fields.find_one({"label": field_label})
else:
print "...found!"
print "Computing ideals..."
ideals_str = F_hmf['ideals']
ideals = [eval(preparse(c)) for c in ideals_str]
ideals_norms = [c[0] for c in ideals]
magma.eval('ideals_str := [' + ''.join(F_hmf['ideals']).replace('][', '], [') + ']')
magma.eval('ideals := [ideal<ZF | {F!x : x in I}> : I in ideals_str];')
# Add the list of primes
print "Computing primes..."
v = hmff.readline() # Skip line
v = hmff.readline()
assert v[:9] == 'PRIMES :='
primes_str = v[10:][:-2]
primes_array = [str(t) for t in eval(preparse(primes_str))]
magma.eval('primes_array := ' + primes_str)
magma.eval('primes := [ideal<ZF | {F!x : x in I}> : I in primes_array];')
magma.eval('primes_indices := [Index(ideals, pp) : pp in primes];')
try:
assert magma('&and[primes_indices[j] gt primes_indices[j-1] : j in [2..#primes_indices]]')
resort = False
except AssertionError:
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Primes reordered!"
resort = True
magma.eval('_, sigma := Sort(primes_indices, func<x,y | x-y>);')
magma.eval('perm := [[xx : xx in x] : x in CycleDecomposition(sigma) | #x gt 1]')
# Check at least they have the same norm
magma.eval('for tau in perm do assert #{Norm(ideals[primes_indices[t]]) : t in tau} eq 1; end for;')
primes_resort = eval(magma.eval('Eltseq(sigma)'))
primes_resort = [c - 1 for c in primes_resort]
primes_indices = eval(magma.eval('primes_indices'))
primes_str = [ideals_str[j - 1] for j in primes_indices]
assert len(primes_array) == len(primes_str)
print "...comparing..."
for i in range(len(primes_array)):
assert magma('ideal<ZF | {F!x : x in ' + primes_array[i] + '}> eq '
+ 'ideal<ZF | {F!x : x in ' + primes_str[i] + '}>;')
if 'primes' in F_hmf: # Nothing smart: make sure it is the same
assert F_hmf['primes'] == primes_str
else:
F_hmf['primes'] = primes_str
hmf_fields.save(F_hmf)
print "...saved!"
# Collect levels
v = hmff.readline()
if v[:9] == 'LEVELS :=':
# skip this line, we do not use the list of levels
v = hmff.readline()
for i in range(3):
if v[:11] != 'NEWFORMS :=':
v = hmff.readline()
else:
break
# Finally, newforms!
print "Starting newforms!"
while v != '':
v = hmff.readline()[1:-3]
if len(v) == 0:
break
data = eval(preparse(v))
level_ideal = data[0]
level_norm = data[0][0]
label_suffix = data[1]
weight = [2 for i in range(n)]
level_ind = int(magma('Index(ideals, ideal<ZF | {F!x : x in ' + str(level_ideal) + '}>)')
) - 1 # Magma counts starting at 1
level_ideal = ideals_str[level_ind]
assert magma('ideal<ZF | {F!x : x in ' + str(level_ideal) + '}> eq '
+ 'ideal<ZF | {F!x : x in ' + str(data[0]) + '}>;')
level_dotlabel = level_ind - ideals_norms.index(level_norm) + 1 # Start counting at 1
assert level_dotlabel > 0
level_label = str(level_norm) + '.' + str(level_dotlabel)
label = parse_label(field_label, weight, level_label, label_suffix)
short_label = level_label + '-' + label_suffix
if len(data) == 3:
hecke_polynomial = x
hecke_eigenvalues = data[2]
else:
hecke_polynomial = data[2]
hecke_eigenvalues = data[3]
if resort:
hecke_eigenvalues = [hecke_eigenvalues[i] for i in primes_resort]
# Sort out Atkin-Lehner involutions
magma.eval('NN := ideal<ZF | {F!x : x in ' + str(level_ideal) + '}>;')
magma.eval('NNfact := Factorization(NN);')
ALind = eval(
preparse(magma.eval('[Index(primes, pp[1])-1 : pp in NNfact | Index(primes,pp[1]) gt 0];')))
AL_eigsin = [hecke_eigenvalues[c] for c in ALind]
try:
assert all([abs(int(c)) <= 1 for c in AL_eigsin])
except:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
AL_eigenvalues = []
ees = eval(preparse(magma.eval('[pp[2] : pp in NNfact]')))
for i in range(len(AL_eigsin)):
if ees[i] >= 2:
if hecke_eigenvalues[ALind[i]] != 0:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
else:
try:
if abs(int(AL_eigsin[i])) != 1:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
else:
AL_eigenvalues.append([primes_str[ALind[i]], -AL_eigsin[i]])
except TypeError:
ferrors.write(str(n) + '/' + str(d) + ' ' + label + '\n')
assert magma('[Valuation(NN, ideal<ZF | {F!x : x in a}>) gt 0 : a in [' +
str(''.join([a[0] for a in AL_eigenvalues])).replace('][', '], [') + ']];')
# Constrain eigenvalues to size 2MB
hecke_eigenvalues = [str(c) for c in hecke_eigenvalues]
leftout = 0
while sum([len(s) for s in hecke_eigenvalues]) > 2000000:
leftout += 1
hecke_eigenvalues = hecke_eigenvalues[:-1]
if leftout > 0:
print "Left out", leftout
info = {"label": label,
"short_label": short_label,
"field_label": field_label,
"level_norm": int(level_norm),
"level_ideal": str(level_ideal),
"level_label": level_label,
"weight": str(weight),
"label_suffix": label_suffix,
"dimension": hecke_polynomial.degree(),
"hecke_polynomial": str(hecke_polynomial),
"hecke_eigenvalues": hecke_eigenvalues,
"AL_eigenvalues": [[str(a[0]), str(a[1])] for a in AL_eigenvalues]}
print info['label']
existing_forms = hmf_forms.find({"label": label})
assert existing_forms.count() <= 1
if existing_forms.count() == 0:
hmf_forms.insert(info)
else:
existing_form = existing_forms.next()
assert info['hecke_eigenvalues'] == existing_form['hecke_eigenvalues']
print "...duplicate"
def import_data_fix_perm_primes(hmf_filename, fileprefix=None, ferrors=None, test=True):
if fileprefix==None:
fileprefix="."
hmff = file(os.path.join(fileprefix,hmf_filename))
if ferrors==None:
ferrors = file('/home/jvoight/lmfdb/backups/import_data.err', 'a')
# Parse field data
v = hmff.readline()
assert v[:9] == 'COEFFS :='
coeffs = eval(v[10:].split(';')[0])
v = hmff.readline()
assert v[:4] == 'n :='
n = int(v[4:][:-2])
v = hmff.readline()
assert v[:4] == 'd :='
d = int(v[4:][:-2])
magma.eval('F<w> := NumberField(Polynomial(' + str(coeffs) + '));')
magma.eval('ZF := Integers(F);')
# Find the corresponding field in the database of number fields
dkey = make_disc_key(ZZ(d))[1]
sig = "%s,%s" % (n,0)
print("Finding field with signature %s and disc_key %s ..." % (sig,dkey))
fields_matching = fields.find({"disc_abs_key": dkey, "signature": sig})
cnt = fields_matching.count()
print("Found %s fields" % cnt)
assert cnt >= 1
field_label = None
co = str(coeffs)[1:-1].replace(" ","")
for i in range(cnt):
nf = fields_matching.next()
print("Comparing coeffs %s with %s" % (nf['coeffs'], co))
if nf['coeffs'] == co:
field_label = nf['label']
assert field_label is not None
print "...found!"
# Find the field in the HMF database
print("Finding field %s in HMF..." % field_label)
F_hmf = hmf_fields.find_one({"label": field_label})
assert F_hmf is not None # only proceed if field already in database
print "...found!"
print "Computing ideals..."
ideals_str = F_hmf['ideals']
# ideals = [eval(preparse(c)) for c in ideals_str] # doesn't appear to be used
# ideals_norms = [c[0] for c in ideals] # doesn't appear to be used
magma.eval('ideals_str := [' + ''.join(F_hmf['ideals']).replace('][', '], [') + ']')
magma.eval('ideals := [ideal<ZF | {F!x : x in I}> : I in ideals_str];')
# Add the list of primes
print "Computing primes..."
v = hmff.readline() # Skip line
v = hmff.readline()
assert v[:9] == 'PRIMES :='
primes_str = v[10:][:-2]
primes_array = [str(t) for t in eval(preparse(primes_str))]
magma.eval('primes_array := ' + primes_str)
magma.eval('primes := [ideal<ZF | {F!x : x in I}> : I in primes_array];')
magma.eval('primes_indices := [Index(ideals, pp) : pp in primes];')
try:
assert magma('&and[primes_indices[j] gt primes_indices[j-1] : j in [2..#primes_indices]]')
resort = False
except AssertionError:
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Primes reordered!"
resort = True
magma.eval('_, sigma := Sort(primes_indices, func<x,y | x-y>);')
magma.eval('perm := [[xx : xx in x] : x in CycleDecomposition(sigma) | #x gt 1]')
# Check at least they have the same norm
magma.eval('for tau in perm do assert #{Norm(ideals[primes_indices[t]]) : t in tau} eq 1; end for;')
primes_resort = eval(magma.eval('Eltseq(sigma)'))
primes_resort = [c - 1 for c in primes_resort]
if resort:
primes_indices = eval(magma.eval('primes_indices'))
primes_str = [ideals_str[j - 1] for j in primes_indices]
assert len(primes_array) == len(primes_str)
print "...comparing..."
for i in range(len(primes_array)):
assert magma('ideal<ZF | {F!x : x in ' + primes_array[i] + '}> eq '
+ 'ideal<ZF | {F!x : x in ' + primes_str[i] + '}>;')
# Important also to resort the list of primes themselves!
# not just the a_pp's
primes_str = [primes_str[i] for i in primes_resort]
print("Compare primes in hmf.fields\n %s\n with NEW primes\n %s" % (F_hmf['primes'], primes_str))
if test:
print("...Didn't do anything! Just a test")
else:
F_hmf['primes'] = primes_str
hmf_fields.save(F_hmf)
print "...saved!"