def _w_tilde(self, u_bar):
"""Compute w_tilde, the threshold for the word-length w such that
MSB = computeNaiveMSB if w >= w_tilde
MSB = computeNaiveMSB+1 if w < w_tilde
(this doesn't count into account the roundoff error, as in FxPF)
See ARITH26 paper
Parameters:
- u_bar: vector of bounds on the inputs of the system
Returns: a vector of thresholds w_tilde
We use: w_tilde = 1 + ceil(log2(zeta_bar)) - floor(log2( 2^ceil(log2(zeta_bar)) - zeta_bar ))
with zeta_bar = <<Hzeta>>.u_bar
"""
#TODO: test if zeta_bar is a power of 2 (should be +Inf in that case)
zeta_bar = self.Hzeta.WCPG() * u_bar
with mpmath.workprec(500): # TODO: compute how many bit we need !!
wtilde = [
int(1 + mpmath.ceil(mpmath.log(x[0], 2)) - mpmath.floor(
mpmath.log(
mpmath.power(2, mpmath.ceil(mpmath.log(x[0], 2))) -
x[0], 2))) for x in zeta_bar.tolist()
]
return wtilde
def documents_partition (time_range=3):
"""
First step of the Temporal Text Mining algorithm. Partition the tweets text based on the day they were posted.
Returns the partition of the tweets and corresponding vectors of the tf-idf matrix.
:param time_range: How many days should be used to form one set in the partition
:type time_range: int
:return: tweets and tf-idf matrix partition
:rtype: tuple
"""
start = timer()
# Load the files
tfidf_matrix = pickle.load(open('TF-IDF Matrix - ' + str(n_data) + ' Tweets.p', 'rb'))
tweets = pickle.load(open('Tweets Data - ' + str(n_data) + ' Tweets.p', 'rb'))
# Partition the data
# Obtain all the days in which a tweet was posted
days = [tweet['date'] for tweet in tweets]
# Count the number of days in the data
number_days = len(set(days))
# Build the partition
tweets_partition = []
dense_tfidf_matrix_partition = []
partition_size = int(math.ceil(number_days/time_range))
for index in range(partition_size):
tweets_partition.append([])
dense_tfidf_matrix_partition.append([])
# Obtain the days in form of index and allocate the tweets and vectors to its partition
minimum_day = min(days).toordinal()
dense_tfidf_matrix = tfidf_matrix.todense().tolist()
for tweet, vector in zip(tweets, dense_tfidf_matrix):
day_index = tweet['date'].toordinal() - minimum_day + 1
partition_index = int(math.ceil(day_index/time_range)) - 1
tweets_partition[partition_index].append(tweet)
dense_tfidf_matrix_partition[partition_index].append(vector)
# Convert the dense tf-idf matrix in the parttion to sparse tf-idf matrix
tfidf_matrix_partition = []
for dense_tfidf_matrix in dense_tfidf_matrix_partition:
tfidf_matrix = csr_matrix(dense_tfidf_matrix)
tfidf_matrix_partition.append(tfidf_matrix)
end = timer()
print('Obtained the partition in %.2f seconds' % (end - start))
txt_file.write('Obtained the partition in %.2f seconds. \n' % (end - start))
# Return the tweets and tf-idf matrix partition
return tweets_partition, tfidf_matrix_partition
def init_tensor(tensor, max_ind=0, init_range=2):
tensor = tensor.clone()
tensor_len = tensor.size(0)
tensor[:int(mpmath.ceil(0.9 * tensor_len))] = tensor[:int(mpmath.ceil(0.9*tensor_len))] * (
2 ** (max_ind - 1 + 1))
tensor[int(mpmath.ceil(0.9*tensor_len)): ] = tensor[int(
mpmath.ceil(0.9*tensor_len)): ] * (2 ** (max_ind -2+ 1))
return tensor
def comp_min_depth(self, t, h):
""" Compute minimal circuit depth needed for FHE,
thus circuit depth needed for decryption circuit.
The depth is applicable only in case when t (plain-text base)
divides q (cypher-text base).
"""
_gamma = 2.01 # constant: 2 < _gamma < 3
_H_f = 1 # computed using equation (5) from the article
v = mpm.ceil(t * _gamma * (_H_f * h + 1))
L = mpm.ceil(mpm.log(v + 0.5, 2))
return L
def hardtanh_tensor(tensor, floor=-1,ceil=1, max_ind=0, hd_range=2):
tensor = tensor.clone()
tensor_len = tensor.size(0)
hd_func = nn.Hardtanh(floor * (2 ** (max_ind - 1 + 1)), ceil * (2 ** (max_ind - 1 + 1)))
tensor[ : int(mpmath.ceil(tensor_len*0.9))] = hd_func(tensor[ :int(mpmath.ceil(tensor_len*0.9))])
hd_func = nn.Hardtanh(floor * (2 ** (max_ind - 2 + 1)), ceil * (2 ** (max_ind - 2 + 1)))
tensor[int(mpmath.ceil(tensor_len*0.9)) : ] = hd_func(tensor[int(mpmath.ceil(tensor_len*0.9)) : ])
'''for i in range(tensor_len):
hd_func = nn.Hardtanh(floor / (2 ** (max_ind - (i%hd_range) + 1)), ceil / (2 ** (max_ind - (i%hd_range) + 1)))
tensor[i] = hd_func(tensor[i])'''
return tensor
def run_optimisation(self, problem: HyperparameterOptimisationProblem) -> None:
R = self.max_iter # maximum amount of resource that can be allocated to a single hyperparameter configuration
eta = self.eta # halving rate
def log_eta(x: int) -> int:
return int(mpmath.log(x) / mpmath.log(eta))
s_max = log_eta(R) # number of unique executions of Successive Halving (minus one)
s_min = 2 if s_max >= 2 else 0 # skip the rest of the brackets after s_min
B = (s_max + 1) * R # total/max resources (without reuse) per execution of Successive Halving
for s in reversed(range(s_min, s_max + 1)):
n = int(mpmath.ceil(int(B / R / (s + 1)) * eta ** s)) # initial number of evaluators/configurations/arms
r = R * eta ** (-s) # initial resources allocated to each evaluator/arm
self._queue.put(Block(bracket=s, i=1, max_i=s+1, n_i=n, r_i=r)) # initial blocks
worker = self._workers[0]
while not self._queue.empty():
block = self._queue.get()
new_block = worker.consume_block(block, problem)
if new_block.i <= new_block.max_i:
self._queue.put(new_block)
else:
best_in_bracket = self.min_or_max(block.evaluations, key=self._get_optimisation_func_val)
print(f'Finished bracket {block.bracket}:\n{block}\n',
best_in_bracket.evaluator.arm, best_in_bracket.optimisation_goals)
def w_tilde(self, u_bar):
"""compute w_tilde, the threshold for the word-length w such that
MSB = computeNaiveMSB if w >= w_tilde
MSB = computeNaiveMSB+1 if w < w_tilde
(this doesn't count into account the roundoff error, as in FxPF"""
zeta_bar = self.Hzeta.WCPG() * u_bar
with workprec(500): # TODO: compute how many bit we need !!
wtilde = [
int(1 + ceil(log(x[0], 2)) -
floor(log(power(2, ceil(log(x[0], 2))) - x[0], 2)))
for x in zeta_bar.tolist()
]
return wtilde
def rand_test(samples, a, k, r):
"""
Tests samples * a >> k == samples // r
:param samples:
:param a:
:param k:
:return:
"""
succ = 0
min_bit_fail, min_bit_fail_val = None, None
max_bit_fail, max_bit_fail_val = None, None
a, k, r = long(a), long(k), long(r)
for x in samples:
if ((x * a) >> k) == (x // r):
succ += 1
else:
was_set = False
if min_bit_fail_val is None or min_bit_fail_val > x:
min_bit_fail_val = x
was_set = True
if max_bit_fail_val is None or max_bit_fail_val < x:
max_bit_fail_val = x
was_set = True
if not was_set:
continue
lg = mpmath.ceil(mpmath.log(x, 2))
if min_bit_fail is None or min_bit_fail > lg:
min_bit_fail = lg
if max_bit_fail is None or max_bit_fail < lg:
max_bit_fail = lg
return succ, min_bit_fail, max_bit_fail
def get_pubmed_central_ids(pubmed_ids, chunk_size=200):
pubmed_id_to_central_id = {}
count = len(pubmed_ids)
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for _ in range(0, num_chunks):
try:
chunk_of_pubmed_ids = pubmed_ids[min_id:min_id+chunk_size]
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&db=pmc&id='
url = url + '&id='.join([str(x) for x in chunk_of_pubmed_ids])
r = requests.get(url)
xml = str(r.text)
pieces = xml.split('<LinkSet>')
j = 0
for piece in pieces[1:]:
pubmed_id = chunk_of_pubmed_ids[j]
linksets = piece.split('<LinkSetDb>')
pubmed_id_to_central_id[pubmed_id] = None
for linkset in linksets[1:]:
if '<LinkName>pubmed_pmc</LinkName>' in linkset:
pubmed_central_id = int(linkset[linkset.index('<Id>')+4:linkset.index('</Id>')])
pubmed_id_to_central_id[pubmed_id] = pubmed_central_id
j = j+1
except:
pass
min_id = min_id + chunk_size
return pubmed_id_to_central_id
def build_samples(R, maxbit):
"""
Builds array of testing samples
:param R:
:param maxbit:
:return:
"""
samples = []
i = 1
while 8 * i <= maxbit:
samples += [random.randint(1, (2**(8 * i)) - 1) for _ in range(100)]
if i < 4:
i += 1
else:
i *= 2
# R-products
Rlog2 = mpmath.ceil(mpmath.log(R, 2))
i = 1
while (8 * i + Rlog2) <= maxbit:
samples += [
R * long(random.randint(1, (2**(8 * i)) - 1)) for _ in range(100)
]
if i < 4:
i += 1
else:
i += 2
return samples
def mpmsb(value, signed):
if isinstance(value, Interval):
return np.max([mpmsb(value.lower_bound, signed=signed),
mpmsb(value.upper_bound, signed=signed)])
if value > 0:
return int(mpmath.floor(mpmath.log(value, 2))) + int(signed)
if value < 0:
return int(mpmath.ceil(mpmath.log(-value, 2)))
return -np.inf
def get_prob_correct_sync(k, pc):
"""prob = \sum_{l=0}^{ceil(k/2)-1} (k_choose_l) * (pc)^l * (1 - pc)^(k-l)
"""
retval = 0
lmax = mpmath.ceil(float(k)/2) - 1
for l in range(0, lmax + 1):
retval += mpmath.binomial(k,l) * mpmath.power(pc, l) * \
mpmath.power(1 - pc, k - l)
return retval
def _mp_round(n):
ceil = mp.ceil(n)
floor = mp.floor(n)
ceil_diff = mp.fabs(n - ceil)
floor_diff = mp.fabs(n - floor)
if ceil_diff <= floor_diff:
return ceil
else:
return floor
def createRange(start, end, step=1):
try:
# attempt to calculate the number of items to be generated
result = RPNGenerator(rangeGenerator(start, end, step),
ceil(fdiv(fadd(fsub(end, start), 1), step)))
except TypeError:
# if that fails, then just do without the count
result = RPNGenerator(rangeGenerator(start, end, step))
return result
def createRange( start, end, step = 1 ):
try:
# attempt to calculate the number of items to be generated
result = RPNGenerator( rangeGenerator( start, end, step ),
ceil( fdiv( fadd( fsub( end, start ), 1 ), step ) ) )
except TypeError:
# if that fails, then just do without the count
result = RPNGenerator( rangeGenerator( start, end, step ) )
return result
def convert_bioentity(sessionmaker, link, chunk_size):
log = logging.getLogger('convert.performance.bioentity')
log.info('begin')
output_creator = OutputCreator(log)
try:
session = sessionmaker()
#Cache current objs
from model_perf_schema.bioentity import Bioentity
current_objs = session.query(Bioentity).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab new objs from backend
objs_json = get_json(link())
min_id = 0
count = len(objs_json)
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
old_objs = objs_json[min_id:min_id+chunk_size]
for obj_json in old_objs:
newly_created_obj = create_bioentity(obj_json)
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, ['format_name', 'class_type', 'dbxref', 'json'], session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Commit
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
session.commit()
min_id = min_id+chunk_size
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
session.close()
log.info('complete')
def comp_all_params_given_degree(self):
log2_modulo_lb = self.comp_log2_modulo_lb_given_degree(
self.poly_degree_log2)
self.log2_q = mpm.ceil(log2_modulo_lb)
#print self.log2_q
log2_sigma_lb = self.comp_log2_sigma_lb_given_modulo_degree(
self.log2_q, self.poly_degree_log2)
self.sigma = mpm.power(2, log2_sigma_lb)
self.B = self._comp_B(self._beta, self.sigma)
self._comp_relin_v2_params()
return self.log2_q, self.sigma, log2_sigma_lb
def mpIntDigits(num):
if not mpmath.almosteq(num,0):
a = mpmath.log(abs(num), b=10)
b = mpmath.nint(a)
if mpmath.almosteq(a,b):
return int(b)+1
else:
c = mpmath.ceil(a)
try:
return int(c)
except:
pass
else:
return 0
def convert_bibentry(sessionmaker, link, cls, chunk_size):
log = logging.getLogger('convert.performance.' + cls.__name__)
log.info('begin')
output_creator = OutputCreator(log)
try:
session = sessionmaker()
#Cache current objs
current_objs = session.query(cls).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab new objs from backend
objs_json = get_json(link())
min_id = 0
count = len(objs_json)
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
old_objs = objs_json[min_id:min_id+chunk_size]
for obj_json in old_objs:
newly_created_obj = cls(obj_json['id'], obj_json['text'])
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_id, ['json'], session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
#Commit
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
session.commit()
min_id = min_id+chunk_size
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
session.close()
log.info('complete')
def mpIntDigits(num):
if not mpmath.almosteq(num, 0):
a = mpmath.log(abs(num), b=10)
b = mpmath.nint(a)
if mpmath.almosteq(a, b):
return int(b) + 1
else:
c = mpmath.ceil(a)
try:
return int(c)
except:
pass
else:
return 0
def isKMorphic(n, k):
'''
Returns true if n to the k power ends with n.
This code won't work correctly for integral powers of 10, but they can
never be morphic anyway, except for 1, which I handle specially.
'''
if n == 1:
return 1
modulo = power(10, ceil(log10(n)))
powmod = getPowMod(n, k, modulo)
return 1 if (n == powmod) else 0
def computeNaiveMSB(self, u_bar, output_info=None):
"""Compute the MSB of t, x and y without taking into account the errors in the filter evaluation, and the
errors in the computation of this MSB (the WCPG computation and the log2 associated)
Returns a vector of MSB"""
# compute the WCPG of Hzeta
zeta_bar = self.Hzeta.WCPG(output_info) * u_bar
with workprec(
500
): # TODO: use right precision !! Or do it as it should be done, as in FxPF (see Nastia thesis p113)
# and then the log2
msb = [int(ceil(log(x[0], 2))) for x in zeta_bar.tolist()]
return msb
def npp(n):
mpmath.mp.dps = len(n)
best = 0
bbase = 0
bk = 0
for k in range(2, int(mpmath.ceil(mpmath.log(n, 2)))):
base = int(mpmath.floor(mpmath.power(n, '1/%d' % (k, ))))
cand = pow(base, k)
if cand > best:
bbase = int(base)
bk = k
best = cand
print bbase
print bk
return best
def find_minimum_k(self, k_last=None):
with workdps(self.dps):
#print 'last_k:', k_last
# delta func
delta_f = lambda k: (self.gamma - 2) / (self.gamma - 1
) * self.gamma**(-k)
#delta_f = lambda k: (self.gamma - 2) * self.gamma**(-self.nth(k + 1))
if k_last == None:
return mpf(self.initial_k)
assert k_last > 0
c_bound = self.sconst
# TODO Ek definition is different!!!
#print 'maxmax:', mp.ceil(-mp.log(delta_f(k_last)/c_bound, 2)), k_last + 1
return max(mp.ceil(-mp.log(delta_f(k_last) / c_bound, 2)),
k_last + 1)
def go_enrichment(self, bioent_ids, callback=None):
from src.sgd.model.perf.core import Bioentity, Bioconcept
bioent_format_names = []
num_chunks = ceil(1.0*len(bioent_ids)/500)
for i in range(num_chunks):
bioent_format_names.extend([json.loads(x.json)['format_name'] for x in DBSession.query(Bioentity).filter(Bioentity.id.in_(bioent_ids[i*500:(i+1)*500])).all()])
enrichment_results = query_batter.query_go_processes(bioent_format_names)
json_format = []
for enrichment_result in enrichment_results:
identifier = 'GO:' + str(int(enrichment_result[0][3:])).zfill(7)
goterm_id = get_obj_id(str(identifier).upper(), 'BIOCONCEPT', 'GO')
goterm = json.loads(get_obj(Bioconcept, 'json', goterm_id))
json_format.append({'go': goterm,
'match_count': enrichment_result[1],
'pvalue': enrichment_result[2]})
return json.dumps(json_format)
def PI(precision):
num_iterations = int(
ceil(precision / 14)
) + 10 #Every iteration in Chudnovsky Algorithm produces approximately 14.18 digits, added 10 more digits for precision
mp.dps = precision + 10 # Added 10 more for more precision
K, M, X, L, S = 6, 1, 1, 13591409, 13591409
for k in range(1, num_iterations + 1):
M = (K**3 - 16 * K) * M / k**3
K += 12
L += 545140134
X *= -262537412640768000
S += mpf(M * L) / X
C = 426880 * sqrt(10005)
pi = C / mpf(S)
print(str(pi)[:precision + 2])
def _compute_LSB(self, l_y_out):
if not isinstance(l_y_out, int):
raise ValueError(
"I implemented the function only for the case of 1 output! \n")
# we need to add one more bit to the account for the
# final rounding error
l_y_out = l_y_out - 1
# construct the error-filter
deltaSIF = self.computeDeltaSIF()
# compute the WCPG of the error filter
wcpgDeltaH = deltaSIF.dSS.WCPG()
# we repartition the error budget equally for all variables
c = self.l + self.n + self.p
# In order to respect the overall error |deltaY(k)| < 2^(l_y_out-1)
# we need to compute the temporary, state and output variables with LSB l_i
# l_i = max(l_y_out) - g_i
# where the correction term g_i is computed via
# g_i = 1 + max_j { ceil( log2 ( c * wcpgDeltaH[j, i] ) )}
g = np.bmat([
1 + max(np.ceil(np.log2(c * wcpgDeltaH[:, i] * 2**-l_y_out)))
for i in range(0, c)
])
# the error budget for the output y(k) that will be later passed on to FloPoCo
error_budget_y = 2**-ceil(
log(c * wcpgDeltaH[0, c - 1] * 2**-l_y_out, 2)) / 2**(l_y_out + 1)
for x in (g == np.inf):
if x.any():
print('Divided by zero\n')
lsb = np.bmat(l_y_out - g - 1)
return lsb, error_budget_y
def removeByRatio(self, head, a, b):
if a < 1 or a > b:
return head
if head is None:
return head
cur = head
n = 1
while cur.next is not None:
n = n + 1
cur = cur.next
ratio = ceil(float(a / b))
if ratio == 1:
head = head.next
if ratio > 0:
cur = head
while --ratio != 1:
cur = cur.next
cur.next = cur.next.next
return head
def apply_rational_with_base(self, n, b, evaluation):
"RealDigits[n_Rational, b_Integer]"
expr = Expression("RealDigits", n)
py_n = abs(n.value)
py_b = b.get_int_value()
if check_finite_decimal(n.denominator().get_int_value()) and not py_b % 2:
return self.apply_2(n, b, evaluation)
else:
exp = int(mpmath.ceil(mpmath.log(py_n, py_b)))
(head, tails) = convert_repeating_decimal(
py_n.as_numer_denom()[0], py_n.as_numer_denom()[1], py_b
)
leaves = []
for x in head:
if not x == "0":
leaves.append(from_python(int(x)))
leaves.append(from_python(tails))
list_str = Expression("List", *leaves)
return Expression("List", list_str, exp)
def computeNaiveMSB(self, u_bar, output_info=None):
"""Compute the MSB of t, x and y without taking into account the errors in the filter evaluation, and the
errors in the computation of this MSB (the WCPG computation and the log2 associated)
Returns a vector of MSB
Parameters:
- u_bar: vector of bounds on the inputs of the system
Returns: a vector of MSB such that the intermediate variables, the states and the output do not overflow
WHEN WE DO NOT THE ROUNDOFF ERRORS INTO ACCOUNT
we just use the equation m = ceil( log2( <<Hzeta>>. u_bar ) )
"""
# compute the WCPG of Hzeta
zeta_bar = self.Hzeta.WCPG(output_info) * u_bar
# TODO: Do it as it should be done, as in FxPF (see Nastia thesis p113)
with mpmath.workprec(500):
# and then the log2
msb = [
int(mpmath.ceil(mpmath.log(x[0], 2)))
for x in zeta_bar.tolist()
]
return msb
def download_review_htmls(restaurant_info):
url = restaurant_info['url']
name = restaurant_info['name']
total_reviews = restaurant_info['total_reviews']
first_file = restaurant_info['first_file']
files = [first_file]
for page_num in range(1, int(mpmath.ceil(total_reviews / 20))): # from page 2 as page 1 is already saved.
page_link = url + "?start=" + str(page_num * 20)
for i in range(5):
try:
response = requests.get(page_link, headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36', })
html = response.content # get the html
filename = 'data/' + name + '/' + str(int(page_num + 1)) + '_' + name + '_' + str(
time.time()) + ".html"
with open(filename, "wb") as file:
file.write(html)
file.close()
files.append(filename)
break
except Exception:
time.sleep(2)
return files
def convert_evidence(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.phenotype import Phenotypeevidence as NewPhenotypeevidence
from model_new_schema.reference import Reference as NewReference
from model_new_schema.evelement import Experiment as NewExperiment, Strain as NewStrain
from model_new_schema.bioentity import Bioentity as NewBioentity
from model_new_schema.misc import Allele as NewAllele
from model_new_schema.phenotype import Phenotype as NewPhenotype
from model_old_schema.reference import Reflink as OldReflink
from model_old_schema.phenotype import PhenotypeFeature as OldPhenotypeFeature
log = logging.getLogger('convert.phenotype.evidence')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
old_session = old_session_maker()
#Values to check
values_to_check = ['experiment_id', 'reference_id', 'strain_id', 'source',
'bioentity_id', 'bioconcept_id', 'date_created', 'created_by',
'reporter', 'reporter_desc', 'strain_details',
'conditions', 'details', 'experiment_details', 'allele_info', 'allele_id']
#Grab cached dictionaries
key_to_experiment = dict([(x.unique_key(), x) for x in new_session.query(NewExperiment).all()])
key_to_phenotype = dict([(x.unique_key(), x) for x in new_session.query(NewPhenotype).all()])
key_to_strain = dict([(x.unique_key(), x) for x in new_session.query(NewStrain).all()])
key_to_allele = dict([(x.unique_key(), x) for x in new_session.query(NewAllele).all()])
bioent_ids = set([x.id for x in new_session.query(NewBioentity).all()])
reference_ids = set([x.id for x in new_session.query(NewReference).all()])
old_reflinks = old_session.query(OldReflink).all()
key_to_reflink = dict([((x.col_name, x.primary_key), x) for x in old_reflinks])
min_id = old_session.query(func.min(OldPhenotypeFeature.id)).first()[0]
count = old_session.query(func.max(OldPhenotypeFeature.id)).first()[0] - min_id
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewPhenotypeevidence).filter(NewPhenotypeevidence.id >= create_evidence_id(min_id)).filter(NewPhenotypeevidence.id < create_evidence_id(min_id+chunk_size)).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldPhenotypeFeature).filter(
OldPhenotypeFeature.id >= min_id).filter(
OldPhenotypeFeature.id < min_id+chunk_size).options(
joinedload('experiment')).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_evidence(old_obj, key_to_reflink, key_to_phenotype, reference_ids,
bioent_ids, key_to_strain, key_to_experiment, key_to_allele)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
''' He let B the number of blue balls and T the total.
We have B = aT, where 0<a<1. The probability is equal to
(B/T) * (B-1)/(T-1) = 1/2, Replacing B = aT, and solving the
second order polynomial we see that a is integer iff
2T^2 + 2T + 1 = q^2, for some q odd. We again solve for integer solutions
and get that 2q^2-1 has to be a square, say k^2
2q^2 = k^2 + 1
Wolfram alpha XD Says that this is a diophantine equation with general solution
equal to
k = -1/2 * [(1+sqrt(2)) * pow(3-2*sqrt(2), n) + (1-sqrt(2)) * pow(3+2*sqrt(2), n)]
and we have the equality k = 2T + 1, we have that
k>= 2 * pow(10, 12) + 1
Also we have the relation B = ceil(T/sqrt(2))
'''
s2 = mp.sqrt(2)
f = lambda n: ((s2 - 1) * mp.power(3 + 2 * s2, n) -
(1 + s2) * mp.power(3 - 2 * s2, n)) / 2 + 2
kmin = 2 * pow(10, 12) + 1
i = 0
while f(i) < kmin:
i += 1
g = lambda i: (f(i) - 1) / 2
h = lambda i: mp.ceil(g(i) / s2)
print("El resultado es: {}".format(h(i)))
print("The time spent is: {}".format(perf_counter() - t))
def apply_2(self, n, b, evaluation, nr_elements=None, pos=None):
"RealDigits[n_?NumericQ, b_Integer]"
expr = Expression("RealDigits", n)
rational_no = (
True if isinstance(n, Rational) else False
) # it is used for checking whether the input n is a rational or not
py_b = b.get_int_value()
if isinstance(n, (Expression, Symbol, Rational)):
pos_len = abs(pos) + 1 if pos is not None and pos < 0 else 1
if nr_elements is not None:
n = Expression(
"N", n,
int(mpmath.log(py_b**(nr_elements + pos_len), 10)) +
1).evaluate(evaluation)
else:
if rational_no:
n = Expression("N", n).evaluate(evaluation)
else:
return evaluation.message("RealDigits", "ndig", expr)
py_n = abs(n.value)
if not py_b > 1:
return evaluation.message("RealDigits", "rbase", py_b)
if isinstance(py_n, complex):
return evaluation.message("RealDigits", "realx", expr)
if isinstance(n, Integer):
display_len = (int(mpmath.floor(mpmath.log(py_n, py_b)))
if py_n != 0 and py_n != 1 else 1)
else:
display_len = int(
Expression(
"N",
Expression(
"Round",
Expression(
"Divide",
Expression("Precision", py_n),
Expression("Log", 10, py_b),
),
),
).evaluate(evaluation).to_python())
exp = int(mpmath.ceil(mpmath.log(
py_n, py_b))) if py_n != 0 and py_n != 1 else 1
if py_n == 0 and nr_elements is not None:
exp = 0
digits = []
if not py_b == 10:
digits = convert_float_base(py_n, py_b, display_len - exp)
# truncate all the leading 0's
i = 0
while digits and digits[i] == 0:
i += 1
digits = digits[i:]
if not isinstance(n, Integer):
if len(digits) > display_len:
digits = digits[:display_len - 1]
else:
# drop any leading zeroes
for x in str(py_n):
if x != "." and (digits or x != "0"):
digits.append(x)
if pos is not None:
temp = exp
exp = pos + 1
move = temp - 1 - pos
if move <= 0:
digits = [0] * abs(move) + digits
else:
digits = digits[abs(move):]
display_len = display_len - move
list_str = Expression("List")
for x in digits:
if x == "e" or x == "E":
break
# Convert to Mathics' list format
list_str.leaves.append(from_python(int(x)))
if not rational_no:
while len(list_str.leaves) < display_len:
list_str.leaves.append(from_python(0))
if nr_elements is not None:
# display_len == nr_elements
if len(list_str.leaves) >= nr_elements:
# Truncate, preserving the digits on the right
list_str = list_str.leaves[:nr_elements]
else:
if isinstance(n, Integer):
while len(list_str.leaves) < nr_elements:
list_str.leaves.append(from_python(0))
else:
# Adding Indeterminate if the length is greater than the precision
while len(list_str.leaves) < nr_elements:
list_str.leaves.append(
from_python(Symbol("Indeterminate")))
return Expression("List", list_str, exp)
def convert_abstract(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.reference import Reference as NewReference, Abstract as NewAbstract
from model_old_schema.reference import Reference as OldReference
log = logging.getLogger('convert.reference_in_depth.abstract')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
old_session = old_session_maker()
#Values to check
values_to_check = ['text']
#Grab cached dictionaries
reference_ids = set([x.id for x in new_session.query(NewReference).all()])
count = old_session.query(func.max(OldReference.id)).first()[0]
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewAbstract).filter(NewAbstract.id >= min_id).filter(NewAbstract.id <= min_id+chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldReference).filter(
OldReference.id >= min_id).filter(
OldReference.id <= min_id+chunk_size).options(
joinedload('abst')).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_abstract(old_obj, reference_ids)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id + chunk_size + 1
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
def convert_interaction_family(new_session_maker, chunk_size):
from model_new_schema.auxiliary import Interaction, InteractionFamily
from model_new_schema.bioentity import Bioentity
log = logging.getLogger('convert.interaction.interaction_family')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
#Values to check
values_to_check = ['bioentity1_id', 'bioentity2_id', 'genetic_ev_count', 'physical_ev_count', 'evidence_count']
#Grab cached dictionaries
id_to_bioent = dict([(x.id, x) for x in new_session.query(Bioentity).all()])
#Grab old objs
interactions = new_session.query(Interaction).filter(or_(Interaction.class_type == 'PHYSINTERACTION', Interaction.class_type == 'GENINTERACTION')).all()
bioent_id_to_evidence_cutoff, bioent_id_to_neighbor_ids, edge_to_counts = interaction_family_precomp(interactions, 100, id_to_bioent)
min_id = 0
count = new_session.query(func.max(Bioentity.id)).first()[0]
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(InteractionFamily).filter(InteractionFamily.bioentity_id >= min_id).filter(InteractionFamily.bioentity_id < min_id + chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
old_objs = new_session.query(Bioentity).filter(Bioentity.id >= min_id).filter(Bioentity.id < min_id+chunk_size).all()
for old_obj in old_objs:
#Convert old objects into new ones
evidence_cutoff = bioent_id_to_evidence_cutoff[old_obj.id]
newly_created_objs = create_interaction_family(old_obj, evidence_cutoff,
bioent_id_to_neighbor_ids, edge_to_counts, id_to_bioent)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def _comp_sigma_k(self, sigma, q, k):
sigma_k = mpm.power(self._alfa, 1 - mpm.sqrt(k))
sigma_k *= mpm.power(q, k - mpm.sqrt(k))
sigma_k *= mpm.power(sigma, mpm.sqrt(k))
sigma_k = mpm.ceil(sigma_k)
return sigma_k
def getCeiling( n ):
if isinstance( n, RPNMeasurement ):
return RPNMeasurement( getCeiling( n.getValue( ) ), n.getUnits( ) )
else:
return ceil( n )
def convert_domain_evidence(new_session_maker, chunk_size):
from model_new_schema.protein import Domain, Domainevidence
from model_new_schema.bioentity import Bioentity
from model_new_schema.reference import Reference
log = logging.getLogger('convert.protein.domain_evidence')
log.info('begin')
output_creator = OutputCreator(log)
try:
#Grab all current objects
new_session = new_session_maker()
current_objs = new_session.query(Domainevidence).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
#Values to check
values_to_check = ['reference_id', 'strain_id', 'source', 'date_created', 'created_by',
'start', 'end', 'evalue', 'status', 'date_of_run', 'protein_id', 'domain_id']
#Grab cached dictionaries
key_to_bioentity = dict([(x.unique_key(), x) for x in new_session.query(Bioentity).all()])
key_to_domain = dict([(x.unique_key(), x) for x in new_session.query(Domain).all()])
pubmed_id_to_reference_id = dict([(x.pubmed_id, x.id) for x in new_session.query(Reference).all()])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
data = break_up_file('/Users/kpaskov/final/yeastmine_protein_domains.tsv')
used_unique_keys = set()
j=0
min_id = 0
count = len(data)
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
old_objs = data[min_id:min_id+chunk_size]
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_domain_evidence(old_obj, j, key_to_bioentity, key_to_domain)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = (newly_created_obj.protein_id, newly_created_obj.domain_id, newly_created_obj.start,
newly_created_obj.end, newly_created_obj.evalue)
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.id not in key_to_current_obj else key_to_current_obj[newly_created_obj.id]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
j = j+1
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
#Grab JASPAR evidence from file
old_objs = break_up_file('/Users/kpaskov/final/TF_family_class_accession04302013.txt')
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_domain_evidence_from_tf_file(old_obj, j, key_to_bioentity, key_to_domain, pubmed_id_to_reference_id)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
j = j+1
output_creator.finished("1/1")
new_session.commit()
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def getFactors( n ):
'''
A factorization of *Nptr into prime and q-prime factors is first obtained.
Selfridge's primality test is then applied to any q-prime factors; the test
is applied repeatedly until either a q-prime is found to be composite or
likely to be composite (in which case the initial factorization is doubtful
and an extra base should be used in Miller's test) or we run out of q-primes,
in which case every q-prime factor of *Nptr is certified as prime.
Returns a list of tuples where each tuple is a prime factor and an exponent.
'''
verbose = g.verbose
if real( n ) < -1:
return [ ( -1, 1 ) ] + getFactors( fneg( n ) )
elif n == -1:
return [ ( -1, 1 ) ]
elif n == 0:
return [ ( 0, 1 ) ]
elif n == 1:
return [ ( 1, 1 ) ]
target = int( n )
dps = ceil( log( n ) )
if dps > mp.dps:
setAccuracy( dps )
if target > g.minValueToCache:
if g.factorCache is None:
loadFactorCache( )
if target in g.factorCache:
if verbose:
print( 'cache hit:', target )
return g.factorCache[ target ]
smallFactors, largeFactors, qPrimes = getPrimeFactors( int( n ), verbose )
if qPrimes:
if verbose:
print( 'testing q-primes for primality' )
print( '--' )
i = 0
for qPrime in qPrimes:
t = doSelfridgeTest( qPrime[ 0 ], verbose )
if not t:
print( 'do FACTOR() again with an extra base' )
return 0
if verbose:
print( 'all q-primes are primes:', n, 'has the following factorization:' )
print( )
for i in smallFactors:
print( 'prime factor:', i[ 0 ], 'exponent:', i[ 1 ] )
for i in largeFactors:
print( 'prime factor:', i[ 0 ], 'exponent:', i[ 1 ] )
else:
if verbose:
print( 'NO Q-PRIMES:' )
print( )
print( n, 'has the following factorization:' )
for i in smallFactors:
print( 'prime factor:', i[ 0 ], 'exponent:', i[ 1 ] )
for i in largeFactors:
print( 'prime factor:', i[ 0 ], 'exponent:', i[ 1 ] )
result = [ ]
result.extend( smallFactors )
result.extend( largeFactors )
if g.factorCache is not None:
product = int( fprod( [ power( i[ 0 ], i[ 1 ] ) for i in largeFactors ] ) )
if product not in g.factorCache:
g.factorCache[ product ] = largeFactors
g.factorCacheIsDirty = True
if n > g.minValueToCache and n not in g.factorCache:
g.factorCache[ n ] = result
g.factorCacheIsDirty = True
return result
def convert_reference(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.reference import Reference as NewReference, Book as NewBook, Journal as NewJournal
from model_old_schema.reference import Reference as OldReference
log = logging.getLogger('convert.reference.reference')
log.info('begin')
output_creator = OutputCreator(log)
try:
#Grab all current objects
new_session = new_session_maker()
#Values to check
values_to_check = ['display_name', 'format_name', 'link', 'source',
'status', 'pubmed_id', 'pubmed_central_id', 'pdf_status', 'year', 'date_published',
'date_revised', 'issue', 'page', 'volume', 'title',
'journal_id', 'book_id', 'doi',
'created_by', 'date_created']
#Grab cached dictionaries
key_to_journal = dict([(x.unique_key(), x) for x in new_session.query(NewJournal).all()])
key_to_book = dict([(x.unique_key(), x) for x in new_session.query(NewBook).all()])
#Grab old objects
old_session = old_session_maker()
used_unique_keys = set()
count = old_session.query(func.max(OldReference.id)).first()[0]
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewReference).filter(NewReference.id >= min_id).filter(NewReference.id <= min_id+chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldReference).filter(
OldReference.id >= min_id).filter(
OldReference.id <= min_id+chunk_size).options(
joinedload('book'),
joinedload('journal')).all()
old_pubmed_ids = [x.pubmed_id for x in old_objs if x.pubmed_id is not None]
pubmed_id_to_pubmed_central_id = get_pubmed_central_ids(old_pubmed_ids)
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_reference(old_obj, key_to_journal, key_to_book, pubmed_id_to_pubmed_central_id)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
used_unique_keys.add(unique_key)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id + chunk_size + 1
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
def convert_domain(new_session_maker, chunk_size):
from model_new_schema.protein import Domain as Domain
log = logging.getLogger('convert.protein.domain')
log.info('begin')
output_creator = OutputCreator(log)
try:
#Grab all current objects
new_session = new_session_maker()
current_objs = new_session.query(Domain).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
#Values to check
values_to_check = ['display_name', 'description', 'interpro_id', 'interpro_description', 'link']
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
data = break_up_file('/Users/kpaskov/final/yeastmine_protein_domains.tsv')
used_unique_keys = set()
min_id = 0
count = len(data)
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
old_objs = data[min_id:min_id+chunk_size]
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_domain(old_obj)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
#Grab JASPAR domains from file
old_objs = break_up_file('/Users/kpaskov/final/TF_family_class_accession04302013.txt')
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_domain_from_tf_file(old_obj)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished("1/1")
new_session.commit()
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def convert_bioentity_reference(new_session_maker, evidence_class, class_type, label, chunk_size, get_bioent_ids_f,
filter_f=None):
from model_new_schema.auxiliary import BioentityReference
from model_new_schema.bioentity import Paragraph
log = logging.getLogger(label)
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
#Values to check
values_to_check = []
#Grab all current objects
current_objs = new_session.query(BioentityReference).filter(BioentityReference.class_type == class_type).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
used_unique_keys = set()
min_id = new_session.query(func.min(evidence_class.id)).first()[0]
count = new_session.query(func.max(evidence_class.id)).first()[0] - min_id
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
old_objs = new_session.query(evidence_class).filter(evidence_class.id >= min_id, evidence_class.id <= min_id+chunk_size).all()
for old_obj in old_objs:
if filter_f is None or filter_f(old_obj):
#Convert old objects into new ones
newly_created_objs = create_bioentity_reference(old_obj, get_bioent_ids_f, class_type)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
#Add paragraph-related bioent_references.
old_objs = new_session.query(Paragraph).filter(Paragraph.class_type == class_type).options(joinedload('paragraph_references')).all()
for old_obj in old_objs:
if filter_f is None or filter_f(old_obj):
#Convert old objects into new ones
newly_created_objs = create_bioentity_reference_from_paragraph(old_obj, class_type)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def _comp_B(self, beta, sigma):
return mpm.ceil(beta * sigma)
def get_list(cls, col_name, obj_ids):
num_chunks = ceil(1.0*len(obj_ids)/500)
objs = []
for i in range(num_chunks):
objs.extend(DBSession.query(cls).filter(cls.id.in_(obj_ids[i*500:(i+1)*500])).all())
return '[' + ', '.join([getattr(obj, col_name) for obj in objs]) + ']'
def convert_interaction(new_session_maker, evidence_class, class_type, label, chunk_size, directed):
from model_new_schema.auxiliary import Interaction
from model_new_schema.bioentity import Bioentity
log = logging.getLogger(label)
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
#Values to check
values_to_check = ['display_name', 'bioentity1_id', 'bioentity2_id', 'evidence_count']
#Grab all current objects
current_objs = new_session.query(Interaction).filter(Interaction.class_type == class_type).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
#Grab cached dictionaries
id_to_bioent = dict([(x.id, x) for x in new_session.query(Bioentity).all()])
untouched_obj_ids = set(id_to_current_obj.keys())
used_unique_keys = set()
#Precomp evidence count
format_name_to_evidence_count = {}
min_id = new_session.query(func.min(evidence_class.id)).first()[0]
count = new_session.query(func.max(evidence_class.id)).first()[0] - min_id
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
more_old_objs = new_session.query(evidence_class).filter(evidence_class.id >= min_id).filter(evidence_class.id < min_id+chunk_size).all()
interaction_precomp(format_name_to_evidence_count, more_old_objs, id_to_bioent, directed)
min_id = min_id + chunk_size
#Create interactions
min_id = new_session.query(func.min(evidence_class.id)).first()[0]
count = new_session.query(func.max(evidence_class.id)).first()[0] - min_id
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
old_objs = new_session.query(evidence_class).filter(evidence_class.id >= min_id).filter(evidence_class.id < min_id+chunk_size).all()
for old_obj in old_objs:
#Convert old objects into new ones
if directed:
format_name = create_directed_key(old_obj)
else:
format_name = create_undirected_interaction_format_name(old_obj, id_to_bioent)
evidence_count = format_name_to_evidence_count[format_name]
newly_created_objs = create_interaction(old_obj, evidence_count, id_to_bioent, directed)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
used_unique_keys.add(unique_key)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def getCeiling( n ):
if isinstance( n, RPNMeasurement ):
return RPNMeasurement( getCeiling( n.value ), n.units )
else:
return ceil( n )
def getLilianDayOperator( n ):
return ceil( n.subtract( RPNDateTime( 1582, 10, 15 ) ).value )
def convert_evidence_chemical(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.phenotype import Phenotypeevidence as NewPhenotypeevidence
from model_new_schema.chemical import Chemical as NewChemical
from model_new_schema.evidence import EvidenceChemical as NewEvidenceChemical
from model_old_schema.phenotype import PhenotypeFeature as OldPhenotypeFeature
log = logging.getLogger('convert.phenotype.evidence_chemical')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
old_session = old_session_maker()
#Values to check
values_to_check = ['chemical_amt']
#Grab cached dictionaries
key_to_chemical = dict([(x.unique_key(), x) for x in new_session.query(NewChemical).all()])
min_id = old_session.query(func.min(OldPhenotypeFeature.id)).first()[0]
count = old_session.query(func.max(OldPhenotypeFeature.id)).first()[0] - min_id
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewEvidenceChemical).filter(NewEvidenceChemical.evidence_id >= create_evidence_id(min_id)).filter(NewEvidenceChemical.evidence_id < create_evidence_id(min_id+chunk_size)).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
id_to_evidence = dict([(x.id, x) for x in new_session.query(NewPhenotypeevidence).filter(NewPhenotypeevidence.id >= create_evidence_id(min_id)).filter(NewPhenotypeevidence.id < create_evidence_id(min_id+chunk_size)).all()])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldPhenotypeFeature).filter(
OldPhenotypeFeature.id >= min_id).filter(
OldPhenotypeFeature.id < min_id+chunk_size).options(
joinedload('experiment')).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_evidence_chemical(old_obj, key_to_chemical, id_to_evidence)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
def convert_url(new_session_maker, chunk_size):
from model_new_schema.reference import Reference, Referenceurl as NewReferenceurl
log = logging.getLogger('convert.reference_in_depth.reference_url')
log.info('begin')
output_creator = OutputCreator(log)
try:
#Grab all current objects
new_session = new_session_maker()
#Values to check
values_to_check = ['display_name', 'category', 'source', 'date_created', 'created_by', 'reference_id', 'url_type']
count = new_session.query(func.max(Reference.id)).first()[0]
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewReferenceurl).filter(NewReferenceurl.reference_id >= min_id).filter(NewReferenceurl.reference_id <= min_id+chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = new_session.query(Reference).filter(
Reference.id >= min_id).filter(
Reference.id <= min_id+chunk_size).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_url(old_obj)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id + chunk_size + 1
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def getNthHexagonalPentagonalNumber( n ):
return nint( ceil( fdiv( fmul( fsub( sqrt( 3 ), 1 ),
power( fadd( 2, sqrt( 3 ) ), fsub( fmul( 4, real_int( n ) ), 2 ) ) ),
12 ) ) )
def convert_author_reference(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.reference import Author as NewAuthor, Reference as NewReference, AuthorReference as NewAuthorReference
from model_old_schema.reference import AuthorReference as OldAuthorReference, Author as OldAuthor
log = logging.getLogger('convert.reference_in_depth.author_reference')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
old_session = old_session_maker()
#Values to check
values_to_check = ['author_type']
#Grab cached dictionaries
reference_ids = set([x.id for x in new_session.query(NewReference).all()])
#Simplify author conversion
old_id_to_key = dict([(x.id, create_format_name(x.name)) for x in old_session.query(OldAuthor).all()])
new_key_to_id = dict([(x.unique_key(), x.id) for x in new_session.query(NewAuthor).all()])
old_id_to_new_id_author = dict([(x, new_key_to_id[y]) for x, y in old_id_to_key.iteritems()])
used_unique_keys = set()
count = old_session.query(func.max(OldAuthorReference.id)).first()[0]
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewAuthorReference).filter(NewAuthorReference.id >= min_id).filter(NewAuthorReference.id < min_id+chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldAuthorReference).filter(
OldAuthorReference.id >= min_id).filter(
OldAuthorReference.id < min_id+chunk_size).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_author_reference(old_obj, old_id_to_new_id_author, reference_ids)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
if unique_key not in used_unique_keys:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
used_unique_keys.add(unique_key)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id + chunk_size
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
def convert_evidence(new_session_maker, chunk_size):
from model_new_schema.regulation import Regulationevidence
from model_new_schema.evelement import Experiment
from model_new_schema.bioentity import Bioentity
from model_new_schema.reference import Reference
log = logging.getLogger('convert.regulation.evidence')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
#Values to check
values_to_check = ['experiment_id', 'reference_id', 'strain_id', 'source', 'conditions',
'bioentity1_id', 'bioentity2_id', 'date_created', 'created_by']
#Grab cached dictionaries
key_to_experiment = dict([(x.unique_key(), x) for x in new_session.query(Experiment).all()])
key_to_bioent = dict([(x.unique_key(), x) for x in new_session.query(Bioentity).all()])
pubmed_to_reference_id = dict([(x.pubmed_id, x.id) for x in new_session.query(Reference).all()])
#Grab old objects
data = break_up_file('/Users/kpaskov/final/yeastmine_regulation.tsv')
count = len(data)
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
j = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(Regulationevidence).filter(Regulationevidence.id >= create_evidence_id(min_id)).filter(Regulationevidence.id < create_evidence_id(min_id+chunk_size)).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
old_objs = data[min_id:min_id+chunk_size]
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_evidence(old_obj, j, key_to_experiment, key_to_bioent, pubmed_to_reference_id)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
unique_key = newly_created_obj.unique_key()
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if unique_key not in key_to_current_obj else key_to_current_obj[unique_key]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
j = j + 1
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def describeIntegerOperator(n):
indent = ' ' * 4
print()
print(int(n), 'is:')
if isOdd(n):
print(indent + 'odd')
elif isEven(n):
print(indent + 'even')
if isPrime(n):
isPrimeFlag = True
print(indent + 'prime')
elif n > 3:
isPrimeFlag = False
print(indent + 'composite')
else:
isPrimeFlag = False
if isKthPower(n, 2):
print(indent + 'the ' + getShortOrdinalName(sqrt(n)) +
' square number')
if isKthPower(n, 3):
print(indent + 'the ' + getShortOrdinalName(cbrt(n)) + ' cube number')
for i in arange(4, fadd(ceil(log(fabs(n), 2)), 1)):
if isKthPower(n, i):
print(indent + 'the ' + getShortOrdinalName(root(n, i)) + ' ' +
getNumberName(i, True) + ' power')
# triangular
guess = findPolygonalNumber(n, 3)
if getNthPolygonalNumber(guess, 3) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' triangular number')
# pentagonal
guess = findPolygonalNumber(n, 5)
if getNthPolygonalNumber(guess, 5) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' pentagonal number')
# hexagonal
guess = findPolygonalNumber(n, 6)
if getNthPolygonalNumber(guess, 6) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' hexagonal number')
# heptagonal
guess = findPolygonalNumber(n, 7)
if getNthPolygonalNumber(guess, 7) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' heptagonal number')
# octagonal
guess = findPolygonalNumber(n, 8)
if getNthPolygonalNumber(guess, 8) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' octagonal number')
# nonagonal
guess = findPolygonalNumber(n, 9)
if getNthPolygonalNumber(guess, 9) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' nonagonal number')
# decagonal
guess = findPolygonalNumber(n, 10)
if getNthPolygonalNumber(guess, 10) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' decagonal number')
# if n > 1:
# for i in range( 11, 101 ):
# if getNthPolygonalNumber( findPolygonalNumber( n, i ), i ) == n:
# print( indent + str( i ) + '-gonal' )
# centered triangular
guess = findCenteredPolygonalNumber(n, 3)
if getNthCenteredPolygonalNumber(guess, 3) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered triangular')
# centered square
guess = findCenteredPolygonalNumber(n, 4)
if getNthCenteredPolygonalNumber(guess, 4) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered square number')
# centered pentagonal
guess = findCenteredPolygonalNumber(n, 5)
if getNthCenteredPolygonalNumber(guess, 5) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered pentagonal number')
# centered hexagonal
guess = findCenteredPolygonalNumber(n, 6)
if getNthCenteredPolygonalNumber(guess, 6) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered hexagonal number')
# centered heptagonal
guess = findCenteredPolygonalNumber(n, 7)
if getNthCenteredPolygonalNumber(guess, 7) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered heptagonal number')
# centered octagonal
guess = findCenteredPolygonalNumber(n, 8)
if getNthCenteredPolygonalNumber(guess, 8) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered octagonal number')
# centered nonagonal
guess = findCenteredPolygonalNumber(n, 9)
if getNthCenteredPolygonalNumber(guess, 9) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered nonagonal number')
# centered decagonal
guess = findCenteredPolygonalNumber(n, 10)
if getNthCenteredPolygonalNumber(guess, 10) == n:
print(indent + 'the ' + getShortOrdinalName(guess) +
' centered decagonal number')
# pandigital
if isPandigital(n):
print(indent + 'pandigital')
# for i in range( 4, 21 ):
# if isBaseKPandigital( n, i ):
# print( indent + 'base ' + str( i ) + ' pandigital' )
# Fibonacci
result = findInput(n, fib, lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Fibonacci number')
# Tribonacci
result = findInput(n, lambda n: getNthKFibonacciNumber(n, 3),
lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Tribonacci number')
# Tetranacci
result = findInput(n, lambda n: getNthKFibonacciNumber(n, 4),
lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Tetranacci number')
# Pentanacci
result = findInput(n, lambda n: getNthKFibonacciNumber(n, 5),
lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Pentanacci number')
# Hexanacci
result = findInput(n, lambda n: getNthKFibonacciNumber(n, 6),
lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Hexanacci number')
# Heptanacci
result = findInput(n, lambda n: getNthKFibonacciNumber(n, 7),
lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Heptanacci number')
# Octanacci
result = findInput(n, lambda n: getNthKFibonacciNumber(n, 8),
lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Octanacci number')
# Lucas numbers
result = findInput(n, getNthLucasNumber, lambda n: fmul(log10(n), 5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Lucas number')
# base-k repunits
if n > 1:
for i in range(2, 21):
result = findInput(n,
lambda x: getNthBaseKRepunit(x, i),
lambda n: log(n, i),
minimum=1)
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' base-' + str(i) + ' repunit')
# Jacobsthal numbers
result = findInput(n, getNthJacobsthalNumber, lambda n: fmul(log(n), 1.6))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Jacobsthal number')
# Padovan numbers
result = findInput(n, getNthPadovanNumber, lambda n: fmul(log10(n), 9))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Padovan number')
# Fibonorial numbers
result = findInput(n, getNthFibonorial, lambda n: sqrt(fmul(log(n), 10)))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Fibonorial number')
# Mersenne primes
result = findInput(n, getNthMersennePrime,
lambda n: fadd(fmul(log(log(sqrt(n))), 2.7), 3))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Mersenne prime')
# perfect number
result = findInput(n, getNthPerfectNumber,
lambda n: fmul(log(log(n)), 2.6))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' perfect number')
# Mersenne exponent
result = findInput(n,
getNthMersenneExponent,
lambda n: fmul(log(n), 2.7),
maximum=50)
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Mersenne exponent')
if not isPrimeFlag and n != 1 and n <= LARGEST_NUMBER_TO_FACTOR:
# deficient
if isDeficient(n):
print(indent + 'deficient')
# abundant
if isAbundant(n):
print(indent + 'abundant')
# k_hyperperfect
for i in sorted(list(set(sorted(
downloadOEISSequence(34898)[:500]))))[1:]:
if i > n:
break
if isKHyperperfect(n, i):
print(indent + str(i) + '-hyperperfect')
break
# smooth
for i in getPrimes(2, 50):
if isSmooth(n, i):
print(indent + str(i) + '-smooth')
break
# rough
previousPrime = 2
for i in getPrimes(2, 50):
if not isRough(n, i):
print(indent + str(previousPrime) + '-rough')
break
previousPrime = i
# is_semiprime
if isSemiprime(n):
print(indent + 'semiprime')
# is_sphenic
elif isSphenic(n):
print(indent + 'sphenic')
elif isSquareFree(n):
print(indent + 'square-free')
# powerful
if isPowerful(n):
print(indent + 'powerful')
# factorial
result = findInput(n, getNthFactorial, lambda n: power(log10(n), 0.92))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' factorial number')
# alternating factorial
result = findInput(n, getNthAlternatingFactorial,
lambda n: fmul(sqrt(log(n)), 0.72))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' alternating factorial number')
# double factorial
if n == 1:
result = (True, 1)
else:
result = findInput(n, getNthDoubleFactorial,
lambda n: fdiv(power(log(log(n)), 4), 7))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' double factorial number')
# hyperfactorial
result = findInput(n, getNthHyperfactorial,
lambda n: fmul(sqrt(log(n)), 0.8))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' hyperfactorial number')
# subfactorial
result = findInput(n, getNthSubfactorial, lambda n: fmul(log10(n), 1.1))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' subfactorial number')
# superfactorial
if n == 1:
result = (True, 1)
else:
result = findInput(n, getNthSuperfactorial,
lambda n: fadd(sqrt(log(n)), 1))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' superfactorial number')
# pernicious
if isPernicious(n):
print(indent + 'pernicious')
# pronic
if isPronic(n):
print(indent + 'pronic')
# Achilles
if isAchillesNumber(n):
print(indent + 'an Achilles number')
# antiharmonic
if isAntiharmonic(n):
print(indent + 'antiharmonic')
# unusual
if isUnusual(n):
print(indent + 'unusual')
# hyperperfect
for i in range(2, 21):
if isKHyperperfect(n, i):
print(indent + str(i) + '-hyperperfect')
# Ruth-Aaron
if isRuthAaronNumber(n):
print(indent + 'a Ruth-Aaron number')
# Smith numbers
if isSmithNumber(n):
print(indent + 'a Smith number')
# base-k Smith numbers
for i in range(2, 10):
if isBaseKSmithNumber(n, i):
print(indent + 'a base-' + str(i) + ' Smith number')
# order-k Smith numbers
for i in range(2, 11):
if isOrderKSmithNumber(n, i):
print(indent + 'an order-' + str(i) + ' Smith number')
# polydivisible
if isPolydivisible(n):
print(indent + 'polydivisible')
# Carol numbers
result = findInput(n, getNthCarolNumber,
lambda n: fmul(log10(n), fdiv(5, 3)))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Carol number')
# Kynea numbers
result = findInput(n, getNthKyneaNumber,
lambda n: fmul(log10(n), fdiv(5, 3)))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Kynea number')
# Leonardo numbers
result = findInput(n, getNthLeonardoNumber,
lambda n: fsub(log(n, phi), fdiv(1, phi)))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Leonardo number')
# Thabit numbers
result = findInput(n, getNthThabitNumber, lambda n: fmul(log10(n), 3.25))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Thabit number')
# Carmmichael
if isCarmichaelNumber(n):
print(indent + 'a Carmichael number')
# narcissistic
if isNarcissistic(n):
print(indent + 'narcissistic')
# PDI
if isPerfectDigitalInvariant(n):
print(indent + 'a perfect digital invariant')
# PDDI
if isPerfectDigitToDigitInvariant(n, 10):
print(indent + 'a perfect digit-to-digit invariant in base 10')
# Kaprekar
if isKaprekarNumber(n):
print(indent + 'a Kaprekar number')
# automorphic
if isAutomorphic(n):
print(indent + 'automorphic')
# trimorphic
if isTrimorphic(n):
print(indent + 'trimorphic')
# k-morphic
for i in range(4, 11):
if isKMorphic(n, i):
print(indent + str(i) + '-morphic')
# bouncy
if isBouncy(n):
print(indent + 'bouncy')
# step number
if isStepNumber(n):
print(indent + 'a step number')
# Apery numbers
result = findInput(n, getNthAperyNumber,
lambda n: fadd(fdiv(log(n), 1.5), 1))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Apery number')
# Delannoy numbers
result = findInput(n, getNthDelannoyNumber, lambda n: fmul(log10(n), 1.35))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Delannoy number')
# Schroeder numbers
result = findInput(n, getNthSchroederNumber, lambda n: fmul(log10(n), 1.6))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Schroeder number')
# Schroeder-Hipparchus numbers
result = findInput(n, getNthSchroederHipparchusNumber,
lambda n: fdiv(log10(n), 1.5))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Schroeder-Hipparchus number')
# Motzkin numbers
result = findInput(n, getNthMotzkinNumber, log)
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Motzkin number')
# Pell numbers
result = findInput(n, getNthPellNumber, lambda n: fmul(log(n), 1.2))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Pell number')
# Sylvester numbers
if n > 1:
result = findInput(n, getNthSylvesterNumber,
lambda n: sqrt(sqrt(log(n))))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' Sylvester number')
# partition numbers
result = findInput(n, getPartitionNumber, lambda n: power(log(n), 1.56))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' partition number')
# menage numbers
result = findInput(n, getNthMenageNumber, lambda n: fdiv(log10(n), 1.2))
if result[0]:
print(indent + 'the ' + getShortOrdinalName(result[1]) +
' menage number')
print()
print(int(n), 'has:')
# number of digits
digits = log10(n)
if isInteger(digits):
digits += 1
else:
digits = ceil(digits)
print(indent + str(int(digits)) + ' digit' + ('s' if digits > 1 else ''))
# digit sum
print(indent + 'a digit sum of ' + str(int(sumDigits(n))))
# digit product
digitProduct = multiplyDigits(n)
print(indent + 'a digit product of ' + str(int(digitProduct)))
# non-zero digit product
if digitProduct == 0:
print(indent + 'a non-zero digit product of ' +
str(int(multiplyNonzeroDigits(n))))
if not isPrimeFlag and n != 1 and n <= LARGEST_NUMBER_TO_FACTOR:
# factors
factors = getFactors(n)
factorCount = len(factors)
print(indent + str(factorCount) + ' prime factor' +
('s' if factorCount > 1 else '') + ': ' +
', '.join([str(int(i)) for i in factors]))
# number of divisors
divisorCount = int(getDivisorCount(n))
print(indent + str(divisorCount) + ' divisor' +
('s' if divisorCount > 1 else ''))
if n <= LARGEST_NUMBER_TO_FACTOR:
print(indent + 'a divisor sum of ' + str(int(getSigma(n))))
print(indent + 'a Stern value of ' + str(int(getNthSternNumber(n))))
calkinWilf = getNthCalkinWilf(n)
print(indent + 'a Calkin-Wilf value of ' + str(int(calkinWilf[0])) +
'/' + str(int(calkinWilf[1])))
print(indent + 'a Mobius value of ' + str(int(getNthMobiusNumber(n))))
print(indent + 'a radical of ' + str(int(getRadical(n))))
print(indent + 'a Euler phi value of ' + str(int(getEulerPhi(n))))
print(indent + 'a digital root of ' + str(int(getDigitalRoot(n))))
if hasUniqueDigits(n):
print(indent + 'unique digits')
print(indent + 'a multiplicative persistence of ' +
str(int(getPersistence(n))))
print(indent + 'an Erdos persistence of ' +
str(int(getErdosPersistence(n))))
if n > 9 and isIncreasing(n):
if not isDecreasing(n):
print(indent + 'increasing digits')
elif n > 9 and isDecreasing(n):
print(indent + 'decreasing digits')
print()
return n
def convert_url(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.bioentity import Bioentity as NewBioentity, Bioentityurl as NewBioentityurl
from model_old_schema.general import WebDisplay as OldWebDisplay, FeatUrl as OldFeatUrl, DbxrefFeat as OldDbxrefFeat
log = logging.getLogger('convert.bioentity_in_depth.bioentity_url')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
old_session = old_session_maker()
#Values to check
values_to_check = ['display_name', 'source', 'created_by', 'date_created']
#Grab cached dictionaries
id_to_bioentity = dict([(x.id, x) for x in new_session.query(NewBioentity).all()])
#Urls of interest
old_web_displays = old_session.query(OldWebDisplay).filter(OldWebDisplay.label_location == 'Interaction Resources').all()
url_to_display = dict([(x.url_id, x) for x in old_web_displays])
count = max(id_to_bioentity.keys())
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewBioentityurl).filter(NewBioentityurl.bioentity_id >= min_id).filter(NewBioentityurl.bioentity_id < min_id+chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldFeatUrl).filter(OldFeatUrl.feature_id >= min_id).filter(OldFeatUrl.feature_id < min_id+chunk_size).options(joinedload('url')).all()
for old_obj in old_objs:
#Convert old objects into new ones
if old_obj.url_id in url_to_display:
newly_created_objs = create_url(old_obj, url_to_display[old_obj.url_id], id_to_bioentity)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Grab old objects (dbxref)
old_objs = old_session.query(OldDbxrefFeat).filter(
OldDbxrefFeat.feature_id >= min_id).filter(
OldDbxrefFeat.feature_id < min_id+chunk_size).options(
joinedload('dbxref'), joinedload('dbxref.dbxref_urls')).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_url_from_dbxref(old_obj, url_to_display, id_to_bioentity)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id + chunk_size
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
def convert_litevidence(old_session_maker, new_session_maker, chunk_size):
from model_new_schema.literature import Literatureevidence as NewLiteratureevidence
from model_new_schema.reference import Reference as NewReference
from model_new_schema.bioentity import Bioentity as NewBioentity
from model_old_schema.reference import LitguideFeat as OldLitguideFeat
log = logging.getLogger('convert.literature.evidence')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
old_session = old_session_maker()
#Values to check
values_to_check = ['experiment_id', 'reference_id', 'class_type', 'strain_id',
'source', 'topic', 'bioentity_id', 'date_created', 'created_by']
#Grab cached dictionaries
bioent_ids = set([x.id for x in new_session.query(NewBioentity).all()])
reference_ids = set([x.id for x in new_session.query(NewReference).all()])
min_id = old_session.query(func.min(OldLitguideFeat.id)).first()[0]
count = old_session.query(func.max(OldLitguideFeat.id)).first()[0] - min_id
num_chunks = ceil(1.0*count/chunk_size)
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewLiteratureevidence).filter(NewLiteratureevidence.id >= create_litevidence_id(min_id)).filter(NewLiteratureevidence.id < create_litevidence_id(min_id+chunk_size)).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = old_session.query(OldLitguideFeat).filter(
OldLitguideFeat.id >= min_id).filter(
OldLitguideFeat.id < min_id+chunk_size).filter(
or_(OldLitguideFeat.topic=='Additional Literature',
OldLitguideFeat.topic=='Primary Literature',
OldLitguideFeat.topic=='Omics',
OldLitguideFeat.topic=='Reviews')).options(
joinedload('litguide')).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_litevidence(old_obj, reference_ids, bioent_ids)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id+chunk_size
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
old_session.close()
log.info('complete')
def getLilianDay( n ):
if not isinstance( n, RPNDateTime ):
raise ValueError( 'a date-time type required for this operator' )
return ceil( n.subtract( RPNDateTime( 1582, 10, 15 ) ).getValue( ) )
def convert_bibentry(new_session_maker, chunk_size):
from model_new_schema.reference import Reference as NewReference, Bibentry as NewBibentry, \
Journal as NewJournal, Book as NewBook, Abstract as NewAbstract, \
Reftype as NewReftype, Author as NewAuthor, AuthorReference as NewAuthorReference
log = logging.getLogger('convert.reference_in_depth.bibentry')
log.info('begin')
output_creator = OutputCreator(log)
try:
new_session = new_session_maker()
#Values to check
values_to_check = ['text']
#Grab cached dictionaries
id_to_journal = dict([(x.id, x) for x in new_session.query(NewJournal).all()])
id_to_book = dict([(x.id, x) for x in new_session.query(NewBook).all()])
id_to_abstract = dict([(x.id, x.text) for x in new_session.query(NewAbstract).all()])
id_to_authors = {}
id_to_author = dict([(x.id, x) for x in new_session.query(NewAuthor).all()])
for ar in new_session.query(NewAuthorReference).all():
reference_id = ar.reference_id
author_name = id_to_author[ar.author_id].display_name
if reference_id in id_to_authors:
id_to_authors[reference_id].add(author_name)
else:
id_to_authors[reference_id] = set([author_name])
id_to_reftypes = {}
reftypes = new_session.query(NewReftype).all()
for reftype in reftypes:
reference_id = reftype.reference_id
reftype_name = reftype.name
if reference_id in id_to_reftypes:
id_to_reftypes[reference_id].add(reftype_name)
else:
id_to_reftypes[reference_id] = set([author_name])
count = new_session.query(func.max(NewReference.id)).first()[0]
num_chunks = ceil(1.0*count/chunk_size)
min_id = 0
for i in range(0, num_chunks):
#Grab all current objects
current_objs = new_session.query(NewBibentry).filter(NewBibentry.id >= min_id).filter(NewBibentry.id <= min_id+chunk_size).all()
id_to_current_obj = dict([(x.id, x) for x in current_objs])
key_to_current_obj = dict([(x.unique_key(), x) for x in current_objs])
untouched_obj_ids = set(id_to_current_obj.keys())
#Grab old objects
old_objs = new_session.query(NewReference).filter(
NewReference.id >= min_id).filter(
NewReference.id <= min_id+chunk_size).options(joinedload('author_references')).all()
for old_obj in old_objs:
#Convert old objects into new ones
newly_created_objs = create_bibentry(old_obj, id_to_journal, id_to_book, id_to_abstract, id_to_reftypes, id_to_authors)
if newly_created_objs is not None:
#Edit or add new objects
for newly_created_obj in newly_created_objs:
current_obj_by_id = None if newly_created_obj.id not in id_to_current_obj else id_to_current_obj[newly_created_obj.id]
current_obj_by_key = None if newly_created_obj.unique_key() not in key_to_current_obj else key_to_current_obj[newly_created_obj.unique_key()]
create_or_update(newly_created_obj, current_obj_by_id, current_obj_by_key, values_to_check, new_session, output_creator)
if current_obj_by_id is not None and current_obj_by_id.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_id.id)
if current_obj_by_key is not None and current_obj_by_key.id in untouched_obj_ids:
untouched_obj_ids.remove(current_obj_by_key.id)
output_creator.finished(str(i+1) + "/" + str(int(num_chunks)))
new_session.commit()
min_id = min_id + chunk_size + 1
#Delete untouched objs
for untouched_obj_id in untouched_obj_ids:
new_session.delete(id_to_current_obj[untouched_obj_id])
output_creator.removed()
#Commit
output_creator.finished()
new_session.commit()
except Exception:
log.exception('Unexpected error:' + str(sys.exc_info()[0]))
finally:
new_session.close()
log.info('complete')
def describeInteger( n ):
if n < 1:
raise ValueError( "'describe' requires a positive integer argument" )
indent = ' ' * 4
print( )
print( real_int( n ), 'is:' )
if isOdd( n ):
print( indent + 'odd' )
elif isEven( n ):
print( indent + 'even' )
if isPrimeNumber( n ):
isPrime = True
print( indent + 'prime' )
elif n > 3:
isPrime = False
print( indent + 'composite' )
else:
isPrime = False
if isKthPower( n, 2 ):
print( indent + 'the ' + getShortOrdinalName( sqrt( n ) ) + ' square number' )
if isKthPower( n, 3 ):
print( indent + 'the ' + getShortOrdinalName( cbrt( n ) ) + ' cube number' )
for i in arange( 4, fadd( ceil( log( fabs( n ), 2 ) ), 1 ) ):
if isKthPower( n, i ):
print( indent + 'the ' + getShortOrdinalName( root( n, i ) ) + ' ' + \
getNumberName( i, True ) + ' power' )
# triangular
guess = findPolygonalNumber( n, 3 )
if getNthPolygonalNumber( guess, 3 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' triangular number' )
# pentagonal
guess = findPolygonalNumber( n, 5 )
if getNthPolygonalNumber( guess, 5 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' pentagonal number' )
# hexagonal
guess = findPolygonalNumber( n, 6 )
if getNthPolygonalNumber( guess, 6 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' hexagonal number' )
# heptagonal
guess = findPolygonalNumber( n, 7 )
if getNthPolygonalNumber( guess, 7 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' heptagonal number' )
# octagonal
guess = findPolygonalNumber( n, 8 )
if getNthPolygonalNumber( guess, 8 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' octagonal number' )
# nonagonal
guess = findPolygonalNumber( n, 9 )
if getNthPolygonalNumber( guess, 9 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' nonagonal number' )
# decagonal
guess = findPolygonalNumber( n, 10 )
if getNthPolygonalNumber( guess, 10 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' decagonal number' )
#if n > 1:
# for i in range( 11, 101 ):
# if getNthPolygonalNumber( findPolygonalNumber( n, i ), i ) == n:
# print( indent + str( i ) + '-gonal' )
# centered triangular
guess = findCenteredPolygonalNumber( n, 3 )
if getNthCenteredPolygonalNumber( guess, 3 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered triangular' )
# centered square
guess = findCenteredPolygonalNumber( n, 4 )
if getNthCenteredPolygonalNumber( guess, 4 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered square number' )
# centered pentagonal
guess = findCenteredPolygonalNumber( n, 5 )
if getNthCenteredPolygonalNumber( guess, 5 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered pentagonal number' )
# centered hexagonal
guess = findCenteredPolygonalNumber( n, 6 )
if getNthCenteredPolygonalNumber( guess, 6 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered hexagonal number' )
# centered heptagonal
guess = findCenteredPolygonalNumber( n, 7 )
if getNthCenteredPolygonalNumber( guess, 7 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered heptagonal number' )
# centered octagonal
guess = findCenteredPolygonalNumber( n, 8 )
if getNthCenteredPolygonalNumber( guess, 8 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered octagonal number' )
# centered nonagonal
guess = findCenteredPolygonalNumber( n, 9 )
if getNthCenteredPolygonalNumber( guess, 9 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered nonagonal number' )
# centered decagonal
guess - findCenteredPolygonalNumber( n, 10 )
if getNthCenteredPolygonalNumber( guess, 10 ) == n:
print( indent + 'the ' + getShortOrdinalName( guess ) + ' centered decagonal number' )
# pandigital
if isPandigital( n ):
print( indent + 'pandigital' )
#for i in range( 4, 21 ):
# if isBaseKPandigital( n, i ):
# print( indent + 'base ' + str( i ) + ' pandigital' )
# Fibonacci
result = findInput( n, fib, lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Fibonacci number' )
# Tribonacci
result = findInput( n, lambda n : getNthKFibonacciNumber( n, 3 ), lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Tribonacci number' )
# Tetranacci
result = findInput( n, lambda n : getNthKFibonacciNumber( n, 4 ), lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Tetranacci number' )
# Pentanacci
result = findInput( n, lambda n : getNthKFibonacciNumber( n, 5 ), lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Pentanacci number' )
# Hexanacci
result = findInput( n, lambda n : getNthKFibonacciNumber( n, 6 ), lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Hexanacci number' )
# Heptanacci
result = findInput( n, lambda n : getNthKFibonacciNumber( n, 7 ), lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Heptanacci number' )
# Octanacci
result = findInput( n, lambda n : getNthKFibonacciNumber( n, 8 ), lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Octanacci number' )
# Lucas numbers
result = findInput( n, getNthLucasNumber, lambda n: fmul( log10( n ), 5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Lucas number' )
# base-k repunits
if n > 1:
for i in range( 2, 21 ):
result = findInput( n, lambda x: getNthBaseKRepunit( x, i ), lambda n: log( n, i ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' base-' + str( i ) + ' repunit' )
# Jacobsthal numbers
result = findInput( n, getNthJacobsthalNumber, lambda n: fmul( log( n ), 1.6 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Jacobsthal number' )
# Padovan numbers
result = findInput( n, getNthPadovanNumber, lambda n: fmul( log10( n ), 9 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Padovan number' )
# Fibonorial numbers
result = findInput( n, getNthFibonorial, lambda n: sqrt( fmul( log( n ), 10 ) ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Fibonorial number' )
# Mersenne primes
result = findInput( n, getNthMersennePrime, lambda n: fadd( fmul( log( log( sqrt( n ) ) ), 2.7 ), 3 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Mersenne prime' )
## perfect number
result = findInput( n, getNthPerfectNumber, lambda n: fmul( log( log( n ) ), 2.6 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' perfect number' )
# Mersenne exponent
result = findInput( n, getNthMersenneExponent, lambda n: fmul( log( n ), 2.7 ), max=50 )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Mersenne exponent' )
if not isPrime and n != 1 and n <= largestNumberToFactor:
# deficient
if isDeficient( n ):
print( indent + 'deficient' )
# abundant
if isAbundant( n ):
print( indent + 'abundant' )
# k_hyperperfect
for i in sorted( list( set( sorted( downloadOEISSequence( 34898 )[ : 500 ] ) ) ) )[ 1 : ]:
if i > n:
break
if isKHyperperfect( n, i ):
print( indent + str( i ) + '-hyperperfect' )
break
# smooth
for i in getPrimes( 2, 50 ):
if isSmooth( n, i ):
print( indent + str( i ) + '-smooth' )
break
# rough
previousPrime = 2
for i in getPrimes( 2, 50 ):
if not isRough( n, i ):
print( indent + str( previousPrime ) + '-rough' )
break
previousPrime = i
# is_semiprime
if isSemiprime( n ):
print( indent + 'semiprime' )
# is_sphenic
elif isSphenic( n ):
print( indent + 'sphenic' )
elif isSquareFree( n ):
print( indent + 'square-free' )
# powerful
if isPowerful( n ):
print( indent + 'powerful' )
# factorial
result = findInput( n, getNthFactorial, lambda n: power( log10( n ), 0.92 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' factorial number' )
# alternating factorial
result = findInput( n, getNthAlternatingFactorial, lambda n: fmul( sqrt( log( n ) ), 0.72 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' alternating factorial number' )
# double factorial
if n == 1:
result = ( True, 1 )
else:
result = findInput( n, getNthDoubleFactorial, lambda n: fdiv( power( log( log( n ) ), 4 ), 7 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' double factorial number' )
# hyperfactorial
result = findInput( n, getNthHyperfactorial, lambda n: fmul( sqrt( log( n ) ), 0.8 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' hyperfactorial number' )
# subfactorial
result = findInput( n, getNthSubfactorial, lambda n: fmul( log10( n ), 1.1 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' subfactorial number' )
# superfactorial
if n == 1:
result = ( True, 1 )
else:
result = findInput( n, getNthSuperfactorial, lambda n: fadd( sqrt( log( n ) ), 1 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' superfactorial number' )
# pernicious
if isPernicious( n ):
print( indent + 'pernicious' )
# pronic
if isPronic( n ):
print( indent + 'pronic' )
# Achilles
if isAchillesNumber( n ):
print( indent + 'an Achilles number' )
# antiharmonic
if isAntiharmonic( n ):
print( indent + 'antiharmonic' )
# unusual
if isUnusual( n ):
print( indent + 'unusual' )
# hyperperfect
for i in range( 2, 21 ):
if isKHyperperfect( n, i ):
print( indent + str( i ) + '-hyperperfect' )
# Ruth-Aaron
if isRuthAaronNumber( n ):
print( indent + 'a Ruth-Aaron number' )
# Smith numbers
if isSmithNumber( n ):
print( indent + 'a Smith number' )
# base-k Smith numbers
for i in range( 2, 10 ):
if isBaseKSmithNumber( n, i ):
print( indent + 'a base-' + str( i ) + ' Smith number' )
# order-k Smith numbers
for i in range( 2, 11 ):
if isOrderKSmithNumber( n, i ):
print( indent + 'an order-' + str( i ) + ' Smith number' )
# polydivisible
if isPolydivisible( n ):
print( indent + 'polydivisible' )
# Carol numbers
result = findInput( n, getNthCarolNumber, lambda n: fmul( log10( n ), fdiv( 5, 3 ) ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Carol number' )
# Kynea numbers
result = findInput( n, getNthKyneaNumber, lambda n: fmul( log10( n ), fdiv( 5, 3 ) ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Kynea number' )
# Leonardo numbers
result = findInput( n, getNthLeonardoNumber, lambda n: fsub( log( n, phi ), fdiv( 1, phi ) ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Leonardo number' )
# Riesel numbers
result = findInput( n, getNthRieselNumber, lambda n: fmul( log( n ), 1.25 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Riesel number' )
# Thabit numbers
result = findInput( n, getNthThabitNumber, lambda n: fmul( log10( n ), 3.25 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Thabit number' )
# Carmmichael
if isCarmichaelNumber( n ):
print( indent + 'a Carmichael number' )
# narcissistic
if isNarcissistic( n ):
print( indent + 'narcissistic' )
# PDI
if isPerfectDigitalInvariant( n ):
print( indent + 'a perfect digital invariant' )
# PDDI
if isPerfectDigitToDigitInvariant( n, 10 ):
print( indent + 'a perfect digit-to-digit invariant in base 10' )
# Kaprekar
if isKaprekar( n ):
print( indent + 'a Kaprekar number' )
# automorphic
if isAutomorphic( n ):
print( indent + 'automorphic' )
# trimorphic
if isTrimorphic( n ):
print( indent + 'trimorphic' )
# k-morphic
for i in range( 4, 11 ):
if isKMorphic( n, i ):
print( indent + str( i ) + '-morphic' )
# bouncy
if isBouncy( n ):
print( indent + 'bouncy' )
# step number
if isStepNumber( n ):
print( indent + 'a step number' )
# Apery numbers
result = findInput( n, getNthAperyNumber, lambda n: fadd( fdiv( log( n ), 1.5 ), 1 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Apery number' )
# Delannoy numbers
result = findInput( n, getNthDelannoyNumber, lambda n: fmul( log10( n ), 1.35 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Delannoy number' )
# Schroeder numbers
result = findInput( n, getNthSchroederNumber, lambda n: fmul( log10( n ), 1.6 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Schroeder number' )
# Schroeder-Hipparchus numbers
result = findInput( n, getNthSchroederHipparchusNumber, lambda n: fdiv( log10( n ), 1.5 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Schroeder-Hipparchus number' )
# Motzkin numbers
result = findInput( n, getNthMotzkinNumber, lambda n: log( n ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Motzkin number' )
# Pell numbers
result = findInput( n, getNthPellNumber, lambda n: fmul( log( n ), 1.2 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Pell number' )
# Sylvester numbers
if n > 1:
result = findInput( n, getNthSylvesterNumber, lambda n: sqrt( sqrt( log( n ) ) ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' Sylvester number' )
# partition numbers
result = findInput( n, getPartitionNumber, lambda n: power( log( n ), 1.56 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' partition number' )
# menage numbers
result = findInput( n, getNthMenageNumber, lambda n: fdiv( log10( n ), 1.2 ) )
if result[ 0 ]:
print( indent + 'the ' + getShortOrdinalName( result[ 1 ] ) + ' menage number' )
print( )
print( int( n ), 'has:' )
# number of digits
digits = log10( n )
if isInteger( digits ):
digits += 1
else:
digits = ceil( digits )
print( indent + str( int( digits ) ) + ' digit' + ( 's' if digits > 1 else '' ) )
# digit sum
print( indent + 'a digit sum of ' + str( int( sumDigits( n ) ) ) )
# digit product
digitProduct = multiplyDigits( n )
print( indent + 'a digit product of ' + str( int( digitProduct ) ) )
# non-zero digit product
if digitProduct == 0:
print( indent + 'a non-zero digit product of ' + str( int( multiplyNonzeroDigits( n ) ) ) )
if not isPrime and n != 1 and n <= largestNumberToFactor:
# factors
factors = getFactors( n )
factorCount = len( factors )
print( indent + str( factorCount ) + ' prime factor' + ( 's' if factorCount > 1 else '' ) + \
': ' + ', '.join( [ str( int( i ) ) for i in factors ] ) )
# number of divisors
divisorCount = int( getDivisorCount( n ) )
print( indent + str( divisorCount ) + ' divisor' + ( 's' if divisorCount > 1 else '' ) )
if n <= largestNumberToFactor:
print( indent + 'a sum of divisors of ' + str( int( getSigma( n ) ) ) )
print( indent + 'a Stern value of ' + str( int( getNthStern( n ) ) ) )
calkin_wilf = getNthCalkinWilf( n )
print( indent + 'a Calkin-Wilf value of ' + str( int( calkin_wilf[ 0 ] ) ) + '/' + str( int( calkin_wilf[ 1 ] ) ) )
print( indent + 'a Mobius value of ' + str( int( getMobius( n ) ) ) )
print( indent + 'a radical of ' + str( int( getRadical( n ) ) ) )
print( indent + 'a Euler phi value of ' + str( int( getEulerPhi( n ) ) ) )
print( indent + 'a digital root of ' + str( int( getDigitalRoot( n ) ) ) )
if hasUniqueDigits( n ):
print( indent + 'unique digits' )
print( indent + 'a multiplicative persistence of ' + str( int( getPersistence( n ) ) ) )
print( indent + 'an Erdos persistence of ' + str( int( getErdosPersistence( n ) ) ) )
if n > 9 and isIncreasing( n ):
if not isDecreasing( n ):
print( indent + 'increasing digits' )
elif n > 9 and isDecreasing( n ):
print( indent + 'decreasing digits' )
print( )
return n
