def df_to_graph(fname: str,verbose=True,weighted=True):
"""
:fname path/to/dataframe
:returns nx graph
"""
if weighted:
df = pd.read_csv(fname,sep='\t').take([0,1,2],axis=1)
else:
df = pd.read_csv(fname,sep='\t').take([0,1],axis=1)
#df = pd.read_csv(fname,sep='\t',skiprows=0,names=['head','tail','weight']).take([0,1,2],axis=1)
#df = df.drop(0)
ff = df
edges = [tuple(x) for x in df.values]
#print(edges)
vertices = list(df.stack())
GR = nx.DiGraph()
for v in vertices:
GR.add_node(v)
for e in edges:
if weighted:
u,v,w = e
GR.add_edge(u,v,weight=float(w),cost=-math_log(max([0.000000001, w]))/math_log(10))
else:
u,v = e
GR.add_edge(u,v)
if verbose:
print('length of edge set: {}'.format(len(set(edges))))
print('number of edges in GR: {}'.format(len(list(GR.edges))))
return GR
def test_log():
log_five = modifiers.log(5)
eq_(solve(log_five), math_log(5))
eq_(solve(pickle.loads(pickle.dumps(log_five))), math_log(5))
eq_(str(log_five), "<log(5)>")
def test_log():
log_five = modifiers.log(5)
eq_(solve(log_five), math_log(5))
eq_(solve(pickle.loads(pickle.dumps(log_five))), math_log(5))
eq_(str(log_five), "<log(5)>")
def test_log():
log_five = modifiers.log(5)
assert solve(log_five) == math_log(5)
assert solve(pickle.loads(pickle.dumps(log_five))) == math_log(5)
assert repr(log_five) == "<feature.log(5)>"
def augment_graph(G, sources, targets, verbose):
if verbose:
print(' augmenting graph...')
## initialize edge variables with capacities, flow_var name, and flow.
for u, v in G.edges():
G[u][v]['cap'] = 1.0 ## edge capacity
G[u][v]['flow_var'] = DELIM.join(['f', u,
v]) ## flow variable name for ILP
G[u][v]['flow'] = None
## (1) Add super source and super sink
G.add_node(SUPERSOURCE)
G.add_node(SUPERSINK)
## (2) Add dir. edges from supersource to receptors and tfs to supersink
## (2a) Add capacities to edges from supersource to receptors
## NOTE: capacities are uniform for both of these types of edges.
## The original paper changed this to the strength of
## each genetic hit. PathLinker used uniform capcities.
for s in sources:
G.add_edge(SUPERSOURCE,
s,
weight=1 / len(sources),
cap=1 / len(sources),
flow_var=DELIM.join(['f', SUPERSOURCE, s]),
flow=None)
G[SUPERSOURCE][s]['cost'] = -math_log(
max([0.000000001, G[SUPERSOURCE][s]['weight']])) / math_log(10)
## (2b) Add capacities to eges from tfs to supersink
for t in targets:
G.add_edge(t,
SUPERSINK,
weight=1 / len(targets),
cap=1 / len(targets),
flow_var=DELIM.join(['f', t, SUPERSINK]),
flow=None)
G[t][SUPERSINK]['cost'] = -math_log(
max([0.000000001, G[t][SUPERSINK]['weight']])) / math_log(10)
orderedvariables = [] # sets the order of flow variables
i = 0
for u, v in G.edges():
G[u][v]['index'] = i
orderedvariables.append(G[u][v]['flow_var'])
i += 1
if verbose:
print('augmented graph has %d nodes and %d edges' %
(G.number_of_nodes(), G.number_of_edges()))
return orderedvariables
def log(v, base=None):
try:
if v == None:
return Null
if v == 0.0:
return -float("inf")
if base == None:
return math_log(v)
return math_log(v, base)
except Exception as e:
from mo_logs import Log
Log.error("error in log", cause=e)
def __pow__(self, other):
if other.units != unity:
raise_QuantError("the exponent can't have units", "%s ^ %s", (self, other))
if self.units == unity:
units = self.units
else:
if not hasattr(other.number, 'denominator'): # not int or fraction
raise_QuantError("can't raise units to irrational exponent", "%s ^ %s", (self, other))
units = tuple([fraction_or_int(u * other.number) for u in self.units])
try:
if hasattr(self.number, 'denominator') and hasattr(other.number, 'denominator') and other.number > 10:
number = self.number ** float(other.number)
else:
number = self.number ** other.number
except ValueError:
raise_QuantError("arithmetic problem, e.g. complex numbers not implemented", "%s ^ %s", (self, other))
except OverflowError:
raise_QuantError("overflow: value too high", "%s ^ %s", (self, other))
# try:
# number = self.number ** mpmath.mpf(other.number)
# except OverflowError:
# raise_QuantError("overflow: value too high", "%s ^ %s", (self, other))
if number.imag:
raise_QuantError("complex numbers not implemented", "%s ^ %s", (self, other))
uncert1 = abs(self.uncert / self.number * number * other.number)
if hasattr(self.number, 'denominator'):
argument = float(self.number)
else:
argument = self.number
uncert2 = abs(other.uncert * math_log(abs(argument)) * number)
uncert = uncert1 + uncert2
return Q(number, '%s ^ %s', units, uncert, self.prefu, (self, other))
def _build_positional_encoding_sublayer(self,
inputs,
seq_len,
scope="PositionalEncoding"):
"""Add positional encoding to the inputs.
Based on the paper "Attention is all you need" by Vaswani et al.
(https://arxiv.org/pdf/1706.03762.pdf).
Code adapted from Google's Tensor2Tensor repo on GitHub.
"""
with tf.variable_scope(scope):
vec_size = inputs.get_shape().as_list()[-1]
positions = tf.expand_dims(tf.to_float(tf.range(seq_len)),
1) # shape (seq_len, 1)
d_model = vec_size // 2
exponent_step = math_log(10000.) / (tf.to_float(d_model) - 1)
pos_multiplier = tf.exp(
tf.to_float(tf.range(d_model)) * -exponent_step)
pos_encoded = positions * tf.expand_dims(
pos_multiplier, 0) # shape (seq_len, d_model)
pos_encoded = tf.concat(
[tf.sin(pos_encoded), tf.cos(pos_encoded)], axis=1)
pos_encoded = tf.pad(pos_encoded,
[[0, 0], [0, tf.mod(vec_size, 2)]])
pos_encoded = tf.reshape(pos_encoded, [1, seq_len, vec_size])
outputs = inputs + pos_encoded
outputs = tf.nn.dropout(outputs, self.keep_prob)
return outputs
def ln_model_prob(self, Xf):
"""Returns the ln of the model probability, which is the
variational bound / K! to account for symmetries in the classifier
matching function.
"""
# we need to use the log function from the math module,
# as the one from the numpy module cannot handle numbers of type 'long'
return self.var_bound(Xf) - math_log(fact(len(self.cls)))
def append_inflection_report(db, row, years, index_roe, total_roe,
total_finish):
row_base_roe = db.get_index_history_minus_years(database2.CONST_INDEX_ROE,
years)
cagr = ((index_roe / row_base_roe.value)**Decimal(1 / years)) - 1
inflect = inflect_years = four_pct_years = five_m_years = float("NaN")
if cagr > 0:
inflect = total_roe * cagr
inflect_years = math_log(CONST_ONE_UNIT / (total_roe * cagr), cagr + 1)
four_pct_years = math_log(total_finish / total_roe, cagr + 1)
five_m_years = math_log(CONST_FIVE_MILLION / total_roe, cagr + 1)
row.append(cagr)
row.append(inflect)
row.append(inflect_years * -1)
row.append(five_m_years * -1)
row.append(four_pct_years * -1)
row.append(row_base_roe.date)
def log(x, b=None):
"""The logarithm function.
Works for both numeric and symbolic (`SymPy.Expr`) inputs.
Default base `b=None` means `e`, i.e. take the natural logarithm.
"""
if isinstance(x, _symExpr):
# https://stackoverflow.com/questions/46129259/how-to-simplify-logarithm-of-exponent-in-sympy
if b is not None:
return _symlog(x, b).expand(force=True)
else:
return _symlog(x).expand(force=True)
if b is not None:
return math_log(x, b)
else:
return math_log(x)
def calc_entropy(byte_dict):
entropy = 0
total = sum(byte_dict.values())
for key in byte_dict:
if byte_dict[key] != 0:
freq = byte_dict[key] / total
entropy = entropy + freq * math_log(freq, 2)
entropy *= -1
return entropy
def get_string_size(bytes):
units = [' b','Kb','Mb','Gb','Tb']
if not bytes:
return '0 b'
exponent = int(math_log(bytes, 1024))
if exponent > 4:
return '%d b' % bytes
value = bytes / 1024.0 ** exponent, units[exponent]
return '%6.2f %s' % value
def count_omx_vol(vol_percent):
try:
float_percent = float(vol_percent) / 100
except Exception as e:
float_percent = 0.01
try:
return int(2000 * (math_log(float_percent, 10)))
except Exception as e:
return -4000
def _safelog(item):
"""Takes the logarithm of `item`. Instead of raising exceptions
like `math.log` does, it returns ``-inf`` for zero and ``nan``
for negative numbers.
"""
if item < 0:
return float("nan")
if item == 0:
return float("-inf")
return math_log(item)
def _safelog(item):
"""Takes the logarithm of `item`. Instead of raising exceptions
like `math.log` does, it returns ``-inf`` for zero and ``nan``
for negative numbers.
"""
if item < 0:
return float('nan')
if item == 0:
return float('-inf')
return math_log(item)
def sizeof_fmt(num):
"""Human friendly sizes"""
if num > 1:
exponent = min(int(math_log(num, 1024)), len(unit_list) - 1)
quotient = float(num) / 1024**exponent
unit, num_decimals = unit_list[exponent]
format_string = '{:.%sf} {}' % (num_decimals)
return format_string.format(quotient, unit)
if num == 0:
return '0 bytes'
if num == 1:
return '1 byte'
def sizeof_fmt(num):
"""Human friendly sizes"""
if num > 1:
exponent = min(int(math_log(num, 1024)), len(unit_list) - 1)
quotient = float(num) / 1024**exponent
unit, num_decimals = unit_list[exponent]
format_string = '{:.%sf} {}' % (num_decimals)
return format_string.format(quotient, unit)
if num == 0:
return '0 bytes'
if num == 1:
return '1 byte'
def _set_target_values(self, feature_map_width, feature_map_height,
image_index, t, target, anchor_index, iou,
ground_truth_center_x_pixel,
ground_truth_center_y_pixel, ground_truth_height,
ground_truth_width, target_center_x_values,
target_center_y_values, target_class_values,
target_confidence_score_values,
target_height_values, target_width_values):
image = target[image_index]
target_center_x_values[image_index][anchor_index][
ground_truth_center_y_pixel][ground_truth_center_x_pixel] = image[
t * 5 + 1] * feature_map_width - ground_truth_center_x_pixel
target_center_y_values[image_index][anchor_index][
ground_truth_center_y_pixel][ground_truth_center_x_pixel] = image[
t * 5 + 2] * feature_map_height - ground_truth_center_y_pixel
target_width_values[image_index][anchor_index][
ground_truth_center_y_pixel][
ground_truth_center_x_pixel] = math_log(
ground_truth_width /
self.anchors[self.anchor_step * anchor_index])
target_height_values[image_index][anchor_index][
ground_truth_center_y_pixel][
ground_truth_center_x_pixel] = math_log(
ground_truth_height /
self.anchors[self.anchor_step * anchor_index + 1])
target_confidence_score_values[image_index][anchor_index][
ground_truth_center_y_pixel][ground_truth_center_x_pixel] = iou
target_class_values[image_index][anchor_index][
ground_truth_center_y_pixel][ground_truth_center_x_pixel] = image[
t * 5]
return target_center_x_values, target_center_y_values, target_width_values, target_height_values, \
target_confidence_score_values, target_class_values
def get_AA_score(networks, target_A, target_B, frequency_dict):
AA_score = 0
A_neighbors = list()
B_neighbors = list()
for edge in networks:
if edge[0] == target_A:
A_neighbors.append(edge[1])
if edge[1] == target_A:
A_neighbors.append(edge[0])
if edge[0] == target_B:
B_neighbors.append(edge[1])
if edge[1] == target_B:
B_neighbors.append(edge[0])
for neighbor in A_neighbors:
if neighbor in B_neighbors:
if frequency_dict[neighbor] > 1:
AA_score += 1 / (math_log(frequency_dict[neighbor]))
return AA_score
def run(G, sources, targets, alpha=0.85, thres=0.80, verbose=False):
print('backward flux calculation...')
personalization = {t:1/len(targets) for t in targets}
G_rev = G.reverse(copy=True)
backward_pagerank = nx.pagerank(G_rev,alpha=alpha,personalization=personalization,weight='weight')
## BACKWARD: We compute BACKWARDS flux score for edge (u, v) by multiplying the visitation probability
## of v by the edge weight and normalizing by the weighted IN degree of v.
for v in backward_pagerank:
denom = len(list(G.predecessors(v)))
for u in G.predecessors(v):
G[u][v]['backward_flux'] = backward_pagerank[v]*G[u][v]['weight']/denom
## combine flux
## take negative logs on the flux scores
print('combining fluxes...')
tot_flux = 0
for u,v in G.edges():
if G[u][v]['backward_flux'] == 0:
G[u][v]['neglog_backward_flux'] = sys.float_info.max ## maximum float value
else:
G[u][v]['neglog_backward_flux'] = - (math_log(G[u][v]['backward_flux']))
tot_flux+=G[u][v]['backward_flux']
print('sorting and taking predictions that comprise %f of total flux' % (thres))
edges = {} #dictionary of (u,v): -log(flux*flux)
running_sum = 0
counter = 0
## iterate through edges sorted by -log(flux*flux)
for u,v,d in sorted(G.edges(data=True), key=lambda t: t[2]['neglog_backward_flux']):
edges[(u,v)] = G[u][v]['neglog_backward_flux']
running_sum+=G[u][v]['backward_flux']
counter+=1
if counter % 1000 == 0:
print('%d of %d: %d of %d (%f)' % (counter,len(G.edges()),running_sum,tot_flux,running_sum/tot_flux))
if running_sum/tot_flux > thres:
print('theshold of %f limits predictions to %d edges' %(thres,len(edges)))
break
return edges
def __pow__(self, other):
if other.units != unity:
raise_QuantError("the exponent can't have units", "%s ^ %s", (self, other))
if self.units == unity:
units = self.units
else:
if not hasattr(other.number, 'denominator'): # not int or fraction
raise_QuantError("can't raise units to irrational exponent", "%s ^ %s", (self, other))
units = tuple([fraction_or_int(u * other.number) for u in self.units])
try:
if hasattr(self.number, 'denominator') and hasattr(other.number, 'denominator') and other.number > 10:
number = self.number ** float(other.number)
else:
number = self.number ** other.number
except ValueError:
raise_QuantError("arithmetic problem, e.g. complex numbers not implemented", "%s ^ %s", (self, other))
except OverflowError:
raise_QuantError("overflow: value too high", "%s ^ %s", (self, other))
if number.imag:
raise_QuantError("complex numbers not implemented", "%s ^ %s", (self, other))
uncert = abs(self.uncert / self.number * number * other.number) + abs(
other.uncert * math_log(abs(self.number)) * number)
return Q(number, '%s ^ %s', units, uncert, self.prefu, (self, other))
def nearestpoweroftwo(num):
"""Helper function to find the next higher power of 2 to a given number.
Such that the power of two is the smallest power of two larger than the
supplied number.
Helps find correct buffer sizes in bytes.
Parameters
----------
num : {int}
The number of bytes for which you wish to find the smallest power of 2
larger than.
2 -> 4
4 -> 8
7 -> 8
Returns
-------
int
The integer value of the nearest power of two larger than num.
"""
return pow(2, int(math_log(num, 2) + 1))
def best_tick(largest, most_ticks):
"""Generate a 'nice' interval for graphs given a max value and most number of intervals
Arguments:
largest: - A number which represents the greatest amount that needs to be displayed.
most_ticks: - A integer which represents the most number of intervals that
should be displayed.
Returns:
A number: that is the best unit of increment for the input data.
"""
minimum = largest / most_ticks#minimum increment
magnitude = 10 ** math_floor(math_log(minimum, 10))
residual = minimum / magnitude#most significant digit of the minimum increment
if residual > 5:
tick = 10 * magnitude
elif residual > 2:
tick = 5 * magnitude
elif residual > 1:
tick = 2 * magnitude
else:
tick = magnitude
return tick
def _row_sax_word_array(self, ntss_tmp, bigger_cardinality, size_word):
"""
Convert all sequences according to the different cardinality of the tree.
For each cardinality, uses :func:`~pyCFOFiSAX.isax.IndexableSymbolicAggregateApproximation.transform_sax`.
:param ntss_tmp: The sequences to be analyzed
:param int bigger_cardinality: The greatest cardinality *i*\ SAX of the tree
:param int size_word: The size of the SAX sequences of the tree
:returns: SAX words from all ``ntss_tmp`` sequences according to all the cardinalities of the tree, a dict returning the cardinality index *i*\ SAX
:rtype: numpy.ndarray, dict
"""
# TODO integrate tree.base_cardinality in the calculation
# si tree.base_cardinality = 3 par exemple...
number_of_card = int(math_log(bigger_cardinality, 2))
row_sax_word = np_zeros(
(number_of_card + 1, len(ntss_tmp), size_word, 1))
# TODO card_current = tree.base_cardinality 1/2
card_current = 1
card_index = 0
card_to_index = dict()
""" TODO np.vectorize
>>> vtransform_sax = np.vectorize(tree.isax.transform_sax)
>>> card_list = 2**np.arange(1,int(np.sqrt(tree.bigger_current_cardinality))-1)
>>> vtransform_sax(ntss_tmp, card_list)"""
while card_current <= bigger_cardinality:
card_to_index[card_current] = card_index
row_sax_word[card_index] = self.transform_sax(
ntss_tmp, card_current)
card_current = card_current * 2
card_index += 1
row_sax_word = row_sax_word.transpose((2, 1, 0, 3))
return row_sax_word.reshape(row_sax_word.shape[:-1]), card_to_index
def _cur_size_limit(self):
len_self, size_factor = len(self), self._size_factor
return int(round(size_factor * math_log(len_self + 2, 2)))
def ln_func(x):
return max(0, float(math_log(x + .1) + 2) / 2.1)
def log(x):
if x < 1e-10:
return 0.0
else:
return math_log(x)
def _process(self, feature_value):
return math_log(feature_value)
def _process(self, feature_value):
return math_log(feature_value)
def calculate_gamma(from_value, to_value=None):
return math_log(
from_value
) / LongExposureStandard.constants.mid_log if to_value is None else math_log(
from_value) / math_log(to_value)
def qln(a):
return Q(math_log(a.number), "ln(%s)", a.units, abs(a.uncert / a.number), a.prefu, (a,))
def qln(a):
return Q(math_log(a.number), "ln(%s)", a.units, abs(a.uncert / a.number), a.prefu, (a,))
def test_prim_log(x):
return math_log(x)
def log(x):
if x == 0:
return -100
else:
return math_log(x)
def log(x):
""" Returns the natural log of x, -inf if x is >= 0 """
return math_log(float(x)) if x > 0.0 else float("-inf")
def get_digits(n):
return int(ceil(math_log(n + 1, 10))) + 1
def log10(v):
try:
return math_log(v, 10)
except Exception as e:
return Null
def ln_func(x):
return max(0, float(math_log(x+.1) + 2)/2.1)
