def _time_sim_run(script,iterations=10):
"""
Execute the script in __main__, then time topo.sim.run(iterations).
Uses the timeit module.
"""
print "Running '%s' for %s iterations"%(script,iterations)
execfile(script,__main__.__dict__)
topo.sim.run(1) # ensure compilations etc happen outside timing
# CB: we enable garbage collection
# (http://docs.python.org/lib/module-timeit.html)
return timeit.Timer('topo.sim.run('+`iterations`+')','gc.enable(); import topo').timeit(number=1)
def time_methods(*fns, n=1):
results = []
for fn in fns:
results.append(timeit(fn, number=n))
# go backwards in case maybe the order of fns affects things, like caching
for i in range(len(fns) - 1, 0 - 1, -1):
fn = fns[i]
results[i] += timeit(fn, number=n)
for fn, t in zip(fns, results):
print(type(fn).__name__, t)
def _time_sim_run(script, iterations=10):
"""
Execute the script in __main__, then time topo.sim.run(iterations).
Uses the timeit module.
"""
print "Running '%s' for %s iterations" % (script, iterations)
execfile(script, __main__.__dict__)
topo.sim.run(1) # ensure compilations etc happen outside timing
# CB: we enable garbage collection
# (http://docs.python.org/lib/module-timeit.html)
return timeit.Timer('topo.sim.run(' + ` iterations ` + ')',
'gc.enable(); import topo').timeit(number=1)
def smart_timeit(stmt, setup, hint):
n = hint
while 1:
v = timeit(stmt, setup, n)
if v[0] * n > MIN_TIME:
return v, n
n <<= 1
def smart_timeit(stmt, setup, hint):
n = hint
while 1:
v = timeit(stmt, setup, n)
if v[0]*n > MIN_TIME:
return v, n
n <<= 1
def main():
"""This is the main function of main.py
Example:
python {{cookiecutter.lib_name}}/main.py -m run_mode_1
-c confs/template_conf.yml
-l logs/output.log
"""
# Initializing
args = get_args()
ColorLogger.setup_logger(log_path=args.log, debug=args.debug, clear_log=True)
# Load the configuration
# configuration = Configuration(config_src=args.config_file,
# config_schema_path='yml_schema_strict.json')
configuration = Configuration(config_src=args.config_file)
# Prints
basic_logger.info("Starting in run mode: {0}".format(args.run_mode))
basic_logger.info("Examples:")
fancy_logger.info("You can customize the logger like this")
fancy_logger.info("You can customize each log message differently",
color="green", on_color="on_white", attrs=[])
basic_logger.info("If you want to print complete blank lines use nl(num_lines=<#>):")
basic_logger.nl(num_lines=2)
# Example timeit code block
basic_logger.info("You can use timeit either as a function Wrapper or a ContextManager:")
for i in range(5):
custom_print = f"{i}: " + "Iterating in a 10,000-number-range took {duration:2.5f} seconds."
skip = i in [1, 2, 3]
with timeit(custom_print=custom_print, skip=skip):
for _ in range(10000):
pass
# Example profileit code block
basic_logger.info(
"Lastly, you can use profileit either as a function Wrapper or a ContextManager:")
with profileit():
# DropboxCloudManager
cloud_conf = configuration.get_config('cloud-filemanager')[0]
if cloud_conf['type'] == 'dropbox' and cloud_conf['config']['api_key'] != 'DROPBOX_API_KEY':
dropbox_obj = DropboxCloudManager(config=cloud_conf['config'])
basic_logger.info(f"Base folder contents in dropbox:\n{dropbox_obj.ls().keys()}")
# HighMySQL
cloud_conf = configuration.get_config('high-sql')[0]
if cloud_conf['type'] == 'mysql' and cloud_conf['config']['username'] != 'MYSQL_USERNAME':
mysql_obj = HighMySQL(config=cloud_conf['config'])
basic_logger.info(f"List of tables in DB:\n{mysql_obj.show_tables()}")
# GmailPyEmailSender
mail_conf = configuration.get_config('pyemail-sender')[0]
if cloud_conf['type'] == 'gmail' and mail_conf['config']['api_key'] != 'GMAIL_API_KEY':
basic_logger.info(f"Sending Sample Email to the email address set..")
gmail_obj = GmailEmailer(config=mail_conf['config'])
gmail_obj.send_email(subject='starter',
to=[gmail_obj.email_address],
text='GmailEmailer works!')
basic_logger.info(f"Done!")
def matchby_hamdist(readtable, bcsmulti, bcs10x):
""" hamming distance on barcodes. runs in roughly N time. hamming for 10x barcodes takes a long time. """
#######this matches each item by the hamming distance threshold to the known barcodes for multi and 10x
####make translation table
trantab = make_trantab()
####apply to multiseq barcodes
if bcsmulti != None:
readtable['multi'] = readtable.apply(
lambda row: find_closest(str(row['multi']), bcsmulti, trantab),
axis=1)
print('finished hamming correction for multiseq.', timeit())
def test_iter_performance():
import utils
init_data()
def a():
for v in utils._iter_verses_in_whole_bible():
pass
def b():
for v in utils.data._all_verses:
pass
a()
def c():
pass
_a = timeit(a, number=10)
_b = timeit(b, number=10)
print(_a, _b)
def get_timing1(limit, label, setup_n, template, typename, use_rep_map):
f = open('dat/%s-%s.dat' % (str(limit), label), 'w')
print('#', label, file=f)
print('#', template.replace('\n', '\\n'), file=f)
if setup_n is None:
setup_n = "x = TypeToTest(range(n))"
else:
setup_n = setup_n
ftimeit = open('fig/%s.txt' % label, 'w')
print('<div class="blist_inner">Setup: <code>%s</code><br/>' %
setup_n.replace('\n', '<br/>'),
file=ftimeit)
print('Timed: <code>%s</code></div>' % template.replace('\n', '<br/>'),
file=ftimeit)
ftimeit.close()
for i in reversed(list(range(len(ns)))):
n = ns[i]
key = (limit, label, setup_n, n, template, typename)
print(n, end=' ')
sys.stdout.flush()
setup2 = '\nTypeToTest = %s\nn = %d\n' % (typename, n)
setup3 = setup + '\n' + setup2 + setup_n
stmt = template
if not use_rep_map:
if i < len(ns) - 1:
rep_map[n] = max(rep_map[n], rep_map[ns[i + 1]])
v, rep = smart_timeit(stmt, setup3, rep_map[n])
if rep_map[n] < rep:
rep_map[n] = rep
else:
k = rep_map[n]
if k * values[key] > MAX_TIME:
k = max(MIN_REPS, int(ceil(MAX_TIME / values[key])))
v = timeit(stmt, setup3, k)
values[key] = v[0]
v = [x * 1000 for x in v]
if typename == 'list':
list_values[n] = v[0]
print(n, file=f, end=' ')
for x in v:
print(x, file=f, end=' ')
for x in v:
print(x / list_values[n], file=f, end=' ')
print(file=f)
print()
f.close()
def get_timing1(limit, label, setup_n, template, typename, use_rep_map):
f = open('dat/%s-%s.dat' % (str(limit), label), 'w')
print('#', label, file=f)
print('#', template.replace('\n', '\\n'), file=f)
if setup_n is None:
setup_n = "x = TypeToTest(range(n))"
else:
setup_n = setup_n
ftimeit = open('fig/%s.txt' % label, 'w')
print('<div class="blist_inner">Setup: <code>%s</code><br/>' % setup_n.replace('\n', '<br/>'),
file=ftimeit)
print('Timed: <code>%s</code></div>' % template.replace('\n', '<br/>'),
file=ftimeit)
ftimeit.close()
for i in reversed(list(range(len(ns)))):
n = ns[i]
key = (limit, label, setup_n, n, template, typename)
print(n, end=' ')
sys.stdout.flush()
setup2 = '\nTypeToTest = %s\nn = %d\n' % (typename, n)
setup3 = setup + '\n' + setup2 + setup_n
stmt = template
if not use_rep_map:
if i < len(ns)-1:
rep_map[n] = max(rep_map[n], rep_map[ns[i+1]])
v, rep = smart_timeit(stmt, setup3, rep_map[n])
if rep_map[n] < rep:
rep_map[n] = rep
else:
k = rep_map[n]
if k * values[key] > MAX_TIME:
k = max(MIN_REPS, int(ceil(MAX_TIME / values[key])))
v = timeit(stmt, setup3, k)
values[key] = v[0]
v = [x*1000 for x in v]
if typename == 'list':
list_values[n] = v[0]
print(n, file=f, end=' ')
for x in v:
print(x, file=f, end=' ')
for x in v:
print(x/list_values[n], file=f, end=' ')
print(file=f)
print()
f.close()
import jinja2
import mold
import timeit
from functools import partial
m = mold.Environment(".")
j = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
r1 = m.get_template("extend_example_child.mold.html").render
r2 = m.get_template("extend_example.mold.html").render
r3 = partial(m.get_template("example.mold.html").render, content="hello!", elements=["a"])
r4 = j.get_template("extend_example_child.jinja.html").render
r5 = j.get_template("extend_example.jinja.html").render
r6 = partial(j.get_template("example.jinja.html").render, content="hello", elements=["a"])
timeit = partial(timeit.timeit, number=int(1e7))
print("Sit back and relax...")
print("Mold times:")
print(timeit(r1))
print(timeit(r2))
print(timeit(r3))
print("")
print("Jinja times:")
print(timeit(r4))
print(timeit(r5))
print(timeit(r6))
"""
created by: Sk Rakeeb
created on:06/11/19
created for: assignment1.
"""
from math import *
from timeit import *
from Fibonacci2 import f
print(f(100))
# time taken to run file:
print()
print('time taken to complete programme: ', timeit())
u[0] = un[0] - un[0] * dt/dx * (un[0]-un[-1]) + nu*dt/dx**2*\ (un[2:]-2*un[0]+un[-1]) u[-1] = un[-1] - un[-1] * dt/dx * (un[-1] - un[-2]) + nu*dt/dx**2*\ (un[0]-2*un[-1]+un[-2]) u_analytical = np.asarray([ufunc(n*dt,xi,nu) for xi in x]) line.set_data(x,u) line2.set_data(x,u_analytical) animation.FuncAnimation(fig,burgers,frames=nt,interval=100) # ------- testing speed for array operation -------- time = timeit() u = np.asarray([ufunc(t,x0,nu) for x0 in x]) for n in range(nt): un = u.copy() for i in range(nx-1): u[i] = un[i] - un[i] * dt/dx *(un[i]-un[i-1]) + nu*dt/dx**2*\ (un[i+1]-2*un[i]+u[i-1]) u[-1] = un[-1] - un[-1]*dt/dx*(un[-1]-un[-2]) + nu*dt/dx**2*\ (un[0] - 2*un[-1] + un[-2]) timeit u = np.asarray([ufunc(t,x0,nu) for x0 in x]) for n in range(nt):
#!/usr/bin/python
# encoding:utf-8
import time
import timeit
# control=1
# while control:
# t = time.time()
# print "%s\r" % t,
def func1():
s = 0
for i in range(1000):
s += i
print(s)
# timeit(函数名_字符串，运行环境_字符串，number=运行次数)
t = timeit('func1()', 'from __main__ import func1', number=1000)
print(t)
import timeit
def sum1(n):
final_sum = 0
for i in xrange(n + 1):
final_sum += i
return final_sum
print sum1(10)
print timeit(sum1)
return p # <markdowncell> # Vejamos quanto tempo leva para calcular alguns fatoriais. # Para isso, utilisamos o módulo `timeit`. # <codecell> import timeit # <codecell> # No ipython, temos o %timeit, um "comando mágico": %timeit(fatorial(10)) %timeit(fatorial(100)) %timeit(fatorial(1000)) %timeit(fatorial(10000)) # <codecell> # Senão, podemos usar o comando Python "puro" timeit.timeit(number=100, setup="from __main__ import fatorial", stmt="x = fatorial(1000)") # <markdowncell> # ## Dois códigos, duas performances # <codecell>
import timeit
y = timeit('"-".join(str(n) for n in range(100))', number=10000)
print(y)
from random import randint import timeit #data = [randint(-10, 10) for _ in range(10)] #print (filter(lambda x: x >= 0, data)) print timeit(x for x in range(10))
from random import randint import timeit #data = [randint(-10, 10) for _ in range(10)] #print (filter(lambda x: x >= 0, data)) print timeit(x for x in range(10))
If -n is not given, a suitable number of loops is calculated by trying
successive powers of 10 until the total time is at least 0.2 seconds.
Note: there is a certain baseline overhead associated with executing a
pass statement. It differs between versions. The code here doesn't try
to hide it, but you should be aware of it. The baseline overhead can be
measured by invoking the program without arguments.
Classes:
Timer
Functions:
timeit(string, string) -> float
repeat(string, string) -> list
default_timer() -> float
CLASSES
builtins.object
Timer
class Timer(builtins.object)
| Class for timing execution speed of small code snippets.
|
| The constructor takes a statement to be timed, an additional
| statement used for setup, and a timer function. Both statements
| default to 'pass'; the timer function is platform-dependent (see
| module doc string). If 'globals' is specified, the code will be
| executed within that namespace (as opposed to inside timeit's
__author__ = 'Jacob Griffiths ([email protected])' __version__ = '0.0.1' ############################################################################## # loops vs. list comprehensions: which is faster? ############################################################################## iters = 1000000 import timeit from profileme import my_squares as my_squares_loops from profileme2 import my_squares as my_squares_lc %timeit my_squares_loops(iters) %timeit my_squares_lc(iters) ############################################################################## # loops vs. the join method for strings: which is faster? ############################################################################## mystring = "my string" from profileme import my_join as my_join_join from profileme2 import my_join as my_join %timeit(my_join_join(iters, mystring)) %timeit(my_join(iters, mystring))
# Row two should be close lets say withing 20%
jump *= NUM_ROWS * distance
row2 = max(min(row1 + jump, NUM_ROWS - 1), 0)
cosine(matrix[row1, :], matrix[row2, :])
avg_dist += abs(row1 - row2)
samples -= 1
print >> stderr, 'Avg Dist: {avg_dist}'.format(avg_dist=avg_dist/tot_samples)
if __name__ == '__main__':
"""Test to see if locality of operations in a matrix affects time.
Build a large matrix, time how long it takes to do millions of operations between _close_
elements and far elements.
Usage:
time python -mtimeit -v -n 5 -r 2 -s 'import matrixtest; m = matrixtest.MatrixTest(num_rows=1000000, num_cols=5000, chunk=100)' 'm.local_samples(100000, .001)'
time python -mtimeit -v -n 5 -r 2 -s 'import matrixtest; m = matrixtest.MatrixTest(num_rows=1000000, num_cols=5000, chunk=100)' 'm.local_samples(100000, .95)'
"""
# matrix = np.random.randn(NUM_ROWS, NUM_COLS)
setup = 'import matrixtest\n'
setup += 'm = matrixtest.MatrixTest(num_rows=1000000, num_cols=5000, chunk=100)'
timeit(stmt='local_samples(100000)', setup=setup)
import timeit
def sum1(n):
'''
Take an input of n and return the sum of the numbers from 0 to n
'''
final_sum = 0
for x in range(n + 1):
final_sum += x
return final_sum
def sum2(n):
"""
Take an input of n and return the sum of the numbers from 0 to n
"""
return (n * (n + 1)) / 2
x = timeit(sum1(100))
import timeit
def FN():
a=10
b=20
c=30
print("A:%s" % (a))
print("B:%s" % (b))
print("B:%s" % (c))
vv = timeit('for x in range(100): print(x)', number=100)
print(vv)
from timeit import *
def fib (n):
if n == 1 or n == 2:
return 1
return fib(n-1) + fib(n-2)
timeit(fib, 30)
# timeit(fib, 35)
def timeit(*args):
"Run the timeit.main function with args, catch and parse the output."
import sys
import re
import timeit
import cStringIO
prev_stdout = sys.stdout
sys.stdout = cStringIO.StringIO()
timeit.main(args)
out = sys.stdout.getvalue()
sys.stdout = prev_stdout
# Parse the output, and apply our own formatting
match = re.search(r"(\d+\.\d*|\d+) usec", out)
time = float(match.group(1))
print "%8.2f us: %s" % (time, args[-1])
if __name__ == "__main__":
timeit("object()")
timeit("list()")
timeit("[]")
timeit("int()")
timeit("-s", "rng = range(32)", "[i for i in rng] # a list comp")
timeit("-s", "class ClassicClass: pass",
"ClassicClass() # create a classic class instance")
timeit("-s", "class NewStyleClass(object): pass",
"NewStyleClass() # create a new style class instance")
self.locales = locales
self.reader = geoip2.database.Reader(dbFilename,
locales=self.locales,
mode=MODE_MEMORY)
# @LruCache(maxsize=2**19, timeout = None)
def getCity(self, ipaddr):
return self.reader.city(ip_address=ipaddr)
def main(argv):
maxmindReader = MaxmindReader(
'/space/storage/site/data/maxmind/GeoIP2/GeoIP2-City.mmdb',
locales=['de', 'en'])
print maxmindReader.reader._db_reader._buffer_size
for i in xrange(2**16):
response = maxmindReader.reader.city('128.101.101.101')
print response
def timeit():
import timeit
print(
timeit.timeit("main(None)",
setup="from __main__ import main",
number=1))
if __name__ == '__main__':
timeit()
def fib(n):
list=[]
a=1
b=1
list.append(a)
list.append(b)
temp = 0
if(n<=2):
return 1
while (b+a < n):
temp = b
b = a + b
list.append(b)
a = temp
print(list)
return list
#for i in range(50):
# main()
global tgraph
tgraph = randomGraph(10)
t=timeit('TSP(graph)', setup='from __main__ import tgraph; from __main__ import TSP; graph = tgraph', number=1)
print("TSP time = ", t)
t=timeit('Zgadn(graph)', setup='from __main__ import tgraph; from __main__ import Zgadn; graph = tgraph', number=1)
print("Zgadn time = ", t)
#t=timeit('Gran(graph)', setup='from __main__ import tgraph; from __main__ import Gran; graph = tgraph', number=0)
#print("Gran time = ", t)
def stringCon_efficentcy():
print("{}\n{}".format(
timeit(string_concatenation, number=1000),
timeit(string_concatenation2, number=1000),
))
import timeit
def factors(n):
if n == 2:
return []
factors = []
p = 2
while True:
if n < p:
return factors
if n % p == 0:
factors.append(p)
p = p + 1
print factors(24)
assert factors(24) == [2, 3, 4, 6, 8, 12, 24]
timeit(factors)
lst = range(100)
def f1(lst):
out = []
for item in lst:
out.append(lst[lst.index(item)])
return out
def f2(lst):
return [item for item in lst]
def test():
c=0
for i in range(100):
c=i-c
return c
print(test())
print(__name__)
TimeIt = timeit("test()", setup="from __main__ import test", number=100000)
Repeat = repeat("test()", setup="from __main__ import test", repeat=3, number=100000)
print(TimeIt)
print(Repeat, min(Repeat))
# %timeit f1(lst)
# %timeit f2(lst)
# we want...
# 0 to be int
try:
df.iloc[:,0] = df.iloc[:,0].astype(int)
except ValueError:
# 12 to be float
df.iloc[:,12] = df.iloc[:,12].astype(float)
# 15 to be str
# 16 to be str
# 17 to be str
# 18 to be datetime
print("converting to datetime...")
timeit(pd.to_datetime(df.iloc[:,18]))
"""
df = df.drop('loc',1)
df = df.dropna(subset=['clearDate'])
df = df.dropna(subset=['lon'])
df = df.dropna(subset=['lat'])
df = df.dropna(subset=['timeAtScene'])
# convert the time strings to datetime objects
df['timeAtScene'] = pd.to_datetime(df['timeAtScene'])
df['clearDate'] = pd.to_datetime(df['clearDate'])
w = np.ones(100)/100
scatter(x, y)
x1 = -0.2
x2 = 0.2
def test1():
return rpf.cubic_fit_ladreg(y, x, x1, x2, w)
def test2():
return rpf.cubic_fit_ladreg2(y, x, x1, x2, w)
print(test1())
print(test2())
print(timeit('test1()', setup='from __main__ import test1', number=10))
print(timeit('test2()', setup='from __main__ import test2', number=10))
#
# res2 = rpf.cubic_fit_ladreg(y, x, x1, x2, w)
#
# res = res1
# xd = np.linspace(-0.5, 0.5, 100)
# yd = np.copy(xd)
# for i in range(0, 100):
# if xd[i] < x1:
# yd[i] = (res[0]+res[4])+(res[1]-3*res[4]/x1)*xd[i]+(res[2]+3*res[4]/(x1**2))*(xd[i]**2)+(res[3]-res[4]/(x1**3))*(xd[i]**3)
# elif xd[i] > x2:
# yd[i] = (res[0]+res[5])+(res[1]-3*res[5]/x2)*xd[i]+(res[2]+3*res[5]/(x2**2))*(xd[i]**2)+(res[3]-res[5]/(x2**3))*(xd[i]**3)
# else:
def main():
url = "http://www.sohu.com"
print(timeit(getHtmlContent(url)))
# externally
#setup = """
#class Test(object):
# def method_a(self, arg):
# some_data = self._method_b(arg)
#
# def _method_b(self, arg):
# return arg+1
#
# def method_a2(self, arg):
#
# def method_b2(self, arg):
# return arg+1
#
# some_data = method_b2(self, arg)
#
#obj = Test()
#"""
#from timeit import Timer
#print min( Timer(stmt='obj.method_a(42)', setup=setup).repeat() )
#print min( Timer(stmt='obj.method_a2(42)', setup=setup).repeat() )
def loop_a( n ):
t = []
for i in range(n):
t.append(i**2)
return t
n = 100
timeit( loop_a(n) )
def timeit(*args):
"Run the timeit.main function with args, catch and parse the output."
import sys
import re
import timeit
import cStringIO
prev_stdout = sys.stdout
sys.stdout = cStringIO.StringIO()
timeit.main(args)
out = sys.stdout.getvalue()
sys.stdout = prev_stdout
# Parse the output, and apply our own formatting
match = re.search(r"(\d+\.\d*|\d+) usec", out)
time = float(match.group(1))
print "%8.2f us: %s" % (time, args[-1])
if __name__ == "__main__":
timeit("object()")
timeit("list()")
timeit("[]")
timeit("int()")
timeit("-s", "rng = range(32)",
"[i for i in rng] # a list comp")
timeit("-s", "class ClassicClass: pass",
"ClassicClass() # create a classic class instance")
timeit("-s", "class NewStyleClass(object): pass",
"NewStyleClass() # create a new style class instance")
def __new__(mcs, class_name, bases, class_dict):
for member_name, member in class_dict.iteritems():
if inspect.isroutine(member):
# class_dict[member_name] = timeit(retryable()(member))
class_dict[member_name] = timeit(retryable()(member))
return type.__new__(mcs, class_name, bases, class_dict)
import timeit mysetup = "from Minimum_weights import Minimum_weights" mycode = ''' Minimum_weights(int(input())).solve() ''' timeit(setup=mysetup,code=mycode)
#factorial with recursion
#Name: Sagar Dam (DNAP)
from timeit import *
mysetup = '''n = int(input("Enter the same no. to calculate the time: "))'''
def fact(n):
if n==1:
return 1
else:
return n*fact(n-1)
mycode = '''
def fact(n):
if n==1:
return 1
else:
return n*fact(n-1)'''
n=int(input("Enter the no: "))
x=fact(n)
print("The factorial: ",x)
print(timeit(stmt = mycode, setup = mysetup, number = 10000)/10000)
# After running this code is taking least time for calculating 200!
# author='lwz'
# coding:utf-8
import time
import timeit
'''
timeit(stmt='pass', setup='pass', timer=<built-in function perf_counter>, number=1000000, globals=None)
Convenience function to create Timer object and call timeit method.
stmt = 主程序 主要测量运行时间的程序
number= 重复次数
setup = 运行主程序之前需要执行的语句如 数据，需要import的包
class Timer(builtins.object)
__init__(self, stmt='pass', setup='pass', timer=<built-in function perf_counter>, globals=None)
| Constructor. See class doc string.
| timeit(self, number=1000000)
'''
t1 = timeit.Timer('sum(x)', 'x = (i for i in range(1000))')
print(t1.timeit(5))
t1 = timeit.Timer(stmt='sum(x)', setup='x = (i for i in range(1000))')
print(t1.timeit(5))
def t():
time.sleep(1)
print(timeit.timeit('x=map(lambda x:x*10,range(32))'))
import timeit
# uses setitem
"""
timeit() runs the setup statement one time,
then calls the main statement count times
"""
t = timeit.Timer('print("main statement")', 'print("setup")')
print('TIMEIT:')
print(t.timeit(2))
print('\nREPEAT:')
print(t.repeat(3, 2))
import timeit
import links_extraction as le
import scrap_and_save as s_s
from preprocessing import preprocessing
from analyse import analyse
"""Order of Execution will be,
1. links_extraction
2. feature_extraction
3. scrap_data
5. if anything left, we can include it in scraper """
def main():
list_of_1100_links = le.get_all_1000_links(
'https://www.goodreads.com/list/show/6.Best_Books_of_the_20th_Century?page='
)
df_all = s_s.create_df_and_save_as_csv(list_of_1100_links)
#### other version
#df_all = s_s.create_df_and_save_as_csv(le.get_all_1000_links('https://www.goodreads.com/list/show/6.Best_Books_of_the_20th_Century?page='))
df_cleaned = preprocessing(df_all)
analyse(df_cleaned, df_all)
if __name__ == '__main__':
timeit(main())
