def get(self):
url = "https://www.cdfangxie.com/Infor/type/typeid/36.html"
html = requests.get(url, verify=False, headers=getHeader())
bs = BeautifulSoup(html.text, "html.parser")
tables = bs.find(class_='right_cont')
tables_a = tables.find_all("a")
db = PickleShareDB('data')
history_list = db.get("history_list", [])
smtpObj = self.get_smtp()
if not smtpObj:
logger.error("smtp error")
exit(-1)
for row in tables_a:
try:
title = row["title"]
url = "https://www.cdfangxie.com" + row["href"]
if title in history_list:
logger.info("Crawl:%s result:pass", title)
else:
logger.info("Crawl:%s result:send_email", title)
if self.send_email(smtpObj, title, url):
logger.info("send_email success")
history_list.append(title)
else:
logger.info("send_email error")
except:
pass
db["history_list"] = history_list
smtpObj.quit()
def __set_memoize_backend__(self):
""" Defines the backend for memoization of the results """
if self.memoized is False:
self._cache = None
elif self.memoized is True:
self._cache = {}
elif isinstance(self.memoized, str):
self._cache = PickleShareDB(self.memoized)
def registBD(username, password):
db = PickleShareDB('miBD')
if username in db.keys():
return False
else:
session['logged'] = True
session['username'] = username
db[username] = {'pass': password}
return True
def registrar_BD(username, password):
db = PickleShareDB('miBD')
if username in db.keys():
return "Incorrecto"
else:
session['logged'] = True
session['username'] = username
db[username] = {'pass': password}
return "Correcto"
def login_BD(username, password):
db = PickleShareDB('miBD')
if username in db.keys(
) and db[username] and password == db[username]["pass"]:
session['logged'] = True
session['username'] = username
return "Credenciales correctas"
else:
return "Credenciales incorrectas"
def register():
form = RegistrationForm()
if form.validate_on_submit():
db = PickleShareDB('miBD')
try:
if db.keys().index(f'{form.username.data}') != ValueError:
flash('User already exist', 'danger')
else:
raise ValueError
except ValueError:
flash(f'Account created for {form.username.data}!', 'success')
db[f'{form.username.data}'] = { 'pass' : f'{form.password.data}' }
return redirect(url_for('home'))
return render_template('register.html', title='Register', form = form)
def login():
form = LoginForm()
if form.validate_on_submit():
db = PickleShareDB('miBD')
try:
if db.keys().index(f'{form.username.data}') != ValueError and form.password.data == db[f'{form.username.data}']['pass']:
flash(f'Hi {form.username.data}!', 'success')
session['username'] = f'{form.username.data}'
return redirect(url_for('home'))
else:
raise ValueError
except ValueError:
flash('Login Unsuccessful. Please check username and password', 'danger')
return render_template('login.html', title='Login', form = form)
def __init__(self, file_name: str, config: Union[list, dict], path: str = config_path):
object.__init__(self)
self.file_name: str = file_name
self.config: Union[list, dict] = config
self.path: str = path
self.db = PickleShareDB(self.path)
if file_name not in self.db:
self.db[file_name] = config
def __init__(self,
configs=None,
path: str = '~/.tkpy2',
file_name='config',
init=False):
self.db = PickleShareDB(path)
self.file_name = file_name
if file_name not in self.db or init:
self.db[file_name] = configs
def signup_user(username,password):
db = PickleShareDB('DAI_database')
user_not_in_database = username not in db
if user_not_in_database:
# Usuario no existía en la base de datos: se registra
today = date.today()
db[username] = {'signup_date': today.strftime("%d/%m/%Y"),
'password': password}
return user_not_in_database
def test_stress(tmpdir):
db = PickleShareDB(tmpdir)
import time, sys
for i in range(100):
for j in range(500):
if i % 15 == 0 and i < 70:
if str(j) in db:
del db[str(j)]
continue
if j % 33 == 0:
time.sleep(0.02)
db[str(j)] = db.get(str(j), []) + [(i, j, "proc %d" % os.getpid())]
db.hset('hash', j, db.hget('hash', j, 15) + 1)
print(i, end=' ')
sys.stdout.flush()
if i % 10 == 0:
db.uncache()
def test_stress(tmpdir):
db = PickleShareDB(tmpdir)
import time,sys
for i in range(100):
for j in range(500):
if i % 15 == 0 and i < 70:
if str(j) in db:
del db[str(j)]
continue
if j%33 == 0:
time.sleep(0.02)
db[str(j)] = db.get(str(j), []) + [(i,j,"proc %d" % os.getpid())]
db.hset('hash',j, db.hget('hash',j,15) + 1 )
print(i, end=' ')
sys.stdout.flush()
if i % 10 == 0:
db.uncache()
def get_preferred_submodules():
"""
Get all submodules of the main scientific modules and others of our
interest
"""
# Path to the modules database
modules_path = get_conf_path('db')
# Modules database
modules_db = PickleShareDB(modules_path)
if 'submodules' in modules_db:
return modules_db['submodules']
submodules = []
for m in PREFERRED_MODULES:
submods = get_submodules(m)
submodules += submods
modules_db['submodules'] = submodules
return submodules
from flask import Flask, request, redirect
from twilio.twiml.messaging_response import MessagingResponse
from pickleshare import PickleShareDB
db = PickleShareDB('db')
app = Flask(__name__)
@app.route("/sms", methods=['GET', 'POST'])
def sms_reply():
from_number = request.values.get('From')
body = request.values.get("Body")
if body.lower() == "stop":
del db[from_number]
return str(MessagingResponse().message("Unregistered!"))
db[from_number] = None
# Could respond with SMS, but just calling will save money
return str(MessagingResponse().message("Registered!"))
if __name__ == "__main__":
app.run(debug=True)
def read_tkpy_file(file_name: str, path: str = config_path):
db = PickleShareDB(path)
return db[file_name]
from time import time
import os
config = RawConfigParser()
if not config.read("credentials.ini"):
print("Could not find credentials. Exiting.")
exit(1)
account_sid = config.get("twilio", "account_sid")
auth_token = config.get("twilio", "auth_token")
call_number = config.get("twilio", "number")
client = Client(account_sid, auth_token)
db = PickleShareDB('db')
for number, nextcall in db.items():
if nextcall is not None and nextcall > time():
continue
sample = choice([f for f in os.listdir("samples") if f.endswith(".mp3")])
response = VoiceResponse()
response.play("samples/" + sample)
# Make the call
call = client.api.account.calls.create(to=line[0],
from_=call_number,
url=response)
def get_users_hashed_password(username):
db = PickleShareDB('DAI_database')
return db[username]['password']
def test_pickleshare(tmpdir):
db = PickleShareDB(tmpdir)
db.clear()
print("Should be empty:", db.items())
assert len(db) == 0
db['hello'] = 15
assert db['hello'] == 15
db['aku ankka'] = [1, 2, 313]
assert db['aku ankka'] == [1, 2, 313]
db['paths/nest/ok/keyname'] = [1, (5, 46)]
assert db['paths/nest/ok/keyname'] == [1, (5, 46)]
db.hset('hash', 'aku', 12)
db.hset('hash', 'ankka', 313)
assert db.hget('hash', 'aku') == 12
assert db.hget('hash', 'ankka') == 313
print("all hashed", db.hdict('hash'))
print(db.keys())
print(db.keys('paths/nest/ok/k*'))
print(dict(db)) # snapsot of whole db
db.uncache() # frees memory, causes re-reads later
# shorthand for accessing deeply nested files
lnk = db.getlink('myobjects/test')
lnk.foo = 2
lnk.bar = lnk.foo + 5
assert lnk.bar == 7
class Task(object):
""" A Task is a Pipe-like function that also propagates job information
note: a task is pickleable
TODO:
* add memoization / conditional work
This should not run the function if not necessary
"""
def __init__(self, func, args=(), name=None, memoized=False, logger=None, **kwargs):
if type(memoized) not in [ bool, str ]:
raise ValueError('memoized value "{}" not understood. Expecting Boolean or str')
self.memoized = memoized
self.func = func
self.name = name or func.__name__
self.args = args
self.kwargs = kwargs
update_wrapper(self, func)
self.__set_memoize_backend__()
self.set_logger(logger)
def set_logger(self, logger, **kwargs):
""" set the logger backend """
if isinstance(logger, Logger):
self.logger = logger
else:
self.logger = Logger(out=logger, **kwargs)
def memoize(self, val=None):
if val is None:
return self.memoized
else:
self.memoized = val
self.__set_memoize_backend__()
def clear_memoized(self):
if self._cache is not None:
self._cache.clear()
def get_job_status(self, job_id, strict=True):
""" Check if previous results are available
if strict, unkown jobs will raise KeyError exceptions, otherwise
return False
"""
key = '{}/{}'.format(job_id, self.name)
if self.memoized is False:
return False
elif (key in self._cache):
return True
else:
if strict:
raise KeyError('Job "{}" not found'.format(job_id))
else:
return False
def get(self, job_id, strict=True):
""" get the result from a previous memoized job
See also get_job_status
"""
if self.get_job_status(job_id, strict=strict):
key = '{}/{}'.format(job_id, self.name)
return self._cache[key]
def __set_memoize_backend__(self):
""" Defines the backend for memoization of the results """
if self.memoized is False:
self._cache = None
elif self.memoized is True:
self._cache = {}
elif isinstance(self.memoized, str):
self._cache = PickleShareDB(self.memoized)
def run(self, key, val, *args, **kwargs):
if hasattr(val, '__iter__'):
_args = tuple(val) + args + self.args
else:
_args = (val, ) + args + self.args
_kwargs = self.kwargs.copy()
_kwargs.update(kwargs)
if self.memoized is False:
self.logger.write('Job {} -- started\n'.format(key))
ret = self.func(*_args, **_kwargs)
self.logger.write('Job {} -- ended\n'.format(key))
return ret
else:
if (key not in self._cache.keys()):
self.logger.write('Job {} -- started\n'.format(key))
self._cache[key] = self.func(*_args, **_kwargs)
self.logger.write('Job {} -- ended\n'.format(key))
return self._cache[key]
else:
self.logger.write('Job {} -- existing result, skipped\n'.format(key))
return self._cache[key]
def __ror__(self, other):
""" Used for the | operator """
job_id = other[0]
key = '{}/{}'.format(job_id, self.name)
return job_id, self.run(key, other[1])
def __call__(self, *args, **kwargs):
if len(args) > 0:
self.args = args
if len(kwargs) > 0:
self.memoized = kwargs.pop('memoized', self.memoized)
self.logger = kwargs.pop('memoized', self.logger)
self.kwargs = kwargs
return self
def __repr__(self):
txt = 'Task: {}'.format(self.func.__repr__())
if self.memoized:
txt += '\n memoized {}'.format(object.__repr__(self._cache))
if self.logger is not None:
txt += '\n Logger {}'.format(self.logger.__repr__())
return txt
def __getitem__(self, job_id):
return self.get(job_id)
def is_user_registered(username):
db = PickleShareDB('DAI_database')
return username in db
from pickleshare import PickleShareDB
db = PickleShareDB('miBD')
db['users'] = []
def check_user(email, passw):
for user in db['users']:
if (user['email'] == email and user['pass'] == passw):
return True
return False
def check_user_exist(email):
for user in db['users']:
if (user['email'] == email):
return True
return False
def registrar(email, passw):
if not check_user_exist(email):
db['users'].append({'email': email, 'pass': passw})
return True
else:
return False
def test_pickleshare(tmpdir):
db = PickleShareDB(tmpdir)
db.clear()
print("Should be empty:",db.items())
assert len(db) == 0
db['hello'] = 15
assert db['hello'] == 15
db['aku ankka'] = [1,2,313]
assert db['aku ankka'] == [1,2,313]
db['paths/nest/ok/keyname'] = [1,(5,46)]
assert db['paths/nest/ok/keyname'] == [1,(5,46)]
db.hset('hash', 'aku', 12)
db.hset('hash', 'ankka', 313)
assert db.hget('hash', 'aku') == 12
assert db.hget('hash', 'ankka') == 313
print("all hashed",db.hdict('hash'))
print(db.keys())
print(db.keys('paths/nest/ok/k*'))
print(dict(db)) # snapsot of whole db
db.uncache() # frees memory, causes re-reads later
# shorthand for accessing deeply nested files
lnk = db.getlink('myobjects/test')
lnk.foo = 2
lnk.bar = lnk.foo + 5
assert lnk.bar == 7
from flask import Flask, render_template, send_from_directory, Response, request, url_for, redirect, session, make_response
from pickleshare import PickleShareDB
from os import urandom
db = PickleShareDB('./pickleDatabase')
db.clear()
app = Flask(__name__)
app.secret_key = urandom(16)
@app.route('/')
@app.route('/health')
def health():
return 'The server is healthy'
@app.route('/static/<path:subpath>', methods=['GET'])
def send_static_content(subpath):
return send_from_directory('static', subpath, mimetype='*')
@app.route('/app', methods=['GET'])
def serve_app():
logged = 'username' in session
username = session['username'] if logged else ''
return render_app(logged=logged, user_name=username)
@app.route('/<path:path>')
def __init__(self, file_name: str, path: str = '~/.tkpy2'):
super(read_tkpy_file, self).__init__()
self.file_name = file_name
self.path = path
self.db = PickleShareDB(path)
def update_fullname(username, new_fullname):
db = PickleShareDB('DAI_database')
db[username]['fullname'] = new_fullname
db[username] = db[username]
def update_email(username, new_email):
db = PickleShareDB('DAI_database')
db[username]['email'] = new_email
db[username] = db[username]
def get_user_info(username):
db = PickleShareDB('DAI_database')
return db[username].copy()
# Py2app only uses .pyc files for the stdlib when optimize=0,
# so we need to add it as another suffix here
if running_in_mac_app():
suffixes = imp.get_suffixes() + [('.pyc', 'rb', '2')]
else:
suffixes = imp.get_suffixes()
# Regular expression for the python import statement
import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)'
r'(?P<package>[/\\]__init__)?'
r'(?P<suffix>%s)$' %
r'|'.join(re.escape(s[0]) for s in suffixes))
# Modules database
modules_db = PickleShareDB(MODULES_PATH)
#-----------------------------------------------------------------------------
# Utility functions
#-----------------------------------------------------------------------------
def module_list(path):
"""
Return the list containing the names of the modules available in the given
folder.
"""
# sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
if path == '':
path = '.'
# A few local constants to be used in loops below
def misDatosBD(username):
db = PickleShareDB('miBD')
return db[username]
def editar_BD(username, password):
db = PickleShareDB('miBD')
if username in db.keys():
del db[username]
registrar_BD(username, password)
def getBD(username):
db = PickleShareDB('miBD')
return db[username]
