def get(self): url = "https://www.cdfangxie.com/Infor/type/typeid/36.html" html = requests.get(url, verify=False, headers=getHeader()) bs = BeautifulSoup(html.text, "html.parser") tables = bs.find(class_='right_cont') tables_a = tables.find_all("a") db = PickleShareDB('data') history_list = db.get("history_list", []) smtpObj = self.get_smtp() if not smtpObj: logger.error("smtp error") exit(-1) for row in tables_a: try: title = row["title"] url = "https://www.cdfangxie.com" + row["href"] if title in history_list: logger.info("Crawl:%s result:pass", title) else: logger.info("Crawl:%s result:send_email", title) if self.send_email(smtpObj, title, url): logger.info("send_email success") history_list.append(title) else: logger.info("send_email error") except: pass db["history_list"] = history_list smtpObj.quit()
def __set_memoize_backend__(self): """ Defines the backend for memoization of the results """ if self.memoized is False: self._cache = None elif self.memoized is True: self._cache = {} elif isinstance(self.memoized, str): self._cache = PickleShareDB(self.memoized)
def registBD(username, password): db = PickleShareDB('miBD') if username in db.keys(): return False else: session['logged'] = True session['username'] = username db[username] = {'pass': password} return True
def registrar_BD(username, password): db = PickleShareDB('miBD') if username in db.keys(): return "Incorrecto" else: session['logged'] = True session['username'] = username db[username] = {'pass': password} return "Correcto"
def login_BD(username, password): db = PickleShareDB('miBD') if username in db.keys( ) and db[username] and password == db[username]["pass"]: session['logged'] = True session['username'] = username return "Credenciales correctas" else: return "Credenciales incorrectas"
def register(): form = RegistrationForm() if form.validate_on_submit(): db = PickleShareDB('miBD') try: if db.keys().index(f'{form.username.data}') != ValueError: flash('User already exist', 'danger') else: raise ValueError except ValueError: flash(f'Account created for {form.username.data}!', 'success') db[f'{form.username.data}'] = { 'pass' : f'{form.password.data}' } return redirect(url_for('home')) return render_template('register.html', title='Register', form = form)
def login(): form = LoginForm() if form.validate_on_submit(): db = PickleShareDB('miBD') try: if db.keys().index(f'{form.username.data}') != ValueError and form.password.data == db[f'{form.username.data}']['pass']: flash(f'Hi {form.username.data}!', 'success') session['username'] = f'{form.username.data}' return redirect(url_for('home')) else: raise ValueError except ValueError: flash('Login Unsuccessful. Please check username and password', 'danger') return render_template('login.html', title='Login', form = form)
def __init__(self, file_name: str, config: Union[list, dict], path: str = config_path): object.__init__(self) self.file_name: str = file_name self.config: Union[list, dict] = config self.path: str = path self.db = PickleShareDB(self.path) if file_name not in self.db: self.db[file_name] = config
def __init__(self, configs=None, path: str = '~/.tkpy2', file_name='config', init=False): self.db = PickleShareDB(path) self.file_name = file_name if file_name not in self.db or init: self.db[file_name] = configs
def signup_user(username,password): db = PickleShareDB('DAI_database') user_not_in_database = username not in db if user_not_in_database: # Usuario no existía en la base de datos: se registra today = date.today() db[username] = {'signup_date': today.strftime("%d/%m/%Y"), 'password': password} return user_not_in_database
def test_stress(tmpdir): db = PickleShareDB(tmpdir) import time, sys for i in range(100): for j in range(500): if i % 15 == 0 and i < 70: if str(j) in db: del db[str(j)] continue if j % 33 == 0: time.sleep(0.02) db[str(j)] = db.get(str(j), []) + [(i, j, "proc %d" % os.getpid())] db.hset('hash', j, db.hget('hash', j, 15) + 1) print(i, end=' ') sys.stdout.flush() if i % 10 == 0: db.uncache()
def test_stress(tmpdir): db = PickleShareDB(tmpdir) import time,sys for i in range(100): for j in range(500): if i % 15 == 0 and i < 70: if str(j) in db: del db[str(j)] continue if j%33 == 0: time.sleep(0.02) db[str(j)] = db.get(str(j), []) + [(i,j,"proc %d" % os.getpid())] db.hset('hash',j, db.hget('hash',j,15) + 1 ) print(i, end=' ') sys.stdout.flush() if i % 10 == 0: db.uncache()
def get_preferred_submodules(): """ Get all submodules of the main scientific modules and others of our interest """ # Path to the modules database modules_path = get_conf_path('db') # Modules database modules_db = PickleShareDB(modules_path) if 'submodules' in modules_db: return modules_db['submodules'] submodules = [] for m in PREFERRED_MODULES: submods = get_submodules(m) submodules += submods modules_db['submodules'] = submodules return submodules
from flask import Flask, request, redirect from twilio.twiml.messaging_response import MessagingResponse from pickleshare import PickleShareDB db = PickleShareDB('db') app = Flask(__name__) @app.route("/sms", methods=['GET', 'POST']) def sms_reply(): from_number = request.values.get('From') body = request.values.get("Body") if body.lower() == "stop": del db[from_number] return str(MessagingResponse().message("Unregistered!")) db[from_number] = None # Could respond with SMS, but just calling will save money return str(MessagingResponse().message("Registered!")) if __name__ == "__main__": app.run(debug=True)
def read_tkpy_file(file_name: str, path: str = config_path): db = PickleShareDB(path) return db[file_name]
from time import time import os config = RawConfigParser() if not config.read("credentials.ini"): print("Could not find credentials. Exiting.") exit(1) account_sid = config.get("twilio", "account_sid") auth_token = config.get("twilio", "auth_token") call_number = config.get("twilio", "number") client = Client(account_sid, auth_token) db = PickleShareDB('db') for number, nextcall in db.items(): if nextcall is not None and nextcall > time(): continue sample = choice([f for f in os.listdir("samples") if f.endswith(".mp3")]) response = VoiceResponse() response.play("samples/" + sample) # Make the call call = client.api.account.calls.create(to=line[0], from_=call_number, url=response)
def get_users_hashed_password(username): db = PickleShareDB('DAI_database') return db[username]['password']
def test_pickleshare(tmpdir): db = PickleShareDB(tmpdir) db.clear() print("Should be empty:", db.items()) assert len(db) == 0 db['hello'] = 15 assert db['hello'] == 15 db['aku ankka'] = [1, 2, 313] assert db['aku ankka'] == [1, 2, 313] db['paths/nest/ok/keyname'] = [1, (5, 46)] assert db['paths/nest/ok/keyname'] == [1, (5, 46)] db.hset('hash', 'aku', 12) db.hset('hash', 'ankka', 313) assert db.hget('hash', 'aku') == 12 assert db.hget('hash', 'ankka') == 313 print("all hashed", db.hdict('hash')) print(db.keys()) print(db.keys('paths/nest/ok/k*')) print(dict(db)) # snapsot of whole db db.uncache() # frees memory, causes re-reads later # shorthand for accessing deeply nested files lnk = db.getlink('myobjects/test') lnk.foo = 2 lnk.bar = lnk.foo + 5 assert lnk.bar == 7
class Task(object): """ A Task is a Pipe-like function that also propagates job information note: a task is pickleable TODO: * add memoization / conditional work This should not run the function if not necessary """ def __init__(self, func, args=(), name=None, memoized=False, logger=None, **kwargs): if type(memoized) not in [ bool, str ]: raise ValueError('memoized value "{}" not understood. Expecting Boolean or str') self.memoized = memoized self.func = func self.name = name or func.__name__ self.args = args self.kwargs = kwargs update_wrapper(self, func) self.__set_memoize_backend__() self.set_logger(logger) def set_logger(self, logger, **kwargs): """ set the logger backend """ if isinstance(logger, Logger): self.logger = logger else: self.logger = Logger(out=logger, **kwargs) def memoize(self, val=None): if val is None: return self.memoized else: self.memoized = val self.__set_memoize_backend__() def clear_memoized(self): if self._cache is not None: self._cache.clear() def get_job_status(self, job_id, strict=True): """ Check if previous results are available if strict, unkown jobs will raise KeyError exceptions, otherwise return False """ key = '{}/{}'.format(job_id, self.name) if self.memoized is False: return False elif (key in self._cache): return True else: if strict: raise KeyError('Job "{}" not found'.format(job_id)) else: return False def get(self, job_id, strict=True): """ get the result from a previous memoized job See also get_job_status """ if self.get_job_status(job_id, strict=strict): key = '{}/{}'.format(job_id, self.name) return self._cache[key] def __set_memoize_backend__(self): """ Defines the backend for memoization of the results """ if self.memoized is False: self._cache = None elif self.memoized is True: self._cache = {} elif isinstance(self.memoized, str): self._cache = PickleShareDB(self.memoized) def run(self, key, val, *args, **kwargs): if hasattr(val, '__iter__'): _args = tuple(val) + args + self.args else: _args = (val, ) + args + self.args _kwargs = self.kwargs.copy() _kwargs.update(kwargs) if self.memoized is False: self.logger.write('Job {} -- started\n'.format(key)) ret = self.func(*_args, **_kwargs) self.logger.write('Job {} -- ended\n'.format(key)) return ret else: if (key not in self._cache.keys()): self.logger.write('Job {} -- started\n'.format(key)) self._cache[key] = self.func(*_args, **_kwargs) self.logger.write('Job {} -- ended\n'.format(key)) return self._cache[key] else: self.logger.write('Job {} -- existing result, skipped\n'.format(key)) return self._cache[key] def __ror__(self, other): """ Used for the | operator """ job_id = other[0] key = '{}/{}'.format(job_id, self.name) return job_id, self.run(key, other[1]) def __call__(self, *args, **kwargs): if len(args) > 0: self.args = args if len(kwargs) > 0: self.memoized = kwargs.pop('memoized', self.memoized) self.logger = kwargs.pop('memoized', self.logger) self.kwargs = kwargs return self def __repr__(self): txt = 'Task: {}'.format(self.func.__repr__()) if self.memoized: txt += '\n memoized {}'.format(object.__repr__(self._cache)) if self.logger is not None: txt += '\n Logger {}'.format(self.logger.__repr__()) return txt def __getitem__(self, job_id): return self.get(job_id)
def is_user_registered(username): db = PickleShareDB('DAI_database') return username in db
from pickleshare import PickleShareDB db = PickleShareDB('miBD') db['users'] = [] def check_user(email, passw): for user in db['users']: if (user['email'] == email and user['pass'] == passw): return True return False def check_user_exist(email): for user in db['users']: if (user['email'] == email): return True return False def registrar(email, passw): if not check_user_exist(email): db['users'].append({'email': email, 'pass': passw}) return True else: return False
def test_pickleshare(tmpdir): db = PickleShareDB(tmpdir) db.clear() print("Should be empty:",db.items()) assert len(db) == 0 db['hello'] = 15 assert db['hello'] == 15 db['aku ankka'] = [1,2,313] assert db['aku ankka'] == [1,2,313] db['paths/nest/ok/keyname'] = [1,(5,46)] assert db['paths/nest/ok/keyname'] == [1,(5,46)] db.hset('hash', 'aku', 12) db.hset('hash', 'ankka', 313) assert db.hget('hash', 'aku') == 12 assert db.hget('hash', 'ankka') == 313 print("all hashed",db.hdict('hash')) print(db.keys()) print(db.keys('paths/nest/ok/k*')) print(dict(db)) # snapsot of whole db db.uncache() # frees memory, causes re-reads later # shorthand for accessing deeply nested files lnk = db.getlink('myobjects/test') lnk.foo = 2 lnk.bar = lnk.foo + 5 assert lnk.bar == 7
from flask import Flask, render_template, send_from_directory, Response, request, url_for, redirect, session, make_response from pickleshare import PickleShareDB from os import urandom db = PickleShareDB('./pickleDatabase') db.clear() app = Flask(__name__) app.secret_key = urandom(16) @app.route('/') @app.route('/health') def health(): return 'The server is healthy' @app.route('/static/<path:subpath>', methods=['GET']) def send_static_content(subpath): return send_from_directory('static', subpath, mimetype='*') @app.route('/app', methods=['GET']) def serve_app(): logged = 'username' in session username = session['username'] if logged else '' return render_app(logged=logged, user_name=username) @app.route('/<path:path>')
def __init__(self, file_name: str, path: str = '~/.tkpy2'): super(read_tkpy_file, self).__init__() self.file_name = file_name self.path = path self.db = PickleShareDB(path)
def update_fullname(username, new_fullname): db = PickleShareDB('DAI_database') db[username]['fullname'] = new_fullname db[username] = db[username]
def update_email(username, new_email): db = PickleShareDB('DAI_database') db[username]['email'] = new_email db[username] = db[username]
def get_user_info(username): db = PickleShareDB('DAI_database') return db[username].copy()
# Py2app only uses .pyc files for the stdlib when optimize=0, # so we need to add it as another suffix here if running_in_mac_app(): suffixes = imp.get_suffixes() + [('.pyc', 'rb', '2')] else: suffixes = imp.get_suffixes() # Regular expression for the python import statement import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)' r'(?P<package>[/\\]__init__)?' r'(?P<suffix>%s)$' % r'|'.join(re.escape(s[0]) for s in suffixes)) # Modules database modules_db = PickleShareDB(MODULES_PATH) #----------------------------------------------------------------------------- # Utility functions #----------------------------------------------------------------------------- def module_list(path): """ Return the list containing the names of the modules available in the given folder. """ # sys.path has the cwd as an empty string, but isdir/listdir need it as '.' if path == '': path = '.' # A few local constants to be used in loops below
def misDatosBD(username): db = PickleShareDB('miBD') return db[username]
def editar_BD(username, password): db = PickleShareDB('miBD') if username in db.keys(): del db[username] registrar_BD(username, password)
def getBD(username): db = PickleShareDB('miBD') return db[username]