def __init__(self): self._db = DB() self._tip = self._db.get('l') if not self._tip: block = Block().pow_of_block() self._db.put(block.hash, pickle.dumps(block)) self._db.put(Blockchain.latest, block.hash) self._tip = block.hash
def __init__(self): self.db = DB() if self.db.get('latest'): self.height = int(self.db.get('latest')) self.prev_block = pickle.loads(self.db.get(int(self.height))) else: new_block = Block(0, "Blockchain initialized!", "") self.prev_block = new_block.pow_on_block() self.db.put(0, pickle.dumps(self.prev_block)) self.db.put('latest', 0) self.height = 0
def iter_uids(limit, ind_cred): API_BLOCKED_STATUS = 429 API_BAD_REQUEST = 400 NOT_FOUND_STATUS = 404 gen_api = get_api(credentials[ind_cred]) api = next(gen_api) def get_fwings(pk): try: return api.getTotalFollowings(pk) except: api.logout() api.login() return api.getTotalFollowings(pk) with DB('instadata') as idb: uids = idb.execute_get( 'SELECT pk FROM uids where is_proceed=FALSE AND is_private=FALSE LIMIT %d OFFSET %d' % (limit, limit * ind_cred)) print(uids) for i, urow in enumerate(uids): pk = urow[0] fwngs = get_fwings(pk) if not fwngs or len(fwngs) == 0: print("Followings Empty", pk) if api.LastResponse.status_code == NOT_FOUND_STATUS: print('NOT_FOUND fwings %s' % pk) continue while api.LastResponse.status_code == API_BLOCKED_STATUS or api.LastResponse.status_code == API_BAD_REQUEST: api.logout() if attempt > 2: print("API stop working on pk=%s" % pk) time.sleep(60) api = next(gen_api) print('Change instagram user') attempt += 1 fwngs = get_fwings(pk) attempt = 0 print('%d %s -> len=%d' % (i, pk, len(fwngs))) try: if fwngs: fwngs_filt = filter_advert(fwngs) if fwngs_filt: proceed_followings(idb, fwngs_filt, pk) set_proceed(idb, pk) except: print('pk = ', pk) raise
def import_session_from_file(filename): db = DB() def get_datetime(cmd, value): return datetime.strptime(cmd[value], '%Y/%m/%d %H:%M:%S') with open(filename, 'r') as file: # load json objects (commands) import json commands = [json.loads(l) for l in file.readlines()] if len(commands) < 1: raise Exception('no command in {}'.format(filename)) last_cmd = commands[-1] # load or create user username = last_cmd['user'] user = db.connetion.query(User).filter(User.username == username).first() if user is None: user = User(username) db.connetion.add(user) db.connetion.commit() # load or create host hostname = last_cmd['host'] host = db.connetion.query(Host).filter(Host.hostname == hostname).first() if host is None: host = Host(hostname) db.connetion.add(host) db.connetion.commit() # check session already exists session_uuid = last_cmd['session'] session = db.connetion.query(Session).filter(Session.uuid == session_uuid).first() if session: raise Exception('session already loaded ({})'.format(filename)) # create session start = get_datetime(last_cmd, 'sessionstart') end = get_datetime(last_cmd, 'sessionend') session = Session(session_uuid, user, host, start, end) db.connetion.add(session) # create commands for command in commands[:-1]: cmd = Command(session, command['cmd'], get_datetime(command, 'timing')) db.connetion.add(cmd) db.connetion.commit() pass
def __init__(self, settings_path: str = "crawler_config.yml"): try: self.config = yaml.safe_load(open(settings_path)) except FileNotFoundError: logging.error(f"settings file cant be fount at: {settings_path}") raise Exception("Crawler settings file cant be found") logging.basicConfig( format=self.config['logger_format'], level=logging.DEBUG, # level=logging.INFO, datefmt=self.config['logging_datefmt']) self.t_formater = TimeFormater(**self.config['time_formatter']) self.to_run = True self.event_loop = asyncio.get_event_loop() self.task_queue = asyncio.Queue() self.db = DB(self.config['db']) self.headers = self.config['browser_headers'] self.schedulers = self.get_schedulers() self.downloaders = self.get_downloaders()
def init_personal_from_file(start_from): API_BLOCKED_STATUS = 429 API_BAD_REQUEST = 400 NOT_FOUND_STATUS = 404 with open('inst_uids_init.csv', 'r') as f: lines = f.readlines() gen_api = get_api(credentials) api = next(gen_api) attempt = 0 with DB('instadata') as idb: for i, pk, _, uname in (l.strip().split() for l in lines[start_from:]): if not pk.isdigit(): print('%s FAIL %s %s' % (i, pk, uname)) continue if not api.getUsernameInfo(pk): if api.LastResponse.status_code == NOT_FOUND_STATUS: print('%s NOT_FOUND %s %s' % (i, pk, uname)) continue while api.LastResponse.status_code == API_BLOCKED_STATUS or api.LastResponse.status_code == API_BAD_REQUEST: api.logout() if attempt > 5: print("API stop working on pk=%s uname=%s (%s)" % (pk, attempt, uname)) time.sleep(60) api = next(gen_api) print('Change instagram user') attempt += 1 api.getUsernameInfo(pk) attemp = 0 fname = api.LastJson['user'].get('full_name', '').strip() fname = fname if fname else None print('%s %s' % (i, pk)) idb.execute("INSERT INTO personal (pk, uname, fname) VALUES " + idb.cur.mogrify("(%s, %s, %s)", (pk, uname, fname)).decode() + "ON CONFLICT DO NOTHING") time.sleep(0.2)
from bots.wxbot import WxBot from bots.telebot import TeleBot from utils import DB import services import asyncio from flask import Flask, request, render_template, session import json import logging app = Flask(__name__) logger = logging.getLogger('flask.app') loop = asyncio.get_event_loop() db = DB('sqlite3', './data/my_fund.db') @app.route('/', methods=['GET', 'POST']) def home(): return render_template('home.html') @app.route('/publish_fund_image/<int:user_id>', methods=['POST']) def publish_user_funds(user_id): user = None try: user = next( db.select_data('select * from users where id = %d' % (user_id))) except StopIteration: return {'status': 'error', 'message': '没有id为%d的用户' % (user_id)}
#!/usr/bin/env python from models import * from utils import DB from db import db as model_db if __name__ == '__main__': db_con = DB() model_db.Model.metadata.create_all(db_con.engine)
from blockchain import Blockchain from utils import DB parser = ArgumentParser() subparsers = parser.add_subparsers(dest='command') parser_addblock = subparsers.add_parser('addblock') parser_addblock.add_argument('-transactions') parser_printchain = subparsers.add_parser('printchain') parser_printblock = subparsers.add_parser('printblock') parser_printblock.add_argument('-height', type=int) parser_resetdb = subparsers.add_parser('resetdb') args = parser.parse_args() bc = Blockchain() if (args.command == 'addblock' and args.transactions): bc.add_block(args.transactions) elif (args.command == 'printchain'): bc.print_chain() elif (args.command == 'printblock' and args.height): bc.print_block(args.height) elif (args.command == 'resetdb'): DB().reset() else: parser.print_help()