def __init__(self):
self._db = DB()
self._tip = self._db.get('l')
if not self._tip:
block = Block().pow_of_block()
self._db.put(block.hash, pickle.dumps(block))
self._db.put(Blockchain.latest, block.hash)
self._tip = block.hash
def __init__(self):
self.db = DB()
if self.db.get('latest'):
self.height = int(self.db.get('latest'))
self.prev_block = pickle.loads(self.db.get(int(self.height)))
else:
new_block = Block(0, "Blockchain initialized!", "")
self.prev_block = new_block.pow_on_block()
self.db.put(0, pickle.dumps(self.prev_block))
self.db.put('latest', 0)
self.height = 0
def iter_uids(limit, ind_cred):
API_BLOCKED_STATUS = 429
API_BAD_REQUEST = 400
NOT_FOUND_STATUS = 404
gen_api = get_api(credentials[ind_cred])
api = next(gen_api)
def get_fwings(pk):
try:
return api.getTotalFollowings(pk)
except:
api.logout()
api.login()
return api.getTotalFollowings(pk)
with DB('instadata') as idb:
uids = idb.execute_get(
'SELECT pk FROM uids where is_proceed=FALSE AND is_private=FALSE LIMIT %d OFFSET %d'
% (limit, limit * ind_cred))
print(uids)
for i, urow in enumerate(uids):
pk = urow[0]
fwngs = get_fwings(pk)
if not fwngs or len(fwngs) == 0:
print("Followings Empty", pk)
if api.LastResponse.status_code == NOT_FOUND_STATUS:
print('NOT_FOUND fwings %s' % pk)
continue
while api.LastResponse.status_code == API_BLOCKED_STATUS or api.LastResponse.status_code == API_BAD_REQUEST:
api.logout()
if attempt > 2:
print("API stop working on pk=%s" % pk)
time.sleep(60)
api = next(gen_api)
print('Change instagram user')
attempt += 1
fwngs = get_fwings(pk)
attempt = 0
print('%d %s -> len=%d' % (i, pk, len(fwngs)))
try:
if fwngs:
fwngs_filt = filter_advert(fwngs)
if fwngs_filt:
proceed_followings(idb, fwngs_filt, pk)
set_proceed(idb, pk)
except:
print('pk = ', pk)
raise
def import_session_from_file(filename):
db = DB()
def get_datetime(cmd, value):
return datetime.strptime(cmd[value], '%Y/%m/%d %H:%M:%S')
with open(filename, 'r') as file:
# load json objects (commands)
import json
commands = [json.loads(l) for l in file.readlines()]
if len(commands) < 1:
raise Exception('no command in {}'.format(filename))
last_cmd = commands[-1]
# load or create user
username = last_cmd['user']
user = db.connetion.query(User).filter(User.username == username).first()
if user is None:
user = User(username)
db.connetion.add(user)
db.connetion.commit()
# load or create host
hostname = last_cmd['host']
host = db.connetion.query(Host).filter(Host.hostname == hostname).first()
if host is None:
host = Host(hostname)
db.connetion.add(host)
db.connetion.commit()
# check session already exists
session_uuid = last_cmd['session']
session = db.connetion.query(Session).filter(Session.uuid == session_uuid).first()
if session:
raise Exception('session already loaded ({})'.format(filename))
# create session
start = get_datetime(last_cmd, 'sessionstart')
end = get_datetime(last_cmd, 'sessionend')
session = Session(session_uuid, user, host, start, end)
db.connetion.add(session)
# create commands
for command in commands[:-1]:
cmd = Command(session, command['cmd'], get_datetime(command, 'timing'))
db.connetion.add(cmd)
db.connetion.commit()
pass
def __init__(self, settings_path: str = "crawler_config.yml"):
try:
self.config = yaml.safe_load(open(settings_path))
except FileNotFoundError:
logging.error(f"settings file cant be fount at: {settings_path}")
raise Exception("Crawler settings file cant be found")
logging.basicConfig(
format=self.config['logger_format'],
level=logging.DEBUG,
# level=logging.INFO,
datefmt=self.config['logging_datefmt'])
self.t_formater = TimeFormater(**self.config['time_formatter'])
self.to_run = True
self.event_loop = asyncio.get_event_loop()
self.task_queue = asyncio.Queue()
self.db = DB(self.config['db'])
self.headers = self.config['browser_headers']
self.schedulers = self.get_schedulers()
self.downloaders = self.get_downloaders()
def init_personal_from_file(start_from):
API_BLOCKED_STATUS = 429
API_BAD_REQUEST = 400
NOT_FOUND_STATUS = 404
with open('inst_uids_init.csv', 'r') as f:
lines = f.readlines()
gen_api = get_api(credentials)
api = next(gen_api)
attempt = 0
with DB('instadata') as idb:
for i, pk, _, uname in (l.strip().split() for l in lines[start_from:]):
if not pk.isdigit():
print('%s FAIL %s %s' % (i, pk, uname))
continue
if not api.getUsernameInfo(pk):
if api.LastResponse.status_code == NOT_FOUND_STATUS:
print('%s NOT_FOUND %s %s' % (i, pk, uname))
continue
while api.LastResponse.status_code == API_BLOCKED_STATUS or api.LastResponse.status_code == API_BAD_REQUEST:
api.logout()
if attempt > 5:
print("API stop working on pk=%s uname=%s (%s)" %
(pk, attempt, uname))
time.sleep(60)
api = next(gen_api)
print('Change instagram user')
attempt += 1
api.getUsernameInfo(pk)
attemp = 0
fname = api.LastJson['user'].get('full_name', '').strip()
fname = fname if fname else None
print('%s %s' % (i, pk))
idb.execute("INSERT INTO personal (pk, uname, fname) VALUES " +
idb.cur.mogrify("(%s, %s, %s)", (pk, uname,
fname)).decode() +
"ON CONFLICT DO NOTHING")
time.sleep(0.2)
from bots.wxbot import WxBot
from bots.telebot import TeleBot
from utils import DB
import services
import asyncio
from flask import Flask, request, render_template, session
import json
import logging
app = Flask(__name__)
logger = logging.getLogger('flask.app')
loop = asyncio.get_event_loop()
db = DB('sqlite3', './data/my_fund.db')
@app.route('/', methods=['GET', 'POST'])
def home():
return render_template('home.html')
@app.route('/publish_fund_image/<int:user_id>', methods=['POST'])
def publish_user_funds(user_id):
user = None
try:
user = next(
db.select_data('select * from users where id = %d' % (user_id)))
except StopIteration:
return {'status': 'error', 'message': '没有id为%d的用户' % (user_id)}
#!/usr/bin/env python
from models import *
from utils import DB
from db import db as model_db
if __name__ == '__main__':
db_con = DB()
model_db.Model.metadata.create_all(db_con.engine)
from blockchain import Blockchain
from utils import DB
parser = ArgumentParser()
subparsers = parser.add_subparsers(dest='command')
parser_addblock = subparsers.add_parser('addblock')
parser_addblock.add_argument('-transactions')
parser_printchain = subparsers.add_parser('printchain')
parser_printblock = subparsers.add_parser('printblock')
parser_printblock.add_argument('-height', type=int)
parser_resetdb = subparsers.add_parser('resetdb')
args = parser.parse_args()
bc = Blockchain()
if (args.command == 'addblock' and args.transactions):
bc.add_block(args.transactions)
elif (args.command == 'printchain'):
bc.print_chain()
elif (args.command == 'printblock' and args.height):
bc.print_block(args.height)
elif (args.command == 'resetdb'):
DB().reset()
else:
parser.print_help()