def collect():
''' Execute inside user container and update database '''
requires = ["fileid"]
if not request.json:
return jsonify({"msg": "not json"}), 400
for req in requires:
if not request.json.get(req):
return jsonify({"msg": "no {}".format(req)}), 400
fileid = request.json['fileid']
user_ip = request.remote_addr
user = db.get_user_by_ip(user_ip)
if user is None:
return jsonify({"msg": "ip switched"})
user.touch()
if user.form == 'ghost':
return jsonify({"msg": "you're a ghost"})
script = db.get("Script", fileid)
if script is None:
return jsonify({"msg":"script not found"})
filename = script.filename
text = script.filetext
row = script.row
col = script.col
is_bad_file = script.material >= 20
author = db.get("User", script.user_id)
if author.id == user.id:
return jsonify({"msg": "script is yours"})
if script.has_collected(user.id):
return jsonify({"msg": "you've already collected this script"})
container = nest.load_container(user.id)
file_obj = create_file(user_ip, filename, text, row, col)
result = nest.run_file(user.id, file_obj)
result['filename'] = filename;
result['filetext'] = text;
if result["has_heart"] == None or result["has_heart"] == False:
user.form = 'ghost'
if is_bad_file:
author.add_material(user.pay_material(script.material))
else:
user.form = user.character
script.collect(user.id)
if is_bad_file:
user.add_material(script.material)
else:
user.add_material(script.material)
author.add_material(script.material)
db.save()
ret = user.to_dict()
del ret["ip"]
return jsonify({"result": result, "user": ret})
def save_container(user_id, container):
'''
Commit and save container to dockerhub
'''
user = db.get("User", user_id)
if user is None:
return None
if container is None:
return None
heal_container(container)
user.container_version += 1
db.save()
repo = "rubyshadows/{}".format(user_id)
container.commit(repository=repo,
author=user.name,
tag=user.container_version)
print("saving container: {}".format(user_id))
# client.images.push(repo)
fundamentals.push_image(user_id, user.container_version)
return True
def dump_scripts():
'''
Upload script from file
'''
user_ip = request.remote_addr
user = db.get_user_by_ip(user_ip)
if user is None:
return jsonify({"msg": "ip not set"}), 401
user.touch()
f = request.files['file']
filename = f.filename
row = 0
col = 0
text = f.read().decode("utf-8")
if user.form == 'ghost':
return jsonify({"msg": "you're a ghost"})
user.material += 1;
file_obj = create_file(user_ip, filename, text, row, col)
material = nest.test_file(file_obj)
new_file = db.create("Script", user_id=user.id, material=material,
filename=filename, filetext=text, filetype=file_obj['filetype'],
row=row, col=col, location=user.location)
db.save()
res = user.to_dict()
del res['ip']
script = new_file.to_dict()
script['user'] = res
return jsonify({"script" : script, "user": res})
async def onnote(client, message):
parts = message.text.split(' ', 2)
if len(parts) == 1:
return
elif len(parts) == 2:
note_key = parts[1]
exists = note_key in db['notes']
if message.reply_to_message:
msg = message.reply_to_message
if msg.text:
note_obj = dict(type='text', value=msg.text)
elif msg.media:
media = msg.audio or msg.document or msg.photo or msg.sticker or msg.video or msg.animation or msg.voice or msg.video_note
if not media:
return await message.edit('Non-supported media')
note_value = {
"file_id": media.file_id,
"file_ref": media.file_ref
}
if msg.caption:
note_value['caption'] = msg.caption
note_obj = dict(type='media', value=note_value)
else:
return await message.edit('Nothing to save here.')
db['notes'][note_key] = note_obj
save(db)
action = 'updated' if exists else 'created'
await message.edit(
f"Note '<code>{html.escape(note_key)}</code>' {action}.")
else:
if exists:
note_obj = db['notes'][note_key]
if note_obj['type'] == 'text':
await message.edit(note_obj['value'])
elif note_obj['type'] == 'media':
await message.delete()
await client.send_cached_media(
message.chat.id,
note_obj['value']['file_id'],
reply_to_message_id=(
message.reply_to_message.message_id
if message.reply_to_message else None))
else:
await message.edit(
f"There isn't a note named '<code>{html.escape(note_key)}</code>'."
)
else:
note_key = parts[1]
note_value = parts[2]
exists = note_key in db['notes']
note_obj = dict(type='text', value=note_value)
db['notes'][note_key] = note_obj
save(db)
action = 'updated' if exists else 'created'
await message.edit(
f"Note '<code>{html.escape(note_key)}</code>' {action}.")
def loadBinDate():
filename = 'data.bin'
test = Bin2Txt()
# default use 'ato_f.xml' which is writen in b2t2.py
# to change the default xml file, write like this:
# recno = test.initFile(filename, 'ato_p.xml')
recno = test.initFile(filename)
#generate formatstring from 'ato_f.xml' file
fmt = test.makeRecordFmt()
vNameList = test.globalVnameList # (v_name, v_type)
#startTime = datetime.now()
for i in xrange(0, recno):
rec = test.loadBinRecord(i)
if rec is None:
break
varData = test.parseRecordFmt(rec, fmt)
record = []
for item in varData:
if type(item) is str:
item = "'" + item.encode('hex') + "'"
else:
item = str(item)
record.append(item)
s = ",".join(record)
db.save(s)
db.safe_exit()
def main():
ads = db.load()
if ads == None:
ads = {}
oldadids = set(ads.keys())
newads = crawler.fetch_new_ads(oldadids)
print "[+] Found", len(newads), "new ads"
# this fills ad.copies for all newads
find_duplicates(newads, ads)
for ad in newads.values():
if ads.get(ad.adid) != None:
# make sure we're not accidentally overwriting an old ad
print "[-] AD %s is already in the DB!"%(ad.adid)
continue
ads[ad.adid] = ad
if len(ad.copies) > 0:
print "[+] Ad copy found!", ad.adid, settings.BASEURL + ad.link, ad.copies
else:
print "[+] NEW AD!", ad.adid, ad.link, ad.copies
body = "%s %s\n"%(ad.adid, ad.link)
send_mail([settings.EMAILADDRESS], "root@localhost", "New real-estate", body)
print "Total ads in DB:", len(ads)
db.save(ads)
def update_user():
'''
Update user in database
'''
user = check_user() # check user
if type(user) == dict:
return jsonify(user)
requires = ["username", "password", "character",
"location"] # # check require - user could be failure object
failed = bad_request(requires)
if failed is not None:
return jsonify(failed)
username = request.json['username'] # # # do algorithm
password = request.json['password']
character = request.json['character']
location = request.json['location']
user = db.update("User",
user.id,
username=username,
password=password,
character=character,
form=character,
location=location)
if user is None:
return jsonify({"msg": "update user error", "error": True})
user.touch()
db.save()
# # # # return user, users, scripts, or outputs
return jsonify({"user": return_user(user)})
def build_city_state(city_name='', state_name=''):
state = find_or_create_by_name(src.State, state_name, conn, cursor)
city = find_by_name(src.City, city_name, cursor)
if not city:
city = City(name=city_name, state_id=state.id)
save(city, conn, cursor)
return city, state
def sell():
data = request.json # data={slug,seed,password}
game_state = GameState.load(data['slug'], data['password'])
message = game_state.sell(data['seed'])
game_state.check_cash()
db.save(data['slug'], game_state.data)
return make_response(game_state, message)
def save_track(track, album_id, image_id):
"""Save track `track` from album `album_id` with image
`image_id`. Return the saved song's ID."""
song = {
'name': track['recording']['title'],
'mbid': track['id'],
'length': get_song_length(track),
'track_number': track['number'],
'album_id': album_id,
'image_id': image_id,
'lyrics': lyrics.get_lyrics(track)
}
song['active'] = song['length'] > 0
song_id = db.save('song', song)
featured = track['recording']['artist-credit'][1:]
for artist in featured:
if isinstance(artist, str):
continue # skip strings like '&' and 'feat.'
artist_id = save_artist(artist['artist'])
db.save('song_featured_artists', (song_id, artist_id))
save_genres(song_id, track['recording'])
return song_id
def unlock():
data = request.json # data={slug,x,y,password}
game_state = GameState.load(data['slug'], data['password'])
message = game_state.unlock(data['x'], data['y'])
game_state.check_cash()
db.save(data['slug'], game_state.data)
return make_response(game_state, message)
def harvest():
data = request.json # data={slug,x,y,password}
game_state = GameState.load(data['slug'], data['password'])
message = game_state.harvest(data['x'], data['y'])
game_state.add_recipes()
db.save(data['slug'], game_state.data)
return make_response(game_state, message)
def run_script():
'''
Run's script inside own container
'''
user = check_user() #
if type(user) == dict:
return jsonify(user)
requires = ["filename", "filetext"] # #
failed = bad_request(requires)
if failed is not None:
return jsonify(failed)
filename = request.json['filename'] # # #
text = request.json['filetext']
row = 0
col = 0
user.touch()
container = nest.load_container(user)
file_obj = create_file(filename, text, row, col)
result = nest.run_file(user, file_obj)
if result["has_heart"] == False:
user.form = 'ghost'
elif result["has_heart"] == None:
pass
else:
user.form = user.character
db.save()
# # # #
return jsonify({"result": result, "user": return_user(user)})
def LogoutAdmin(username):
if username[1] in VG.ADMIN:
info = registered(username[0])
if info[0] is True:
if 'ADMIN' in VG.db_users[info[1]]['other']:
VG.db_users[info[1]]['other'].remove('ADMIN')
save('usersdata', VG.db_users)
return True if not 'ADMIN' in VG.db_users[info[1]]['other'] else False
def add(self):
name = self.add_input.text()
if name:
self.items_view.addItem(QListWidgetItem(name))
self.add_input.setText('')
self.items.append({'name': name, 'done': False})
# self.save()
db.save(name, False)
def _save_size(img, image_id, size):
"""Save the image identified by `image_id` in size `size.`"""
filename = _save(img)
db.save('image_size', {
'image_id': image_id,
'size': size.upper(),
'path': IMG_DIR + filename
}, (image_id, size))
def collect():
''' Execute inside user container and update database '''
user = check_user() #
if type(user) == dict:
return jsonify(user)
requires = ["fileid"] # #
failed = bad_request(requires)
if failed is not None:
return jsonify(failed)
fileid = request.json['fileid'] # # #
if user.form == 'ghost':
return jsonify({"msg": "you're a ghost", "error": True})
script = db.get("Script", fileid)
if script is None:
return jsonify({"msg": "script not found", "error": True})
filename = script.filename
text = script.filetext
row = script.row
col = script.col
author = db.get("User", script.user_id)
if author.id == user.id:
return jsonify({"msg": "script is yours", "error": True})
if script.has_collected(user.id):
return jsonify({
"msg": "you've already collected this script",
"error": True
})
is_bad_file = script.material >= 20
container = nest.load_container(user)
file_obj = create_file(filename, text, row, col)
result = nest.run_file(user, file_obj)
if result["has_heart"] == None or result["has_heart"] == False:
user.form = 'ghost'
if is_bad_file:
author.add_material(user.pay_material(script.material))
else:
user.form = user.character
script.collect(user.id)
if is_bad_file:
user.add_material(script.material)
else:
user.add_material(script.material)
author.add_material(script.material)
result['script'] = return_script(user, script)
result['material'] = script.material
user.touch()
db.save()
# # # #
return jsonify({"user": return_user(user), "result": result})
def check_expired():
if db.schedule_form and 'unlock_date' in db.schedule_form:
unlock_time = datetime.strptime(db.schedule_form['unlock_date'], '%Y-%m-%d %H:%M')
if unlock_time <= datetime.now():
logging.info("scheduled unlock (#s)".format(unlock_time))
db.lock_state.unlock(db.schedule_user_id)
db.schedule_form = None
db.schedule_user_id = None
db.save()
def LoginAdmin(username, password):
if username[1] in VG.ADMIN:
if VG.ADMIN[username[1]] == hashlib.md5(password).hexdigest():
info = registered(username[0])
if info[0] is True:
if not 'ADMIN' in VG.db_users[info[1]]['other']:
VG.db_users[info[1]]['other'].append('ADMIN')
save('usersdata', VG.db_users)
return True if 'ADMIN' in VG.db_users[info[1]]['other'] else False
def process(file, q):
q.put(file)
if file == 'B':
time.sleep(6)
db.save(file)
if file == 'F':
raise TimeoutError
print(file, os.getpid())
return file
def refresh_token(user_id, db):
b = requests.post(
"https://accounts.spotify.com/api/token",
headers=dict(Authorization=f"Basic {keys['basic']}"),
data=dict(grant_type="refresh_token",
refresh_token=db[str(user_id)]["refresh_token"])).json()
db[str(user_id)]['access_token'] = b['access_token']
save(db)
return b['access_token']
def lock():
if db.lock_state.is_locked():
db.lock_state.sync()
flash("Already locked")
else:
db.lock_state.lock(current_user_id())
db.save()
return redirect("/config")
def main():
if st.button("show map with both risk Layers"):
show_maps_complete()
st.write('This map uses coordinate format WGS84-EPSG4326')
else:
show_maps_Flood()
st.write('This map uses coordinate format WGS84-EPSG4326')
# Text labels to enter the lat & long coordinates once you read them on the map
lat_long = st.text_input(
'Insert Latitude,Longitude (without spaces) format WGS84-EPSG4326 (DD.dddd) for example: 15.2533,-61.3164',
max_chars=16)
if lat_long != '':
latitude = float(lat_long.split(',')[0])
longitude = float(lat_long.split(',')[1])
name = st.text_input(
' Give a TAG to this point for the DB (not mandatory)', max_chars=15)
if name == '':
name = 'None'
else:
name = str(name)
if st.button(
'Analyse Lat & Long'
): # this is if you want to add a button to launch the analysis (without this, it does automatically when there's lat & long values in the cell)
st.header('Extracting Results for the location selected:\n(Lat: ' +
str(latitude) + ' & Long: ' + str(longitude) + ')')
try:
landslide_code, new_risk = risk_prediction(longitude, latitude)
st.markdown('**-Landslide Risk: **' + str(landslide_code) +
' ---> ' + dict2(landslide_code))
st.markdown('**-Flood risk: **' + str(new_risk) + '---> ' +
dict1(new_risk))
landslide_rclass = dict2(landslide_code)
flood_rclass = dict1(new_risk)
url1 = 'tablerisk.png'
image1 = Image.open(url1)
st.image(image1, caption='', width=350)
##### save DB
save(latitude, longitude, landslide_code, new_risk, name,
landslide_rclass, flood_rclass)
except:
st.markdown(
"The coordinate insert are outside Dominica Island. Please insert correct coordinates!"
)
if st.button("Clear Result Database"):
conn = get_connection("test1.db")
delete_all_tasks(conn)
st.markdown('Result Database Cleaned!')
def state(slug):
valid_chars(slug)
body = request.json
if body['newOrLoad'] == 'new':
game_state = GameState.new(slug, body['password'])
if body['newOrLoad'] == 'load':
game_state = GameState.load(slug, body['password'])
if game_state is None:
raise NotFound("Game %s does not exist" % slug)
db.save(slug, game_state.data)
return make_response(game_state)
def updateTasks():
# iterate through the containers and update the state
for task_id, task in db.tasks().items():
if task.container.type == messages_pb2.ContainerInfo.Type.DOCKER:
task.state = dockerhelper.getContainerStatus(task_id)
if task.state == messages_pb2.TaskInfo.ERRORED:
task.error_message = dockerhelper.getContainerLogs(task_id)
elif task.container.type == messages_pb2.ContainerInfo.Type.WASM:
# get task state
task.state = wasmhelper.getTaskStatus(task_id)
db.save()
def get_token(user_id, auth_code, db):
b = requests.post("https://accounts.spotify.com/api/token",
headers=dict(Authorization=f"Basic {keys['basic']}"),
data=dict(grant_type="authorization_code",
code=auth_code,
redirect_uri="https://lyricspy.ml/go")).json()
if b.get("error"):
return False, b['error']
else:
db[str(user_id)] = b
save(db)
return True, b['access_token']
def uprade(client, message):
if message.from_user.id in sudos:
a = message.reply('Atualizando...')
out = subprocess.getstatusoutput('git pull {}'.format(
' '.join(git_repo)))[1]
a.edit(f'Resultado da atualização:\n{out}')
sent = message.reply('Reiniciando...')
db["restart"] = {'cid': message.chat.id, 'mid': sent.message_id}
save(db)
time.sleep(1)
os.execl(sys.executable, sys.executable, *sys.argv)
del threading.Thread
def search():
city = ui.get_city()
restaurants = rest.get_restaurants(city)
news_articles = news.get_articles(city)
weather_forecast = weather.get_forecast(city)
ui.display(restaurants, news_articles, weather_forecast)
# figure out if user wants to save
if user_wants_to_save:
db.save(restaurants, news_articles, weather_forecast)
def updateDevice(userid, deviceid, sensorid, state=0, value=0):
if (len(findSensor(userid, deviceid, sensorid)) > 0):
sql_update = ''' UPDATE devices SET state = ?, value=?
WHERE (userID=? and deviceID=? and sensorID=?) '''
db.update(db.get_conn(DBPath), sql_update,
[(state, value, userid, deviceid, sensorid)])
else:
sql_insert = ''' insert into devices (userID, deviceID, sensorID, state, value)
VALUES (?,?,?,?,?);
'''
db.save(db.get_conn(DBPath), sql_insert,
[(userid, deviceid, sensorid, state, value)])
def _save_orig(img):
"""Save an image."""
filename = _save(img)
image_id = db.save('image', {
'width': img.size[0],
'height': img.size[1]
}, filename)
db.save('image_size', {
'image_id': image_id,
'size': 'FULL',
'path': IMG_DIR + filename
}, (image_id, 'full'))
return image_id
def signup():
if current_user.is_authenticated:
return redirect('home')
message = ''
if request.method == 'POST':
try:
db.save(request.form)
return redirect(url_for('/'))
except pymongo.errors.DuplicateKeyError:
message = 'User already exists.'
return render_template('signup.html', message=message)
def get_it(stock_id):
print 'Fetch stock ', stock_id
total = fetch_total(stock_id)
a_list = list()
i_index = 1
i_size = 200
i_total = math.ceil(total / i_size)
while i_index <= i_total:
new_list = fetch_list(stock_id, i_index, i_size)
save_to(new_list, a_list)
i_index = i_index + 1
db.save(stock_id, a_list)
print 'Save %s successfully.' % (stock_id, )
def process_item(text, path, fanhao):
'''
process item page
'''
global counter
counter += 1
print(f'process item {fanhao}')
url = router.get_full_url(path)
meta, tags = parse_item(text)
meta.update(url=url)
print('meta keys', len(meta.keys()))
print('tag count', len(tags))
save(meta, tags)
check_exit()
def unlock():
if not db.lock_state.can_unlock(current_user_id()):
flash(
"You can't unlock the box because it was locked by someone else.")
return redirect("/config")
if not db.lock_state.is_locked():
db.lock_state.sync()
flash("Already unlocked")
else:
db.lock_state.unlock(current_user_id())
db.save()
return redirect("/config")
def login(username, hostname, password):
try:
if hashlib.md5(password).hexdigest() == userinfo.info(username, 'password'):
if userinfo.info(username, 'status') is 'desconectado':
index = userinfo.registered(username)[1]
VG.db_users[index]['status'] = 'conectado'
VG.db_users[index]['hostname'] = hostname
db.save('DB_Users', VG.db_users)
logger.log('Usuario logeado a: '+username).Logger()
else:
raise UnboundLocalError(username + '. Ya se encuentra conectado.')
else:
raise ValueError(username + '. Contraseña erronea.')
except (TypeError, ValueError, UnboundLocalError) as e:
logger.log(e).LogError()
return e
def logout(username, hostname):
try:
if userinfo.info(username, 'hostname') == hostname:
if userinfo.info(username, 'status') is 'conectado':
index = userinfo.registered(username)[1]
VG.db_users[index]['status'] = 'desconectado'
VG.db_users[index]['hostname'] = ''
db.save('DB_Users', VG.db_users)
logger.log('Usuario deslogueado: ' + username).Logger()
else:
raise UnboundLocalError(username + '. Ya se enuentra desconectado.')
else:
raise ValueError(hostname + '. El host no coincide.')
except (TypeError, ValueError, UnboundLocalError) as e:
logger.log(e).LogError()
return e
def add_toBuffer(post):
if post not in buffer:
post['sentiment_result'] = sentiment_analisys.process(post['text'])
post['owner_info'] = vk_util.get_profile(post['owner_id'])
if len(buffer) >= 10000:
logging.info('deleting ' + str(buffer[0]))
del buffer[0]
post["added_time"] = int(datetime.now().strftime("%s"))
try:
post['zone'] = poligons.get_zone(post['lat'], post['long'])
except:
post['zone'] = "nozone"
buffer.add(post)
db.save(post.data)
logging.info('buffer len = {}'.format(len(buffer)))
def start(): pool = Pool(processes=16) queue = [] websites = db.getJob() if websites: queue.extend(websites) while len(queue): print "Browsing", len(queue), 'urls' result = pool.map(getLinks, queue) print "Adding links" newQueue = [] assert len(queue) == len(result) for i in xrange(len(queue)): for path in result[i]: url = urljoin(queue[i][1], path) result = db.addUrl(url, queue[i][0]) if result: newQueue.append(result) queue = newQueue db.save() print "No more job."
def parse_data():
url = "http://wetter.rzbd.haw-hamburg.de/php/aktuell_old.php3"
wetter = urlopen(url).read()
soup = BeautifulSoup(wetter)
table = soup.find('table')
rows = table.findAll('tr')
for n in xrange(1, 11):
try:
cols = rows[n].findAll('td')
dtstring = cols[7].string + cols[8].string
snapshot = {'temperature': float(cols[0].string),
'humidity': int(cols[1].string),
'airpressure': float(cols[2].string),
'precipitation': float(cols[3].string),
'irradiance': int(cols[4].string),
'windspeed': float(cols[5].string),
'winddirection': int(cols[6].string),
'timestamp': datetime.datetime.strptime(
dtstring, "%d.%m.%Y%H:%M")}
db.save(snapshot)
except:
main.log.warning("parsing error")
def save(data):
db.save(data, problem_db_file)
def process_item(self, item, spider):
db.save(dict(item))
return item
def test_log(self):
tmp = {'t': 1, 'e': 233, 's': 577}
db.save(tmp, 'tmpdata')
self.assertEqual(tmp, db.get('tmpdata'))
def get(self):
db.save()
self.write({
'result': True,
'msg': 'Database save success',
})
import app
import db
import menu
import sys
database_location = "database.txt"
def init():
app.make_context()
db.load(database_location)
if __name__ == "__main__":
init()
context = app.get_context()
while True:
menu.menu()
if input("Save changes? ").lower().startswith('y'):
db.save(database_location)
if context.quitting:
sys.exit(0)
def exit(message=""):
print message
db.save()
sys.exit()
def work():
tweets,finished = queue.pop(100)
if not tweets: return
db.save(*filter(None,map(analyze, tweets)))
finished()
return len(tweets)
def preload():
stock_block = ts.get_industry_classified()
db.save(STOCK_BLOCK_INDUSTRY, stock_block)
stock_block = ts.get_concept_classified()
db.save(STOCK_BLOCK_CONCEPT, stock_block)
stock_block = ts.get_area_classified()
db.save(STOCK_BLOCK_AREA, stock_block)
stock_block = ts.get_sme_classified()
db.save(STOCK_BLOCK_SME, stock_block)
stock_block = ts.get_gem_classified()
db.save(STOCK_BLOCK_GEM, stock_block)
stock_block = ts.get_st_classified()
db.save(STOCK_BLOCK_ST, stock_block)
stock_block = ts.get_hs300s()
db.save(STOCK_BLOCK_HS300S, stock_block)
stock_block = ts.get_sz50s()
db.save(STOCK_BLOCK_SZ50S, stock_block)
stock_block = ts.get_zz500s()
db.save(STOCK_BLOCK_ZZ500S, stock_block)
def save_solution_db_file(problem_id, data):
solution_db_file = 'problem_' + str(problem_id) + '_solution.db.txt'
db.save(data, solution_db_file)
def main(pages):
global _IMAGE_CNT, _VIDEO_CNT
for i in range(pages):
url = _GRAP_URL + '&page=' + str(i)
html_doc = urllib.urlopen(url).read()
if html_doc:
soup = BeautifulSoup(html_doc)
try:
# ============================================================
# 采集视频
# ============================================================
target_divs = soup.find_all('article', {'class': 'post-video'})
if target_divs and len(target_divs):
for td in target_divs:
# 资源标题
title = td.find('h2')['title']
video_src = td.find('embed')['src'].strip()
# FIXME url
video_id = _get_video_id('http://player.56.com/renrenshare_(.*).swf/1030_.*.swf', video_src)
if not video_id:
continue
# 获取视频信息
video_info = _get_video_info(video_id)
if video_info.get('info', None):
video_img = video_info['info']['bimg']
# 不抓取重复资源
if db.exists(video_src):
continue
resource_info = _deal_resource(video_img)
resource_info['url'] = video_src
resource = db.SpideResource(title, 'video', json.dumps(resource_info), video_src)
db.save(resource)
print '成功爬取视频资源!'
_VIDEO_CNT = _VIDEO_CNT + 1
else:
print 'Fail to get video_info by: %s %s' % (video_src, video_id)
except Exception, err:
print '[VideoError] ' + str(err)
try:
# ============================================================
# 采集图片
# ============================================================
target_divs = soup.find_all('article', {'class': ['post-photo', 'post-article']})
if target_divs and len(target_divs):
for td in target_divs:
post_content = td.find('div', {'class': 'post-content'})
# 资源标题
title = post_content.find('h2')['title']
images = post_content.find_all('img')
# 抓取只有一副图片的资源
if images and len(images) == 1:
image = images[0]
if image.get('data-src', None):
image_src = image['data-src'].strip()
else:
image_src = image['src'].strip()
# 不抓取重复资源
if db.exists(image_src):
continue
resource_info = _deal_resource(image_src)
resource = db.SpideResource(title, 'image', json.dumps(resource_info), image_src)
db.save(resource)
print '成功爬取图片资源!'
_IMAGE_CNT = _IMAGE_CNT + 1
except Exception, err:
print '[ImageError] ' + str(err)
def process_msg(msg , conn, socket, runtimeData):
paras = msg.split(MESSAGE_SPLIT_TAG)
msg_head = paras[0]
tobekicked = []
if(msg_head == "LOGIN"):
user = paras[1]
password = paras[2]
u = db.getUser(user, conn)
if(u != None and u[2] == password):
send(socket,"LOGIN" + MESSAGE_SPLIT_TAG + "true")
runtimeData["online"][socket] = u
#踢掉当前在线的
for u in runtimeData["online"]:
username = runtimeData["online"][u][1]
if(username==user and u != socket):
tobekicked.append(u)
else:
send(socket,"LOGIN" + MESSAGE_SPLIT_TAG + "false")
tobekicked.append(socket)
if(msg_head == "LOGOUT"):
runtimeData["online"].remove(socket)
tobekicked.append(socket)
if(msg_head == "JOIN_CHANNEL"):
channels = paras[1].split("#")
runtimeData["channel"][socket] = channels
send(socket, "JOIN_CHANNEL" + MESSAGE_SPLIT_TAG + "true")
if(msg_head == "GET_ONLINE_USERS"):
ret = "GET_ONLINE_USERS"
for u in runtimeData["online"]:
username = runtimeData["online"][u][1]
score = int(runtimeData["online"][u][3])
ret = ret + MESSAGE_SPLIT_TAG + username + "," + str(score)
send(socket,ret)
if(msg_head == "CHAT"):
channel = paras[1]
info = paras[2]
me_username = runtimeData["online"][socket][1]
me_score = int(runtimeData["online"][socket][3])
msg = "CHAT" + MESSAGE_SPLIT_TAG + me_username + "," + str(me_score) + MESSAGE_SPLIT_TAG + channel + MESSAGE_SPLIT_TAG + info.decode("utf-8")
for u in runtimeData["channel"]:
try:
if channel in runtimeData["channel"][u] and u!=socket:
send(u, msg)
except:
pass
if(msg_head == "GET_SAVES"):
me_username = runtimeData["online"][socket][1]
saves = []
for s in db.getSaves(me_username, conn):
if(s[1] != None):
saves.append(s[1])
else:
saves.append("Empty")
msg = "GET_SAVES"
for s in saves:
msg = msg + MESSAGE_SPLIT_TAG + s
send(socket,msg)
if(msg_head == "SAVE"):
me_username = runtimeData["online"][socket][1]
index = paras[1]
content = paras[2]
ret = db.save(me_username, index, content, conn)
if ret :
send(socket,"SAVE" + MESSAGE_SPLIT_TAG + "true")
else:
send(socket,"SAVE" + MESSAGE_SPLIT_TAG + "false")
if(msg_head == "BATTLE_RESULT"):
channel = paras[1]
winner = paras[2]
loser = paras[3]
uwin = db.getUser(winner, conn)
ulose = db.getUser(loser, conn)
winner_score = (int)(uwin[3])
loser_score = (int)(ulose[3])
#积分规则
if(winner_score - loser_score > 100): #差距100以上，忽略
pass
elif(winner_score >= loser_score):
delta = winner_score - loser_score
winner_score += 10 - delta / 10
loser_score -= 10 - delta / 20
elif(loser_score > winner_score):
delta = loser_score - winner_score
winner_score += 10
loser_score -= 10
db.updateScore(winner, winner_score, conn)
db.updateScore(loser, loser_score, conn)
for u in runtimeData["online"]:
username = runtimeData["online"][u][1]
if username == winner:
runtimeData["online"][u] = db.getUser(winner, conn)
if username == loser:
runtimeData["online"][u] = db.getUser(loser, conn)
send(socket, "BATTLE_RESULT" + MESSAGE_SPLIT_TAG + "true")
return tobekicked
def list(self, stock_block_type):
stock_block = None
if stock_block_type == self.industry:
stock_block = db.get(STOCK_BLOCK_INDUSTRY)
if stock_block is None:
stock_block = ts.get_industry_classified()
db.save(STOCK_BLOCK_INDUSTRY, stock_block)
elif stock_block_type == self.concept:
stock_block = db.get(STOCK_BLOCK_CONCEPT)
if stock_block is None:
stock_block = ts.get_concept_classified()
db.save(STOCK_BLOCK_CONCEPT, stock_block)
elif stock_block_type == self.area:
stock_block = db.get(STOCK_BLOCK_AREA)
if stock_block is None:
stock_block = ts.get_area_classified()
db.save(STOCK_BLOCK_AREA, stock_block)
elif stock_block_type == self.sme:
stock_block = db.get(STOCK_BLOCK_SME)
if stock_block is None:
stock_block = ts.get_sme_classified()
db.save(STOCK_BLOCK_SME, stock_block)
elif stock_block_type == self.gem:
stock_block = db.get(STOCK_BLOCK_GEM)
if stock_block is None:
stock_block = ts.get_gem_classified()
db.save(STOCK_BLOCK_GEM, stock_block)
elif stock_block_type == self.st:
stock_block = db.get(STOCK_BLOCK_ST)
if stock_block is None:
stock_block = ts.get_st_classified()
db.save(STOCK_BLOCK_ST, stock_block)
elif stock_block_type == self.hs300s:
stock_block = db.get(STOCK_BLOCK_HS300S)
if stock_block is None:
stock_block = ts.get_hs300s()
db.save(STOCK_BLOCK_HS300S, stock_block)
elif stock_block_type == self.sz50s:
stock_block = db.get(STOCK_BLOCK_SZ50S)
if stock_block is None:
stock_block = ts.get_sz50s()
db.save(STOCK_BLOCK_SZ50S, stock_block)
elif stock_block_type == self.zz500s:
stock_block = db.get(STOCK_BLOCK_ZZ500S)
if stock_block is None:
stock_block = ts.get_zz500s()
db.save(STOCK_BLOCK_ZZ500S, stock_block)
else:
return None
return stock_block
def new_roi():
roiData = request.json
db.save(roiData)