def main():
db.connect()
hosts = db.get_online()
shells = ['sh', 'bash', '']
options = ['', 'y', 'n']
interpreter = ""
execute = ""
url = input(printer.color_red("Enter url > "))
dest = input(printer.color_cyan("Enter The Directory To Save > "))
execute = input(
printer.color_cyan(
"Do you wish to execute Enter y or n ?Default is n press enter to leave default > "
))
if not execute == "":
while not execute.lower() in options:
execute = input("Enter y or n > ")
if execute == "y":
interpreter = input(
printer.color_cyan(
"Enter the interpreter to use... Default is sh ..press enter to leave default > "
))
while not interpreter.lower() in shells:
interpreter = input("please specify bash or sh > ")
if execute == "n":
interpreter = "n"
for ip in hosts:
id, host, port, username, password, pkey, session = ip
printer.print_success("Connecting to Bot:%s" % host)
if (str(pkey) == 'None'):
pkey = ''
if (bot.connect(host, port, username, password, pkey)):
bot.downloader(dest, url, execute, interpreter)
def final ():
"""
Final actions for after the detailed fund data is added to the
Postgres database.
"""
import os
import psycopg2.extras
from modules import db
# Copy table fundsnew to funds
conn = db.connect (60) # Start database connection
cur = conn.cursor () # Start cursor for the database connection
db.update_table (conn, cur)
conn.close () # Close database connection
dir_output = os.environ ['BSF_OUTPUT']
file_csv_output = dir_output + '/fund_profiles.csv'
conn = db.connect (60) # Start database connection
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
print "Creating output CSV file at:"
print file_csv_output
db.print_csv (conn, cur, file_csv_output)
conn.close () # Close database connection
print "Copying the output CSV file to the Rails site at:"
dir_rails_public = os.environ ['BSF_RAILS_PUBLIC']
file_rails = dir_rails_public + '/fund_profiles.csv'
print file_rails
try:
os.system ('cp ' + file_csv_output + ' ' + file_rails)
except:
print "Could not find " + dir_rails_public
print "End of script - MISSION ACCOMPLISHED!"
def scrape_all ():
"""
Scrape the downloaded Yahoo Finance pages for detailed fund data.
"""
import time, os
import psycopg2.extras
from modules import db
from modules import common
print "*********************************************"
print "Scraping the detailed data on all stock funds"
# Get list of symbols
conn = db.connect (60) # Start database connection
cur1 = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
list_symbols = db.get_symbols (cur1)
conn.close () # Close database connection
conn2 = db.connect (1800) # Start database connection
cur2 = conn2.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
i = 0 # Number of funds completed
i_max = len (list_symbols) # Total number of funds
start = time.time ()
for symbol in list_symbols:
try:
scrape_fund (conn2, cur2, symbol)
except:
pass
i += 1
now = time.time ()
t_elapsed = now - start
try:
rate_s = i / t_elapsed # Stocks/second
remain_s = (i_max - i)/rate_s
remain_m = round(remain_s/60, 1)
if i == 10 or i % 100 == 0:
print "Scraping completion: " + str(i) + '/' + str(i_max)
print "Minutes remaining: " + str(remain_m)
except:
pass
conn2.close () # Close database connection
now = time.time ()
t_elapsed_sec = now - start
t_elapsed_min = t_elapsed_sec/60
print "Finished scraping the detailed stock fund data"
print "Completed in " + str(t_elapsed_min) + " minutes"
print "**********************************************"
def updateMot(uuid, lettre):
jeu = json.loads(getJeu(uuid))
mot = getMot(jeu["mot_id"])
splitEtat = list(jeu["etat_mot"])
splitMot = list(mot)
for index, char in enumerate(splitMot):
if char == lettre:
splitEtat[index] = char
updateMot = ''.join(splitEtat)
connection = db.connect()
# On insert le nouveau jeu
req = "UPDATE jeu SET etat_mot = %s WHERE uuid = %s"
try:
cursor = connection.cursor()
cursor.execute(req, (updateMot, uuid))
connection.commit()
finally:
connection.close()
def getMot(mot_id):
connection = db.connect()
# On récupère un mot de la longueur souhaitée
req = "SELECT * FROM mot WHERE id = %s"
try:
cursor = connection.cursor()
cursor.execute(req, mot_id)
return cursor.fetchone()['mot']
finally:
connection.close()
def nouveauMot(length):
connection = db.connect()
# On récupère un mot de la longueur souhaitée
req = "SELECT * FROM mot WHERE RAND() > 0.9 AND longueur = %s ORDER BY RAND() LIMIT 1"
try:
cursor = connection.cursor()
cursor.execute(req, length)
return cursor.fetchone()['id']
finally:
connection.close()
def getJeu(uuid):
connection = db.connect()
# On récupère un mot de la longueur souhaitée
req = "SELECT * FROM jeu WHERE uuid = %s"
try:
cursor = connection.cursor()
cursor.execute(req, uuid)
return json.dumps(cursor.fetchone())
finally:
connection.close()
def getLettreTrouvee(uuid, lettre):
connection = db.connect()
# On récupère un mot de la longueur souhaitée
req = "SELECT COUNT(*) AS count FROM lettre_trouve WHERE uuid = %s AND lettre = %s"
try:
cursor = connection.cursor()
cursor.execute(req, (uuid, lettre))
return cursor.fetchone()["count"]
finally:
connection.close()
def sauvegardeNouvellePartie(uuid, mot_id, coup_restant, etat_mot, statut):
connection = db.connect()
# On insert le nouveau jeu
req = "INSERT INTO jeu(uuid, mot_id, coup_restant, etat_mot, statut) VALUES(%s, %s, %s, %s, %s)"
try:
cursor = connection.cursor()
cursor.execute(req, (uuid, mot_id, coup_restant, etat_mot, statut))
connection.commit()
finally:
connection.close()
def terminePartie(uuid, statut):
connection = db.connect()
# On insert le nouveau jeu
req = "UPDATE jeu SET statut = %s WHERE uuid = %s"
try:
cursor = connection.cursor()
cursor.execute(req, (statut, uuid))
connection.commit()
finally:
connection.close()
def addLettre(uuid, char):
connection = db.connect()
# On insert le nouveau jeu
req = "INSERT INTO lettre_trouve(uuid, lettre) VALUES(%s, %s)"
try:
cursor = connection.cursor()
cursor.execute(req, (uuid, char))
connection.commit()
finally:
connection.close()
def decrementePartie(uuid):
connection = db.connect()
# On insert le nouveau jeu
req = "UPDATE jeu SET coup_restant = coup_restant - 1 WHERE uuid = %s"
req1 = "SELECT coup_restant FROM jeu WHERE uuid = %s"
try:
cursor = connection.cursor()
cursor.execute(req, uuid)
connection.commit()
cursor.execute(req1, uuid)
if cursor.fetchone()['coup_restant'] == 0:
terminePartie(uuid, 1)
finally:
connection.close()
def update_session():
"""This module runs automatically in the background and updates ip addresses that are alive or dead"""
global hosts
while (True):
db.connect()
hosts = db.get_online()
ipaddr = db.get_ip()
for ip in ipaddr:
addr = ip.__getitem__(0)
if (portscanner(addr)):
db.connect()
db.update_db(addr, 1)
else:
db.connect()
db.update_db(addr, 0)
sleep(30)
def download ():
"""
Download detailed data on funds from Yahoo Finance
"""
import time, os
import psycopg2.extras
from modules import db
from modules import common
dir_output = os.environ ['BSF_OUTPUT']
file_csv_output = dir_output + '/fund_unfiltered.csv'
conn = db.connect (60) # Start database connection
cur2 = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
print "Creating unfiltered list of funds at:"
print file_csv_output
db.print_csv (conn, cur2, file_csv_output)
conn.close () # Close database connection
print "Filtering the list of funds"
conn = db.connect (60) # Start database connection
cur1 = conn.cursor ()
# Filter the list of funds
db.filter_by_fundtype (conn, cur1)
db.filter_by_obj (conn, cur1)
db.filter_by_name (conn, cur1)
db.renumber (conn, cur1) # Reset ID numbers, DOES NOT WORK
conn.close () # Close database connection
print "************************************************"
print "Downloading the detailed data on all stock funds"
print "NOTE: This may be a VERY long process."
# Get list of symbols
conn = db.connect (60) # Start database connection
cur2 = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
list_symbols = db.get_symbols (cur2)
conn.close () # Close database connection
i = 0 # Number of funds completed
i_max = len (list_symbols) # Total number of funds
start = time.time ()
for symbol in list_symbols:
dir_detailed = os.environ ['BSF_DETAILED']
common.create_dir (dir_detailed + '/' + symbol)
url1 = 'http://finance.yahoo.com/q/pr?s=' + symbol + '+Profile'
url2 = 'http://finance.yahoo.com/q/hl?s=' + symbol + '+Holdings'
url3 = 'http://finance.yahoo.com/d/quotes.csv?s=' + symbol + '&f=l1'
file1 = dir_detailed + '/' + symbol + '/profile.html'
file2 = dir_detailed + '/' + symbol + '/holdings.html'
file3 = dir_detailed + '/' + symbol + '/quote.csv'
common.download_file (url1, file1, 164, .2)
common.download_file (url2, file2, 164, .2)
common.download_file (url3, file3, 20, .002)
i += 1
now = time.time ()
t_elapsed = now - start
try:
rate_s = i / t_elapsed # Stocks/second
remain_s = (i_max - i)/rate_s
remain_m = round(remain_s/60, 1)
if i == 10 or i % 100 == 0:
print "Download completion: " + str(i) + '/' + str(i_max)
print "Minutes remaining: " + str(remain_m)
except:
pass
print "Finished downloading detailed data on stock funds"
print "*************************************************"
def get_database(self, server_file):
"""Create the Lan Nanny database if it's not existent, then return the MySql connection."""
conn, cursor = db.connect(server_file)
logging.info('Database connection successful')
return conn, cursor
def __init__(self, configs):
self.conn, self.cursor = db.connect(configs.KIO_SERVER_DB)
self.config = configs