def main():
# Start logging
start_log()
# Set up command-line parser
cmdparser = argparse.ArgumentParser()
cmdparser.add_argument("configpath",
help="Path to configuration file (.cfg); defaults to using Templates/default.cfg",
nargs='?',
default="Templates/default.cfg")
args = cmdparser.parse_args()
if args.configpath == "Templates/default.cfg":
log_it("No configuration file was selected; using " + args.configpath)
log_it("=====\n Pipeline started\n")
# Genreal-purpose dictionary that gets passed everywhere
hotpotato = {}
# Read the config file
hotpotato = cfg.read_config(args.configpath)
# Ensure proper conversion of config file and header parameters in dictionary
hotpotato = cfg.convert_values(hotpotato)
# Dynamically import and call the main function of each method defined in cfg
for x in get_value(hotpotato, 'methods'):
if (x == 'data'):
continue
temp = __import__(x + '_method')
hotpotato = temp.main(hotpotato)
# Exit cleanly
log_it("\n Pipeline tasks completed \n=====")
sys.exit()
def main(date):
if date == None:
date = datetime.datetime.now().strftime('%Y-%m-%d')
date_time = datetime.datetime.strptime(date, '%Y-%m-%d')
start_t = (date_time - datetime.timedelta(days=90)).strftime('%Y-%m-%d')
# day_delta = 40
db_config = read_config('db_config')
print('db_config:', db_config)
db = pymysql.connect(host=db_config["host"],
user=db_config["user"],
password=db_config["password"],
database=db_config["database"])
# db = pymysql.connect(host="192.168.1.6", user="******", password="******", database="stockdb")
# cursor = db.cursor()
#test 作为单个账号历史数据测试
# sql = "select stock_id,stock_name,trade_date,close_price,increase from stock_history_trade{0} " \
# "where trade_date <= '{1}' and stock_id not like '688%' " \
# "and stock_id = '002407' order by trade_date DESC limit {2} ".format(h_tab,date,day_delta)
# sql = "select stock_id,stock_name,trade_date,close_price,increase,turnover_rate from stock_history_trade{0} " \
# "where trade_date >= '{1}' and trade_date <= '{2}' and stock_id not like '688%' ".format(h_tab,start_t,date)#and stock_id in ('002940','000812')
# sql = "select stock_id,stock_name,trade_date,close_price,increase,turnover_rate from stock_trade_data " \
# "where trade_date >= '{0}' and trade_date <= '{1}' and stock_id not like '688%' ".format(start_t,date)#and stock_id in ('002940','000812')
sql = "select stock_id,stock_name,trade_date,close_price,increase,turnover_rate from stock_trade_data " \
"where stock_id not like '688%' and stock_id not like '300%' and trade_date >= '{0}' and trade_date <= '{1}' " \
"and stock_name not like 'ST%' and stock_name not like '%ST%' ".format(start_t,date)#and stock_id in ('002940','000812')
time_start = datetime.datetime.now()
df = get_df_from_db(sql, db)
time_end = datetime.datetime.now()
print('df_len:', len(df))
print('time_delta:', time_end - time_start)
df = core(df, date)
save(db, df)
def creat_db(self):
db_config = read_config('db_config')
db = pymysql.connect(host=db_config["host"],
user=db_config["user"],
password=db_config["password"],
database=db_config["database"])
return db
def __init__(self):
db_config = read_config('db_config')
host = db_config["host"],
user = db_config["user"],
password = db_config["password"],
database = db_config["database"]
self.conf = "mysql+pymysql://{user}:{password}@{host}:3306/{database}".format(
user=user, password=password, host=host, database=database)
self.conf = "mysql+pymysql://user1:[email protected]:3306/stockdb"
self.engine = create_engine(self.conf)
def twitter_api_config():
api_config = readconfig.read_config('auth_info')
consumer_key = api_config['consumer_key']
consumer_secret = api_config['consumer_secret']
access_key = api_config['token']
access_secret = api_config['token_secret']
# Create Auth object
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def history_com(start_date, end_date):
db_config = read_config('db_config')
db = pymysql.connect(host=db_config["host"],
user=db_config["user"],
password=db_config["password"],
database=db_config["database"])
cursor = db.cursor()
sql = "select distinct(trade_date) from stock_trade_data where trade_date >= '{0}' and trade_date <= '{1}'".format(
start_date, end_date)
cursor.execute(sql)
date_list = cursor.fetchall()
#print('date_list:',date_list)
p = Pool(8)
for date_t in date_list:
date = date_t[0].strftime('%Y-%m-%d')
print('date:', date)
# main(date)
p.apply_async(main, args=(date, ))
p.close()
p.join()
def clear_main(h_table,start_date,end_date):
db_config = read_config('db_config')
db = pymysql.connect(host=db_config["host"], user=db_config["user"],
password=db_config["password"], database=db_config["database"])
cursor = db.cursor()
sql = "select distinct stock_id from stock_trade_data where stock_id like '%{}'".format(h_table)
time1 = datetime.datetime.now()
cursor.execute(sql) # 执行SQL语句
id_list = cursor.fetchall()
print('id select time:',datetime.datetime.now() - time1)
cursor.close()
sql = "select stock_id,trade_code,close_price,trade_date " \
"from stock_trade_data " \
"where stock_id not like '688%' and trade_date >= '{0}' and trade_date <= '{1}'".format(start_date,end_date)
time2 = datetime.datetime.now()
df = get_df_from_db(sql,db)
print('df select time:', datetime.datetime.now() - time2)
# print(df)
time3 = datetime.datetime.now()
make_data(df, db, id_list)
print('deal df select time:', datetime.datetime.now() - time3)
def main(update_flag=0):
db_config = read_config('db_config')
db = pymysql.connect(host=db_config["host"],
user=db_config["user"],
password=db_config["password"],
database=db_config["database"])
# cursor = db.cursor()
#get_data(stock_id='603828')#000790
#get_data(stock_id='000790')
# stock_id_list=select_info(db)
#stock_id_list = [('002038',)]
# for stock in stock_id_list:
# print('stock[0]:',stock[0])
# get_data(stock[0],db)
if update_flag == 1:
git_base_info(db)
update_other_tab(db)
elif update_flag == 0:
git_base_info(db)
elif update_flag == 2:
update_other_tab(db)
def main():
'''
Launches all programs and setups
'''
config = readconfig.read_config()
# config contains : number_sensors, sen_id, save_dir, graph_name
# Create files
script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'create_files.sh')
print(script_path)
subprocess32.call([script_path, config[2], '/var/www', config[3]])
# Opens them
file_config = set_dir_dev.open_files(config)
# Opens devices
dev = set_dir_dev.open_dev(config[7])
# Prints IP address
ip_local = [(s.connect(('8.8.8.8', 80)), s.getsockname()[0], s.close())
for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
][0][1]
dev[0].clear()
dev[0].message('Adresse IP :\n{IP}'.format(IP=ip_local))
while not (dev[0].buttonPressed(dev[0].RIGHT) or dev[0].buttonPressed(
dev[0].LEFT) or dev[0].buttonPressed(dev[0].UP)
or dev[0].buttonPressed(dev[0].DOWN)):
pass # Wait for an entry
# dev contains : lcd, board, iter8
set_dir_dev.set_arduino(config[0], dev[1])
# Create threads
data2Graph = AnalogGraphThreads.AnalogGraphThreads(config[0], config[6])
data2Graph.startThreads(config, dev, file_config)
return 0
parser.add_argument('-m', '--messages',
help='path to the Messages binary')
parser.add_argument('-b', '--batch', action='store_true',
help='batch mode - will log into a file')
parser.add_argument('-B', '--backupcount', default=8,
help='how many log files to keep around in batch mode')
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig()
logger = logging.getLogger(os.path.basename(sys.argv[0]))
config = read_config(logger, args.config)
if config is None:
logger.error("Cannot read config file from {}".format(args.config))
sys.exit(1)
# Make sure the log directory exists.
try:
logdir = config["logdir"]
except:
logger.error("'logdir' does not exist in configuration")
sys.exit(1)
else:
check_create_dir(logdir)
if args.messages:
messages_file = which(args.messages)
def main():
args=sys.argv
if len(args) < 3:
print "usage %s url metric_id" % args[0]
sys.exit()
url = "http://%s/getValue?id=%s{0}" % (
os.getenv("metricsinyourfaceurl", args[1]),
args[2])
# url = "http://%s/%s/{0}" % (
# os.getenv("metricsinyourfaceurl", args[1]),
# args[2])
hostname = socket.gethostname()
# Note: if config circuit is not present, import readconfig_fake instead
readconfig.setup()
readconfig.load_data()
config = readconfig.read_config()
print config
disp = make_displays_shift(config)
display_config(disp, config)
# Blink last decimal point to indicate data is fresh
blink = False
disp_data = []
disp_lock = Lock()
disp_thread = Thread(target=display_rolling)
disp_thread.start()
def display_rolling():
disp_lock.acquire()
for i in range(len(disp_data)):
disp.set(i, data[i] + ('.' * blink))
disp_lock.release()
disp.display()
time.sleep(0.5)
while (True):
data = get_values(url, hostname, config)
if data:
disp_lock.acquire()
disp_data = data
disp_lock.release()
blink = not blink
else:
# Blink if cannot connect
print("could not reach server")
disp.blank()
time.sleep(.2)
disp.display()
# check whether configuration changed (new display, different ID)
readconfig.load_data() # remove with shift register displays, display clock loads
c = readconfig.read_config()
if c and not c == config:
config = c
disp = make_displays_shift(c)
print "new config: %i digits, ID = %i" % (config[0][0], config[0][1])
else:
time.sleep(2)
sevenseg.cleanup()