def main(*args, **kwargs):
######################################################################
# MLS DATA LOAD #
# ----------------------------------------------------------- #
# Program will load info from downloaded MLS data into DB #
# ----------------------------------------------------------- #
# * Pull info using mech_it.mls_data() #
# * Load information into Postgress database #
# db_utils.load_db(csv_file) #
######################################################################
#Pull config information from config.py
config_info = {
'l_url': config.get_l_url(),
'q_url': config.get_q_url(),
'cs_url': config.get_cs_url(),
'user': config.get_user(),
'password': config.get_password(),
'folder_out': config.get_folder_out(),
}
#Get mls data using mech_it
mech_it.get_mls_data(config_info)
''' ToDo -- add filename pull '''
#Load data into database
db_utils.load_db(filename)
def nav_login_page(driver):
"""
:param driver: selenium driver already setup and set to Sporcle website
Procedure navagates home sporcle page and logs in using the login javascript
frame. Should end on home page and logged in.
"""
#get login information from config class
user_email = config.get_username()
user_password = config.get_password()
#HTML constants for element id/class names
login_btn_id = "user-not-logged-in"
username_c_name = "usernameInput"
password_c_name = "passwordInput"
submit_btn_c_name = "log-in-button"
#click login button to open new frame to login
driver.find_element_by_id(login_btn_id).click()
#wait until login frame has loaded
WebDriverWait(driver, 30).until(
EC.presence_of_element_located((By.CLASS_NAME, username_c_name)))
#enter login information and submit login
driver.find_element_by_class_name(username_c_name).send_keys(user_email)
driver.find_element_by_class_name(password_c_name).send_keys(user_password)
submit = driver.find_element_by_class_name(submit_btn_c_name)
#submits and loads new page
timeout_catcher(driver=driver, func=submit.click)
def create_connection():
cluster = MongoClient('mongodb+srv://' + get_username() + ':' +
get_password() + '@cluster0.mfsww.mongodb.net/' +
get_db_name() + '?retryWrites=true&w=majority')
db = cluster.CleanData
col = db.tradeCandlesticks
return col
def sendEmail(toaddr: str, subject: str, body: str, filepath: str):
msg = MIMEMultipart()
msg['From'] = config.sender_email
msg['To'] = toaddr
msg['Subject'] = subject
msg.attach(MIMEText(body, 'plain'))
attachment = open(filepath, 'rb')
fileName = attachment.name.split('/')[-1]
part = MIMEBase('application', 'octet-streaam')
part.set_payload((attachment).read())
encoders.encode_base64(part)
part.add_header('Content-Disposition',
'attachment; fileName= %s' % fileName)
msg.attach(part)
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(config.sender_email, config.get_password())
text = msg.as_string()
server.sendmail(config.sender_email, toaddr, text)
server.quit()
def __init__(self, server, args):
print("Attempting to connect...")
self.server = server
self.server.login(login["user"], login["password"])
print("SMTP connected for sending mails!")
self.inbox = imaplib.IMAP4_SSL("imap.gmail.com")
self.inbox.login(login["user"], login["password"])
print("IMAP connected for receiving mails!")
self.mails = []
self.workers = {}
self.tasks_done = {}
self.key = get_password(key, 1)['password']
self.white_list = white_list
self.identifier = make_email_identifier()
self.login_adrr = login['user']
self.safebox = login
self.mail_addr = make_email_address(login['user'], self.identifier)
self.idle_time = 5
# self.tasks = {}
print('Identifier: ' + self.identifier)
print('Mail: ' + self.mail_addr)
self.do_ip = SendIP(self, self.identifier)
self.do_exec = FalseSSH(self, self.mail_addr, self.key)
self.do_tobe = MailTask(self)
self.do_update = SelfUpdate(self)
# Register an handler for the timeout
def handler(signum, frame):
print("Forever is over!")
raise Exception("end of time")
# Register the signal function handler
signal.signal(signal.SIGALRM, handler)
def __init__(self):
self.host = config.get_host()
self.port = config.get_port()
self.username = config.get_username()
self.password = config.get_password(self.host, self.port, self.username)
self.remember = True
if all((self.host, self.port, self.username, self.password)):
if connect(self.host, self.port, self.username, self.password):
sys.exit()
self.construct()
if self.host is not None:
self.host_entry.set_text(self.host)
if self.port is not None:
self.port_entry.set_text(str(self.port))
if self.username is not None:
self.username_entry.set_text(self.username)
if self.password is not None:
self.password_entry.set_text(self.password)
self.remember_checkbox.set_active(self.remember)
def login():
#Get config info
url = config.get_url()
username = config.get_user()
password = config.get_password()
driver.get(url)
username = driver.find_element_by_id('LoginCtrl_txtLoginUsername')
username.send_keys(username)
password = driver.find_element_by_id('LoginCtrl_txtPassword')
password.send_keys(password)
login = driver.find_element_by_id('LoginCtrl_btnLogin').click()
def query(sql, db=None, user=None):
conn = pymysql.connect(host=config.db_host,
user=user,
password=config.get_password(user),
db=db)
with conn.cursor() as cursor:
try:
cursor.execute(sql)
rv = (cursor.fetchall(), True)
except Exception as e:
rv = ([['Error', str(e)]], False)
conn.close()
return rv
def mega_upload_files(root, files, dest=None):
mega = Mega()
m = mega.login(config.get_username(), config.get_password())
for filename in files:
current = os.path.join(root, filename)
print("Uploading: " + str(current))
uploaded = m.upload(current)
print(m.get_upload_link(uploaded))
if dest != None:
fileRef = m.find(filename)
folder = m.find(dest)
m.move(fileRef[0], folder)
def __init__(self):
try:
print(config.get_user(), config.get_password())
self.conn = psycopg2.connect('dbname=test user={} password={}'.
format(config.get_user(), config.get_password()))
self.conn.autocommit = True
except:
print('I am unable to connect to the database.')
print("Index won't be creating")
return
cur = self.conn.cursor()
cur.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'""")
tables = cur.fetchall()
if (self.TABLE_NAME,) not in tables:
try:
cur.execute(self.command_create_table)
except Exception as err:
print('Could not create a table {}.'.format(self.TABLE_NAME))
print("PageFileAttribute index won't be creating")
print(err)
return
def load(self):
wf = self.wf
response = web.get(
'{0}/rest/api/latest/project/{1}.json?expand=plans'.format(
wf.stored_data(KEY_BAMBOO_SERVER),
wf.stored_data(KEY_BAMBOO_PROJECT)
),
auth=(wf.stored_data(KEY_USERNAME), get_password(wf, KEY_PASSWORD))
).json()
return map(
lambda plan: {
'title': plan['key'],
'subtitle': plan['name'],
'arg': plan['key'],
'copytext': '{0}: {1}'.format(plan['key'], plan['name']),
'autocomplete': plan['key'],
'valid': True
},
response['plans']['plan']
)
def load(self):
wf = self.wf
response = web.get(
'{0}/rest/api/latest/result/{1}.json'.format(
wf.stored_data(KEY_BAMBOO_SERVER),
self.branch_key
),
auth=(wf.stored_data(KEY_USERNAME), get_password(wf, KEY_PASSWORD))
).json()
return map(
lambda result: {
'title': result_key(result),
'subtitle': result['buildState'],
'arg': result_key(result),
'copytext': result_key(result),
'autocomplete': result_key(result),
'valid': True
},
response['results']['result']
)
def load(self):
wf = self.wf
project = wf.stored_data(KEY_BAMBOO_PROJECT)
response = web.get(
'{0}/rest/api/latest/plan/{1}/branch.json'.format(
wf.stored_data(KEY_BAMBOO_SERVER),
self.plan_key
),
auth=(wf.stored_data(KEY_USERNAME), get_password(wf, KEY_PASSWORD))
).json()
return map(
lambda branch: {
'title': branch_key(project, branch),
'subtitle': branch['shortName'],
'arg': branch_key(project, branch),
'copytext': branch_key(project, branch),
'autocomplete': branch_key(project, branch),
'valid': True
},
response['branches']['branch']
)
def load(self):
wf = self.wf
url = '{0}/rest/api/2/search'.format(wf.stored_data(KEY_JIRA_SERVER))
params = {'jql': 'filter={0}'.format(wf.stored_data(KEY_JIRA_FILTER))}
logger.debug('URL: %s, params: %s', url, params)
wf.logger.debug('URL: %s, params: %s', url, params)
response = web.get(
url,
params=params,
headers={
'Content-Type': 'application/json',
'Authorization': 'Basic {0}'.format(
b64encode('{0}:{1}'.format(
wf.stored_data(KEY_USERNAME),
get_password(wf, KEY_PASSWORD))))
}
).json()
# wf.logger.debug('Response: %s', response)
return map(
lambda issue: {
'title': issue['fields']['summary'],
'subtitle': issue['key'],
'arg': u'{0}: {1}'.format(
issue['key'], issue['fields']['summary']),
'copytext': u'{0}: {1}'.format(
issue['key'], issue['fields']['summary']),
'valid': True
},
response['issues']
)
parser = argparse.ArgumentParser()
parser.add_argument('root_username')
parser.add_argument('root_password')
argp = parser.parse_args()
mysql_handle = pymysql.connect(host=config.db_host,
port=3306,
user=argp.root_username,
password=argp.root_password,
database='mysql')
for name, spec in config.db_setup.items():
print(f'Setting up [{name}]')
user = spec['user']
password = config.get_password(user)
database = spec['database']
schema = spec['schema']
with mysql_handle.cursor() as cursor:
cursor.execute(f"DROP USER IF EXISTS '{user}'")
cursor.execute(f"CREATE USER '{user}'@'%' IDENTIFIED BY '{password}'")
cursor.execute(f"DROP DATABASE IF EXISTS {database}")
cursor.execute(f"CREATE DATABASE {database}")
cursor.execute(f"GRANT SELECT ON {database}.* TO '{user}'@'%'")
db_handle = pymysql.connect(host=config.db_host,
port=3306,
user=argp.root_username,
password=argp.root_password,
database=database)
from flask import Flask, render_template, redirect, request
import config
# Flask初期設定
app = Flask(__name__)
# Twitter API 初期設定
CK = config.get_consumer_key()
CS = config.get_consumer_secret()
AT = config.get_access_token()
AS = config.get_access_secret()
twitter = Twitter(auth=OAuth(AT, AS, CK, CS))
# Watson Personality Insights API 初期設定
UN = config.get_username()
PS = config.get_password()
personality_insights = PersonalityInsightsV3(version='2017-10-13',
username=UN,
password=PS)
# json一時ファイル格納場所
json_folder = join(dirname(abspath('__file__')), 'tmp/')
if not exists(json_folder):
mkdir(json_folder)
# jsonファイル名を返す
def get_file_name(type, user_name):
if type == 'tw':
return 'tweets-' + user_name + '.json'
elif type == 'an':
import sys
import smtplib
from config import get_password, get_email
msg = sys.argv[1]
smtpObj = smtplib.SMTP('smtp.gmail.com',587)
smtpObj.ehlo()
smtpObj.starttls()
smtpObj.login('*****@*****.**',get_password())
smtpObj.sendmail('*****@*****.**',get_email(),"Subject: [{}] is done!".format(msg))
smtpObj.quit()
def mega_file_listing(where=None):
mega = Mega()
m = mega.login(config.get_username(), config.get_password())
files = m.get_files()
"""
From http://julien-marchand.fr/blog/using-mega-api-with-python-examples:
h: The ID of the node ;
p: The ID of the parent node (directory) ;
u: The owner of the node ;
t: The type of the node:
0: File
1: Directory
2: Special node: Root ("Cloud Drive")
3: Special node: Inbox
4: Special node: Trash Bin
a: The attributes of the node. Currently only contains its name.
k: The key of the node (used to encrypt its content and its attributes) ;
s: The size of the node ;
ts: The time of the last modification of the node.
# Example of how the output of Mega looks like using https://github.com/richardasaurus/mega.py
files = {
u'fU8iHSqb': {u'a': {u'n': u'20130504-timelapse.mpg'}, u'h': u'fU8iHSqb', u'k': (89707223, 161142552, 573402337L, 1470622432L), u'ts': 1370116008, 'iv': (415650236, 706183423, 0, 0), u'p': u'OVV1jSKJ', u's': 2351010, 'meta_mac': (2158143175L, 4255881983L), u'u': u'BVs8yisX5G4', u't': 0, 'key': (496927595, 596465639, 2727332390L, 2852941855L, 415650236, 706183423, 2158143175L, 4255881983L)},
u'Kc91ADhR': {u'a': {u'n': u'Fraggle'}, u'h': u'Kc91ADhR', u'k': (502465564, 1834626310, 1381829007, 76144220), u'ts': 1369248504, u'p': u'uBUzRayb', u'u': u'BVs8yisX5G4', u't': 1, 'key': (502465564, 1834626310, 1381829007, 76144220)},
u'DB8QBbDY': {u'a': {'n': 'Rubbish Bin'}, u'h': u'DB8QBbDY', u'k': u'', u'ts': 1361396027, u'p': u'', u'u': u'BVs8yisX5G4', u't': 4},
u'GYky1L7I': {u'a': {u'n': u'nop'}, u'h': u'GYky1L7I', u'k': (1062046627, 1804392549, 144264946, 7166313), u'ts': 1369248720, u'p': u'uBUzRayb', u'u': u'BVs8yisX5G4', u't': 1, 'key': (1062046627, 1804392549, 144264946, 7166313)},
u'uA0WwDDC': {u'a': {u'n': u'FR1X09.avi'}, u'h': u'uA0WwDDC', u'k': (1419330019, 761322817, 2094550110L, 230172585L), u'ts': 1370113885, 'iv': (769934036, 1468969908, 0, 0), u'p': u'Kc91ADhR', u's': 266778624, 'meta_mac': (2710469947L, 2279034223L), u'u': u'BVs8yisX5G4', u't': 0, 'key': (2038235959, 2062446325, 3713413477L, 2322557638L, 769934036, 1468969908, 2710469947L, 2279034223L)},
u'LZ0wkRiL': {u'a': {u'n': u'FR1X04.avi'}, u'h': u'LZ0wkRiL', u'k': (673266462, 1332906007, 1031887772L, 2010887260L), u'ts': 1369254134, 'iv': (1641340076, 154211803, 0, 0), u'p': u'Kc91ADhR', u's': 279056384, 'meta_mac': (4128886324L, 3650644366L), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1240852402, 1178836428, 3415774632L, 2923683282L, 1641340076, 154211803, 4128886324L, 3650644366L)},
u'uBUzRayb': {u'a': {'n': 'Cloud Drive'}, u'h': u'uBUzRayb', u'k': u'', u'ts': 1361396027, u'p': u'', u'u': u'BVs8yisX5G4', u't': 2},
u'XQkWzCzD': {u'a': {u'n': u'FR1X03.avi'}, u'h': u'XQkWzCzD', u'k': (1822030122, 1461683990, 104100297, 171436116L), u'ts': 1369251984, 'iv': (739456022, 1878020475, 0, 0), u'p': u'Kc91ADhR', u's': 281018368, 'meta_mac': (172116505, 3745444068L), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1082838332, 955242093, 209075152, 3574143152L, 739456022, 1878020475, 172116505, 3745444068L)},
u'6ZtkhaQD': {u'a': {u'n': u'20130505-timelapse.mpg'}, u'h': u'6ZtkhaQD', u'k': (1828039708, 1316190495, 1207177820L, 407323452), u'ts': 1370116209, 'iv': (967685081, 75438788, 0, 0), u'p': u'OVV1jSKJ', u's': 9489254, 'meta_mac': (4200315324L, 1578827400), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1431838661, 1242325979, 3182406624L, 1180548532, 967685081, 75438788, 4200315324L, 1578827400)},
u'bMdEhZ5Q': {u'a': {'n': 'Inbox'}, u'h': u'bMdEhZ5Q', u'k': u'', u'ts': 1361396027, u'p': u'', u'u': u'BVs8yisX5G4', u't': 3},
u'jVlyVIqb': {u'a': {u'n': u'FR1X05.avi'}, u'h': u'jVlyVIqb', u'k': (112256182, 218491874, 982809646, 1581866576), u'ts': 1369255313, 'iv': (1723776206, 1447320173, 0, 0), u'p': u'Kc91ADhR', u's': 281335808, 'meta_mac': (889945617, 2012844929), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1611552888, 1531031951, 262079039, 699452881, 1723776206, 1447320173, 889945617, 2012844929)},
u'nEtiSDbb': {u'a': {u'n': u'FR1X07.avi'}, u'h': u'nEtiSDbb', u'k': (225167755, 1671501548, 591928764, 63089283), u'ts': 1370111321, 'iv': (1300145638, 2013187368, 0, 0), u'p': u'Kc91ADhR', u's': 281380864, 'meta_mac': (979318925, 1617691340), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1075145837, 341826500, 420947249, 1672390735, 1300145638, 2013187368, 979318925, 1617691340)},
u'XA0CmS5R': {u'a': {u'n': u'FR1X06.avi'}, u'h': u'XA0CmS5R', u'k': (1811000465, 757679140, 1277418342L, 1150075587L), u'ts': 1370110241, 'iv': (1120378468, 1265751793, 0, 0), u'p': u'Kc91ADhR', u's': 281528320, 'meta_mac': (3583748394L, 3453581643L), u'u': u'BVs8yisX5G4', u't': 0, 'key': (691417845, 1717080789, 2579001932L, 2304097160L, 1120378468, 1265751793, 3583748394L, 3453581643L)},
u'nNV3wb6L': {u'a': {u'n': u'FR1X08.avi'}, u'h': u'nNV3wb6L', u'k': (857321979, 113754691, 550525577L, 2005802334), u'ts': 1370112438, 'iv': (2091818972, 474711454, 0, 0), u'p': u'Kc91ADhR', u's': 280852480, 'meta_mac': (3944887044L, 997593366), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1337404967, 445401053, 3421661581L, 1291324488, 2091818972, 474711454, 3944887044L, 997593366)},
u'bRlQGKaa': {u'a': {u'n': u'FR1X02.avi'}, u'h': u'bRlQGKaa', u'k': (1834413953, 323343779, 1972206936L, 926199209), u'ts': 1369250867, 'iv': (1923007029, 226044668, 0, 0), u'p': u'Kc91ADhR', u's': 278165504, 'meta_mac': (3813458802L, 2051077149), u'u': u'BVs8yisX5G4', u't': 0, 'key': (533212596, 507314015, 2529279530L, 1299470772, 1923007029, 226044668, 3813458802L, 2051077149)},
u'OIcyBYBI': {u'a': {u'n': u'20130506-timelapse.mpg'}, u'h': u'OIcyBYBI', u'k': (1803067817, 1556254904, 1135565474L, 1855076844), u'ts': 1370116238, 'iv': (7431726, 229302727, 0, 0), u'p': u'OVV1jSKJ', u's': 8058367, 'meta_mac': (2271367677L, 1318499050), u'u': u'BVs8yisX5G4', u't': 0, 'key': (1795803015, 1365799295, 3301772127L, 537168646, 7431726, 229302727, 2271367677L, 1318499050)},
u'3FlxEKjb': {u'a': {u'n': u'FR1X01.avi'}, u'h': u'3FlxEKjb', u'k': (1928705133, 1361926709, 871495869L, 2008746031), u'ts': 1369249819, 'iv': (1906187233, 153356539, 0, 0), u'p': u'Kc91ADhR', u's': 283379712, 'meta_mac': (2759251385L, 2095529633), u'u': u'BVs8yisX5G4', u't': 0, 'key': (57388940, 1477006030, 2542218500L, 190592654, 1906187233, 153356539, 2759251385L, 2095529633)},
u'OVV1jSKJ': {u'a': {u'n': u'timelapse'}, u'h': u'OVV1jSKJ', u'k': (1075652685, 899584718, 471199899, 32827924), u'ts': 1370115910, u'p': u'uBUzRayb', u'u': u'BVs8yisX5G4', u't': 1, 'key': (1075652685, 899584718, 471199899, 32827924)},
u'CBEHyJQC': {u'a': {u'n': u'FR1X10.avi'}, u'h': u'CBEHyJQC', u'k': (1095843221, 846021889, 1205770022L, 1642369272), u'ts': 1370115188, 'iv': (1137674974, 26639331, 0, 0), u'p': u'Kc91ADhR', u's': 266668032, 'meta_mac': (2177042245L, 418704470), u'u': u'BVs8yisX5G4', u't': 0, 'key': (43956043, 872103650, 3323826275L, 2031123630, 1137674974, 26639331, 2177042245L, 418704470)}
}
"""
listing = []
directoryDict = {}
# Get directories
for index in files:
currentFile = files[index]
try:
filename = currentFile['a']['n']
directory = currentFile['t']
current = currentFile['h']
if directory != int(1): # Save only directories
continue
else:
#print("SAVING " + str(filename) + " in dict " + str(current))
directoryDict[current] = filename
except:
pass
for index in files:
currentFile = files[index]
parent = currentFile['p']
current = currentFile['h']
filename = currentFile['a']['n']
#print("PARENT ID: " + str(parent))
#print("CURRENT ID: " + str(current))
try:
if where == None:
listing.append(str(directoryDict[parent]) + "/" + str(filename))
else:
if directoryDict[parent] == where:
listing.append(str(filename))
except:
pass
return listing
