def __init__(self):
self.TRACK_FILE_INI = "track.ini"
self.SETTINGS_SECTION = "SETTINGS"
self.TRACK_SECTION = "TRACK"
self.BEST_MODEL_FILE_INI = "best_model.ini"
self.BEST_MODEL_SECTION = "BEST_MODEL"
self.TRACK_DATA_DIR = "track_data"
self.TRACK_DATA_FILE_DIR = "." + os.sep + self.TRACK_DATA_DIR
self.converter = PointConverter()
self.ini_cfg_track = configparser.RawConfigParser()
self.ini_track_file = self.TRACK_DATA_FILE_DIR + os.sep + self.TRACK_FILE_INI
self.ini_cfg_best = configparser.RawConfigParser()
self.ini_best_file = self.TRACK_DATA_FILE_DIR + os.sep + self.BEST_MODEL_FILE_INI
self.make_dir(self.TRACK_DATA_FILE_DIR)
if not self.exist_ini(self.ini_cfg_track, self.ini_track_file):
self.add_settings_section(self.ini_cfg_track, self.ini_track_file)
if not self.exist_ini(self.ini_cfg_best, self.ini_best_file):
self.add_best_model_section(self.ini_cfg_best, self.ini_best_file)
def write(self, amazon_object):
self.ensure_config_files_exist()
assert (
self.profile is not None
), "Can not store config/credentials if the AWS_PROFILE is None."
# Write to the configuration file
profile = Configuration.config_profile(self.profile)
config_parser = configparser.RawConfigParser()
config_parser.read(self.config_file)
if not config_parser.has_section(profile):
config_parser.add_section(profile)
config_parser.set(profile, 'region', self.region)
config_parser.set(profile, 'google_config.ask_role', self.ask_role)
config_parser.set(profile, 'google_config.keyring', self.keyring)
config_parser.set(profile, 'google_config.duration', self.duration)
config_parser.set(profile, 'google_config.google_idp_id', self.idp_id)
config_parser.set(profile, 'google_config.role_arn', self.role_arn)
config_parser.set(profile, 'google_config.google_sp_id', self.sp_id)
config_parser.set(profile, 'google_config.u2f_disabled',
self.u2f_disabled)
config_parser.set(profile, 'google_config.google_username',
self.username)
config_parser.set(profile, 'google_config.bg_response',
self.bg_response)
with open(self.config_file, 'w+') as f:
config_parser.write(f)
# Write to the credentials file (only if we have credentials)
if amazon_object is not None:
credentials_parser = configparser.RawConfigParser()
credentials_parser.read(self.credentials_file)
if not credentials_parser.has_section(self.profile):
credentials_parser.add_section(self.profile)
credentials_parser.set(self.profile, 'aws_access_key_id',
amazon_object.access_key_id)
credentials_parser.set(self.profile, 'aws_secret_access_key',
amazon_object.secret_access_key)
credentials_parser.set(self.profile, 'aws_security_token',
amazon_object.session_token)
credentials_parser.set(
self.profile, 'aws_session_expiration',
amazon_object.expiration.strftime('%Y-%m-%dT%H:%M:%S%z'))
credentials_parser.set(self.profile, 'aws_session_token',
amazon_object.session_token)
with open(self.credentials_file, 'w+') as f:
credentials_parser.write(f)
if self.__saml_cache is not None:
with open(self.saml_cache_file, 'w') as f:
f.write(self.__saml_cache.decode("utf-8"))
def packages(self):
"""Return a list of :class:`.package.Package` in the source."""
rval = []
# Use raw parser so no value interpolation takes place.
parser = configparser.RawConfigParser()
aggregate_file = os.path.join(self.clone.working_dir,
AGGREGATE_DATA_FILE)
parser.read(aggregate_file)
for index_file in self.package_index_files():
relative_path = index_file[len(self.clone.working_dir) + 1:]
directory = os.path.dirname(relative_path)
lines = []
with open(index_file) as f:
lines = [line.rstrip('\n') for line in f]
for url in lines:
pkg_name = name_from_path(url)
agg_key = os.path.join(directory, pkg_name)
metadata = {}
if parser.has_section(agg_key):
metadata = {
key: value
for key, value in parser.items(agg_key)
}
package = Package(git_url=url,
source=self.name,
directory=directory,
metadata=metadata)
rval.append(package)
return rval
def checkAuthenticatedSender(sender, body, asenderRegEx):
pLog("Check Authenticated Sender")
#print(asenderRegEx)
asender = re.findall(r'\(Authenticated\ssender\:\s{0,10}([^)]*)\)\n\s*',
body)
if (len(asender) > 0):
asender = asender[0]
else:
asender = None
if asenderRegEx != None and re.match(asenderRegEx, str(asender)) is None:
pLog("AuthenticatedSender is required but failed")
#Send Abuse Mail
config = ConfigParser.RawConfigParser()
config.read('defaults.cfg')
msg = "Mail from %s send from Authenticated Sender %s\r\n\r\nThe Mail Header must match!\r\nMail don't send, please check the Config or Contact your Mail-Server-Admin per E-Mail to %s" % (
sender, asender, config.get('SendAbuse', 'from'))
sendMail(
sender,
"Mail from " + sender + " with wrong Authenticated Sender Header",
msg)
pLog("AuthenticatedSender %s dont match on %s" %
(asender, asenderRegEx))
return False
pLog("AuthenticatedSender Match")
return True
def read(self, profile):
self.ensure_config_files_exist()
# Shortening Convenience functions
coalesce = util.Util.coalesce
unicode_to_string = util.Util.unicode_to_string_if_needed
profile_string = Configuration.config_profile(profile)
config_parser = configparser.RawConfigParser()
config_parser.read(self.config_file)
if config_parser.has_section(profile_string):
self.profile = profile
# Ask Role
read_ask_role = config_parser[profile_string].getboolean('google_config.ask_role', None)
self.ask_role = coalesce(read_ask_role, self.ask_role)
# Keyring
read_keyring = config_parser[profile_string].getboolean('google_config.keyring', None)
self.keyring = coalesce(read_keyring, self.keyring)
# Duration
read_duration = config_parser[profile_string].getint('google_config.duration', None)
self.duration = coalesce(read_duration, self.duration)
# IDP ID
read_idp_id = unicode_to_string(config_parser[profile_string].get('google_config.google_idp_id', None))
self.idp_id = coalesce(read_idp_id, self.idp_id)
# Region
read_region = unicode_to_string(config_parser[profile_string].get('region', None))
self.region = coalesce(read_region, self.region)
# Role ARN
read_role_arn = unicode_to_string(config_parser[profile_string].get('google_config.role_arn', None))
self.role_arn = coalesce(read_role_arn, self.role_arn)
# SAML Cache
read_saml_cache = unicode_to_string(config_parser[profile_string].get('google_config.google_saml_cache', None))
self.__saml_cache = coalesce(read_saml_cache, self.__saml_cache)
# SP ID
read_sp_id = unicode_to_string(config_parser[profile_string].get('google_config.google_sp_id', None))
self.sp_id = coalesce(read_sp_id, self.sp_id)
# U2F Disabled
read_u2f_disabled = config_parser[profile_string].getboolean('google_config.u2f_disabled', None)
self.u2f_disabled = coalesce(read_u2f_disabled, self.u2f_disabled)
# Username
read_username = unicode_to_string(config_parser[profile_string].get('google_config.google_username', None))
self.username = coalesce(read_username, self.username)
# SAML Cache
with open(self.saml_cache_file, 'r') as f:
self.__saml_cache = f.read().encode("utf-8")
def addReceived(body, ruleId):
config = ConfigParser.RawConfigParser()
config.read('defaults.cfg')
start = body.find("Received")
receivedmsg = "Received: mailChain (" + config.get(
'Mail', 'ReceivedName') + ") #" + str(ruleId) + "\r\n"
body = body[0:start] + receivedmsg + body[start:]
pLog("Add Received Header to Mail")
return body
def handle(to, sender, subject, body):
try:
return runBasic(to, sender, subject, body)
except:
pLog("Something go Wrong, Mail sendet")
errorStr = str(sys.exc_info())
config = ConfigParser.RawConfigParser()
config.read('defaults.cfg')
sendMail(
sender, "Mail cant not sendet",
"Hello, you try to send a Mail, we cant send this Mail because an error appear. Our Team get the Information, please try it again later!"
)
sendMail(config.get('SendAbuse', 'from'), "MailChain Error",
"So following error appear:\r\n\r\n" + errorStr)
return "550 Something go wrong"
def sendMail(to, subject, msgContext):
pLog("Send Server Mail")
config = ConfigParser.RawConfigParser()
config.read('defaults.cfg')
msg = Message(subject, fromaddr=config.get('SendAbuse', 'from'), to=to)
if (to != config.get('SendAbuse', 'from')):
msg.bcc = config.get('SendAbuse', 'from')
msg.body = msgContext
msg.date = time.time()
msg.charset = "utf-8"
mail = Mail(config.get('SendAbuse', 'server'),
port=config.get('SendAbuse', 'port'),
username=config.get('SendAbuse', 'user'),
password=config.get('SendAbuse', 'pass'),
use_tls=False,
use_ssl=False,
debug_level=None)
mail.send(msg)
def run_pants_with_workdir_without_waiting(self, command, workdir, config=None, extra_env=None,
build_root=None, print_exception_stacktrace=True,
**kwargs):
args = [
'--no-pantsrc',
'--pants-workdir={}'.format(workdir),
'--kill-nailguns',
'--print-exception-stacktrace={}'.format(print_exception_stacktrace),
]
if self.hermetic():
args.extend(['--pants-config-files=[]',
# Turn off cache globally. A hermetic integration test shouldn't rely on cache,
# or we have no idea if it's actually testing anything.
'--no-cache-read', '--no-cache-write',
# Turn cache on just for tool bootstrapping, for performance.
'--cache-bootstrap-read', '--cache-bootstrap-write'
])
if config:
config_data = config.copy()
# TODO(python3port): RawConfigParser is legacy. Investigate updating to modern API.
ini = configparser.RawConfigParser(defaults=config_data.pop('DEFAULT', None))
for section, section_config in config_data.items():
ini.add_section(section)
for key, value in section_config.items():
ini.set(section, key, value)
ini_file_name = os.path.join(workdir, 'pants.ini')
with safe_open(ini_file_name, mode='w') as fp:
ini.write(fp)
args.append('--pants-config-files=' + ini_file_name)
pants_script = os.path.join(build_root or get_buildroot(), self.PANTS_SCRIPT_NAME)
# Permit usage of shell=True and string-based commands to allow e.g. `./pants | head`.
if kwargs.get('shell') is True:
assert not isinstance(command, list), 'must pass command as a string when using shell=True'
pants_command = ' '.join([pants_script, ' '.join(args), command])
else:
pants_command = [pants_script] + args + command
# Only whitelisted entries will be included in the environment if hermetic=True.
if self.hermetic():
env = dict()
# With an empty environment, we would generally get the true underlying system default
# encoding, which is unlikely to be what we want (it's generally ASCII, still). So we
# explicitly set an encoding here.
env['LC_ALL'] = 'en_US.UTF-8'
for h in self.hermetic_env_whitelist():
value = os.getenv(h)
if value is not None:
env[h] = value
hermetic_env = os.getenv('HERMETIC_ENV')
if hermetic_env:
for h in hermetic_env.strip(',').split(','):
env[h] = os.getenv(h)
else:
env = os.environ.copy()
if extra_env:
env.update(extra_env)
# Don't overwrite the profile of this process in the called process.
# Instead, write the profile into a sibling file.
if env.get('PANTS_PROFILE'):
prof = '{}.{}'.format(env['PANTS_PROFILE'], self._get_profile_disambiguator())
env['PANTS_PROFILE'] = prof
# Make a note the subprocess command, so the user can correctly interpret the profile files.
with open('{}.cmd'.format(prof), 'w') as fp:
fp.write(' '.join(pants_command))
return PantsJoinHandle(
pants_command,
subprocess.Popen(
pants_command,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kwargs
),
workdir
)
# Written by https://www.reddit.com/user/CJDAM/
# Written in Python 3
import format_func
import pip
pip.main(['install', 'configparser==3.5.0'])
from backports import configparser
def install(package):
pip.main(['install', package])
reqParse = configparser.RawConfigParser()
reqFilePath = r'required_modules.txt'
reqParse.read(reqFilePath)
for each_section in reqParse.sections():
for (each_key, each_val) in reqParse.items(each_section):
install(each_val)
from filecmp import dircmp
from logging import getLogger, ERROR
getLogger('googleapiclient.discovery_cache').setLevel(
ERROR) # Disables annoying Google API logging errors
import time
import sys
def add_stadia_data(update_external=True):
# Variables
url_for_login = '******'
url_stadia_store = 'https://stadia.google.com/store/list/3'
# Get credentials to login
config = configparser.RawConfigParser()
config.read('config.properties')
username = config.get('Google Account', 'google_user')
password = config.get('Google Account', 'google_pass')
# WebDriver Initialization. Stadia only works with Chrome
if platform == 'darwin':
driver = webdriver.Chrome('drivers/chromedriver')
elif platform == 'win32':
driver = webdriver.Chrome('drivers/chromedriver.exe')
# Login through a website that allows you to log in with Google data
driver.get(url_for_login)
driver.find_element_by_xpath('//*[@id="openid-buttons"]/button[1]').click()
driver.find_element_by_xpath('//input[@type="email"]').send_keys(username)
driver.find_element_by_xpath('//*[@id="identifierNext"]').click()
time.sleep(5)
driver.find_element_by_xpath('//input[@type="password"]').send_keys(
password)
driver.find_element_by_xpath('//*[@id="passwordNext"]').click()
time.sleep(3)
# Open Stadia Store
driver.get(url_stadia_store)
driver.maximize_window()
time.sleep(3)
# SCRAPER
games_info = {'title': [], 'price': [], 'type_game': []}
items = driver.find_elements_by_class_name('h6J22d.QAAyWd')
print('Tenim {} elements per extreure la informació.'.format(len(items)))
for item in items:
title = item.find_element_by_class_name('T2oslb.zHGix').text
price = item.find_element_by_class_name('eoQBOd').text
typee = item.find_element_by_class_name('vaa0f.eoQBOd').text
games_info['title'].append(title)
games_info['price'].append(price)
games_info['type_game'].append(typee)
# Close browser
driver.quit()
# Update file with external data
df = pd.DataFrame(games_info)
if update_external:
df = df[df['type_game'].isin(['Juego', 'Game', 'Joc'])]
df['price'] = df['price'].apply(lambda x: x.split('\n')[0])
df.loc[df['price'].isin([
'Game', 'Juego', 'Joc', 'Claimed', 'Reclamat', 'Obtenido', 'Free',
'Gratis', 'Purchased', 'Comprado', 'S\'ha comprat'
]), 'price'] = 'Unknown'
df_external = pd.read_csv('output_data/stadia_games_info.csv')
if 'price_stadia' in df_external.columns:
df_external.drop('price_stadia', axis=1, inplace=True)
df_stadia_titles_match = pd.DataFrame(
columns=['title', 'price_stadia'])
for idx, row in df.iterrows():
sim = 0
tit_sim = ''
for tit2 in df_external['title'].unique():
new_score = SequenceMatcher(None, row['title'].lower(),
tit2.lower()).ratio()
if new_score > sim:
sim = new_score
tit_sim = tit2
if sim >= .5:
df_stadia_titles_match = df_stadia_titles_match.append(
pd.DataFrame({
'title': [tit_sim],
'price_stadia': row['price']
}))
else:
df_stadia_titles_match = df_stadia_titles_match.append(
pd.DataFrame({
'title': [row['title']],
'price_stadia': [row['price']]
}))
df_external.merge(df_stadia_titles_match, on='title',
how='outer').to_csv(
'output_data/stadia_games_info.csv',
index=False)
else:
df.to_csv('output_data/stadia_games_info.csv', index=False)
def runBasic(to, sender, subject, body):
pLog("Incoming mail from %s" % sender)
#Read Config
config = ConfigParser.RawConfigParser()
config.read('defaults.cfg')
#Dump Mail to File System
if config.get('Mail', 'dump') == True:
dumpMail(to, sender, subject, body)
#Connect to Database
try:
con = mdb.connect(config.get('MYSQL', 'host'),
config.get('MYSQL', 'user'),
config.get('MYSQL', 'pass'),
config.get('MYSQL', 'db'))
cur = con.cursor()
except:
pLog("FATAL!!! MYSQL Connection cant create, can`t run without MYSQL")
pLog(str(sys.exc_info()))
sys.exit()
#Get Rules
cur.execute("SELECT * FROM `mailChain` ORDER BY `mailChain`.`prio` ASC")
chains = cur.fetchall()
send = False
for rule in chains:
if send == True:
continue
check = checkChain(rule, to, sender, subject)
pLog("Chain %s is %s" % ((rule[0]), (check)))
if check == True:
#Send Mail
send = True #Mail alwady send
tosend = True #Send Mail?
#Do Stuff with Mail
#Add Received Header to Mail
if config.get('Mail', 'addReceived'):
body = addReceived(body, rule[0])
#Check Authenticated Header
if config.get('Mail', 'mailAuthenticatedSender'):
pLog("Check Authentication Sender for %s" % sender)
cur.execute(
'SELECT * FROM `mailAuthenticatedSender` WHERE `from` = "%s"'
% sender)
asenderregex = cur.fetchone()
if asenderregex != None:
retAuth = checkAuthenticatedSender(sender, body,
str(asenderregex[2]))
if retAuth == False:
tosend = False
return "503 Autoriced Header is wrong"
if config.get('Mail', 'spamc') == True:
res = checkSpamc(sender, body)
if res == False:
pLog("Mail as Spam detectet and reject")
tosend = False
return "550 Spam detect"
pLog("No Spam, send mail")
tosend = False
if tosend == True:
pLog("Redirect Mail")
if config.get('Mail', 'sendLog'):
logMail(to, sender, subject, body, con)
#Remove Authenticate Header:
if rule[11] == True:
body = removeAuthenticatedSende(body)
if rule[6] != None:
pLog("Send Mail ofer SMTP Relay %s" % rule[6])
smtp = SMTP()
port = 25
if rule[7] != None:
port = int(rule[7])
smtp.connect(str(rule[6]), int(port))
if rule[8] != None and rule[9] != None:
smtp.login(rule[8], rule[9])
smtp.sendmail(sender, to, body)
smtp.quit()
return "250 OK"
if rule[10] != None:
pLog("Make HTTP Call to %s" % rule[10])
payload = {
'to': to,
'sender': sender,
'subject': subject,
'body': body
}
r = requests.post(rule[10], data=payload)
return "250 OK - POST Request send"
else:
return "550 Something go wrong"
