def parse_cfg_with_sections(stream):
"""Return as dict of dict of ...
"""
#Experimental:
"""
ConfigParser sections become sub-sub sections when separated by dots.
[foo.bar]
baz = 42
is equivalent to JSON
{"foo": {"bar": {"baz": 42}}}
"""
content = stream.read()
result = dict()
try:
jdict = json.loads(NativeIO(content).read())
return jdict
except ValueError:
pass #logger.exception('Could not parse stream as JSON.')
try:
config = ConfigParser(strict=False)
config.optionxform = str
config.read_file(NativeIO(content))
sections = config.sections()
for sec in sections:
result[sec] = dict(config.items(sec))
return result
except:
raise
def main():
# Logging configuration
logging.config.fileConfig('/app/config/log_config.ini')
# Reading configuration file
global config
config._interpolation = configparser.ExtendedInterpolation()
f = open('/app/config/app_config.ini')
try:
config.read_file(f)
finally:
f.close()
start_http_server(8000)
# Generate Prometheus Metrics
metrics = get_available_metrics()
prometheus_metrics = generate_prometheus_metrics(metrics=metrics)
# Endless loop gathering metrics (sleep's for time defined on config file)
while True:
get_name_mapping()
metrics = get_available_metrics()
get_metric_value(prometheus_metrics=prometheus_metrics,
metrics=metrics)
time.sleep(float(config.get('EXPORTER_CONFIG', 'refresh_time')))
def main():
logging.config.fileConfig('conf/logging.conf')
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('key', help='key of JIRA issue')
parser.add_argument('--type',
choices=['task', 'bug'],
default='task',
help='type to print')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--mode',
choices=['preview', 'print'],
default='preview',
help='just save or print directly')
args = parser.parse_args()
config = configparser.ConfigParser()
with open('conf/live.conf') as f:
config.read_file(f)
try:
pdf = create_card(args.key, args.type, config['jira'], args.debug)
if args.mode == 'print':
printer_name = config['printer']['name']
printer_tray = config['printer']['tray']
logger.info('Printing card for %s(%s) on %s: "%s"', args.key,
args.type, printer_name, pdf)
print_pdf(pdf, printer_name, printer_tray)
except Exception as e:
logger.exception(e)
sys.exit(1)
def main():
logging.config.fileConfig('conf/logging.conf')
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('key', help='key of JIRA issue')
parser.add_argument('--type', choices=['task', 'bug'],
default='task', help='type to print')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--mode', choices=['preview', 'print'],
default='preview', help='just save or print directly')
args = parser.parse_args()
config = configparser.ConfigParser()
with open('conf/live.conf') as f:
config.read_file(f)
try:
pdf = create_card(args.key, args.type, config['jira'], args.debug)
if args.mode == 'print':
printer_name = config['printer']['name']
printer_tray = config['printer']['tray']
logger.info('Printing card for %s(%s) on %s: "%s"',
args.key,
args.type,
printer_name,
pdf)
print_pdf(pdf, printer_name, printer_tray)
except Exception as e:
logger.exception(e)
sys.exit(1)
def play_game():
config = configparser.ConfigParser()
with open(LOG_CONFIG_FILE) as f:
config.read_file(f)
logging.config.fileConfig(config)
logging.info('Logger configured')
game = akeldama.game.window.GameWindow()
pyglet.app.run()
def read_config(logger):
try:
config = ConfigParser(interpolation=ExtendedInterpolation(),
delimiters=(':', ))
config.read_file(open(os.path.join('settings', 'config.cfg')))
logger.info(config.get('info', 'config_read'))
return config
except Exception:
logger.error('Config is not declared.')
raise Exception
def collate_configs(filenames, defaults):
_log.debug("Loading configuration files: %r", filenames)
config = ConfigParser(defaults)
for filename in filenames:
with open(filename, 'r') as config_file:
config.read_file(config_file)
return config
def get_config(file_=CONFIG):
"""Get configuration.
:param str file_: file
:returns: config
:rtype: ConfigParser
"""
config = configparser.ConfigParser(allow_no_value=True)
config.read_file(open(file_))
return config
def create_app():
application = Flask(__name__)
logging.config.fileConfig('conf/logging.conf')
config = configparser.ConfigParser()
with open('conf/live.conf') as f:
config.read_file(f)
application.config['file'] = config
return application
def create_app():
application = Flask(__name__)
logging.config.fileConfig('conf/logging.conf')
config = configparser.ConfigParser()
with open('conf/live.conf') as f:
config.read_file(f)
application.config['file'] = config
return application
def parse_config(config_file: Iterable[str]) -> configparser.ConfigParser:
"""Parse the configuration file.
Args:
config_file:
The file to parse
"""
_logger.debug("Reading config file %s", config_file)
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
config.read_file(config_file)
_logger.debug("Parsed config file: %s", config)
return config
def _config_provided(filepath):
config = configparser.ConfigParser()
try:
with open(filepath) as f_prov:
config.read_file(f_prov)
if not config.options('Microsoft Teams'):
raise MissingConnectorConfigKeyException('missing connector key in provided config')
except configparser.NoSectionError:
raise MissingConnectorConfigKeyException('missing required Microsoft Teams / '
'connector key in provided config')
return config
def __init__(self, config_path):
# Setup config with defaults.
config = configparser.ConfigParser()
config['myapp'] = {}
config['myapp']['fullscreen'] = 'false'
config['myapp']['timeout'] = '60'
# Update config from file.
with open(config_path, 'r', encoding='utf-8') as config_file:
config.read_file(config_file)
self.connection = config.get('myapp', 'connection')
self.is_fullscreen = config.getboolean('myapp', 'fullscreen')
self.timeout = config.getint('myapp', 'timeout')
def load_config(f: TextIO) -> argparse.Namespace:
"""Load the configuration file with correct parameter data types.
Args:
f: a config file opened in text mode
Returns:
conf: a Namespace object with the loaded settings
"""
# Interpolation is used e.g. for expanding the log file name
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation()
)
config.read_file(f)
# Read regex as a byte-string
regex = literal_eval("b'{}'".format(config.get("parser", "regex", raw=True)))
variables = validate_regex(regex)
# Load group_by related options
group = Group.from_config(config.get("parser", "group_by", fallback=None))
group.validate(variables)
# Hardcode the filename template, with {group} and {date} to be substituted when
# writing to disk.
config["DEFAULT"][
"filename"
] = "${device:station}_${device:name}{group}_{date:%Y-%m-%d_%H-%M-%S}.npz"
# Flatten the structure and convert the types of the parameters
conf = dict(
station=config.get("device", "station"),
device=config.get("device", "name"),
host=config.get("device", "host"),
port=config.getint("device", "port"),
timeout=config.getint("device", "timeout", fallback=None),
regex=regex,
group=group,
pack_length=config.getint("parser", "pack_length"),
dest_dir=config.get("parser", "destination"),
filename=config.get("DEFAULT", "filename"),
log_level=config.get("logging", "level"),
log_file=config.get("logging", "file"),
)
# Convert the dictionary to a Namespace object, to enable .attribute access
conf = argparse.Namespace(**conf)
return conf
def try_conf_file(file_path: pathlib.Path) -> Optional[Dict[str, Any]]:
output: Dict[str, Any] = {}
config = configparser.ConfigParser()
try:
with file_path.open() as fh:
config.read_file(fh)
logger.info("%s found", file_path)
for key, value in config["Default"].items():
output[key] = value
except FileNotFoundError as e:
logger.info("%s not found: %s", file_path, str(e))
return None
return output
def execute():
path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'config.cfg')
print('Loading config file from {}'.format(path))
try:
config = configparser.ConfigParser()
with open(path) as f:
config.read_file(f)
except:
print('Failed to load config.')
exit(1)
logging.config.fileConfig(config)
arguments = gather_arguments(config)
arguments.entry_point(arguments)
def main():
parser = argparse.ArgumentParser(description="BPCBot: to make our digital lives easier")
parser.add_argument("-c", "--config", default="./bpcbot.conf", help="Configuration file")
parser.add_argument("-d", "--daemon", action="store_true", help="Run as daemon")
args = parser.parse_args()
config = configparser.ConfigParser()
config.read_file(open(args.config))
bpcbot_start_dir = os.path.dirname(os.path.realpath(__file__))
log_conf_file = config.get('bpcbot', 'log-configuration-file', fallback=bpcbot_start_dir + '/bpcbot-log.conf')
skills = config.get('bpcbot', 'skills', fallback=bpcbot_start_dir + '/skills')
api_token = config.get('bpcbot', 'api-token')
bpcbot = BPCBot(skills, api_token)
run_in_background(bpcbot, log_conf_file, bpcbot_start_dir) if args.daemon else run_in_foreground(bpcbot, log_conf_file)
def _main(argv=None):
# read initial config file
# logging.config.fileConfig('logging.conf')
args = docopt(__doc__, argv=argv, help=True)
# Make a dictionary of supplied option values
# Take care to remove None values without removing False's or O's
siphon_off = ['--help']
opt_values = [
(k.replace('-', ''), v) for k, v in args.items()
if v is not None and k.startswith('--') and k not in siphon_off
]
opt_values = dict(opt_values)
cfg_file_name = 'email_gen.cfg'
try:
config = ConfigParser()
# Definitely use with-blocks to limit resource acquisition.
with open(cfg_file_name, 'r') as cf:
config.read_file(cf)
cfg_values = dict(config['Default'])
except Exception:
logging.warn("Could not read configuration file", exc_info=True)
# We will exit without the cfg.
# IT IS BEST TO FAIL-FAST HERE. You don't want to think too much about
# second-guessing the contents of a missing CFG without somehow involving
# a human.
# You can always put in code to auto-generate a defaulted CFG TEMPLATE
# to help the human.
sys.exit()
# Now we are happy from operational perspective: there are two ways to source
# configuration settings: via command line options, and via a configuration file.
# You'll find that having both is necessary when you have highly configurable
# software.
cfg = ChainMap(opt_values, cfg_values)
sim = EmailSimulator(cfg)
sim.generate()
def read_config():
"""Read configuration file.
:returns: (email, send_from) tuple or None
:raises: ConfigError
"""
if not os.path.isfile(CONFIG_FILE):
return None
with open(CONFIG_FILE, mode="r") as f:
config = configparser.ConfigParser(default_section="url2kindle")
config.read_file(f)
try:
email = config.get("url2kindle", "email")
except configparser.Error as e:
raise ConfigError(
"Kindle email address not found in configuration file") from e
send_from = config.get("url2kindle", "from", fallback=None)
return (email, send_from)
def initialisation():
try:
config = ConfigParser(interpolation=ExtendedInterpolation(),
delimiters=(':'))
config.read_file(open(path.join('settings', 'config.cfg')))
except Exception:
logger.error('Config is not declared.')
raise Exception
logger.info(config.get('info', 'config_read'))
if not os.path.exists(config.get('path', 'mapping')):
logger.error(
config.get('error', 'output_columns_path').format(
config.get('path', 'mapping')))
raise Exception
try:
output_columns_map = get_output_map(config)
except Exception:
logger.error(
config.get('error',
'output_columns').format(config.get('path', 'mapping')))
raise Exception
delete_temp(config)
return config, output_columns_map
from pyspark.sql import SparkSession
from transform import Transform_data
from s3_utils import S3Module
import logging
import logging.config
import configparser
config = configparser.ConfigParser()
config.read_file(open("config.cfg"))
def create_sparksession():
return SparkSession.builder.master("yarn").appName(
"etl").enableHiveSupport().getOrCreate()
def main():
logging.debug("\n\nSetting up Spark Session...")
spark = create_sparksession()
TA = Transform_data(spark)
# Modules in the project
modules = {
"books.csv": TA.transform_book_data,
"user.csv": TA.transform_user_data,
"ratings.csv": TA.transform_book_ratings_data,
}
logging.debug("\n\nCopying data from s3 landing zone to ...")
S3M = S3Module()
S3M.MoveData(source_bucket=config.get('BUCKET', 'LANDING_ZONE1999'),
# Processing Countries
import boto3
import configparser
import logging
import logging.config
from pathlib import Path
from pyspark.sql.types import StructField, StructType, StringType as Str, IntegerType as Int
from helpers import country_working_zone_insert
config = configparser.ConfigParser()
config.read_file(open("review_project.cfg"))
logging.config.fileConfig(f"{Path(__file__).parents[0]}/logging.ini")
logger = logging.getLogger(__name__)
class CountryWrangling:
def __init__(self, spark, landing_zone, working_zone):
"""
Init function
:param spark: SparkSession
:param landing_zone: name of the S3 bucket for Landing Zone
:param working_zone: name of the S3 bucket for Working Zone
"""
self._spark = spark
self._landing_zone = landing_zone
self._working_zone = working_zone
self._file_mask = "country/"
self._s3_resource = boto3.resource(
"s3",
# Command line arguments
parser = argparse.ArgumentParser(description='Regenerate the fractions HTML for development buildings.')
parser.add_argument('-c', '--config', required=True, help='Configuration file', metavar='CONFIGFILE')
parser.add_argument('-l', '--logger-config', required=True, help='Logger dict configuration file', metavar='LOGGERDICTCONFIGFILE')
parser_result = parser.parse_args()
arguments = vars(parser_result)
# Configuration files
config_filename = arguments['config']
logger_config_filename = arguments['logger_config']
# Load logging configuration
source_file_loader = SourceFileLoader('logsettings', logger_config_filename)
logsettings = source_file_loader.load_module()
logging.config.dictConfig(logsettings.LOGGING)
# Load configuration from file
config = configparser.ConfigParser(interpolation=None)
with io.open(config_filename) as config_file_handle:
config.read_file(config_file_handle)
# Initialize odoo connector
common_proxy = xmlrpc.client.Server('{}/xmlrpc/2/common'.format(config['odoo']['api_url']))
models_proxy = xmlrpc.client.Server('{}/xmlrpc/2/object'.format(config['odoo']['api_url']))
odoo = Connector(common_proxy, models_proxy, config['odoo']['database'], config['odoo']['username'], config['odoo']['password'])
# Go
odoo.regenerate_fractions_html()
def parse_config(config_file):
_logger.debug('Reading config file %s', config_file)
config = configparser.ConfigParser()
config.read_file(config_file)
_logger.debug('Parsed config file: %s', config)
return config
options = config_object.options(section_name)
for option in options:
try:
section_dictionary[option] = config_object.get(
section_name, option)
if section_dictionary[option] == -1:
logger.debug('Skipping %s.' % option)
except Exception as err:
logger.error('Exception %s on %s.' % (err, option))
section_dictionary[option] = None
return section_dictionary
# Create config object in memory, read config from file into it.
config = configparser.ConfigParser()
config.read_file(codecs.open(app_path + 'settings.ini', 'rU', 'utf8'))
database_config = config_load_section(config, 'Database')
if database_config['type'] == 'sqlite':
engine = create_engine('sqlite:///' + database_config['filename'],
encoding='utf8')
elif database_config['type'] == 'postgresql':
engine = create_engine('postgresql://' +
database_config['connection_string'],
client_encoding='utf8')
else:
engine = None
Base = declarative_base()
def go(name, configfile, debug, execute):
# turn on debug mode (mainly for boto)
if debug is True:
logging.basicConfig(level=logging.DEBUG)
logger.setLevel(logging.DEBUG)
else:
logging.basicConfig(level=logging.ERROR)
logger.setLevel(logging.ERROR)
# we need the config file
try:
config = configparser.ConfigParser()
config.read_file(open(configfile))
access_key_id = config.get('runabove', 'access_key_id')
if access_key_id is None or len(access_key_id) == 0:
raise UsageError('access_key_id not found in %s' % configfile)
secret_access_key = config.get('runabove', 'secret_access_key')
if secret_access_key is None or len(secret_access_key) == 0:
raise UsageError('secret_access_key not found in %s' % configfile)
ssh_user = config.get('runabove', 'ssh_user')
ssh_ports = config.get('runabove', 'ssh_ports').split(',')
try:
consumer_key = config.get('runabove', 'consumer_key')
except configparser.NoOptionError:
consumer_key = None
if consumer_key is None or len(consumer_key) == 0:
# we don't have a consumer yet, let's generate one
# Create an instance of Runabove SDK interface
run = Runabove(access_key_id, secret_access_key)
# Request an URL to securely authenticate the user
print("You should login here: %s" % run.get_login_url())
input("When you are logged, press Enter")
# Show the consumer key
print("Your consumer key is: %s" % run.get_consumer_key())
print("Please report this key to your config file : %s" % configfile)
sys.exit(0)
try:
region = config.get('runabove', 'region')
except configparser.NoOptionError:
region = 'SBG-1'
try:
key_path = config.get('runabove', 'key_path')
except configparser.NoOptionError:
key_path = '~/.ssh/'
except IOError:
raise UsageError('%s config file not found' % configfile)
except configparser.NoOptionError as e:
raise UsageError('%s in config file %s' % (e, configfile))
except configparser.NoSectionError:
raise UsageError('section %s not found in config file %s' % ('runabove', configfile))
# now we can try to connect
try:
ssh = CourirSsh(access_key_id=access_key_id,
secret_access_key=secret_access_key,
consumer_key=consumer_key,
region=region,
key_path=key_path,
log_level=logger.getEffectiveLevel())
instances = ssh.get_instances_by_name(name)
# if we have more than one instance, we need to make a choice
if len(instances) > 1:
count_instance = 1
click.echo('0) None, I will filter more')
for instance in instances:
click.echo('%s) %s - %s' % (count_instance,
instance.id,
instance.ip))
count_instance += 1
choosen_one = int(click.prompt('Please choose an instance', type=int))
if choosen_one < 0 or choosen_one > len(instances):
raise UsageError('You have to choose a correct instance'
' between %s and %s' % (1, len(instances)))
# if we have one instance only
elif len(instances) == 1:
choosen_one = 1
# if we have no instance at all
else:
raise UsageError('Name %s not found' % (name, ))
if choosen_one == 0:
sys.exit(0)
instance_chosen = instances[choosen_one - 1]
# we have to list ssh_key before getting the good one, don't know why
ssh.list_ssh_key()
ssh_key_name = instance_chosen.ssh_key.name
return ssh.connect(instance=instance_chosen,
ssh_user=ssh_user,
ssh_ports=ssh_ports,
ssh_key_name=ssh_key_name,
cmd=execute)
except CourirSshException as e:
raise UsageError(str(e))
def main():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(levelname)s[%(name)s]: %(message)s',
)
# set the log level for aiosmtpd to warning
# they spam a bunch of stuff at info that should be debug
logging.config.dictConfig({
'version': 1,
'incremental': True,
'loggers': {
'mail.log': {
'level': logging.WARNING
},
},
})
cli = argparse.ArgumentParser(description="LMTP IMAP Delivery Agent", )
cli.add_argument(
'-f',
'--config-file',
default='/etc/imapda.conf',
help="load an alternative configuration file",
)
args = cli.parse_args()
log.info('imapda %s starting', __version__)
config_file = os.path.abspath(args.config_file)
config = ConfigParser()
try:
with open(config_file, 'rt', encoding='utf-8') as reader:
config.read_file(reader)
except OSError as caught:
log.error("can't read config file %s: %s\n", config_file,
caught.strerror)
sys.exit(1)
log.info("loaded config file %s", config_file)
# validate and normalize address config sections
addresses = []
addr_fail = False
for section in config.sections():
if '@' not in section:
continue
try:
addr = validate_email(
section,
allow_smtputf8=False,
check_deliverability=False,
)
except EmailNotValidError as err:
log.error("invalid address in config: %s", section)
addr_fail = True
continue
addresses.append(addr['email'])
if addr_fail:
sys.exit(2)
factory = LMTPFactory(addresses=addresses)
loop = asyncio.get_event_loop()
# signal handlers aren't supported on Windows
with suppress(NotImplementedError):
loop.add_signal_handler(signal.SIGINT, loop.stop)
loop.add_signal_handler(signal.SIGTERM, loop.stop)
proto = config['server'].get('protocol', 'tcp')
if 'tcp' == proto:
host = config['server'].get('host', '127.0.0.1')
port = config['server'].get('port', 8025)
try:
server = loop.run_until_complete(
loop.create_server(factory, host, int(port)))
except Exception as caught:
log.error(
"can't bind to TCP port %s:%s: %s",
host,
port,
getattr(caught, 'strerror', str(caught)),
)
sys.exit(2)
log.info("listening on TCP port %s:%s", host, port)
elif 'unix' == proto:
path = config['server'].get('path')
if path is None:
log.error("path is required when protocol=unix")
sys.exit(1)
path = os.path.abspath(path)
try:
server = loop.run_until_complete(
loop.create_unix_server(factory, path=path))
except Exception as caught:
log.error(
"can't bind to UNIX socket %s: %s",
path,
getattr(caught, 'strerror', str(caught)),
)
sys.exit(2)
log.info("listening on UNIX socket %s", path)
else:
log.info("unsupported protocol %s", proto)
sys.exit(1)
with suppress(KeyboardInterrupt):
loop.run_forever()
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
import configparser
from goodreads import client
from goodreads import collectdata
from datetime import datetime, timedelta
import logging
import logging.config
from pathlib import Path
import time
# Setting up logger, Logger properties are defined in logging.ini file
logging.config.fileConfig(f"{Path(__file__).parents[0]}/logging.ini")
logger = logging.getLogger(__name__)
# Reading configurations
config = configparser.ConfigParser()
config.read_file(open('config.cfg'))
def main():
logging.debug("Creating good reads client.")
grclient = client.GoodreadsClient(config['GOODREADKEYS']['KEY'], config['GOODREADKEYS']['SECRET'])
grcollector = collectdata.GoodreadsCollect(grclient, config['DATA_DIR_PATH']['PATH'])
end_after = int(config['TIMEPARAM']['END_TIME'])
wait_time = int(config['TIMEPARAM']['WAIT_TIME'])
logging.debug(f"Execution started at {datetime.now()}")
end_time = datetime.now() + timedelta(seconds=end_after)
logging.debug(f"Execution will end at : {end_time}")
while(datetime.now() < end_time):
from pyspark.sql import SparkSession
from etl_helper import ETLTransform
from s3_helper import S3Module
from pathlib import Path
import logging
import logging.config
import configparser
from data_warehouse_start.pipeline_start import WarehouseDriver
import time
# Setting configurations. Look config.cfg for more details
config = configparser.ConfigParser()
config.read_file(open(f'{Path(__file__).parents[0]}/config.yml'))
# Setting up logger, Logger properties are defined in logging.ini file
logging.config.fileConfig(f'{Path(__file__).parents[0]}/logging.ini')
logger = logging.getLogger(__name__)
def create_sparksession():
"""
Initialize a spark session
"""
return SparkSession.builder.master('yarn').appName('') \
.config('','') \
.config('', '') \
.enableHiveSupport().getOrCreate()
def main():
import logging
import logging.config
from pathlib import Path
import argparse
import time
#To create redshift cluster, you need ec2 security group, IAM role
#redshift cluster runs on a EC2 VPC instance and need IAM role to access other aws services
#set up logger with logging.conf file
logging.config.fileConfig('logging.conf')
logger = logging.getLogger()
#read cluster configs from file cluster.conf
config = configparser.ConfigParser()
with open('cluster.conf', 'r') as f:
config.read_file(f)
def create_IAM_role(iam_client):
'''
create IAM (identity and access management) role based on cluster.conf
:param -> iam_client: IAM service client instance
:return -> True if IAM role created and policy applied successfully
'''
role_name = config.get('IAM_ROLE', 'NAME')
role_desc = config.get('IAM_ROLE', 'DESCRIPTION')
role_policy_arn = config.get('IAM_ROLE', 'POLICY_ARN')
logger.info(
'Creating IAM role with name: {},description: {} and policy: {}'.
format(role_name, role_desc, role_policy_arn))
with open('role_policy_doc.json') as f:
import logging
import logging.config
import os
from pathlib import Path
# import time
from remindo_api import client
from remindo_api import collectdata
# Setting up logger, Logger properties are defined in logging.ini file
logging.config.fileConfig(f"{Path(__file__).parents[0]}/logging.ini")
logger = logging.getLogger(__name__)
# Reading configurations
config = configparser.ConfigParser()
config.read_file(
open(os.path.join(Path(__file__).parents[0], "/config/prod.cfg")))
# TODO: fix manual change when retrieval breaks
# Either: 1) delete what was retrieved and restart
# select only the missing recipes and moments and continue
# TODO: display total time at the end of the retrieval
def _open_from_temp(directory, name):
file = os.path.join(directory, f"{name}.txt")
with open(file, "r") as f:
result = list()
for line in f:
result.append(int(line.strip()))
return result
from pyspark.sql import SparkSession
from immigration_transform import ImmigrationDatasetTransform
from s3_module import ImmigrationS3Module
from pathlib import Path
import logging
import logging.config
import time
# Setting configurations. Look config.cfg for more details
config = configparser.ConfigParser()
config.read_file(open(f"{Path(__file__).parents[0]}/config.cfg"))
# Setting up logger, Logger properties are defined in logging.ini file
logging.config.fileConfig(f"{Path(__file__).parents[0]}/logging.ini")
logger = logging.getLogger(__name__)
def create_sparksession():
"""
Initialize a spark session
"""
return SparkSession.builder.master('yarn').appName("goodreads") \
.config("spark.jars.packages","saurfang:spark-sas7bdat:2.0.0-s_2.11") \
.config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:2.7.2") \
.enableHiveSupport().getOrCreate()
import threading
import glob
import boto3
import logging.config
import configparser
from pathlib import Path
# Setting up logger, logger properties are defined in logging.ini file
logging.config.fileConfig(f"{Path(__file__).parents[0]}/logging.ini"
) # get the path of logging.ini file
logger = logging.getLogger(__name__) # setup logger
# Loading configuration from s3_buckets.cfg
config = configparser.ConfigParser()
config.read_file(open('s3_buckets.cfg'))
class ProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify, assume this is hooked up to a single filename
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
def _ini_load(fp: IO) -> dict:
config: ConfigParser = ConfigParser()
config.read_file(fp)
return {s: dict(config[s]) for s in config.sections()}
def parse_config(config_file):
_logger.debug('Reading config file %s', config_file)
config = configparser.ConfigParser()
config.read_file(config_file)
_logger.debug('Parsed config file: %s', config)
return config
import configparser
from botocore.exceptions import ClientError
import json
import logging
import logging.config
from pathlib import Path
import argparse
import time
# Setting up logger, Logger properties are defined in logging.ini file
logging.config.fileConfig(f"{Path(__file__).parents[0]}/logging.ini")
logger = logging.getLogger(__name__)
# Loading cluster configurations from cluster.config
config = configparser.ConfigParser()
config.read_file(open('cluster.config'))
def create_IAM_role(iam_client):
"""
Create and IAM_role, Define configuration in cluster.config
:param iam_client: an IAM service client instance
:return: True if IAM role created and policy applied successfully.
"""
role_name = config.get('IAM_ROLE', 'NAME')
role_description = config.get('IAM_ROLE', 'DESCRIPTION')
role_policy_arn = config.get('IAM_ROLE', 'POLICY_ARN')
logging.info(
f"Creating IAM role with name : {role_name}, description : {role_description} and policy : {role_policy_arn}"
