def __init__(self, params):
"""
Constructor
"""
Base_Type.__init__(self)
self.print_only = False
if params:
if 'files' in params:
self.addall_config_files(params['files'])
if 'file' in params:
self.add_config_file(params['file'])
if 'run_date' in params:
self.run_date = params['run_date']
if params.get('print_only',False):
self.print_only = True
self.force_name = params.get('force',None)
if 'log_file' in params:
directory = os.path.dirname(os.path.realpath(params['log_file']))
if not os.path.exists(directory):
os.makedirs(directory)
self.log_file = params['log_file']
else:
run_path = os.path.dirname(os.path.realpath(sys.argv[0]))
directory = os.path.join(run_path,'log')
if not os.path.exists(directory):
os.makedirs(directory)
self.log_file = os.path.join(directory,'rawdata.log')
self.setup_logging()
self.log_message("Kicking Off Raw Data Processor", log_type='main', status='start', step='root', name='raw_processor')
def __init__(self, params):
"""
Constructor
"""
Base_Type.__init__(self)
self.return_status = 0
self.return_results = None
self.entry = params.get('entry',None)
self.name = params.get('name',None)
self.message = None
self.csv_files = {}
self.param_dict = {}
self.temp_writer = None
self.temp_csvfile = None
def __init__(self, params):
"""
Constructor
"""
Base_Type.__init__(self)
if 'run_date' in params:
self.run_date = RawDataUtilities.date_from_string(params['run_date'])
self.entry_name = params.get('entry_name',None)
self.total_time = 0.0
self.entries[0] = []
self.entries[1] = []
#self.set_uuid(params.get('uuid','---none---'))
self.log_message("Initialization Complete", log_type='main', status='running', step='load configs',name='config_file_reader',log_level=self.log_info())
def __init__(self, params):
"""
Constructor
"""
Base_Type.__init__(self)
self.print_only = False
if params:
if 'files' in params:
self.addall_config_files(params['files'])
if 'file' in params:
self.add_config_file(params['file'])
if 'run_date' in params:
self.run_date = params['run_date']
if params.get('print_only', False):
self.print_only = True
self.force_name = params.get('force', None)
if 'log_file' in params:
directory = os.path.dirname(
os.path.realpath(params['log_file']))
if not os.path.exists(directory):
os.makedirs(directory)
self.log_file = params['log_file']
else:
run_path = os.path.dirname(os.path.realpath(sys.argv[0]))
directory = os.path.join(run_path, 'log')
if not os.path.exists(directory):
os.makedirs(directory)
self.log_file = os.path.join(directory, 'rawdata.log')
self.setup_logging()
self.log_message("Kicking Off Raw Data Processor",
log_type='main',
status='start',
step='root',
name='raw_processor')
def __init__(self, params):
"""
Constructor
"""
Base_Type.__init__(self)
## class variables
self.updates={}
self.return_val = None
self.description = None
self.connection_type = None
self.last_processed = None
self.num_run = 0
self.message = None
self.no_run_reason = 'Schedule Not Ready'
### valid options... Default is Append
# Append - Add records to the end
# Overlay - replace the current with the new
# New - Add a new file with the run number on the end Most current result has the name with no number
self.file_write=None
## Valid Options... Default is Temp CSV file
# Dataframe
# ListList
# DictionaryList
# TempCSV
# None
self.temp_results=None
self.out_filename = None
self.src_implementation = None
self.run_frequency = None
## location where results are written
self.working_directory = None
self.options = {}
self.today_dt = None
self.source_file = None
self.instance = None
self.temp_csv = None
self.csv = None
## required fields
self.today_dt = params['run_date']
self.name = params['name']
self.source_file = params['source_file']
self.description = params['description']
self.src_implementation = params['src_implementation']
self.working_directory = params['working_directory']
## optional fields with defaults
self.dependencies = [] if (params.get('dependencies', None)) == None else params['dependencies'].split(",")
self.dependencies.append('kicker')
self.successors= [] if (params.get('successors', None)) == None else params['successors'].split(",")
self.connection_type = params.get('connection_type','none')
# if it hasn't run before it will be empty
self.first_run = False
if params.get('last_processed',None):
self.last_processed = params['last_processed']
else:
self.last_processed = RawDataUtilities.string_from_date(self.today_dt)
self.first_run = True
self.last_processed = RawDataUtilities.date_from_string(self.last_processed)
self.num_run = int(params.get('num_run',0))
self.out_filename = params.get('out_filename',self.name)
self.run_frequency = params.get('run_frequency','Daily')
self.temp_results = params.get('temp_results','TempCSV')
self.file_write = params.get('file_write','Append')
#self.entry_type = params.get('entry_type', 'Connector')
self.last_run = params.get('last_run','success')
self.instantiate_instance = True if params.get("instantiate","true") == "true" else False
## parameters to pass down to the entry implementation
self.options = params.get('non_std',{})
self.updates['last_processed'] = RawDataUtilities.string_from_date(self.today_dt)
self.updates['num_run'] = str(self.num_run + 1)
self.order = 0
self.ready = True
if self.instantiate_instance and self.get_instance():
self.log_message("Initialization Complete (success): "+self.name, log_type='entry', status='running', step='load configs',name='config_entry',log_level=self.log_info())
else:
if self.instantiate_instance:
self.ready = False
self.log_message("Initialization Complete (failure): "+self.name, log_type='entry', status='running', step='load configs',name='config_entry',log_level=self.log_info())
def __init__(self, params):
"""
Constructor
"""
Base_Type.__init__(self)
## class variables
self.updates = {}
self.return_val = None
self.description = None
self.connection_type = None
self.last_processed = None
self.num_run = 0
self.message = None
self.no_run_reason = 'Schedule Not Ready'
### valid options... Default is Append
# Append - Add records to the end
# Overlay - replace the current with the new
# New - Add a new file with the run number on the end Most current result has the name with no number
self.file_write = None
## Valid Options... Default is Temp CSV file
# Dataframe
# ListList
# DictionaryList
# TempCSV
# None
self.temp_results = None
self.out_filename = None
self.src_implementation = None
self.run_frequency = None
## location where results are written
self.working_directory = None
self.options = {}
self.today_dt = None
self.source_file = None
self.instance = None
self.temp_csv = None
self.csv = None
## required fields
self.today_dt = params['run_date']
self.name = params['name']
self.source_file = params['source_file']
self.description = params['description']
self.src_implementation = params['src_implementation']
self.working_directory = params['working_directory']
## optional fields with defaults
self.dependencies = [] if (params.get(
'dependencies',
None)) == None else params['dependencies'].split(",")
self.dependencies.append('kicker')
self.successors = [] if (params.get(
'successors', None)) == None else params['successors'].split(",")
self.connection_type = params.get('connection_type', 'none')
# if it hasn't run before it will be empty
self.first_run = False
if params.get('last_processed', None):
self.last_processed = params['last_processed']
else:
self.last_processed = RawDataUtilities.string_from_date(
self.today_dt)
self.first_run = True
self.last_processed = RawDataUtilities.date_from_string(
self.last_processed)
self.num_run = int(params.get('num_run', 0))
self.out_filename = params.get('out_filename', self.name)
self.run_frequency = params.get('run_frequency', 'Daily')
self.temp_results = params.get('temp_results', 'TempCSV')
self.file_write = params.get('file_write', 'Append')
#self.entry_type = params.get('entry_type', 'Connector')
self.last_run = params.get('last_run', 'success')
self.instantiate_instance = True if params.get(
"instantiate", "true") == "true" else False
## parameters to pass down to the entry implementation
self.options = params.get('non_std', {})
self.updates['last_processed'] = RawDataUtilities.string_from_date(
self.today_dt)
self.updates['num_run'] = str(self.num_run + 1)
self.order = 0
self.ready = True
if self.instantiate_instance and self.get_instance():
self.log_message("Initialization Complete (success): " + self.name,
log_type='entry',
status='running',
step='load configs',
name='config_entry',
log_level=self.log_info())
else:
if self.instantiate_instance:
self.ready = False
self.log_message("Initialization Complete (failure): " +
self.name,
log_type='entry',
status='running',
step='load configs',
name='config_entry',
log_level=self.log_info())
