class Transformer:
def __init__(self):
self.__data = Extractor().extract_csv_data()
self.__loader = Loader()
# save all extracted DataFrames from csv files to parquet files
for k, v in self.__data.items():
self.__loader.save_to_parquet(k, v)
# reads all saved parquet files
data_files = self.__loader.read_parquets("weather")
# combines all DataFrames into one to get the highest temp from all records
self.__df = pd.concat(data_files, ignore_index=True)
def find_hottest_day(self):
"""
Gets a subset of the combined data containing only the columns we need.
Then finds the row that equals to the maximum ScreenTemperature and returns it.
:return: a DataFrame row containing the result of the query.
"""
# creates a subset of the data with only the columns we need
df_subset = self.__df[[
'ObservationDate', 'ScreenTemperature', 'Region'
]]
# find the row with max temperature
return df_subset[df_subset['ScreenTemperature'] ==
df_subset['ScreenTemperature'].max()]
def __init__(self):
self.__data = Extractor().extract_csv_data()
self.__loader = Loader()
# save all extracted DataFrames from csv files to parquet files
for k, v in self.__data.items():
self.__loader.save_to_parquet(k, v)
# reads all saved parquet files
data_files = self.__loader.read_parquets("weather")
# combines all DataFrames into one to get the highest temp from all records
self.__df = pd.concat(data_files, ignore_index=True)
def editMap():
global mapEditor, ctrlLayer, mapLayer
mapfile = "./map/%s"%(editMapPanel.listBox.selectedtext())
ctrlLayer = mapEditor
mapEditor.mapfile = mapfile
mapLayer = Map(mapwidth,mapheight)
mapEditor.map = mapLayer
mapLayer.read(mapfile)
loader = Loader()
loader.load(images["loadmap"],screen,mapLayer.load,\
[images["bar"].subsurface(0,0,barlength,barheight),(barx,bary)])
ctrlLayer.minimap = mapLayer.minimap
def startNewGame():
global ctrlLayer, startMenu, mapLayer, characterLayer
ctrlLayer = gameController
mapLayer = Map(mapwidth,mapheight)
characterLayer = Game(mapLayer)
characterLayer.initNewGame(defaultplayers)
ctrlLayer.takeOverGame(characterLayer,characterLayer.players[0])
ctrlLayer.map = mapLayer
mapLayer.read("./map/%s"%(startMenu.mapfilename))
loader = Loader()
loader.load(images["loadmap"],screen,mapLayer.load,\
[images["bar"].subsurface(0,0,barlength,barheight),(barx,bary)])
ctrlLayer.minimap = mapLayer.minimap
characterLayer.running = True
class LoaderTest(TestCase):
def setUp(self):
self.loader = Loader()
self.project_dir = os.path.abspath(__file__ + "/../../")
for f in glob.glob(self.project_dir + "/resources/*.parquet.gzip"):
os.remove(f)
def tearDown(self):
for f in glob.glob(self.project_dir + "/resources/*.parquet.gzip"):
os.remove(f)
def test_save_parquet(self):
data = DataFrame([{"key1": "value1", "key2": "value2"}])
self.loader.save_to_parquet("test", data)
result = os.path.isfile(self.project_dir +
"/resources/test.parquet.gzip")
self.assertTrue(result)
def test_read_parquets(self):
data = DataFrame([{"key1": "value1", "key2": "value2"}])
self.loader.save_to_parquet("test", data)
result = self.loader.read_parquets("test")
self.assertCountEqual(data, result[0])
return self.data.to_json(**params)
if __name__ == "__main__":
from load import Loader
# Load the data
url = 'http://apps.nccd.cdc.gov/brfss/list.' \
'asp?cat=HI&yr=2013&qkey=8671&state=All'
attrs = {'border': 1, 'cellpadding': 5, 'cellspacing': 0}
params = {
'url': url,
'header': 0,
'attrs': attrs
}
load = Loader()
load.setParams(params)
data = load.loadHtml().data
# Transform the data
jsonFp = './states.json'
colName = 'State:'
# initiate transformer object
transform = Transformer(data)
transform.setTransformDictionary(jsonFp)
# get long values of the states
statesDict = transform.flipTransformDictionary()
states = statesDict.keys()
cor_forms = target_dict_list[6]['records']
cap_projects = target_dict_list[3]['records']
expense_builders = target_dict_list[4]['records']
npv_tasks = target_dict_list[1]['records']
print(
'All data successfully queried. Any errors after this point are due to DATA VALIDATION ONLY.'
)
t = Transformer(opps, service_orders, quotes, cor_forms, cap_projects,
expense_builders)
valid_opp_to_service_orders = t.validate_opp_to_service_order()
valid_opp_to_quote_or_cor_form = t.validate_opp_to_quote_or_cor_form(
valid_opp_to_service_orders)
standardized_opp_to_cp_or_eb = t.standardize_opp_to_cp_or_eb(
valid_opp_to_quote_or_cor_form)
valid_opp_to_cp_or_eb = t.validate_opp_to_cp_or_eb(
valid_opp_to_quote_or_cor_form, standardized_opp_to_cp_or_eb)
# print(valid_opp_to_cp_or_eb)
## All validation stages passed, applicable NPV tasks can now be closed:
l = Loader(valid_opp_to_cp_or_eb, npv_tasks) if full_soql_query_mode else None
tasks_closed = l.load_tasks() if full_soql_query_mode else []
if len(tasks_closed) == 0:
print('0 NPV tasks validated by automation.')
elif len(tasks_closed) > 0:
print(
'{} NPV tasks validated by automation. Please move to Stage 5 via the corresponding report queue: \n \
https://zayo.my.salesforce.com/00O0z000005btK4'.format(
len(tasks_closed)))
from load import Loader
from CCE.loadCCE_2015 import CCE_2015_Campaign
from DAPloaders import runGliderLoader, runLrauvLoader
class Campaigns():
pass
# Reuse CCELoader and Loader code to create our test db and load a
# small amount of data for testing of the loading code
db_alias = 'stoqs'
campaign_name = 'Loading test database'
campaign_description = 'Test database for all kinds of data: EPIC from CCE, Glider, LRAUV, etc.'
campaign = CCE_2015_Campaign(db_alias, campaign_name)
loader = Loader()
campaigns = Campaigns()
loader.args = Namespace()
loader.args.test = False
loader.args.clobber = True
loader.args.db = db_alias
loader.args.drop_indexes = False
campaigns.campaigns = {db_alias: 'CCE/loadCCE_2015.py'}
loader.load(campaigns, create_only=True)
# Load only the March 2016 event lores Mooring data for ms2
campaign.hires_event_times = []
campaign.lores_event_times = [campaign.lores_event_times[1]]
campaign.cl.ccems2_start_datetime, campaign.cl.ccems2_end_datetime = campaign.lores_event_times[
def load():
global screen
loader = Loader()
loader.loadimg("loadimg")
loader.load(images["loadimg"],screen,loadImages)
def setUp(self):
self.loader = Loader()
self.project_dir = os.path.abspath(__file__ + "/../../")
for f in glob.glob(self.project_dir + "/resources/*.parquet.gzip"):
os.remove(f)
#!/usr/bin/env python3
from fetch import Fetcher
from load import Loader
if __name__ == '__main__':
options = {
# Prices to fetch list
'data_file': "fetch_list.txt",
# Price data file
'file_name': "watch_prices.txt",
# Data directory
'output_directory': "./",
}
l = Loader()
data = l.load_data(options['data_file'])
options['data'] = data
w = Fetcher(options)
w.run()
sys.path.insert(0, parent_dir) # So that CCE & DAPloaders are found
from argparse import Namespace
from load import Loader
from CCE.loadCCE_2015 import CCE_2015_Campaign
from DAPloaders import runGliderLoader
class Campaigns():
pass
# Reuse CCELoader and Loader code to create our test db and load a
# small amount of data for testing of the loading code
db_alias = 'stoqs'
campaign_name = 'Loading test database'
campaign = CCE_2015_Campaign(db_alias, campaign_name)
loader = Loader()
campaigns = Campaigns()
loader.args = Namespace()
loader.args.test = False
loader.args.clobber = True
loader.args.db = db_alias
loader.args.drop_indexes = False
campaigns.campaigns = {db_alias: 'CCE/loadCCE_2015.py'}
loader.load(campaigns, create_only=True)
# Load only the March 2016 event lores Mooring data for ms2
campaign.hires_event_times = []
campaign.lores_event_times = [campaign.lores_event_times[1]]
campaign.cl.ccems2_start_datetime, campaign.cl.ccems2_end_datetime = campaign.lores_event_times[0]
from load import Loader
from CCE.loadCCE_2015 import CCE_2015_Campaign
from DAPloaders import runGliderLoader, runLrauvLoader
class Campaigns():
pass
# Reuse CCELoader and Loader code to create our test db and load a
# small amount of data for testing of the loading code
db_alias = 'stoqs'
campaign_name = 'Loading test database'
campaign_description = 'Test database for all kinds of data: EPIC from CCE, Glider, LRAUV, etc.'
campaign = CCE_2015_Campaign(db_alias, campaign_name)
loader = Loader()
campaigns = Campaigns()
loader.args = Namespace()
loader.args.test = False
loader.args.clobber = True
loader.args.db = db_alias
loader.args.drop_indexes = False
loader.args.noinput = False
campaigns.campaigns = {db_alias: 'CCE/loadCCE_2015.py'}
try:
loader._dropdb(db_alias)
except KeyError:
pass
loader.load(campaigns, create_only=True)
