class Transformer: def __init__(self): self.__data = Extractor().extract_csv_data() self.__loader = Loader() # save all extracted DataFrames from csv files to parquet files for k, v in self.__data.items(): self.__loader.save_to_parquet(k, v) # reads all saved parquet files data_files = self.__loader.read_parquets("weather") # combines all DataFrames into one to get the highest temp from all records self.__df = pd.concat(data_files, ignore_index=True) def find_hottest_day(self): """ Gets a subset of the combined data containing only the columns we need. Then finds the row that equals to the maximum ScreenTemperature and returns it. :return: a DataFrame row containing the result of the query. """ # creates a subset of the data with only the columns we need df_subset = self.__df[[ 'ObservationDate', 'ScreenTemperature', 'Region' ]] # find the row with max temperature return df_subset[df_subset['ScreenTemperature'] == df_subset['ScreenTemperature'].max()]
def __init__(self): self.__data = Extractor().extract_csv_data() self.__loader = Loader() # save all extracted DataFrames from csv files to parquet files for k, v in self.__data.items(): self.__loader.save_to_parquet(k, v) # reads all saved parquet files data_files = self.__loader.read_parquets("weather") # combines all DataFrames into one to get the highest temp from all records self.__df = pd.concat(data_files, ignore_index=True)
def editMap(): global mapEditor, ctrlLayer, mapLayer mapfile = "./map/%s"%(editMapPanel.listBox.selectedtext()) ctrlLayer = mapEditor mapEditor.mapfile = mapfile mapLayer = Map(mapwidth,mapheight) mapEditor.map = mapLayer mapLayer.read(mapfile) loader = Loader() loader.load(images["loadmap"],screen,mapLayer.load,\ [images["bar"].subsurface(0,0,barlength,barheight),(barx,bary)]) ctrlLayer.minimap = mapLayer.minimap
def startNewGame(): global ctrlLayer, startMenu, mapLayer, characterLayer ctrlLayer = gameController mapLayer = Map(mapwidth,mapheight) characterLayer = Game(mapLayer) characterLayer.initNewGame(defaultplayers) ctrlLayer.takeOverGame(characterLayer,characterLayer.players[0]) ctrlLayer.map = mapLayer mapLayer.read("./map/%s"%(startMenu.mapfilename)) loader = Loader() loader.load(images["loadmap"],screen,mapLayer.load,\ [images["bar"].subsurface(0,0,barlength,barheight),(barx,bary)]) ctrlLayer.minimap = mapLayer.minimap characterLayer.running = True
class LoaderTest(TestCase): def setUp(self): self.loader = Loader() self.project_dir = os.path.abspath(__file__ + "/../../") for f in glob.glob(self.project_dir + "/resources/*.parquet.gzip"): os.remove(f) def tearDown(self): for f in glob.glob(self.project_dir + "/resources/*.parquet.gzip"): os.remove(f) def test_save_parquet(self): data = DataFrame([{"key1": "value1", "key2": "value2"}]) self.loader.save_to_parquet("test", data) result = os.path.isfile(self.project_dir + "/resources/test.parquet.gzip") self.assertTrue(result) def test_read_parquets(self): data = DataFrame([{"key1": "value1", "key2": "value2"}]) self.loader.save_to_parquet("test", data) result = self.loader.read_parquets("test") self.assertCountEqual(data, result[0])
return self.data.to_json(**params) if __name__ == "__main__": from load import Loader # Load the data url = 'http://apps.nccd.cdc.gov/brfss/list.' \ 'asp?cat=HI&yr=2013&qkey=8671&state=All' attrs = {'border': 1, 'cellpadding': 5, 'cellspacing': 0} params = { 'url': url, 'header': 0, 'attrs': attrs } load = Loader() load.setParams(params) data = load.loadHtml().data # Transform the data jsonFp = './states.json' colName = 'State:' # initiate transformer object transform = Transformer(data) transform.setTransformDictionary(jsonFp) # get long values of the states statesDict = transform.flipTransformDictionary() states = statesDict.keys()
cor_forms = target_dict_list[6]['records'] cap_projects = target_dict_list[3]['records'] expense_builders = target_dict_list[4]['records'] npv_tasks = target_dict_list[1]['records'] print( 'All data successfully queried. Any errors after this point are due to DATA VALIDATION ONLY.' ) t = Transformer(opps, service_orders, quotes, cor_forms, cap_projects, expense_builders) valid_opp_to_service_orders = t.validate_opp_to_service_order() valid_opp_to_quote_or_cor_form = t.validate_opp_to_quote_or_cor_form( valid_opp_to_service_orders) standardized_opp_to_cp_or_eb = t.standardize_opp_to_cp_or_eb( valid_opp_to_quote_or_cor_form) valid_opp_to_cp_or_eb = t.validate_opp_to_cp_or_eb( valid_opp_to_quote_or_cor_form, standardized_opp_to_cp_or_eb) # print(valid_opp_to_cp_or_eb) ## All validation stages passed, applicable NPV tasks can now be closed: l = Loader(valid_opp_to_cp_or_eb, npv_tasks) if full_soql_query_mode else None tasks_closed = l.load_tasks() if full_soql_query_mode else [] if len(tasks_closed) == 0: print('0 NPV tasks validated by automation.') elif len(tasks_closed) > 0: print( '{} NPV tasks validated by automation. Please move to Stage 5 via the corresponding report queue: \n \ https://zayo.my.salesforce.com/00O0z000005btK4'.format( len(tasks_closed)))
from load import Loader from CCE.loadCCE_2015 import CCE_2015_Campaign from DAPloaders import runGliderLoader, runLrauvLoader class Campaigns(): pass # Reuse CCELoader and Loader code to create our test db and load a # small amount of data for testing of the loading code db_alias = 'stoqs' campaign_name = 'Loading test database' campaign_description = 'Test database for all kinds of data: EPIC from CCE, Glider, LRAUV, etc.' campaign = CCE_2015_Campaign(db_alias, campaign_name) loader = Loader() campaigns = Campaigns() loader.args = Namespace() loader.args.test = False loader.args.clobber = True loader.args.db = db_alias loader.args.drop_indexes = False campaigns.campaigns = {db_alias: 'CCE/loadCCE_2015.py'} loader.load(campaigns, create_only=True) # Load only the March 2016 event lores Mooring data for ms2 campaign.hires_event_times = [] campaign.lores_event_times = [campaign.lores_event_times[1]] campaign.cl.ccems2_start_datetime, campaign.cl.ccems2_end_datetime = campaign.lores_event_times[
def load(): global screen loader = Loader() loader.loadimg("loadimg") loader.load(images["loadimg"],screen,loadImages)
def setUp(self): self.loader = Loader() self.project_dir = os.path.abspath(__file__ + "/../../") for f in glob.glob(self.project_dir + "/resources/*.parquet.gzip"): os.remove(f)
#!/usr/bin/env python3 from fetch import Fetcher from load import Loader if __name__ == '__main__': options = { # Prices to fetch list 'data_file': "fetch_list.txt", # Price data file 'file_name': "watch_prices.txt", # Data directory 'output_directory': "./", } l = Loader() data = l.load_data(options['data_file']) options['data'] = data w = Fetcher(options) w.run()
sys.path.insert(0, parent_dir) # So that CCE & DAPloaders are found from argparse import Namespace from load import Loader from CCE.loadCCE_2015 import CCE_2015_Campaign from DAPloaders import runGliderLoader class Campaigns(): pass # Reuse CCELoader and Loader code to create our test db and load a # small amount of data for testing of the loading code db_alias = 'stoqs' campaign_name = 'Loading test database' campaign = CCE_2015_Campaign(db_alias, campaign_name) loader = Loader() campaigns = Campaigns() loader.args = Namespace() loader.args.test = False loader.args.clobber = True loader.args.db = db_alias loader.args.drop_indexes = False campaigns.campaigns = {db_alias: 'CCE/loadCCE_2015.py'} loader.load(campaigns, create_only=True) # Load only the March 2016 event lores Mooring data for ms2 campaign.hires_event_times = [] campaign.lores_event_times = [campaign.lores_event_times[1]] campaign.cl.ccems2_start_datetime, campaign.cl.ccems2_end_datetime = campaign.lores_event_times[0]
from load import Loader from CCE.loadCCE_2015 import CCE_2015_Campaign from DAPloaders import runGliderLoader, runLrauvLoader class Campaigns(): pass # Reuse CCELoader and Loader code to create our test db and load a # small amount of data for testing of the loading code db_alias = 'stoqs' campaign_name = 'Loading test database' campaign_description = 'Test database for all kinds of data: EPIC from CCE, Glider, LRAUV, etc.' campaign = CCE_2015_Campaign(db_alias, campaign_name) loader = Loader() campaigns = Campaigns() loader.args = Namespace() loader.args.test = False loader.args.clobber = True loader.args.db = db_alias loader.args.drop_indexes = False loader.args.noinput = False campaigns.campaigns = {db_alias: 'CCE/loadCCE_2015.py'} try: loader._dropdb(db_alias) except KeyError: pass loader.load(campaigns, create_only=True)