def run(self): step = 1 util = JobConfig(0, 0, None, None) best_util = JobConfig(0, 0, None, None) state = self.init_state() if self.check_constraint: while self.check_constraint(state): state = self.init_state() while step < self.max_step and util.get_util() < self.max_util: temperature = self.find_temperature(step) new_state = self.generate_neighbor(state) if self.check_constraint: while self.check_constraint(new_state): new_state = self.generate_neighbor(new_state) new_util = self.compute_util(new_state) # Utilization = 0 -> very undesirable -> reset if new_util.get_util() == 0: state = self.init_state() elif self.transition(util, new_util, temperature) >= random(): state = new_state util = new_util if new_util.compare(best_util): best_util = new_util step += 1 return best_util
def run(self): step = 1 util = JobConfig(0, 0, None, None) best_util = JobConfig(0, 0, None, None) state = self.init_state() if self.check_constraint: while self.check_constraint(state): state = self.init_state() while step < self.max_step and util.get_util() < self.max_util: temperature = self.find_temperature(step) new_state = self.generate_neighbor(state) if self.check_constraint: while self.check_constraint(new_state): new_state = self.generate_neighbor(new_state) new_util = self.compute_util(new_state) # Utilization = 0 -> very undesirable -> reset if new_util.get_util() == 0: state = self.init_state() elif self.transition(util, new_util, temperature) >= random(): state = new_state util = new_util if new_util.compare(best_util): best_util = new_util step += 1 return best_util
def routing_compute_util(self, state): self.add_previous_jobs() cloned_links = self.graph.copy_links() valid = True paths_used = [] util = 0 job_config = JobConfig() for p in state: bw, links = p[0], p[1] for l in range(len(links) - 1): link_id = Link.get_id(links[l], links[l + 1]) link = cloned_links[link_id] link_bandwidth = link.get_bandwidth() if link_bandwidth < bw: valid = False break link.set_bandwidth(link_bandwidth - bw) if not valid: # logging.debug(str(state) + " cannot be built") break else: paths_used.append(p) if valid: self.graph.set_links(cloned_links) all_paths_used = deepcopy(paths_used) for job in self.jobs_config.values(): all_paths_used.extend(job.get_used_paths()) self.graph.set_flow(all_paths_used) util = 0 total_util = 0 for p in paths_used: flow = self.graph.get_flow(Flow.get_id(p[1][0], p[1][-1])) util += (flow.get_requested_bandwidth() + flow.get_effective_bandwidth()) for p in all_paths_used: flow = self.graph.get_flow(Flow.get_id(p[1][0], p[1][-1])) total_util += (flow.get_requested_bandwidth() + flow.get_effective_bandwidth()) job_config = JobConfig(util, total_util, copy_links(cloned_links), paths_used) self.reset() return job_config
def test_case_save_resolved_ipdata(self): """Test the process of ip resolution and file placement in S3.""" config = JobConfig().getconfig() utility.check_s3path(config) ipResolver = IPResolver() panoplyreader = PanoplyImport() datawriter = S3Writer() ipdata_geoservice = IPDataService() query_panoply = config["panoplydatabase"]["readQuery"] for dataframe_ip_address in panoplyreader.getbatch_pandas_dataframe( query_panoply): dataframes = utility.split_dataframe(dataframe_ip_address) processNo = 0 processList = [] for frame in enumerate(dataframes): processNo = processNo + 1 process_ipresolve = processes.Process( target=ipResolver.resolve_ipaddress, args=(frame[1], ipdata_geoservice, datawriter, processNo)) processList.append(process_ipresolve) process_ipresolve.start() logger.info('processNo-' + str(process_ipresolve.pid)) for p in processList: p.join() for i in range(1, config['processConfig']['noOfParallelProcess']): s3file_url = datawriter.getfileurl(i) self.assertTrue(datawriter.fileWriter.exists(s3file_url)) print('Ran test_case_save_resolved_ipdata test case')
def __init__(self): """Initialize panoply connection.""" self.config = JobConfig().getconfig() self.logger = utility.getlogger('ip_resolution', 'ip_resolution') self.filePath = self.config['storageDetails']['filePath'] self.fileName = self.config['storageDetails']['fileName'] self.fileExtension = self.config['storageDetails']['fileExtension'] self.directoryName = date.today().strftime("%m/%d/%y").replace("/", "_") self.awsKey = self.config['storageDetails']['awsKey'] self.secretKey = self.config['storageDetails']['secretKey'] self.s3file_url = 's3://' + str(self.filePath) + '/' + str(self.directoryName) + '/' + str(self.fileName) self.tableName = self.config['storageDetails']['tableName'] self.region = self.config['storageDetails']['region'] username = self.config['panoplydatabase']['user'] password = self.config['panoplydatabase']['password'] db = self.config['panoplydatabase']['database'] host = self.config['panoplydatabase']['host'] port = self.config['panoplydatabase']['port'] self.connection = psycopg2.connect(user=username, password=password, host=host, port=port, database=db) self.write_command = """BEGIN; truncate """ + self.tableName + """ ; copy """ + self.tableName + """ from '""" + self.s3file_url + """' access_key_id '""" + self.awsKey + """' secret_access_key '""" + self.secretKey + """' region '""" + self.region + """' ignoreheader 1 null as 'NA' removequotes delimiter ','; COMMIT;""" self.append_command = """BEGIN; copy """ + self.tableName + """ from '""" + self.s3file_url + """' access_key_id '""" + self.awsKey + """' secret_access_key '""" + self.secretKey + """' region '""" + self.region + """'
def __init__(self, processNo=0): """Initialize IP Data API object connection.""" self.config = JobConfig().getconfig() self.processNo = processNo self.logger = utility.getlogger('ip_resolution', 'ip_resolution') url = self.config['geoservice']['url'] apikey = self.config['geoservice']['apikey'] self.connection_url = url.replace("userkey", apikey)
def compute_route(self): util = JobConfig() chosen_paths = [] if self.build_paths(): for path in self.valid_paths.values(): chosen_paths.append(choice(path)) util = self.routing_compute_util(chosen_paths) return util
def test_case_panoplyImport_dataframes(self): """Test the import of dataframe from Panoply.""" connector = PanoplyImport() config = JobConfig().getconfig() query_panoply = 'select distinct ipaddress from activity_us (nolock) where ipaddress is not null limit 40' for panda_df in connector.getbatch_pandas_dataframe(query_panoply): self.assertTrue( panda_df.shape[0] <= config['panoplydatabase']['chunksize'] and (panda_df['ipaddress'].iloc[0] is not None)) print('Ran test_case_panoplyImport_dataframes test case')
def split_dataframe(dataframe): """Split a Pandas dataframe in sets.""" size = dataframe.shape[0] config = JobConfig().getconfig() max_rows = size // config['processConfig']['noOfParallelProcess'] list_df = [ dataframe[i:i + max_rows] for i in range(0, dataframe.shape[0], max_rows) ] return list_df
def __init__(self): """Initialize connection to S3.""" self.config = JobConfig().getconfig() self.logger = utility.getlogger('ip_resolution', 'ip_resolution') self.filePath = self.config['storageDetails']['filePath'] self.fileName = self.config['storageDetails']['fileName'] self.fileExtension = self.config['storageDetails']['fileExtension'] self.directoryName = date.today().strftime("%m/%d/%y").replace("/", "_") self.awsKey = self.config['storageDetails']['awsKey'] self.secretKey = self.config['storageDetails']['secretKey'] self.s3file_url = 's3://' + str(self.filePath) + '/' + str(self.directoryName) \ + '/' + str(self.fileName) + str(self.fileExtension) self.fileWriter = s3fs.S3FileSystem(self.awsKey, self.secretKey)
def splitdataframe1(dataframe): """Alternative approach, but not correct.""" size = dataframe.shape[0] config = JobConfig().getconfig() max_rows = size // config['processConfig']['noOfParallelProcess'] print(max_rows) dataframes = [] while size > max_rows: top = dataframe.loc[0:max_rows - 1] dataframes.append(top) dataframe = dataframe.loc[max_rows:size - 1] size = dataframe.shape[0] else: dataframes.append(dataframe) return dataframes
def compute_route(self): util = JobConfig() max_step = self.max_step max_util = self.num_mappers * self.num_reducers * self.bandwidth if self.build_paths(): # Executing simulated annealing for map-reduce routing simulated_annealing = SimulatedAnnealing(max_util, \ max_step, \ self.routing_init_state, \ self.routing_generate_neighbor, \ self.routing_compute_util) util = simulated_annealing.run() # print "util: ", util.get_util() return util
def _execute_job(self): available_hosts = [h for h in self.graph.get_hosts() if h.is_free()] hosts = [] util = JobConfig() # There are enough nodes to run the job if len(available_hosts) > (self.num_mappers + self.num_reducers): for i in range(self.num_mappers + self.num_reducers): host_to_add = choice(available_hosts) while host_to_add in hosts: host_to_add = choice(available_hosts) hosts.append(host_to_add) util = self.placement_compute_util(hosts) return util
def compute_route(self): util = JobConfig() if self.build_paths(): max_step = self.max_step max_util = self.cur_demand.get_net() # Executing simulated annealing for map-reduce routing simulated_annealing = SimulatedAnnealing(max_util, \ max_step, \ self.routing_init_state, \ self.routing_generate_neighbor, \ self.routing_compute_util, \ self.check_constraint) util = simulated_annealing.run() # print "util: ", util.get_util() return util
def test_case_ipresolution_pipeline(self): """Test the complete flow of ip resolution and result load in panoply.""" config = JobConfig().getconfig() utility.check_s3path(config) ip_resolver = IPResolver() panoplyreader = PanoplyImport() datawriter = S3Writer() ipdata_geoservice = IPDataService() if ResolveIp.process_chunks(config, ip_resolver, ipdata_geoservice, panoplyreader, datawriter): panoplywriter = PanoplyWriter() panoplywriter.save_data() query_panoply = 'select max(createdAt) as created from test_sp_ipaddress_parsed_us (nolock) limit 1' df = panoplyreader.get_pandas_dataframe(query_panoply) now = datetime.now() dt_string = now.strftime("%d/%m/%Y") df_string = df['created'].iloc[0] self.assertTrue(df_string[0:10] == dt_string) print('Ran test_case_ipresolution_pipeline test case')
def __init__(self): """Initialize Panoply connection.""" self.config = JobConfig().getconfig() self.logger = utility.getlogger('ip_resolution', 'ip_resolution') username = self.config['panoplydatabase']['user'] password = self.config['panoplydatabase']['password'] db = self.config['panoplydatabase']['database'] host = self.config['panoplydatabase']['host'] port = self.config['panoplydatabase']['port'] self.connection_url = 'postgresql://' + str(username) + ':' + str( password) + '@' + str(host) + ':' + str(port) + '/' + str(db) self.readQuery = self.config['panoplydatabase']['readQuery'] self.chunksize = self.config['panoplydatabase']['chunksize'] try: self.connection_panoply = create_engine(self.connection_url, echo=False) self.logger.info('Initialized Panoply connection') except Exception as ex: self.logger.info('Issue with panoply connection:' + str(ex)) self.logger.error(utility.print_exception())
def test_case_s3_writer(self): """Test file write to S3 bucket.""" config = JobConfig().getconfig() utility.check_s3path(config) panoplyreader = PanoplyImport() datawriter = S3Writer() # print(dataframe_results) query_panoply = 'select distinct ipaddress from activity_us (nolock) where ipaddress is not null limit 10' dataframe_results = panoplyreader.get_pandas_dataframe(query_panoply) datawriter.append_data(dataframe_results, 0) datawriter = S3Writer() s3file_url = datawriter.getfileurl(0) with datawriter.fileWriter.open(s3file_url, mode='rb') as pointer: file_bytes = pointer.read() data_bytes = dataframe_results[['ipaddress' ]].to_csv(None, header=False, index=False).encode() self.assertEqual(file_bytes[11:], data_bytes) print('Ran test_case_s3_writer test case')
def execute_job(self, job): available_hosts = [h for h in self.graph.get_hosts() if h.is_free()] util = JobConfig() # There are enough nodes to run the job if len(available_hosts) > (self.num_mappers + self.num_reducers): max_util = self.num_mappers * self.num_reducers * self.bandwidth max_step = self.max_step # Executing simulated annealing for map-reduce placement simulated_annealing = SimulatedAnnealing(max_util, \ max_step, \ self.placement_init_state, \ self.placement_generate_neighbor, \ self.placement_compute_util) util = simulated_annealing.run() return util
def __init__(self): """Initialize instance of IP Resolver.""" self.config = JobConfig().getconfig() self.logger = utility.getlogger('ip_resolution', 'ip_resolution')
target=ipResolver.resolve_ipaddress, args=(frame[1], geoservice, datawriter, processNo)) processList.append(process_ipresolve) process_ipresolve.start() logger.info('processNo-' + str(process_ipresolve.pid)) for p in processList: p.join() # print(str(p.exitcode)) except Exception as ex: logger.info('Issue in fetching data from Panoply:' + str(ex)) logger.error(utility.print_exception()) return False logger.info("Finished the batch job in %s seconds" % str( (time.time() - seconds) // 1)) return True if __name__ == "__main__": logger = utility.getlogger('ip_resolution', 'ip_resolution') logger.info('Starting ip resolution job') config = JobConfig().getconfig() utility.check_s3path(config) ip_resolver = IPResolver() panoplyreader = PanoplyImport() datawriter = S3Writer() ipdata_geoservice = IPDataService() if process_chunks(config, ip_resolver, ipdata_geoservice, panoplyreader, datawriter): panoplywriter = PanoplyWriter() panoplywriter.save_data()