def main(argv): logger.init_logger() logging.info('Welcome to GOGetter 3000 :) \nType -h for help') input_path, output_path = arguments_parser.parse_input(argv) fh = file_handler.FileHandler(input_path, output_path) dm = data_manipulate.DataManipulate(fh) res = dm.get_host_dict_from_csv() if res: fh.save_to_csv(res)
def __init__(self, orch_filename): # Initialise the thread Thread.__init__(self) # Set up the log file for logging logging.basicConfig(format='%(asctime)s %(message)s', filename='logs/log1.log', filemode='w', level=logging.DEBUG) # load orch file info, these methods are static so easy testing self.orch_dict = Member._get_orch_parameters(orch_filename) logging.info('MEMBER: Orch Dict %s', self.orch_dict) # Make a dummy file if the file does not exist Member._write_dummy_composition(self.orch_dict.get('composition_name'), self.orch_dict.get('total_bytes')) # self.parts_dict = Member._get_parts_dict( self.orch_dict.get('composition_name'), self.orch_dict.get('bytes_per_part'), self.orch_dict.get('parts_checksum_dict')) # Initialise dicts that will be for quick lookup of ips, parts lists, transfers etc... self.connections_ip_dict = {} self.connections_parts_dict = {} self.active_transfers = {} self.connections_queue_dict = {} self.list_of_orch_ips = {} # The "director" queue for recieving messages, we add a message to it to give the conductor self.director_queue = queue.Queue() message = {'msg': 'CONDUCTOR'} self.director_queue.put(message) # Initialise the connection handler self.connect_queue = queue.Queue() self.con_handler = connection_handler.ConnectionHandler( self.connect_queue, self.director_queue, self.orch_dict) # Starts a thread that handles reading & writing parts to disk self.file_queue = queue.Queue() self.f_handler = file_handler.FileHandler(self.orch_dict, self.file_queue, self.director_queue) # Starts a thread that handles monitoring progress for output in terminal # First count to see how many parts we have already has_parts = len([i for i in self.parts_dict.values() if i is True]) # A monitor thread the shows a progress bar in the terminal self.monitor_queue = queue.Queue() self.mon = monitor.Monitor(has_parts, self.orch_dict['num_parts'], self.monitor_queue, self.director_queue)
'w') as fp: # first, get rid of any previous log fp.write('Log\r\n') autonomous = True except OSError as e: pass # fake drive system if config.DEBUG is set and we're teathered. system = System(drive.make_drive(config.DEBUG and not autonomous)) system.ir.threshold = 25 system.power_on() #set the indicator and log to a file if autonomous: system.indicate(colours.RED) import file_handler logger.addHandler(file_handler.FileHandler(config.LOG_FILE)) else: system.indicate(colours.GREEN) # Setup behaviours behaviours = Behaviours() behaviours.add(ForwardBehaviour(system)) behaviours.add(WanderBehaviour(system)) # behaviours.add(ChaseBehaviour(system)) # behaviours.add(RunAwayBehaviour(system)) behaviours.add(WhiskerBehaviour(system)) # Timing control update_time = 0
return text if __name__ == '__main__': in_directory = r"E:\Corpora\PII_Directory_20190507" out_directory = r"E:\Corpora\PII_Directory_20190507\export_documents" label_dictionary = { 'PII|Health|condition_treatment': 0, 'PII|Health|health_payment': 1, 'PII|Health|applications_and_claims': 2, 'PII|Employment|performance_review': 3 } fh = file_handler.FileHandler(in_directory=in_directory) # Guid / path dictionary. fh.make_file_dictionary() csv_file = r"E:\Corpora\PII_Directory_20190507\Paragraph_Reports\health_payment\combined.csv" df = pd.read_csv(csv_file, usecols=['guid', 'tag']) df.drop_duplicates(inplace=True) df = df.loc[df['tag'].isin([ 'PII|Health|condition_treatment', 'PII|Health|health_payment', 'PII|Health|applications_and_claims', 'PII|Employment|performance_review' ])] df['content'] = '' df['content'] = df.apply(get_text_from_row, axis=1) df['category_id'] = 0
def get_nett_for_range(): handler = file_handler.FileHandler() df = handler.get_classified('transactions.csv')
def __init__(self, options=None, parser=None): self.options = options self.parser = parser self.handle = file_handler.FileHandler() self.config_data = None
import file_handler debug = 0 fh = file_handler.FileHandler() fh.set_subdir("Vzorování") # defines name of subfolder fh.make_dirs(600, 10) # creates folders 000600 to 000609
# configure logging if args.log_file is None: # use stdout by default logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') else: logging.basicConfig(filename=args.log_file, level=logging.INFO, format='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') # setup event handler event_handler = file_handler.FileHandler(args.fhicl_configuration, args.input_dir) # setup observer for file system changes observer = Observer() observer.schedule(event_handler, args.input_dir) observer.start() # Sleep Forever try: while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join()