def main(argv):
logger.init_logger()
logging.info('Welcome to GOGetter 3000 :) \nType -h for help')
input_path, output_path = arguments_parser.parse_input(argv)
fh = file_handler.FileHandler(input_path, output_path)
dm = data_manipulate.DataManipulate(fh)
res = dm.get_host_dict_from_csv()
if res:
fh.save_to_csv(res)
def __init__(self, orch_filename):
# Initialise the thread
Thread.__init__(self)
# Set up the log file for logging
logging.basicConfig(format='%(asctime)s %(message)s',
filename='logs/log1.log',
filemode='w',
level=logging.DEBUG)
# load orch file info, these methods are static so easy testing
self.orch_dict = Member._get_orch_parameters(orch_filename)
logging.info('MEMBER: Orch Dict %s', self.orch_dict)
# Make a dummy file if the file does not exist
Member._write_dummy_composition(self.orch_dict.get('composition_name'),
self.orch_dict.get('total_bytes'))
#
self.parts_dict = Member._get_parts_dict(
self.orch_dict.get('composition_name'),
self.orch_dict.get('bytes_per_part'),
self.orch_dict.get('parts_checksum_dict'))
# Initialise dicts that will be for quick lookup of ips, parts lists, transfers etc...
self.connections_ip_dict = {}
self.connections_parts_dict = {}
self.active_transfers = {}
self.connections_queue_dict = {}
self.list_of_orch_ips = {}
# The "director" queue for recieving messages, we add a message to it to give the conductor
self.director_queue = queue.Queue()
message = {'msg': 'CONDUCTOR'}
self.director_queue.put(message)
# Initialise the connection handler
self.connect_queue = queue.Queue()
self.con_handler = connection_handler.ConnectionHandler(
self.connect_queue, self.director_queue, self.orch_dict)
# Starts a thread that handles reading & writing parts to disk
self.file_queue = queue.Queue()
self.f_handler = file_handler.FileHandler(self.orch_dict,
self.file_queue,
self.director_queue)
# Starts a thread that handles monitoring progress for output in terminal
# First count to see how many parts we have already
has_parts = len([i for i in self.parts_dict.values() if i is True])
# A monitor thread the shows a progress bar in the terminal
self.monitor_queue = queue.Queue()
self.mon = monitor.Monitor(has_parts, self.orch_dict['num_parts'],
self.monitor_queue, self.director_queue)
'w') as fp: # first, get rid of any previous log
fp.write('Log\r\n')
autonomous = True
except OSError as e:
pass
# fake drive system if config.DEBUG is set and we're teathered.
system = System(drive.make_drive(config.DEBUG and not autonomous))
system.ir.threshold = 25
system.power_on()
#set the indicator and log to a file
if autonomous:
system.indicate(colours.RED)
import file_handler
logger.addHandler(file_handler.FileHandler(config.LOG_FILE))
else:
system.indicate(colours.GREEN)
# Setup behaviours
behaviours = Behaviours()
behaviours.add(ForwardBehaviour(system))
behaviours.add(WanderBehaviour(system))
# behaviours.add(ChaseBehaviour(system))
# behaviours.add(RunAwayBehaviour(system))
behaviours.add(WhiskerBehaviour(system))
# Timing control
update_time = 0
return text
if __name__ == '__main__':
in_directory = r"E:\Corpora\PII_Directory_20190507"
out_directory = r"E:\Corpora\PII_Directory_20190507\export_documents"
label_dictionary = {
'PII|Health|condition_treatment': 0,
'PII|Health|health_payment': 1,
'PII|Health|applications_and_claims': 2,
'PII|Employment|performance_review': 3
}
fh = file_handler.FileHandler(in_directory=in_directory)
# Guid / path dictionary.
fh.make_file_dictionary()
csv_file = r"E:\Corpora\PII_Directory_20190507\Paragraph_Reports\health_payment\combined.csv"
df = pd.read_csv(csv_file, usecols=['guid', 'tag'])
df.drop_duplicates(inplace=True)
df = df.loc[df['tag'].isin([
'PII|Health|condition_treatment', 'PII|Health|health_payment',
'PII|Health|applications_and_claims',
'PII|Employment|performance_review'
])]
df['content'] = ''
df['content'] = df.apply(get_text_from_row, axis=1)
df['category_id'] = 0
def get_nett_for_range():
handler = file_handler.FileHandler()
df = handler.get_classified('transactions.csv')
def __init__(self, options=None, parser=None):
self.options = options
self.parser = parser
self.handle = file_handler.FileHandler()
self.config_data = None
import file_handler
debug = 0
fh = file_handler.FileHandler()
fh.set_subdir("Vzorování") # defines name of subfolder
fh.make_dirs(600, 10) # creates folders 000600 to 000609
# configure logging
if args.log_file is None:
# use stdout by default
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
else:
logging.basicConfig(filename=args.log_file,
level=logging.INFO,
format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
# setup event handler
event_handler = file_handler.FileHandler(args.fhicl_configuration,
args.input_dir)
# setup observer for file system changes
observer = Observer()
observer.schedule(event_handler, args.input_dir)
observer.start()
# Sleep Forever
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
