def createFile( self, content, rootPath=c.PATH_OUTPUT, ext=config.EXT_DEFAULT_OUTPUT ): pathAndFileName= rootPath + h.generateIdentify() + ext with open(pathAndFileName, 'w+') as f: for item in content: f.write("%s\n" % item) f.close h.logger(f"Created file {pathAndFileName}")
def on_any_event(event): try: if event.event_type == 'created' and not event.is_directory: pathAndFilename = event.src_path h.logger(f"File {h.removePath( pathAndFilename )} has been added.") main.run( pathAndFilename ) except Exception as e: h.logger(f"Something wrong on watcher. Erro: {str(e)}")
def run(self): event_handler = Handler() h.logger(f"Watcher has started") self.observer.schedule(event_handler, self.DIRECTORY_TO_WATCH, recursive=True) self.observer.start() try: while True: time.sleep(1) except: self.observer.stop() h.logger(f"File watcher has been finalized") self.observer.join()
def getBlockIntervalPositions( self, fileName ): positions = list() fileName = c.PATH_INPUT+fileName STRING_BLOCK_SEPARATOR = config.STRING_BLOCK_SEPARATOR if config.STRING_BLOCK_SEPARATOR == None: h.logger(f"Unidentified environment variables") else: if os.path.isfile(fileName): with open(fileName) as f: for num, line in enumerate(f, 1): if STRING_BLOCK_SEPARATOR in line: positions.append(str(num)) positions.append(str(num)) return positions
def getContentFileByInterval( self, fileName, positions, maxLine, batch=1 ): blocks = list() running = True listCycle = cycle(positions) nextItem = next(listCycle) fileName = c.PATH_INPUT+fileName for item in range(1, maxLine): while running: thisItem, nextItem = nextItem, next(listCycle) block = self.extractContentBlockFileByInterval( fileName, start=thisItem, end=nextItem ) self.register( content=block ) blocks.append( block ) h.logger(f"Extracted the file detail #{str(item+1)}/{maxLine} of batch {batch}: StartLine: {str(thisItem)} - EndLine: {str(nextItem)}") break return blocks
class handleWatchFileSystem( PatternMatchingEventHandler ): patterns = c.EXT_ALLOWED_INPUT try: def process(self, event): # the file will be processed there if event.event_type == 'created': pathAndFilename = event.src_path h.logger(f"File {h.removePath(pathAndFilename)} has been added.") # print now only for degug main.run( pathAndFilename ) def on_modified(self, event): self.process(event) def on_created(self, event): self.process(event) except: h.logger(f"Something wrong")
def run( pathAndFileName ): startTime = dt.datetime.now().strftime("%Y/%m/%d at %H:%M:%S") h.logger(f"Initiating file data extraction process.") inputFileName = h.removePath(pathAndFileName) ios = io.IOStreams() positions = ios.getBlockIntervalPositions( inputFileName ) #sys.exit("Finally process") packages = h.splitList( positions, 2 ) if len(packages) > 0: for num, package in enumerate(packages.items()): ios.getContentFileByInterval( inputFileName, positions=package[1], maxLine=len(package[1]), batch=num+1 ) endTime = dt.datetime.now().strftime("%Y/%m/%d at %H:%M:%S") h.logger(f"Extraction process closed.") h.logger(f"Started at {startTime} and finished at {endTime}") #run('.\\storage\\input\\text-small.txt') # call test
def execute(self, query, params): try: self.connector() self.cursor.execute(query, params) self.connection.commit() h.logger( f"Record successfully into table. ID: {self.cursor.lastrowid}") return self.cursor.lastrowid except mysql.connector.Error as error: self.connection.rollback() #rollback if any exception occured h.logger(f"Failed record into table {format(error)}") finally: #closing database connection. if (self.connection.is_connected()): self.cursor.close() self.connection.close() h.logger("MySQL connection is closed")
def process(self, event): # the file will be processed there if event.event_type == 'created': pathAndFilename = event.src_path h.logger(f"File {h.removePath(pathAndFilename)} has been added.") # print now only for degug main.run( pathAndFilename )
try: def process(self, event): # the file will be processed there if event.event_type == 'created': pathAndFilename = event.src_path h.logger(f"File {h.removePath(pathAndFilename)} has been added.") # print now only for degug main.run( pathAndFilename ) def on_modified(self, event): self.process(event) def on_created(self, event): self.process(event) except: h.logger(f"Something wrong") if __name__ == '__main__': args = sys.argv[1:] observer = Observer() observer.schedule( handleWatchFileSystem(), path=args[0] if args else '.' ) observer.start() h.logger(f"Watcher has started") try: while True: time.sleep(1) except KeyboardInterrupt: h.logger(f"File watcher has been finalized") observer.stop() observer.join()