def createFile( self, content, rootPath=c.PATH_OUTPUT, ext=config.EXT_DEFAULT_OUTPUT ):
pathAndFileName= rootPath + h.generateIdentify() + ext
with open(pathAndFileName, 'w+') as f:
for item in content:
f.write("%s\n" % item)
f.close
h.logger(f"Created file {pathAndFileName}")
def on_any_event(event):
try:
if event.event_type == 'created' and not event.is_directory:
pathAndFilename = event.src_path
h.logger(f"File {h.removePath( pathAndFilename )} has been added.")
main.run( pathAndFilename )
except Exception as e:
h.logger(f"Something wrong on watcher. Erro: {str(e)}")
def run(self):
event_handler = Handler()
h.logger(f"Watcher has started")
self.observer.schedule(event_handler, self.DIRECTORY_TO_WATCH, recursive=True)
self.observer.start()
try:
while True:
time.sleep(1)
except:
self.observer.stop()
h.logger(f"File watcher has been finalized")
self.observer.join()
def getBlockIntervalPositions( self, fileName ):
positions = list()
fileName = c.PATH_INPUT+fileName
STRING_BLOCK_SEPARATOR = config.STRING_BLOCK_SEPARATOR
if config.STRING_BLOCK_SEPARATOR == None:
h.logger(f"Unidentified environment variables")
else:
if os.path.isfile(fileName):
with open(fileName) as f:
for num, line in enumerate(f, 1):
if STRING_BLOCK_SEPARATOR in line:
positions.append(str(num))
positions.append(str(num))
return positions
def getContentFileByInterval( self, fileName, positions, maxLine, batch=1 ):
blocks = list()
running = True
listCycle = cycle(positions)
nextItem = next(listCycle)
fileName = c.PATH_INPUT+fileName
for item in range(1, maxLine):
while running:
thisItem, nextItem = nextItem, next(listCycle)
block = self.extractContentBlockFileByInterval( fileName, start=thisItem, end=nextItem )
self.register( content=block )
blocks.append( block )
h.logger(f"Extracted the file detail #{str(item+1)}/{maxLine} of batch {batch}: StartLine: {str(thisItem)} - EndLine: {str(nextItem)}")
break
return blocks
class handleWatchFileSystem( PatternMatchingEventHandler ):
patterns = c.EXT_ALLOWED_INPUT
try:
def process(self, event):
# the file will be processed there
if event.event_type == 'created':
pathAndFilename = event.src_path
h.logger(f"File {h.removePath(pathAndFilename)} has been added.") # print now only for degug
main.run( pathAndFilename )
def on_modified(self, event):
self.process(event)
def on_created(self, event):
self.process(event)
except:
h.logger(f"Something wrong")
def run( pathAndFileName ):
startTime = dt.datetime.now().strftime("%Y/%m/%d at %H:%M:%S")
h.logger(f"Initiating file data extraction process.")
inputFileName = h.removePath(pathAndFileName)
ios = io.IOStreams()
positions = ios.getBlockIntervalPositions( inputFileName )
#sys.exit("Finally process")
packages = h.splitList( positions, 2 )
if len(packages) > 0:
for num, package in enumerate(packages.items()):
ios.getContentFileByInterval( inputFileName, positions=package[1], maxLine=len(package[1]), batch=num+1 )
endTime = dt.datetime.now().strftime("%Y/%m/%d at %H:%M:%S")
h.logger(f"Extraction process closed.")
h.logger(f"Started at {startTime} and finished at {endTime}")
#run('.\\storage\\input\\text-small.txt') # call test
def execute(self, query, params):
try:
self.connector()
self.cursor.execute(query, params)
self.connection.commit()
h.logger(
f"Record successfully into table. ID: {self.cursor.lastrowid}")
return self.cursor.lastrowid
except mysql.connector.Error as error:
self.connection.rollback() #rollback if any exception occured
h.logger(f"Failed record into table {format(error)}")
finally:
#closing database connection.
if (self.connection.is_connected()):
self.cursor.close()
self.connection.close()
h.logger("MySQL connection is closed")
def process(self, event):
# the file will be processed there
if event.event_type == 'created':
pathAndFilename = event.src_path
h.logger(f"File {h.removePath(pathAndFilename)} has been added.") # print now only for degug
main.run( pathAndFilename )
try:
def process(self, event):
# the file will be processed there
if event.event_type == 'created':
pathAndFilename = event.src_path
h.logger(f"File {h.removePath(pathAndFilename)} has been added.") # print now only for degug
main.run( pathAndFilename )
def on_modified(self, event):
self.process(event)
def on_created(self, event):
self.process(event)
except:
h.logger(f"Something wrong")
if __name__ == '__main__':
args = sys.argv[1:]
observer = Observer()
observer.schedule( handleWatchFileSystem(), path=args[0] if args else '.' )
observer.start()
h.logger(f"Watcher has started")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
h.logger(f"File watcher has been finalized")
observer.stop()
observer.join()
