class WordsWriterTest(unittest.TestCase):
def setUp(self):
self.textfile = 'in'
self.emptyfile = 'out'
self.nonemptyfile = 'nonempty'
self.sample = 'no_file_disk'
self.data = 'qwerty \n !,.?:QweRty;[] \n q1W34erT78y \t soFTeq 9Softeq9 sofTEQ\n'
self.len_data = len(self.data)
with open(self.textfile, 'w') as f:
f.write(self.data)
with open(self.emptyfile, 'w') as f:
f.write('')
with open(self.nonemptyfile, 'w') as f:
f.write('data')
self.file_ext = Analyzer(self.textfile, self.emptyfile)
self.file_ext_empty = Analyzer(self.emptyfile, self.textfile)
self.file_nonempty = Analyzer(self.textfile,self.nonemptyfile)
self.file_sample = Analyzer(self.textfile, self.sample)
def tearDown(self):
os.unlink(self.textfile)
os.unlink(self.emptyfile)
os.unlink(self.nonemptyfile)
try:
os.unlink(self.sample)
except FileNotFoundError:
pass
def test_EmptyFileWriteOnDisk(self):
self.file_sample.words = []
assert os.path.exists(self.sample) is False
self.file_sample.words_writer()
assert os.path.exists(self.sample) is True
def test_NonEmptyFileWriteOnDisk(self):
self.file_sample.words = [self.data]
assert os.path.exists(self.sample) is False
self.file_sample.words_writer()
assert os.path.exists(self.sample) is True
def test_ValidDataWrite(self):
self.file_sample.words_processing()
self.filesize = os.path.getsize(self.sample)
self.lenght = 0
for item in self.file_sample.words:
self.lenght +=len(item)
self.lenght +=1
self.assertEqual(self.lenght,self.filesize)
def test_AddDataToExistFile(self):
self.file_ext.words_processing()
self.filesize_before = os.path.getsize(self.emptyfile)
self.file_ext.words_processing()
self.filesize_after = int(os.path.getsize(self.emptyfile))
self.lenght = 0
for item in self.file_ext.words:
self.lenght +=len(item)
self.lenght +=1
self.assertEqual(self.filesize_before + self.lenght, self.filesize_after)
def setUp(self):
self.textfile = 'in'
self.emptyfile = 'out'
self.data = 'qwerty \n Qwerty \n !,.?:QweRty;[] \n q1W34erT78y'
with open(self.textfile, 'w') as f:
f.write(self.data)
with open(self.emptyfile, 'w') as f:
f.write('')
self.words_ext = Analyzer(self.textfile, self.emptyfile)
self.words_ext_empty = Analyzer(self.emptyfile, self.textfile)
def test_WriteValidDataToNonEmptyFile(self):
assert os.path.exists(self.file_out) is False
self.tmp = Analyzer(self.textfile, self.file_out).lines_processing()
assert os.path.exists(self.file_out) is True
self.tmp = Analyzer(self.textfile, self.file_out).lines_processing()
assert os.path.exists(self.file_out) is True
with open(self.file_out) as f:
self.lines_in_file = sum(1 for line in f)
self.lines_in_data = len((self.data+self.data).split('\n'))
self.assertEqual(self.lines_in_data, self.lines_in_file)
class LineProcessingTest(unittest.TestCase):
def setUp(self):
self.textfile = 'in'
self.emptyfile = 'empty'
self.file_out = 'out'
self.data = 'qwerty \n !,.?:QweRty;[] \n q1W34erT78y \n soFTeq \n 9Sof \n teq9 sofTEQ'
with open(self.textfile, 'w') as f:
f.write(self.data)
with open(self.emptyfile, 'w') as f:
f.write('')
def tearDown(self):
os.unlink(self.textfile)
os.unlink(self.emptyfile)
try:
os.unlink(self.file_out)
except FileNotFoundError:
pass
def test_FileCreateToWriteLines(self):
self.tmp = Analyzer(self.textfile, self.file_out)
assert os.path.exists(self.file_out) is False
self.tmp.lines_processing()
assert os.path.exists(self.file_out) is True
def test_FileOpenToWriteLines(self):
self.tmp = Analyzer(self.textfile, self.emptyfile)
assert os.path.exists(self.emptyfile) is True
self.tmp.lines_processing()
assert os.path.exists(self.emptyfile) is True
def test_WriteValidDataToEmptyFile(self):
self.tmp = Analyzer(self.textfile, self.emptyfile).lines_processing()
self.lines_in_data = len(self.data.split('\n'))
with open(self.emptyfile) as f:
self.lines_in_file = sum(1 for line in f)
self.assertEqual(self.lines_in_data, self.lines_in_file)
def test_WriteValidDataToNonEmptyFile(self):
assert os.path.exists(self.file_out) is False
self.tmp = Analyzer(self.textfile, self.file_out).lines_processing()
assert os.path.exists(self.file_out) is True
self.tmp = Analyzer(self.textfile, self.file_out).lines_processing()
assert os.path.exists(self.file_out) is True
with open(self.file_out) as f:
self.lines_in_file = sum(1 for line in f)
self.lines_in_data = len((self.data+self.data).split('\n'))
self.assertEqual(self.lines_in_data, self.lines_in_file)
def main(args):
log_level = logging.INFO
if args.debug:
log_level = logging.DEBUG
logging.basicConfig(filename=None if args.verbose else "reylgan.log",
level=log_level)
workers = []
if args.worker:
workers.extend([[Worker(), Worker]
for _ in range(0, int(args.worker))])
logging.info("add %s crawlers." % args.worker)
if args.analyzer:
workers.extend([[Analyzer(), Analyzer]
for _ in range(0, int(args.analyzer))])
logging.info("add %s analyzers." % args.analyzer)
if args.frontend:
"""
@todo: start frontend worker
"""
raise NotImplementedError
[w[0].start() for w in workers]
while True:
for i, worker in enumerate(workers):
if not worker[0].is_alive():
worker[0] = worker[1]()
logging.info("Starting a new worker %s" % worker[0].name)
worker[0].start()
time.sleep(5)
def runner():
Log.logger.debug('Start application.... ')
try:
args = parser.parse_args()
f_path, f_out, mode = (args.file, args.out, args.mode)
except:
Log.logger.error('Missing required parameters')
Log.logger.debug('Stop application....')
exit()
if mode != 'lines' and mode != 'words':
Log.logger.error('Invalid mode!')
Log.logger.debug('Stop application....')
exit()
if not os.path.exists(f_path):
Log.logger.error('File not found!')
Log.logger.debug('Stop application....')
exit()
elif f_path == f_out:
Log.logger.error('File must be different!')
Log.logger.debug('Stop application....')
exit()
elif os.path.exists(f_out):
Log.logger.debug('Output file {} exists'.format(f_out))
Log.logger.debug('Waiting for user confirmation...')
choise = input(
'Data will be added to the existing file. Conitnue [y/n] ?\n')
if choise.lower() == 'y':
Log.logger.debug('Data will be added to the existing file')
pass
elif choise.lower() == 'n':
Log.logger.debug(
'Negative answer is selected. The application will be closed')
Log.logger.debug('Stop application....')
exit()
elif choise.lower() != 'n' and choise.lower() != 'y':
Log.logger.error('Invalid choise')
Log.logger.debug('Stop application....')
exit()
if mode == 'lines':
Log.logger.debug(
'User choise: find lines in {} and put them in {}'.format(
f_path, f_out))
obj = Analyzer(args.file, args.out)
obj.lines_processing()
Log.logger.debug('Stop application....')
elif mode == 'words':
Log.logger.debug(
'User choise: find unique words in {} and put them in {}'.format(
f_path, f_out))
obj = Analyzer(args.file, args.out)
obj.words_processing()
Log.logger.debug('Stop application....')
class WordExtractorTest(unittest.TestCase):
def setUp(self):
self.textfile = 'in'
self.emptyfile = 'out'
self.data = 'qwerty \n Qwerty \n !,.?:QweRty;[] \n q1W34erT78y'
with open(self.textfile, 'w') as f:
f.write(self.data)
with open(self.emptyfile, 'w') as f:
f.write('')
self.words_ext = Analyzer(self.textfile, self.emptyfile)
self.words_ext_empty = Analyzer(self.emptyfile, self.textfile)
def tearDown(self):
os.unlink(self.textfile)
os.unlink(self.emptyfile)
def test_IsInstanceCreate(self):
self.assertIsInstance(self.words_ext, Analyzer)
def test_WordsListInit(self):
self.words_ext_empty.words_extractor()
self.assertListEqual(self.words_ext_empty.words, [])
def test_EmptyFile(self):
self.words_ext_empty.words_extractor()
self.assertEqual(len(self.words_ext_empty.words), 0)
def test_CounterSuccess(self):
self.words_ext.words_extractor()
self.assertEqual(len(self.words_ext.words), 4)
def test_CounterFailed(self):
self.words_ext.words_extractor()
self.assertNotEqual(len(self.words_ext.words), not 4)
def test_ValidData(self):
self.words_ext.words_extractor()
self.assertListEqual(self.words_ext.words, ['qwerty', 'qwerty', 'qwerty', 'q1w34ert78y'])
def test_analyzer():
"""
collection users need some data before test
"""
import logging
from worker import Analyzer
logging.basicConfig(level=logging.DEBUG)
analyzer = Analyzer()
analyzer.start()
analyzer.join()
def setUp(self):
self.textfile = 'in'
self.emptyfile = 'out'
self.sample = 'no_file_disk'
self.data = 'qwerty \n !,.?:QweRty;[] \n q1W34erT78y \t soFTeq 9Softeq9 sofTEQ\n'
with open(self.textfile, 'w') as f:
f.write(self.data)
with open(self.emptyfile, 'w') as f:
f.write('')
self.words_ext = Analyzer(self.textfile, self.emptyfile)
self.words_ext_empty = Analyzer(self.emptyfile, self.textfile)
self.words_sample = Analyzer(self.textfile, self.sample)
def test_WriteValidDataToEmptyFile(self):
self.tmp = Analyzer(self.textfile, self.emptyfile).lines_processing()
self.lines_in_data = len(self.data.split('\n'))
with open(self.emptyfile) as f:
self.lines_in_file = sum(1 for line in f)
self.assertEqual(self.lines_in_data, self.lines_in_file)
def test_FileOpenToWriteLines(self):
self.tmp = Analyzer(self.textfile, self.emptyfile)
assert os.path.exists(self.emptyfile) is True
self.tmp.lines_processing()
assert os.path.exists(self.emptyfile) is True
def test_FileCreateToWriteLines(self):
self.tmp = Analyzer(self.textfile, self.file_out)
assert os.path.exists(self.file_out) is False
self.tmp.lines_processing()
assert os.path.exists(self.file_out) is True
def test_ReturnStr(self):
self.assertMultiLineEqual(str(6), Analyzer.filesize(self.file))
def test_Failed(self):
self.assertNotEqual('8', Analyzer.filesize(self.file))
def test_Success(self):
self.assertEqual(self.len_data, Analyzer.filesize(self.file))
def test_InstanceCreate(self):
self.inst = Analyzer(self.file, self.data)
self.assertIsInstance(self.inst, Analyzer)
class WordProcessingTest(unittest.TestCase):
def setUp(self):
self.textfile = 'in'
self.emptyfile = 'out'
self.sample = 'no_file_disk'
self.data = 'qwerty \n !,.?:QweRty;[] \n q1W34erT78y \t soFTeq 9Softeq9 sofTEQ\n'
with open(self.textfile, 'w') as f:
f.write(self.data)
with open(self.emptyfile, 'w') as f:
f.write('')
self.words_ext = Analyzer(self.textfile, self.emptyfile)
self.words_ext_empty = Analyzer(self.emptyfile, self.textfile)
self.words_sample = Analyzer(self.textfile, self.sample)
def tearDown(self):
os.unlink(self.textfile)
os.unlink(self.emptyfile)
try:
os.unlink(self.sample)
except FileNotFoundError:
pass
def test_WordsExtractorRun(self):
self.words_ext_empty.words_processing()
self.assertListEqual(self.words_ext_empty.words, [])
def test_WordsWriterRun(self):
self.words_sample.words_processing()
assert os.path.exists(self.sample) is True
def test_WordsExtractorReturn(self):
self.words_ext.words_extractor()
self.assertEqual(len(self.words_ext.words), 6)
def test_EmptyFileProcessing(self):
self.words_ext_empty.words_processing()
self.assertEqual(len(self.words_ext_empty.words), 0)
def test_EmptyFileProcessingList(self):
self.words_ext_empty.words_processing()
self.assertListEqual(self.words_ext_empty.words, [])
def test_RemoveNonAlphaWords(self):
self.words_ext.words_processing()
self.assertEqual(len(self.words_ext.words), 2)
