"""
Utility method tests
"""
from tempfile import mkdtemp
import unittest
from gbots.tests import skip, TestCase
from gbots.util.web import save_web_page_complete, convert_to_posix_args
from gbots.util.loggers import getLogger
logger = getLogger(__name__)
class PynixTest(unittest.TestCase):
"""
@note: inheriting from a simple unittest.TestCase to avoid loading fixtures
"""
def testConvertToPosixArgsSimpleArguments(self):
"""
convert_to_posix_args uses proper '--' for verbose arguments
"""
args = convert_to_posix_args(a=True)
self.assertEqual('-a', args[0],
"simple posix arguments must start with '-'")
def testConvertToPosixArgsVerboseArguments(self):
"""
convert_to_posix_args uses proper '--' for verbose arguments
"""
args = convert_to_posix_args(aa=True)
self.assertEqual('--aa', args[0],
"verbose posix arguments must start with '--'")
from django.db import models
from django.db.models import Manager
from dynamic_scraper.models import Scraper, SchedulerRuntime
from gbots.util import fields, loggers
from gbots.scraping.processors import process
import re
logger = loggers.getLogger(__name__)
# Sample code to dynamically delete active scrapers when the scraper is deleted from the admin interface
#@receiver(pre_delete)
#def pre_delete_handler(sender, instance, using, **kwargs):
# ....
#
# if isinstance(instance, Article):
# if instance.checker_runtime:
# instance.checker_runtime.delete()
#
#pre_delete.connect(pre_delete_handler)
##### Sources ####
class SourceModel(models.Model):
class Meta:
abstract = True
scraper = fields.WeakForeignKey(Scraper)
scraper_runtime = fields.WeakForeignKey(SchedulerRuntime)
alias = models.CharField(max_length=20)
description = models.CharField(max_length=200)
import re
from gbots.util.loggers import getLogger
logger = getLogger(__name__)
__author__ = 'jeffmay'
class UnrecognizedProcessor(ValueError):
pass
class IllegalProcessorCommand(ValueError):
pass
def process(command, value):
def combine_ending_backslashes(seq, next):
last = seq.pop() if len(seq) > 0 else None
if last is not None and last.endswith('\\'):
seq.append("%s/%s" % (last, next))
else:
if last is not None:
seq.append(last)
seq.append(next)
return seq
chunks = reduce(combine_ending_backslashes, command.split('/'), [])
try:
processor, find, repl, flags = chunks
except ValueError, e:
logger.warning("Could not parse processor command: %s, expected %d unpacked %d" % (e, 4, len(chunks)))
raise IllegalProcessorCommand(command, e)
if processor == 's':