def setUp(self):
self.mock_exec_commit = mock.Mock(return_value=100)
handler = mock.Mock()
handler.exec_commit = self.mock_exec_commit
operator_klass = import_string(self.operate_cls_str)
self.operator = operator_klass(handler)
def parse(self, kwargs):
# Gets config from the kwargs
config = self.get_config(kwargs)
# Initialize the fetcher
if self.fetcher_class:
self.fetcher = self.fetcher_class(self.query_class,
self.query_context)
# If data_model was defined as a string with dot,
# for example 'appname.models.AppnameModel'
if self.data_model:
self.data_model_cls = import_string(self.data_model)
# Sets base environment
environment = {
'root': config.common['root'],
'relpath': config.common.get('relpath', config.common['root']),
'task_id': config.upload['task_id'],
'title': config.export['title']
}
# Sets custom environment
self.set_environment(environment, config, kwargs)
return environment
def __load_or_import(self, handler_cls, base_cls, context):
if isinstance(handler_cls, six.string_types):
handler_cls = import_string(handler_cls)
if issubclass(handler_cls, base_cls):
return handler_cls(context)
else:
raise ImproperlyConfigured(\
"Database handlers must be a subclass of {}.".format(base_cls))
def configure_logging(logging_config, logging_settings):
if logging_config:
# First find the logging configuration function ...
logging_config_func = import_string(logging_config)
logging.config.dictConfig(DEFAULT_LOGGING)
# ... then invoke it with the custom logging settings
if logging_settings:
logging_config_func(logging_settings)
def create_from_options(cls, pool_cls_str, options, cache_path):
topics_repeat_table = {o[0]:o[-1] for o in options}
pool_cls = import_string(pool_cls_str)
topics_feeder = cls(topics_repeat_table, pool_cls, cache_path)
for option in options:
topic, path, proportion, repeat = option
topics_feeder.feed(topic, path)
return topics_feeder
def create_from_context(cls, query_context):
handler_cls = query_context['sql_handler']
handler_settings = query_context['sql_context']
if isinstance(handler_cls, six.string_types):
handler_cls = import_string(handler_cls)
if issubclass(handler_cls, cls.base_handler_cls):
handler = handler_cls(handler_settings)
else:
raise ImproperlyConfigured(\
"Database handlers must be a subclass of {}.".format(base_cls))
table_alias = handler_settings['TABLE_ALIAS']
return cls(handler, table_alias)
def run(self, **kwargs):
context = self.get_context(kwargs)
queryset = self.fetch(context)
output = []
neffective = 0
data_model_cls = import_string(self.data_model)
for item in queryset:
dm = data_model_cls(item)
if dm.is_effective():
neffective += 1
self.handle(dm, context)
output.append("%d results processed." % neffective)
return '\n'.join(output)
def __init__(self, app_config, stdout=None, stderr=None, style=None): super(BaseAction, self).__init__() # Sets app_config self.app = app_config # Sets stdout and stderr, which were supposed to passed from command self.stdout = stdout or sys.stdout self.stderr = stderr or sys.stderr self.style = style or color_style() # Imports stats class self.stats = import_string(self.stats_class)(self) # String to record and display after all works done self.output = []
def __init__(self,
callback,
stats,
worker_cls=None,
timeout=None,
overwrite=False,
nworkers=10):
self.queue = Queue.Queue()
self.callback = callback
self.stats = stats
self.timeout = timeout or settings.DEFAULT_TIMEOUT
self.overwrite = overwrite
# Run in one loop if setting DEBUG mode
self.nworkers = nworkers
worker_cls_str = worker_cls if worker_cls else self.DEFAULT_WORKER_CLASS
self.worker_cls = import_string(worker_cls_str)
def create_from_context(cls, query_context):
handler_cls = query_context['sql_handler']
handler_settings = query_context['sql_context']
if isinstance(handler_cls, six.string_types):
try:
handler_cls = import_string(handler_cls)
except ImportError as e:
raise ImproperlyConfigured(\
"SQL handler specified is not an importable module: {}.".format(handler_cls))
if issubclass(handler_cls, cls.base_handler_cls):
handler = handler_cls(handler_settings)
else:
raise ImproperlyConfigured(\
"Database handlers must be a subclass of {}.".format(cls.base_handler_cls))
return cls(handler)
def run(self, **kwargs):
context = self.get_context(kwargs)
queryset = self.fetch(context)
output = []
urls = []
neffective = 0
data_model_cls = import_string(self.data_model)
for item in queryset:
dm = data_model_cls(item)
if dm.is_effective():
neffective += 1
self.handle(dm, context)
urls.append((dm.filelink(context['task_id']), dm.filepath))
dst = os.path.join(self.app.data_dirname, context['title'])
stat = DownloadStat()
download(urls, dst, stat, overwrite=self.overwrite_conflict)
output.append("%d results processed." % neffective)
output.append(str(stat))
return '\n'.join(output)
def __init__(self, feeder_cls, maker_cls, cache_path):
self.feeder_cls = import_string(feeder_cls)
# self.maker_cls = import_string(maker_cls)
self.cache_path = cache_path
def setUp(self):
self.mock_exec_query = mock.Mock(return_value="test")
handler = mock.Mock()
handler.exec_query = self.mock_exec_query
query_klass = import_string(self.query_cls_str)
self.querier = query_klass(handler)