def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection, download_func=None):
self.style = color.color_style()
##########from MediaPipeline###########
self.spiderinfo = {}
self.download_func = download_func
##########from MediaPipeline###########
self.store = self._get_store(shard_server, shard_port, shard_db, shard_gridfs_collection)
self.item_download = {}
def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection):
self.style = color.color_style()
try:
client = MongoClient(shard_server, shard_port)
self.db = client[shard_db]
self.fs = gridfs.GridFS(self.db, shard_gridfs_collection)
except Exception as e:
print self.style.ERROR("ERROR(MongodbBookFilesStore): %s" % (str(e),))
traceback.print_exc()
def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection, download_func=None):
self.style = color.color_style()
##########from MediaPipeline###########
self.spiderinfo = {}
self.download_func = download_func
##########from MediaPipeline###########
self.store = self._get_store(shard_server, shard_port, shard_db, shard_gridfs_collection)
self.item_download = {}
def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection):
self.style = color.color_style()
self.shard_gridfs_collection = shard_gridfs_collection
try:
self.client = MongoClient(shard_server, shard_port)
# self.inti_fs(shard_db)
except Exception as e:
print self.style.ERROR("ERROR(MongodbFilesStore): %s" % (str(e),))
traceback.print_exc()
def __init__(self, crawler):
super(GraphiteStatsCollector, self).__init__(crawler)
host = crawler.settings.get("GRAPHITE_HOST", self.GRAPHITE_HOST)
port = crawler.settings.get("GRAPHITE_PORT", self.GRAPHITE_PORT)
self.ignore_keys = crawler.settings.get("GRAPHITE_IGNOREKEYS",
self.GRAPHITE_IGNOREKEYS)
self.style = color.color_style()
self._sock = socket()
self._sock.connect((host, port))
def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection):
self.style = color.color_style()
try:
client = MongoClient(shard_server, shard_port)
self.db = client[shard_db]
self.fs = gridfs.GridFS(self.db, shard_gridfs_collection)
except Exception as e:
print (self.style.ERROR("ERROR(MongodbBookFilesStore): %s" % (str(e),)))
traceback.print_exc()
def __init__(self, shard_server, shard_port, shard_db,
shard_gridfs_collection):
self.style = color.color_style()
self.shard_gridfs_collection = shard_gridfs_collection
try:
self.client = MongoClient(shard_server, shard_port)
# self.inti_fs(shard_db)
except Exception as e:
print self.style.ERROR("ERROR(MongodbFilesStore): %s" % (str(e), ))
traceback.print_exc()
def __init__(self):
"""
The only async framework that PyMongo fully supports is Gevent.
Currently there is no great way to use PyMongo in conjunction with Tornado or Twisted. PyMongo provides built-in connection pooling, so some of the benefits of those frameworks can be achieved just by writing multi-threaded code that shares a MongoClient.
"""
self.style = color.color_style()
try:
client = MongoClient(self.MONGODB_SERVER, self.MONGODB_PORT)
self.db = client[self.MONGODB_DB]
except Exception as e:
print self.style.ERROR("ERROR(ShardMongodbPipeline): %s" % (str(e),))
traceback.print_exc()
def __init__(self):
"""
The only async framework that PyMongo fully supports is Gevent.
Currently there is no great way to use PyMongo in conjunction with Tornado or Twisted. PyMongo provides built-in connection pooling, so some of the benefits of those frameworks can be achieved just by writing multi-threaded code that shares a MongoClient.
"""
self.style = color.color_style()
try:
client = MongoClient(self.MONGODB_SERVER, self.MONGODB_PORT)
self.db = client[self.MONGODB_DB]
except Exception as e:
print self.style.ERROR("ERROR(ShardMongodbPipeline): %s" %
(str(e), ))
traceback.print_exc()
def __init__(self, store_uri, download_func=None):
if not store_uri:
raise NotConfigured
self.store = self._get_store(store_uri)
self.style = color.color_style()
super(FilePipeline, self).__init__(download_func=download_func)
def __init__(self, original_url="", *args):
self.original_url = original_url
self.style = color.color_style()
DropItem.__init__(self, *args)
def __init__(self):
self.style = color.color_style()
def __init__(self, host="127.0.0.1", port=2003):
self.style = color.color_style()
self._sock = socket()
self._sock.connect((host, port))
def __init__(self, file_url=None, *args):
self.file_url = file_url
self.style = color.color_style()
Exception.__init__(self, *args)
def __init__(self, store_uri, download_func=None):
if not store_uri:
raise NotConfigured
self.store = self._get_store(store_uri)
self.style = color.color_style()
super(FilePipeline, self).__init__(download_func=download_func)
def __init__(self, file_url=None, *args):
self.file_url = file_url
self.style = color.color_style()
Exception.__init__(self, *args)
def __init__(self, host="127.0.0.1", port=2003):
self.style = color.color_style()
self._sock = socket()
self._sock.connect((host, port))
def __init__(self, original_url="", *args):
self.original_url = original_url
self.style = color.color_style()
DropItem.__init__(self, *args)
