def open_spider(self, spider):
self.load_spider(spider)
# Configure the connection
self.configure()
if self.config['replica_set'] is not None:
connection = MongoReplicaSetClient(
self.config['uri'],
replicaSet=self.config['replica_set'],
w=self.config['write_concern'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY_PREFERRED)
else:
# Connecting to a stand alone MongoDB
connection = MongoClient(
self.config['uri'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY)
try:
connection.server_info()
except ServerSelectionTimeoutError:
self.logger.error(
u'Connection refused {0}'.format(
self.config['uri']))
raise NotConfigured(u'Connection refused: {0}'.format(
self.config['uri']))
# Set up the collection
database = connection[self.config['database']]
self.collection = database[self.config['collection']]
self.logger.info(u'Connected to MongoDB {0}, using "{1}/{2}"'.format(
self.config['uri'],
self.config['database'],
self.config['collection']))
# Ensure unique index
if self.config['unique_key']:
self.collection.ensure_index(self.config['unique_key'], unique=True)
self.logger.info(u'Ensuring index for key {0}'.format(
self.config['unique_key']))
# Get the duplicate on key option
if self.config['stop_on_duplicate']:
tmpValue = self.config['stop_on_duplicate']
if tmpValue < 0:
self.logger.error(
u'Negative values are not allowed for'
u' MONGODB_STOP_ON_DUPLICATE option.'
)
raise SyntaxError(
(
'Negative values are not allowed for'
' MONGODB_STOP_ON_DUPLICATE option.'
)
)
self.stop_on_duplicate = self.config['stop_on_duplicate']
else:
self.stop_on_duplicate = 0
