def parse_args(self, argv=None):
"""Parses command line arguments from stdin (or given argv).
Does the following:
1. Parses command line arguments
2. Loads config file into options (if config file specified)
3. calls option.apply_function with the parsed cli_values
"""
# parse the command line options
parser = optparse.OptionParser()
for option in self.options:
for args, kwargs in option.cli_options:
cli_option = parser.add_option(*args, **kwargs)
option.cli_names.append(cli_option.dest)
parsed_options, args = parser.parse_args(argv)
# load the config file
if parsed_options.config_file:
try:
with open(parsed_options.config_file) as f:
self.load_json(f.read())
except (OSError, IOError, ValueError):
reraise(errors.InvalidConfiguration, *sys.exc_info()[1:])
# apply the command line arguments
values = parsed_options.__dict__
for option in self.options:
option.apply_function(
option, dict((k, values.get(k)) for k in option.cli_names))
def parse_args(self, argv=None):
"""Parses command line arguments from stdin (or given argv).
Does the following:
1. Parses command line arguments
2. Loads config file into options (if config file specified)
3. calls option.apply_function with the parsed cli_values
"""
# parse the command line options
parser = optparse.OptionParser()
for option in self.options:
for args, kwargs in option.cli_options:
cli_option = parser.add_option(*args, **kwargs)
option.cli_names.append(cli_option.dest)
parsed_options, args = parser.parse_args(argv)
if args:
raise errors.InvalidConfiguration(
'The following command line arguments are not recognized: ' +
', '.join(args))
# load the config file
if parsed_options.config_file:
try:
with open(parsed_options.config_file) as f:
self.load_json(f.read())
except (OSError, IOError, ValueError):
reraise(errors.InvalidConfiguration, *sys.exc_info()[1:])
# apply the command line arguments
values = parsed_options.__dict__
for option in self.options:
option.apply_function(
option, dict((k, values.get(k)) for k in option.cli_names))
def wrap(*args, **kwargs):
try:
return func(*args, **kwargs)
except es_exceptions.ConnectionError:
exc_type, exc_value, exc_tb = sys.exc_info()
reraise(errors.ConnectionFailed, exc_value, exc_tb)
except es_exceptions.TransportError:
exc_type, exc_value, exc_tb = sys.exc_info()
reraise(errors.OperationFailed, exc_value, exc_tb)
def wrap(*args, **kwargs):
try:
return func(*args, **kwargs)
except es_exceptions.ConnectionError:
exc_type, exc_value, exc_tb = sys.exc_info()
reraise(errors.ConnectionFailed, exc_value, exc_tb)
except es_exceptions.TransportError:
exc_type, exc_value, exc_tb = sys.exc_info()
reraise(errors.OperationFailed, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
new_type = mapping.get(exc_type)
if new_type is None:
raise
reraise(new_type, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
new_type = mapping.get(exc_type)
if new_type is None:
raise
reraise(new_type, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
new_type = mapping.get(exc_type)
if new_type is None:
# Look for a superclass of the Exception.
for exc_class in mapping:
if isinstance(e, exc_class):
reraise(mapping[exc_class], exc_value, exc_tb)
# Otherwise raise the original Exception.
raise
reraise(new_type, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
new_type = mapping.get(exc_type)
if new_type is None:
# Look for a superclass of the Exception.
for exc_class in mapping:
if isinstance(e, exc_class):
reraise(mapping[exc_class], exc_value, exc_tb)
# Otherwise raise the original Exception.
raise
reraise(new_type, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
new_type = None
for src_type in mapping:
if issubclass(exc_type, src_type):
new_type = mapping[src_type]
break
if new_type is None:
raise
reraise(new_type, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
new_type = None
for src_type in mapping:
if issubclass(exc_type, src_type):
new_type = mapping[src_type]
break
if new_type is None:
raise
reraise(new_type, exc_value, exc_tb)
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
LOG.error("GOT EXCEPTION::", (exc_type, exc_value, exc_tb))
new_type = None
for src_type in mapping:
if issubclass(exc_type, src_type):
new_type = mapping[src_type]
break
if new_type is None:
raise
reraise(new_type, exc_value, exc_tb)
def parse_args(self, argv=None):
"""Parses command line arguments from stdin (or given argv).
Does the following:
1. Parses command line arguments
2. Loads config file into options (if config file specified)
3. calls option.apply_function with the parsed cli_values
"""
# parse the command line options
parser = optparse.OptionParser(version='%prog version: ' + __version__)
for option in self.options:
for args, kwargs in option.cli_options:
cli_option = parser.add_option(*args, **kwargs)
option.cli_names.append(cli_option.dest)
# argv = ["--auto-commit-interval=1", "-dsolr_doc_manager", "-m192.168.1.100:27017", "-nkxlist_list.quotelist", "-t http://192.168.1.100:8983/solr/user"]
parsed_options, args = parser.parse_args(argv)
if args:
raise errors.InvalidConfiguration(
'The following command line arguments are not recognized: ' +
', '.join(args))
# load the config file
if parsed_options.config_file:
try:
with open(parsed_options.config_file) as f:
self.load_json(f.read())
except (OSError, IOError, ValueError):
reraise(errors.InvalidConfiguration, *sys.exc_info()[1:])
# apply the command line arguments
values = parsed_options.__dict__
for option in self.options:
option.apply_function(
option, dict((k, values.get(k)) for k in option.cli_names))
def apply_update(self, doc, update_spec):
"""Apply an update operation to a document."""
# Helper to cast a key for a list or dict, or raise ValueError
def _convert_or_raise(container, key):
if isinstance(container, dict):
return key
elif isinstance(container, list):
return int(key)
else:
raise ValueError
# Helper to retrieve (and/or create)
# a dot-separated path within a document.
def _retrieve_path(container, path, create=False):
looking_at = container
for part in path:
if isinstance(looking_at, dict):
if create and not part in looking_at:
looking_at[part] = {}
looking_at = looking_at[part]
elif isinstance(looking_at, list):
index = int(part)
# Do we need to create additional space in the array?
if create and len(looking_at) <= index:
# Fill buckets with None up to the index we need.
looking_at.extend(
[None] * (index - len(looking_at)))
# Bucket we need gets the empty dictionary.
looking_at.append({})
looking_at = looking_at[index]
else:
raise ValueError
return looking_at
def _set_field(doc, to_set, value):
if '.' in to_set:
path = to_set.split(".")
where = _retrieve_path(doc, path[:-1], create=True)
index = _convert_or_raise(where, path[-1])
wl = len(where)
if isinstance(where, list) and index >= wl:
where.extend([None] * (index + 1 - wl))
where[index] = value
else:
doc[to_set] = value
def _unset_field(doc, to_unset):
try:
if '.' in to_unset:
path = to_unset.split(".")
where = _retrieve_path(doc, path[:-1])
index_or_key = _convert_or_raise(where, path[-1])
if isinstance(where, list):
# Unset an array element sets it to null.
where[index_or_key] = None
else:
# Unset field removes it entirely.
del where[index_or_key]
else:
del doc[to_unset]
except (KeyError, IndexError, ValueError):
source_version = get_mininum_mongodb_version()
if source_version is None or source_version.at_least(2, 6):
raise
# Ignore unset errors since MongoDB 2.4 records invalid
# $unsets in the oplog.
LOG.warning("Could not unset field %r from document %r. "
"This may be normal when replicating from "
"MongoDB 2.4 or the destination could be out of "
"sync." % (to_unset, doc))
# wholesale document replacement
if not "$set" in update_spec and not "$unset" in update_spec:
# update spec contains the new document in its entirety
return update_spec
else:
try:
# $set
for to_set in update_spec.get("$set", []):
value = update_spec['$set'][to_set]
_set_field(doc, to_set, value)
# $unset
for to_unset in update_spec.get("$unset", []):
_unset_field(doc, to_unset)
except (KeyError, ValueError, AttributeError, IndexError):
exc_t, exc_v, exc_tb = sys.exc_info()
reraise(UpdateDoesNotApply,
"Cannot apply update %r to %r" % (update_spec, doc),
exc_tb)
return doc
def apply_update(self, doc, update_spec):
"""Apply an update operation to a document."""
# Helper to cast a key for a list or dict, or raise ValueError
def _convert_or_raise(container, key):
if isinstance(container, dict):
return key
elif isinstance(container, list):
return int(key)
else:
raise ValueError
# Helper to retrieve (and/or create)
# a dot-separated path within a document.
def _retrieve_path(container, path, create=False):
looking_at = container
for part in path:
if isinstance(looking_at, dict):
if create and not part in looking_at:
looking_at[part] = {}
looking_at = looking_at[part]
elif isinstance(looking_at, list):
index = int(part)
if create and len(looking_at) < index:
looking_at.extend([None] *
(len(looking_at) - index - 1))
looking_at.append({})
looking_at = looking_at[index]
else:
raise ValueError
return looking_at
# wholesale document replacement
if not "$set" in update_spec and not "$unset" in update_spec:
# update spec contains the new document in its entirety
update_spec['_ts'] = doc['_ts']
update_spec['ns'] = doc['ns']
return update_spec
else:
try:
# $set
for to_set in update_spec.get("$set", []):
value = update_spec['$set'][to_set]
if '.' in to_set:
path = to_set.split(".")
where = _retrieve_path(doc, path[:-1], create=True)
where[_convert_or_raise(where, path[-1])] = value
else:
doc[to_set] = value
# $unset
for to_unset in update_spec.get("$unset", []):
if '.' in to_unset:
path = to_unset.split(".")
where = _retrieve_path(doc, path[:-1])
where.pop(_convert_or_raise(where, path[-1]))
else:
doc.pop(to_unset)
except (KeyError, ValueError, AttributeError, IndexError):
exc_t, exc_v, exc_tb = sys.exc_info()
reraise(UpdateDoesNotApply,
"Cannot apply update %r to %r" % (update_spec, doc),
exc_tb)
return doc
def apply_update(self, doc, update_spec):
"""Apply an update operation to a document."""
# Helper to cast a key for a list or dict, or raise ValueError
def _convert_or_raise(container, key):
if isinstance(container, dict):
return key
elif isinstance(container, list):
return int(key)
else:
raise ValueError
# Helper to retrieve (and/or create)
# a dot-separated path within a document.
def _retrieve_path(container, path, create=False):
looking_at = container
for part in path:
if isinstance(looking_at, dict):
if create and not part in looking_at:
looking_at[part] = {}
looking_at = looking_at[part]
elif isinstance(looking_at, list):
index = int(part)
# Do we need to create additional space in the array?
if create and len(looking_at) <= index:
# Fill buckets with None up to the index we need.
looking_at.extend([None] * (index - len(looking_at)))
# Bucket we need gets the empty dictionary.
looking_at.append({})
looking_at = looking_at[index]
else:
raise ValueError
return looking_at
def _set_field(doc, to_set, value):
if '.' in to_set:
path = to_set.split(".")
where = _retrieve_path(doc, path[:-1], create=True)
index = _convert_or_raise(where, path[-1])
wl = len(where)
if isinstance(where, list) and index >= wl:
where.extend([None] * (index + 1 - wl))
where[index] = value
else:
doc[to_set] = value
def _unset_field(doc, to_unset):
try:
if '.' in to_unset:
path = to_unset.split(".")
where = _retrieve_path(doc, path[:-1])
index_or_key = _convert_or_raise(where, path[-1])
if isinstance(where, list):
# Unset an array element sets it to null.
where[index_or_key] = None
else:
# Unset field removes it entirely.
del where[index_or_key]
else:
del doc[to_unset]
except (KeyError, IndexError, ValueError):
source_version = get_mininum_mongodb_version()
if source_version is None or source_version.at_least(2, 6):
raise
# Ignore unset errors since MongoDB 2.4 records invalid
# $unsets in the oplog.
LOG.warning("Could not unset field %r from document %r. "
"This may be normal when replicating from "
"MongoDB 2.4 or the destination could be out of "
"sync." % (to_unset, doc))
# wholesale document replacement
if not "$set" in update_spec and not "$unset" in update_spec:
# update spec contains the new document in its entirety
return update_spec
else:
try:
# $set
for to_set in update_spec.get("$set", []):
value = update_spec['$set'][to_set]
_set_field(doc, to_set, value)
# $unset
for to_unset in update_spec.get("$unset", []):
_unset_field(doc, to_unset)
except (KeyError, ValueError, AttributeError, IndexError):
exc_t, exc_v, exc_tb = sys.exc_info()
reraise(UpdateDoesNotApply,
"Cannot apply update %r to %r" % (update_spec, doc),
exc_tb)
return doc
def apply_update(self, doc, update_spec):
"""Apply an update operation to a document."""
# Helper to cast a key for a list or dict, or raise ValueError
def _convert_or_raise(container, key):
if isinstance(container, dict):
return key
elif isinstance(container, list):
return int(key)
else:
raise ValueError
# Helper to retrieve (and/or create)
# a dot-separated path within a document.
def _retrieve_path(container, path, create=False):
looking_at = container
for part in path:
if isinstance(looking_at, dict):
if create and not part in looking_at:
looking_at[part] = {}
looking_at = looking_at[part]
elif isinstance(looking_at, list):
index = int(part)
# Do we need to create additional space in the array?
if create and len(looking_at) <= index:
# Fill buckets with None up to the index we need.
looking_at.extend(
[None] * (index - len(looking_at)))
# Bucket we need gets the empty dictionary.
looking_at.append({})
looking_at = looking_at[index]
else:
raise ValueError
return looking_at
# wholesale document replacement
if not "$set" in update_spec and not "$unset" in update_spec:
# update spec contains the new document in its entirety
return update_spec
else:
try:
# $set
for to_set in update_spec.get("$set", []):
value = update_spec['$set'][to_set]
if '.' in to_set:
path = to_set.split(".")
where = _retrieve_path(doc, path[:-1], create=True)
index = _convert_or_raise(where, path[-1])
wl = len(where)
if isinstance(where, list) and index >= wl:
where.extend([None] * (index + 1 - wl))
where[index] = value
else:
if isinstance(value, dict) and isinstance(doc[to_set], dict):
doc[to_set] = dict(doc[to_set], **value)
else:
doc[to_set] = value
# $unset
for to_unset in update_spec.get("$unset", []):
if '.' in to_unset:
path = to_unset.split(".")
where = _retrieve_path(doc, path[:-1])
where.pop(_convert_or_raise(where, path[-1]))
else:
doc.pop(to_unset)
except (KeyError, ValueError, AttributeError, IndexError):
exc_t, exc_v, exc_tb = sys.exc_info()
reraise(UpdateDoesNotApply,
"Cannot apply update %r to %r" % (update_spec, doc),
exc_tb)
return doc
def apply_update(self, doc, update_spec):
"""Apply an update operation to a document."""
# Helper to cast a key for a list or dict, or raise ValueError
def _convert_or_raise(container, key):
if isinstance(container, dict):
return key
elif isinstance(container, list):
return int(key)
else:
raise ValueError
# Helper to retrieve (and/or create)
# a dot-separated path within a document.
def _retrieve_path(container, path, create=False):
looking_at = container
for part in path:
if isinstance(looking_at, dict):
if create and not part in looking_at:
looking_at[part] = {}
looking_at = looking_at[part]
elif isinstance(looking_at, list):
index = int(part)
if create and len(looking_at) < index:
looking_at.extend(
[None] * (len(looking_at) - index - 1))
looking_at.append({})
looking_at = looking_at[index]
else:
raise ValueError
return looking_at
# wholesale document replacement
if not "$set" in update_spec and not "$unset" in update_spec:
# update spec contains the new document in its entirety
update_spec['_ts'] = doc['_ts']
update_spec['ns'] = doc['ns']
return update_spec
else:
try:
# $set
for to_set in update_spec.get("$set", []):
value = update_spec['$set'][to_set]
if '.' in to_set:
path = to_set.split(".")
where = _retrieve_path(doc, path[:-1], create=True)
where[_convert_or_raise(where, path[-1])] = value
else:
doc[to_set] = value
# $unset
for to_unset in update_spec.get("$unset", []):
if '.' in to_unset:
path = to_unset.split(".")
where = _retrieve_path(doc, path[:-1])
where.pop(_convert_or_raise(where, path[-1]))
else:
doc.pop(to_unset)
except (KeyError, ValueError, AttributeError, IndexError):
exc_t, exc_v, exc_tb = sys.exc_info()
reraise(UpdateDoesNotApply,
"Cannot apply update %r to %r" % (update_spec, doc),
exc_tb)
return doc