def callWrapper(sh, rows, methodName, function, *args):
"""
This function should be used by registrars to wrap calls to
registrar-specific create/update/delete functions. It hides all
transient errors (by retrying indefinitely) and raises all others.
'sh' and 'rows' are supplied by this module and should simply be
passed through. 'function' is the function to call; 'methodName' is
its name for error reporting purposes. Any additional arguments are
passed through to 'function'.
"""
while True:
_checkAbort(sh)
try:
return function(*args)
except Exception, e:
if ((isinstance(e, urllib2.HTTPError) and e.code >= 500)
or (isinstance(e, IOError)
and not isinstance(e, urllib2.HTTPError))
or isinstance(e, httplib.HTTPException)):
for r in rows:
r.error = util.formatException(e)
_checkAbort(sh)
with django.db.transaction.atomic():
for r in rows:
r.save()
_sleep(sh, sh.reattemptDelay)
else:
raise Exception("%s error: %s" %
(methodName, util.formatException(e)))
def from_db_value(self, value, *args, **kwargs):
if value is None:
return None
else:
try:
si = django.core.serializers.deserialize(
"json", zlib.decompress(value)).next().object
# The citation metadata, being a dictionary and not a type the
# Django serializer understands, appears to get serialized as
# though by calling repr() and then base64-ing that. Thus we
# must eval() it to return it to its dictionary form.
# (There's a way to inform the serializer of new types, but
# that's not supported until Django 1.11.)
si.cm = ast.literal_eval(str(si.cm))
# Replace subservient objects referenced by foreign keys with
# pointers to cached copies to avoid database lookups.
if si.owner_id != None:
si.owner = store_user.getById(si.owner_id)
if si.ownergroup_id != None:
si.ownergroup = store_group.getById(si.ownergroup_id)
if si.datacenter_id != None:
si.datacenter = shoulder.getDatacenterById(
si.datacenter_id)
si.profile = store_profile.getById(si.profile_id)
return (si, value)
except Exception, e:
raise django.core.exceptions.ValidationError(
"Exception encountered unpacking StoreIdentifier database value: " +\
util.formatException(e))
def from_db_value(self, value, *args, **kwargs):
if value is None:
return None
else:
try:
return json.loads(zlib.decompress(value))
except Exception, e:
raise django.core.exceptions.ValidationError(
"Exception encountered unpacking compressed JSON database value: " +\
util.formatException(e))
def get_db_prep_save(self, value, *args, **kwargs):
if value is None:
return None
else:
try:
return super(CompressedJsonField, self).get_db_prep_save(
zlib.compress(json.dumps(value, separators=(",", ":"))),
*args, **kwargs)
except Exception, e:
raise django.core.exceptions.ValidationError(
"Exception encountered packing compressed JSON database value: " +\
util.formatException(e))
def _workerThread(sh):
# Sleep between 1x and 2x the idle sleep, to give the main daemon a
# chance to load the row cache and to prevent the workers from
# running synchronously.
time.sleep(sh.idleSleep * (random.random() + 1))
while True:
try:
while True:
rows = _nextUnprocessedLoadedRows(sh)
if len(rows) > 0:
break
_sleep(sh)
try:
if len(rows) == 1:
f = sh.functions["single"][rows[0].operation]
f(sh, rows, rows[0].identifier,
util.deblobify(rows[0].metadata))
else:
f = sh.functions["batch"][rows[0].operation]
f(
sh,
rows,
[(r.identifier, util.deblobify(r.metadata))
for r in rows],
)
except _AbortException:
raise
except Exception, e:
# N.B.: on the assumption that the registrar-specific function
# used callWrapper defined above, the error can only be
# permanent.
for r in rows:
r.error = util.formatException(e)
r.errorIsPermanent = True
_checkAbort(sh)
with django.db.transaction.atomic():
for r in rows:
r.save()
log.otherError("register_async._workerThread/" + sh.registrar,
e)
else:
_checkAbort(sh)
with django.db.transaction.atomic():
for r in rows:
# Django "helpfully" sets seq, the primary key, to None
# after deleting a row. But we need the seq value to
# delete the row out of sh.loadedRows, ergo...
t = r.seq
r.delete()
r.seq = t
finally:
def kmlPolygonToDatacite(kml):
"""
Converts a polygon defined in a KML
<http://www.opengeospatial.org/standards/kml> version 2.2 or 2.3
document to a DataCite 4.0 <geoLocationPolygon> element. The return
is a pair (lxml.etree.Element, [warning, ...]) if successful or a
string error message if not. The conversion fails for the usual
reasons (malformed KML, etc.) but also if the document defines more
than one geometry or does not define a polygon. Polygon holes and
non-zero altitude coordinates are ignored and result in warnings.
"""
try:
root = util.parseXmlString(kml)
except Exception, e:
return "XML parse error: " + util.formatException(e)
def get_db_prep_save(self, value, *args, **kwargs):
if value is None:
return None
else:
try:
if type(value) in [str, buffer]:
v = value
else:
v = zlib.compress(
django.core.serializers.serialize("json", [value]))
return super(StoreIdentifierObjectField,
self).get_db_prep_save(v, *args, **kwargs)
except Exception, e:
raise django.core.exceptions.ValidationError(
"Exception encountered packing StoreIdentifier database value: " +\
util.formatException(e))
def get_db_prep_save(self, value, *args, **kwargs):
if value is None:
return None
# When the DB is populated via a JSON fixture, using the loaddata management
# command, the values arrive here as strings wrapped in native C buffer objects
# instead of the expected Python container type. This only occurs when invoking
# loaddata using the Django call_command() method, not when invoking loaddata
# via manage.py.
if isinstance(value, buffer):
value = eval(str(value))
try:
json_str = json.dumps(value, separators=(",", ":"))
return super(CompressedJsonField, self).get_db_prep_save(
zlib.compress(json_str), *args, **kwargs
)
except Exception, e:
raise django.core.exceptions.ValidationError(
"Exception encountered packing compressed JSON database value: "
+ util.formatException(e)
)
def geojsonPolygonToDatacite (geojson):
"""
Converts a polygon defined in a GeoJSON <RFC 7946,
https://tools.ietf.org/html/rfc7946> document to a DataCite 4.0
<geoLocationPolygon> element. The return is a pair
(lxml.etree.Element, [warning, ...]) if successful or a string error
message if not. The conversion fails for the usual reasons
(malformed JSON, etc.) but also if the document defines more than
one geometry or does not define a polygon. Polygon holes and
non-zero altitude coordinates are ignored and result in warnings.
"""
objects = []
def objectHandler (d):
if d.get("type", "unknown") in _geojsonTypes and\
d["type"] != "GeometryCollection":
objects.append(d)
return d
try:
root = json.loads(geojson, object_hook=objectHandler)
except Exception, e:
return "JSON parse error: " + util.formatException(e)
def _reconcileShoulders():
global _shoulders, _datacenters
import log
try:
stage = "loading"
f = None
try:
r = urllib2.Request(_url)
if _username != None:
r.add_header("Authorization", "Basic " +\
base64.b64encode("%s:%s" % (_username, _password)))
f = urllib2.urlopen(r)
fc = f.read().decode("UTF-8")
finally:
if f: f.close()
entries, errors, warnings = shoulder_parser.parse(fc)
assert len(errors) == 0, "file validation error(s): " +\
", ".join("(line %d) %s" % e for e in errors)
newDatacenters = dict((e.key, e) for e in entries\
if e.type == "datacenter" and e.manager == "ezid" and e.active)
newShoulders = dict((e.key, e) for e in entries\
if e.type == "shoulder" and e.manager == "ezid" and e.active)
# The following operations must be performed in exactly this order
# because MySQL enforces integrity constraints after every
# operation.
stage = "reconciling with"
with django.db.transaction.atomic():
datacenters = dict((d.symbol, d) for d in\
store_datacenter.StoreDatacenter.objects.all())
shoulders = dict((s.prefix, s) for s in\
Shoulder.objects.select_related("datacenter").all())
# 1. For modified shoulders, replace fields that have UNIQUE
# constraints with random unique values and foreign keys with
# NULL; delete shoulders that no longer exist.
shoulderFixups = []
for prefix, s in shoulders.items():
if prefix in newShoulders:
ns = newShoulders[prefix]
if s.name != ns.name or s.minter != ns.minter or\
((s.datacenter == None and "datacenter" in ns) or\
(s.datacenter != None and\
s.datacenter.symbol != ns.get("datacenter", ""))) or\
(s.crossrefEnabled ^\
(ns.get("registration_agency", "") == "crossref")):
shoulderFixups.append(
(s, ns.name, ns.minter, ns.get("datacenter", None),
ns.get("registration_agency", "") == "crossref"))
s.name = str(uuid.uuid1())
s.datacenter = None
s.save()
else:
try:
# Unfortunately, Django doesn't offer on_delete=PROTECT on
# many-to-many relationships, so we have to check
# manually.
if s.storegroup_set.count() > 0:
raise django.db.IntegrityError(
"shoulder is referenced by group")
s.delete()
except django.db.IntegrityError, e:
raise django.db.IntegrityError(
"error deleting shoulder %s, shoulder is in use: %s" %\
(s.prefix, util.formatException(e)))
del shoulders[prefix]
# 2. Similarly for datacenters.
datacenterFixups = []
for symbol, d in datacenters.items():
if symbol in newDatacenters:
nd = newDatacenters[symbol]
if d.name != nd.name:
datacenterFixups.append((d, nd.name))
d.name = str(uuid.uuid1())
d.save()
else:
try:
d.delete()
except django.db.IntegrityError, e:
raise django.db.IntegrityError(
"error deleting datacenter %s, datacenter is in use: %s" %\
(d.symbol, str(e)))
del datacenters[symbol]
# 3. Now apply datacenter fixups.
for d, name in datacenterFixups:
d.name = name
d.full_clean(validate_unique=False)
d.save()
# 4. Add new datacenters.
for symbol, nd in newDatacenters.items():
if symbol not in datacenters:
d = store_datacenter.StoreDatacenter(symbol=symbol,
name=nd.name)
d.full_clean(validate_unique=False)
d.save()
datacenters[symbol] = d
# 5. Now apply shoulder fixups.
for s, name, minter, datacenter, crossrefEnabled in shoulderFixups:
s.name = name
s.minter = minter
if datacenter != None: s.datacenter = datacenters[datacenter]
s.crossrefEnabled = crossrefEnabled
s.full_clean(validate_unique=False)
s.save()
# 6. Finally, add new shoulders.
for prefix, ns in newShoulders.items():
if prefix not in shoulders:
s = Shoulder(prefix=prefix,
name=ns.name,
minter=ns.minter,
crossrefEnabled=(ns.get(
"registration_agency", "") == "crossref"))
if "datacenter" in ns:
s.datacenter = datacenters[ns.datacenter]
else:
s.datacenter = None
s.full_clean(validate_unique=False)
s.save()
shoulders[prefix] = s
crossrefEnabled=(ns.get(
"registration_agency", "") == "crossref"))
if "datacenter" in ns:
s.datacenter = datacenters[ns.datacenter]
else:
s.datacenter = None
s.full_clean(validate_unique=False)
s.save()
shoulders[prefix] = s
except Exception, e:
# Log the error, but otherwise continue to run with the shoulders
# and datacenters we have.
log.otherError(
"shoulder._reconcileShoulders",
Exception("error %s external shoulder file: %s" %
(stage, util.formatException(e))))
with django.db.transaction.atomic():
# In all cases, to fill the in-memory caches do fresh queries to
# get proper dependent datacenter objects.
_shoulders = dict((s.prefix, s) for s in Shoulder.objects.\
select_related("datacenter").all())
dc = dict((d.symbol, d) for d in\
store_datacenter.StoreDatacenter.objects.all())
_datacenters = (dc, dict((d.id, d) for d in dc.values()))
def _lockAndLoad(f):
# Decorator.
def wrapped(*args, **kwargs):
_lock.acquire()
try:
