def __call__(self, data: dict):
filename, partition = filename_for(data)
factory = data['_CROM_FACTORY']
model_object = data['_LOD_OBJECT']
dr = self.dr
if self.partition_directories:
dr = os.path.join(dr, partition)
with ExclusiveValue(dr):
fn = os.path.join(dr, filename)
if os.path.exists(fn):
m = self.merge(model_object, fn)
if m:
d = factory.toString(m, self.compact)
else:
d = None
else:
d = factory.toString(model_object, self.compact)
if d:
with open(fn, 'w', encoding='utf-8') as fh:
fh.write(d)
if getsize(fn) == 0:
warnings.warn(f'*** Wrote empty file: {fn}')
return NOT_MODIFIED
def set_properties(self, data, thing):
name = data.get('name')
data.setdefault('names', [name])
super().set_properties(data, thing)
type_name = data.get('type', 'place').lower()
label = name
parent_data = data.get('part_of')
place_type = MakeLinkedArtPlace.TYPES.get(type_name)
parent = None
if parent_data:
parent_data = self(parent_data)
parent = get_crom_object(parent_data)
if label:
try:
label = f'{label}, {parent._label}'
except AttributeError:
print('*** NO LABEL IN PARENT:' +
factory.toString(parent, False))
placeargs = {'label': label}
if data.get('uri'):
placeargs['ident'] = data['uri']
if place_type:
thing.classified_as = place_type
for c in data.get('classified_as', []):
thing.classified_as = c
if not name:
warnings.warn(f'Place with missing name on {thing.id}')
if parent:
# print(f'*** Setting parent on place object: {parent}')
thing.part_of = parent
def generate_example(self, egtext, resource):
# Yes really...
exec(egtext)
# Now in scope should be a top resource
factory.pipe_scoped_contexts = False
factory.toFile(top, compact=False)
factory.pipe_scoped_contexts = True
jsstr = factory.toString(top, compact=False, collapse=80)
factory.pipe_scoped_contexts = False
js = factory.toJSON(top)
# Generate all our serializations
nq = to_rdf(js, {"format": "application/nquads"})
g = ConjunctiveGraph()
for ns in ['crm', 'dc', 'schema', 'dcterms', 'skos', 'la']:
g.bind(ns, ctxt[ns])
g.parse(data=nq, format="nt")
out = g.serialize(format="turtle")
fp = js['id'][len(factory.base_url):]
fp2 = fp + ".ttl"
fh = open(os.path.join(factory.base_dir, fp2), 'w')
fh.write(out)
fh.close()
# And build mermaid description
mermaid = self.build_mermaid(js)
# Build index references
self.traverse(js, top.id, resource)
# And return the JSON plus links, to be substed by the top level filter
raw = top.id + ".json"
self.example_list.append(raw)
rawq = urllib.quote(raw).replace('/', "%2F")
playground = "http://json-ld.org/playground-dev/#startTab=tab-expanded©Context=true&json-ld=%s" % rawq
turtle = top.id + ".ttl"
turtle_play = "http://cdn.rawgit.com/niklasl/ldtr/v0.2.2/demo/?edit=true&url=%s" % turtle
egid = fp.replace('/', '_')
resp = """
<a id="%s"></a>
```json
%s
```
<div class="mermaid">
%s
</div>
Other Representations: [JSON-LD (Raw)](%s) |
[JSON-LD (Playground)](%s) |
[Turtle (Raw)](%s) |
[Turtle (Styled)](%s)
""" % (egid, jsstr, mermaid, raw, playground, turtle, turtle_play)
return resp
def merge(self, model_object, fn):
r = reader.Reader()
merger = self.merger
with open(fn, 'r') as fh:
content = fh.read()
try:
m = r.read(content)
if m == model_object:
return None
else:
merger.merge(m, model_object)
return m
except model.DataError as e:
print(f'Exception caught while merging data from {fn} ({str(e)}):')
print(factory.toString(model_object, False))
print(content)
raise
def merge_objects(self, objects):
r = JSONValueRewriter(self.prev_post_sales_map)
for k in list(objects.keys()):
data = objects[k]
updated = r.rewrite(data)
ident = updated['id']
if k != ident:
if ident in objects:
read = reader.Reader()
m = read.read(json.dumps(objects[ident]))
n = read.read(json.dumps(updated))
merger = CromObjectMerger()
m = merger.merge(m, n)
objects[ident] = json.loads(factory.toString(m, False))
else:
objects[ident] = updated
del (objects[k])
def __call__(self, data: dict, *args, **kwargs):
d = data['_OUTPUT']
dd = json.loads(d)
dr = data['_ARCHES_MODEL']
if dr not in self.output:
self.output[dr] = {}
uu = data.get('uuid')
if 'id' in dd:
uu = hashlib.sha256(dd['id'].encode('utf-8')).hexdigest()
elif not uu and 'uri' in data:
uu = hashlib.sha256(data['uri'].encode('utf-8')).hexdigest()
# print(f'*** No UUID in top-level resource. Using a hash of top-level URI: {uu}')
if not uu:
uu = str(uuid.uuid4())
# print(f'*** No UUID in top-level resource;')
# print(f'*** Using an assigned UUID filename for the content: {uu}')
fn = '%s.json' % uu
data = json.loads(d)
if fn in self.output[dr]:
r = reader.Reader()
model_object = r.read(d)
merger = self.merger
content = self.output[dr][fn]
try:
m = r.read(content)
if m == model_object:
self.output[dr][fn] = data
return
else:
merger.merge(m, model_object)
self.output[dr][fn] = json.loads(factory.toString(
m, False))
return
except model.DataError:
print(f'Exception caught while merging data from {fn}:')
print(d)
print(content)
raise
else:
self.output[dr][fn] = data
add_art_setter()
acq = Purchase()
lot = Aggregation(label="Verrue sale, lot 87")
acq2 = Purchase()
obj = Painting("http://www.getty.edu/art/collection/objects/882/rembrandt-harmensz-van-rijn-the-abduction-of-europa-dutch-1632/", label="The Abduction of Europa", art=1)
est = Actor(label="The estate of Jeanne Baptiste d'Albert de Luynes, comtesse de Verrue")
lot.identified_by = LotNumber(value="87")
lot.aggregates = obj
# Date of Acquisition
date = TimeSpan()
date.begin_of_the_begin = "1737-03-27T00:00:00Z"
date.end_of_the_end = "1737-03-27T23:59:59Z"
# Description
des = Description(value="1736 - 1737: Estate of Jeanne Baptiste d'Albert de Luynes, comtesse de Verrue, 1670 - 1736 [sold, Verrue sale, Paris, March 27, 1737, lot 87.]")
acq.used_specific_object = lot
acq.part = acq2
acq2.timespan = date
acq2.transferred_title_from = est
acq2.transferred_title_of = obj
acq2.referred_to_by = des
print(factory.toString(acq, compact=False))
# print(factory.toString(obj, compact=False))
# Example of Ownership
# from cromulent.model import factory, Acquisition
# from cromulent.vocab import Painting, MuseumOrg, add_art_setter
# add_art_setter()
# acq = Acquisition("https://linked.art/example/activity/28", "Acquisition of Painting")
# org = MuseumOrg("https://linked.art/example/group/16", "Museum")
# obj = Painting("https://linked.art/example/object/62", "Painting", art=1)
# obj.current_owner = org
# org.acquired_title_through = acq
# print(factory.toString(obj, compact=False))
# Example of Location
from cromulent.model import factory, Place
from cromulent.vocab import Painting, add_art_setter
add_art_setter()
loc = Place("https://linked.art/example/place/8", "Gallery W6")
obj = Painting("https://linked.art/example/object/63", "Painting", art=1)
obj.current_location = loc
print(factory.toString(obj, compact=False))
def main():
f = open("ref_col.csv", "r")
r = csv.reader(f)
next(r) #Skip header
headers = ["acq_by", "acq_from", "acq_date", "add_names", "avai_data", "cat",
"certified", "chem_comp", "CAS", "chem_form", "chem_name", "col_name", "color",
"CI", "name", "comp_type", "contact", "email", "experiments", "fire", "formulation",
"fbarcode", "geo_org", "grid_loc", "health", "manufacturer", "mass_vol", "mix_pigment",
"mix_type", "MSDS", "nat_syn", "notes", "obarcode", "org_date", "other_safe",
"part_col", "phone", "phys_form", "prep", "reactivity", "samp_type", "typ_use",
"warning", "borrower", "inv_status"]
for counter, row in enumerate(r):
rec = dict(zip(headers, row))
# Testing first two rows
if counter < 2:
s = ManMadeObject(label="Sample Object")
# Sample Identification/General Information
# AccessionNumber is used for now, will a new class Barcode - aat:300343361
s.identified_by = Barcode(label="Full Barcode", value=rec['fbarcode'])
s.identified_by = Barcode(label="Old Barcode", value=rec['obarcode'])
s.identified_by = Name(label="Common Name", value=rec['name'])
s.identified_by = Name(label="Additional Names", value=rec['add_names'])
# Sample Type
if rec['samp_type']:
s.classified_as = instances[rec['samp_type'].lower()]
# Typical use
if rec['typ_use']:
s.as_general_use = instances[rec['typ_use'].lower()]
# Physical Form
if rec['phys_form']:
pf = rec['phys_form'].split()
if len(pf) < 4:
i = 0
while i <= 2:
s.classified_as = instances[pf[i].lower().replace(',', '')]
i += 1
break
else:
s.referred_to_by = Description(label="Sample Type", value=rec['phys_form'])
# Color
if rec['color']:
s.classified_as = instances[rec['color']]
# Natural/Synthetic
if rec['nat_syn']:
s.classified_as = instances[rec['nat_syn'].lower()]
elif rec[nat_syn] == "Unknown":
s.classified_as = Type(label="Natural/Synthetic", value="Unknown")
# Grid Location: lab shelf/storage?
loc = Place()
if rec['grid_loc']:
loc.identified_by = Identifier(label="Grid Location", value=rec['grid_loc'])
s.current_location = loc
# Index (CI) No., Preparation, Certified Standard
# Acquisition Information
acq = Acquisition()
gci = Department("http://www.getty.edu/conservation/", label="Getty Research Institute")
s.changed_ownership_through = acq
acq.transferred_title_to = gci
# Acquisition Date
if rec['acq_date']:
acq.timespan = TimeSpan(label=rec['acq_date'])
# Acquired by
if rec['acq_by']:
emp = Actor(label=rec['acq_by'])
acq.carried_out_by = emp
emp.member_of = gci
# Acquired from, debating whether it should be Actor or Group,
# so info of person in contact can be linked
if rec['acq_from']:
acq.transferred_title_from = Actor(label=rec['acq_from'])
# Geographic Origin
if rec['geo_org']:
prod = Production()
origin = Place(label="Geographic Origin")
origin.identified_by = Name(value=rec['geo_org'])
prod.took_place_at = origin
s.produced_by = prod
# Catalog No.
if rec['cat']:
s.identified_by = CatalogNumber(label="Catalog No.", value=rec['cat'])
# Miscellaneous
# Notes
if rec['notes']:
io = InformationObject(value=rec['notes'])
io.classified_as = instances["notes"]
print(factory.toString(s, compact=False))
def main():
f = open("ref_col.csv", "r")
r = csv.reader(f)
next(r) #Skip header
headers = [
"acq_by", "acq_from", "acq_date", "add_names", "avai_data", "cat",
"certified", "chem_comp", "CAS", "chem_form", "chem_name", "coll_name",
"color", "CI", "name", "comp_type", "contact", "email", "experiments",
"fire", "formulation", "fbarcode", "geo_org", "grid_loc", "health",
"manufacturer", "mass_vol", "mix_pigment", "mix_type", "MSDS",
"nat_syn", "notes", "obarcode", "org_date", "other_safe", "part_coll",
"phone", "phys_form", "prep", "reactivity", "samp_type", "typ_use",
"warning", "borrower", "inv_status"
]
for counter, row in enumerate(r):
rec = dict(zip(headers, row))
# Testing first two rows
if counter < 2:
s = ManMadeObject(label=rec['name'])
# Sample Identification/General Information
s.identified_by = Barcode(label="Full Barcode",
value=rec['fbarcode'])
s.identified_by = Barcode(label="Old Barcode",
value=rec['obarcode'])
s.identified_by = PrimaryName(label="Common Name",
value=rec['name'])
s.identified_by = Name(label="Additional Names",
value=rec['add_names'])
# Sample Type
if rec['samp_type']:
try:
s.classified_as = instances[rec['samp_type'].lower()]
except:
s.referred_to_by = Description(label="Sample Type",
value=rec['samp_type'])
# Typical use
if rec['typ_use']:
use = rec['typ_use'].split('/')
for i in range(len(use)):
try:
s.as_general_use = instances[use[i].lower().strip()]
except:
s.referred_to_by = Description(label="Typical Use",
value=use[i])
# Physical Form
if rec['phys_form']:
pf = rec['phys_form'].split()
if len(pf) < 4:
l = []
for i in range(len(pf)):
try:
s.classified_as = instances[pf[i].lower().replace(
',', '').strip()]
except:
l.append(pf[i])
l_join = " ".join(l)
s.referred_to_by = Description(label="Physical Form",
value=l_join)
else:
s.referred_to_by = Description(label="Physical Form",
value=rec['phys_form'])
# Color
if rec['color']:
try:
s.classified_as = instances[rec['color'].lower()]
except:
s.referred_to_by = Description(label="Color",
value=rec['color'])
# Index (CI) Color
# Natural/Synthetic
if rec['nat_syn']:
s.classified_as = instances[rec['nat_syn'].lower()]
elif rec['nat_syn'] == "Unknown":
s.classified_as = Type(label="Natural/Synthetic",
value="Unknown")
# Preparation : go to Production
# Certified Standard
# Grid Location: lab shelf/storage?
loc = Place()
if rec['grid_loc']:
loc.identified_by = Identifier(label="Grid Location",
value=rec['grid_loc'])
s.current_location = loc
# Chemical Information
# Chemical Formula
if rec['chem_form']:
s.identified_by = Formula(value=rec['chem_form'])
if rec['chem_name']:
s.identified_by = Name(label="Chemical Name",
value=rec['chem_name'])
if rec['CAS']:
s.identified_by = Identifier(label="Chemical (CAS) No.",
value=rec['CAS'])
if rec['comp_type']:
comp_type = rec['comp_type'].lower()
try:
s.classified_as = instances[comp_type]
except:
s.classified_as = Type(label="Compound Type",
value=rec['comp_type'])
if rec['mix_type']:
mix_type = rec['mix_type'].split('-')
for i in mix_type:
s.classified_as = instances[i.lower().strip()]
# Acquisition Information
acq = Acquisition()
s.changed_ownership_through = acq
acq.transferred_title_to = v.dept['GCI']
# Acquisition Date
if rec['acq_date']:
adate = rec['acq_date']
tspan = TimeSpan(label=adate)
if len(adate) == 4:
tspan.begin_of_the_begin = str(
parse(adate + 'January',
settings={'PREFER_DAY_OF_MONTH': 'first'}))
tspan.end_of_the_end = str(
parse(adate + 'January',
settings={'PREFER_DAY_OF_MONTH': 'first'}) +
timedelta(days=365))
elif 'ca.' in adate or 'Spring' in adate:
y = adate.split()[1]
tspan.begin_of_the_begin = str(
parse(y + 'January',
settings={'PREFER_DAY_OF_MONTH': 'first'}))
tspan.end_of_the_end = str(
parse(y + 'January',
settings={'PREFER_DAY_OF_MONTH': 'first'}) +
timedelta(days=365))
elif '-' in adate:
y = adate.split('-')
start_year = y[0].strip()
end_year = y[1].strip()
if len(start_year) == 4:
tspan.begin_of_the_begin = str(
parse(start_year + 'January',
settings={'PREFER_DAY_OF_MONTH': 'first'}))
tspan.end_of_the_end = str(
parse(end_year + 'January',
settings={'PREFER_DAY_OF_MONTH': 'first'}) +
timedelta(days=365))
else:
tspan.begin_of_the_begin = str(
parse(adate,
settings={'PREFER_DAY_OF_MONTH': 'first'}))
tspan.end_of_the_end = str(
parse(adate,
settings={'PREFER_DAY_OF_MONTH': 'last'}) +
timedelta(days=1))
else:
tspan.begin_of_the_begin = str(
parse(adate, settings={'PREFER_DAY_OF_MONTH':
'first'}))
tspan.end_of_the_end = str(
parse(adate, settings={'PREFER_DAY_OF_MONTH': 'last'})
+ timedelta(days=1))
acq.timespan = tspan
# Acquired by
if rec['acq_by']:
p = rec['acq_by'].split('-')
pname = p[0].split()
agent = " ".join(pname)
dpt = p[-1]
if agent in v.person:
emp = v.person[agent]
else:
emp = Person(label=agent)
acq.carried_out_by = emp
if dpt in v.dept:
emp.member_of = v.dept[dpt]
# Part of Collection
if rec['part_coll']:
s.referred_to_by = LinguisticObject(label='Part of Collection',
value=rec['part_coll'])
# Acquired from, debating whether it should be Actor or Group,
# so info of person in contact can be linked
global seller
if rec['acq_from']:
if rec['acq_from'] in v.sellers:
seller = v.sellers[rec['acq_from'].strip()]
else:
seller = Actor(label=rec['acq_from'].strip())
acq.transferred_title_from = seller
# Collection Name
if rec['coll_name']:
coll = v.collection[rec['coll_name']]
s.aggregated_by = coll
coll_creation = Creation()
coll.created_by = coll_creation
coll_creation.carried_out_by = seller
# Contact name
if rec['contact']:
if rec['contact'] in v.contact:
contact_name = v.contact[rec['contact'].strip()]
else:
s.related_entity = contact_name
# Email
if rec['email']:
contact_name.contact_point = EMail(label="E-Mail",
value=rec['email'])
# Phone number: could be either the number of the contact person,
# or of the acquired_from
# if rec['phone']:
prod = Production()
s.produced_by = prod
# Preparation
if rec['prep']:
prep = SamplePreparation(value=rec['prep'])
prod.technique = prep
if rec['formulation']:
prep.referred_to_by = Formulation(value=rec['formulation'])
# Geographic Origin
if rec['geo_org']:
origin = Place(label="Geographic Origin")
origin.identified_by = Name(value=rec['geo_org'])
prod.took_place_at = origin
# Manufacturer
if rec['manufacturer']:
if rec['manufacturer'] in v.manufacturers:
mfr = v.manufacturers[rec['manufacturer']]
elif rec['manufacturer'] in v.sellers:
mfr = v.sellers[rec['manufacturer']]
else:
mfr = Group(label=rec['manufacturer'])
prod.carried_out_by = mfr
# Catalog No.
if rec['cat']:
s.identified_by = CatalogNumber(label="Catalog No.",
value=rec['cat'])
# Miscellaneous
# Origination Date
# if rec['org_date']:
# odate = rec['org_date']
# t_span = TimeSpan(label=odate)
# if len(adate) == 4:
# t_span.begin_of_the_begin = str(parse(odate + 'January', settings={'PREFER_DAY_OF_MONTH': 'first'}))
# t_span.end_of_the_end = str(parse(odate + 'January', settings={'PREFER_DAY_OF_MONTH': 'first'}) + timedelta(days=365))
# elif 'ca.' in adate or 'pre' in adate:
# y = odate.split()[1]
# t_span.begin_of_the_begin = str(parse(y + 'January', settings={'PREFER_DAY_OF_MONTH': 'first'}))
# t_span.end_of_the_end = str(parse(y + 'January', settings={'PREFER_DAY_OF_MONTH': 'first'}) + timedelta(days=365))
# elif '-' in adate:
# y = odate.split('-')
# start_year = y[0].strip()
# end_year = y[1].strip()
# if len(start_year) == 4:
# t_span.begin_of_the_begin = str(parse(start_year + 'January', settings={'PREFER_DAY_OF_MONTH': 'first'}))
# t_span.end_of_the_end = str(parse(end_year + 'January', settings={'PREFER_DAY_OF_MONTH': 'first'}) + timedelta(days=365))
# else:
# t_span.begin_of_the_begin = str(parse(odate, settings={'PREFER_DAY_OF_MONTH': 'first'}))
# t_span.end_of_the_end = str(parse(odate, settings={'PREFER_DAY_OF_MONTH': 'last'}) + timedelta(days=1))
# else:
# t_span.begin_of_the_begin = str(parse(odate, settings={'PREFER_DAY_OF_MONTH': 'first'}))
# t_span.end_of_the_end = str(parse(odate, settings={'PREFER_DAY_OF_MONTH': 'last'}) + timedelta(days=1))
# acq.timespan = t_span
# Mass or Volume
if rec['mass_vol']:
mv = rec['mass_vol'].replace('appx.',
'').replace('approx.',
'').split()
if len(mv) == 2:
dim = MassVolume(value=mv[0])
s.dimension = dim
dim.unit = instances[mv[1].replace('.', '')]
else:
try:
u = instances[(mv[1] + " " + mv[2]).replace('.', '')]
dim = MassVolume(value=mv[0])
s.dimension = dim
dim.unit = u
except:
s.referred_to_by = DimensionStatement(
label="Mass or Volume", value=rec['mass_vol'])
# Experiments on Sample
if rec['experiments']:
s.referred_to_by = Description(label="Experiments on Sample",
value=rec['experiments'])
# Notes
if rec['notes']:
s.referred_to_by = Notes(value=rec['notes'])
# Available Data
# MSDS & Safety
# Safety: classified_as or referred_to_by properties
if rec['fire']:
s.classified_as = FireSafety(value=rec['fire'])
if rec['health']:
health_safe = Safety(label="Health Safety",
value=rec['health'])
s.classified_as = health_safe
health_safe.classified_as = instances['health']
if rec['reactivity']:
react_safe = Safety(label="Reactivity Safety",
value=rec['reactivity'])
s.classified_as = react_safe
react_safe.classified_as = instances['reactivity']
if rec['other_safe']:
other_safe = Safety(label="Other Safety",
value=rec['other_safe'])
s.classified_as = other_safe
other_safe.classified_as = instances['other']
print(factory.toString(s, compact=False))
# p = Person("https://linked.art/example/person/35", "Copyist")
# prod.produced = copy
# prod.influenced_by = org
# prod.carried_out_by = p
# print(factory.toString(prod, compact=False))
# from cromulent.model import factory, Production, VisualItem
# from cromulent.vocab import Negative
# prod = Production("https://linked.art/example/activity/72", "Printing of Photograph")
# src = Negative("https://linked.art/example/object/89", "Negative")
# vi = VisualItem("https://linked.art/example/VisualItem/6", "Visual Content of Photographs and Negative")
# prod.used_specific_object = src
# src.shows = vi
# print(factory.toString(prod, compact=False))
# Example of Destructiom
from cromulent.model import factory, Destruction, TimeSpan
from cromulent.vocab import Painting, add_art_setter
add_art_setter()
prod = Destruction("https://linked.art/example/activity/80")
obj = Painting("https://linked.art/example/object/94",
"Example Destroyed Painting",
art=1)
tspan = TimeSpan("https://linked.art/example/time/22")
prod.timespan = tspan
tspan.begin_of_the_begin = "1823-03-01T00:00:00Z"
tspan.end_of_the_end = "1823-03-31T00:00:00Z"
prod.destroyed = obj
print(factory.toString(prod, compact=False))
def _rewrite_output_files(files, r, update_filename, worker_id, total_workers,
kwargs):
i = 0
if not files:
return
print(
f'rewrite worker partition {worker_id} called with {len(files)} files [{files[0]} .. {files[-1]}]'
)
start = time.time()
rewritten_count = 0
processed_count = 0
ignore_errors = kwargs.get('ignore_errors', False)
for i, f in enumerate(files):
processed_count += 1
# print(f'{i} {f}', end="\r", flush=True)
with open(f) as data_file:
try:
bytes = data_file.read()
if 'content_filter_re' in kwargs:
filter_re = kwargs['content_filter_re']
if not re.search(filter_re, bytes):
pass
# print(f'skipping {f}')
continue
else:
pass
# print(f'processing {f}')
data = json.loads(bytes)
except json.decoder.JSONDecodeError:
sys.stderr.write(
f'Failed to load JSON during rewriting of {f}\n')
if ignore_errors:
continue
else:
raise
d = r.rewrite(data, file=f)
if update_filename:
newfile = filename_for(d, original_filename=f, **kwargs)
else:
newfile = f
if d == data and f == newfile:
# nothing changed; do not rewrite the file
continue
else:
pass
# print(f'*** rewrote data in {f} --> {newfile}')
if newfile != f:
if os.path.exists(newfile):
read = reader.Reader()
merger = CromObjectMerger()
with open(newfile, 'r') as fh:
content = fh.read()
try:
m = read.read(content)
n = read.read(d)
# print('========================= MERGING =========================')
# print('merging objects:')
# print(f'- {m}')
# print(f'- {n}')
merger.merge(m, n)
# except model.DataError as e:
except Exception as e:
print(
f'Exception caught while merging data from {newfile} ({str(e)}):'
)
print(d)
print(content)
if ignore_errors:
continue
else:
raise
data = factory.toString(m, False)
d = json.loads(data)
with open(newfile, 'w') as data_file:
rewritten_count += 1
json.dump(d, data_file, indent=2, ensure_ascii=False)
if newfile != f:
os.remove(f)
end = time.time()
elapsed = end - start
if rewritten_count:
print(
f'worker partition {worker_id}/{total_workers} finished with {rewritten_count}/{processed_count} files rewritten in %.1fs'
% (elapsed, ))
else:
print(
f'worker partition {worker_id}/{total_workers} finished in %.1fs' %
(elapsed, ))
canon_file = seen[id]
# print(f'*** {id} already seen in {canon_file} ; merging {filename}')
merger = CromObjectMerger()
with open(canon_file, 'r') as cfh:
canon_content = cfh.read()
n = read.read(canon_content)
try:
merger.merge(m, n)
except model.DataError as e:
print(
f'Exception caught while merging data from {newfile} ({str(e)}):'
)
print(d)
print(content)
raise
merged_data = factory.toString(m, False)
d = json.loads(merged_data)
with open(canon_file, 'w') as data_file:
json.dump(d,
data_file,
indent=2,
ensure_ascii=False)
os.remove(filename)
coalesce_count += 1
else:
seen[id] = filename
except model.DataError as e:
print(f'*** Failed to read CRM data from {filename}: {e}')
print(f'{filename}:\n=======\n{content}')
print(f'{canon_file}:\n=======\n{canon_content}')
print(f'Coalesced {coalesce_count} JSON files in {path}')
def test_add_classification(self):
amnt = model.MonetaryAmount(ident='')
amnt.value = 7.0
self.assertNotIn('Asking Price', factory.toString(amnt))
vocab.add_classification(amnt, vocab.AskingPrice)
self.assertIn('Asking Price', factory.toString(amnt))
