def process(self, item, url):
pdf_dir = os.path.join(DIR_PATH, 'pdf')
image_dir = os.path.join(DIR_PATH, 'images')
inspection_dir = os.path.join(DIR_PATH, 'inspection')
data_dir = os.path.join(DIR_PATH, 'data')
out_dir = os.path.join(data_dir, item.form, item.true_name)
os.makedirs(pdf_dir, exist_ok=True)
os.makedirs(image_dir, exist_ok=True)
os.makedirs(inspection_dir, exist_ok=True)
os.makedirs(out_dir, exist_ok=True)
pdf_path = Crawler.download(url, item.true_name, pdf_dir)
if not pdf_path:
return
try:
im = Oratory1990Crawler.pdf_to_image(
os.path.join(pdf_dir, f'{item.true_name}.pdf'),
os.path.join(image_dir, f'{item.true_name}.png'))
except ValueError as err:
if str(err) == 'Measured by Crinacle':
ignored = item.copy()
ignored.form = 'ignore'
self.name_index.update(ignored,
false_name=item.false_name,
true_name=item.true_name,
form=item.form)
self.write_name_index()
print(
f'Ignored {item.false_name} because it is measured by Crinacle.'
)
return
fr, inspection = Oratory1990Crawler.parse_image(im, item.true_name)
inspection.save(os.path.join(inspection_dir, f'{item.true_name}.png'))
fr.write_to_csv(os.path.join(out_dir, f'{item.true_name}.csv'))
def process(self, item, url):
pdf_dir = os.path.join(DIR_PATH, 'pdf')
image_dir = os.path.join(DIR_PATH, 'images')
inspection_dir = os.path.join(DIR_PATH, 'inspection')
data_dir = os.path.join(DIR_PATH, 'data')
out_dir = os.path.join(data_dir, item.form, item.true_name)
os.makedirs(pdf_dir, exist_ok=True)
os.makedirs(image_dir, exist_ok=True)
os.makedirs(inspection_dir, exist_ok=True)
os.makedirs(out_dir, exist_ok=True)
Crawler.download(url, item.true_name, pdf_dir)
im = Oratory1990Crawler.pdf_to_image(
os.path.join(pdf_dir, f'{item.true_name}.pdf'),
os.path.join(image_dir, f'{item.true_name}.png'))
if im is None:
return
fr, inspection = Oratory1990Crawler.parse_image(im, item.true_name)
inspection.save(os.path.join(inspection_dir, f'{item.true_name}.png'))
fr.write_to_csv(os.path.join(out_dir, f'{item.true_name}.csv'))
def process(item, link):
if not Crawler.download(link, item.true_name,
os.path.join(DIR_PATH, 'json')):
return
with open(os.path.join(DIR_PATH, 'json', f'{item.true_name}.json'),
'r') as fh:
json_data = json.load(fh)
fr, target = RtingsCrawler.parse_json(json_data)
fr.name = item.true_name
fr.interpolate()
if np.std(fr.raw) == 0:
# Frequency response data has non-zero target response, use that
target.interpolate()
target = target
print(f'Using target for {fr.name}')
elif item.form == 'inear':
# Using in-ear target response
target = INEAR_TARGET
else:
# Using on-ear or earbud target response
target = ONEAR_TARGET
target.center()
fr.raw += target.raw
fr.center()
# Inspection
dir_path = os.path.join(DIR_PATH, 'inspection')
os.makedirs(dir_path, exist_ok=True)
file_path = os.path.join(dir_path, f'{fr.name}.png')
fr.plot_graph(file_path=file_path)
# Write to file
dir_path = os.path.join(DIR_PATH, 'data', item.form, fr.name)
os.makedirs(dir_path, exist_ok=True)
file_path = os.path.join(dir_path, fr.name + '.csv')
fr.write_to_csv(file_path)
print(f'Saved "{fr.name}" to "{file_path}"')
def process(self, item, url):
json_file = Crawler.download(url, item.true_name,
os.path.join(DIR_PATH, 'json'))
if json_file is not None:
with open(os.path.join(DIR_PATH, 'json', f'{item.true_name}.json'),
'r',
encoding='utf-8') as fh:
json_data = json.load(fh)
fr, target = RtingsCrawler.parse_json(json_data)
fr.name = item.true_name
else:
# No frequency response available, download bass, mid and treble
# Bass
Crawler.download(
url.replace('frequency-response-14.json', 'bass.json'),
f'{item.true_name}-bass', os.path.join(DIR_PATH, 'json'))
with open(os.path.join(DIR_PATH, 'json',
f'{item.true_name}-bass.json'),
'r',
encoding='utf-8') as fh:
bass_fr, bass_target = self.parse_json(json.load(fh))
# Mid
Crawler.download(
url.replace('frequency-response-14.json', 'mid.json'),
f'{item.true_name}-mid', os.path.join(DIR_PATH, 'json'))
with open(os.path.join(DIR_PATH, 'json',
f'{item.true_name}-mid.json'),
'r',
encoding='utf-8') as fh:
mid_fr, mid_target = self.parse_json(json.load(fh))
# Treble
Crawler.download(
url.replace('frequency-response-14.json', 'treble.json'),
f'{item.true_name}-treble', os.path.join(DIR_PATH, 'json'))
with open(os.path.join(DIR_PATH, 'json',
f'{item.true_name}-treble.json'),
'r',
encoding='utf-8') as fh:
treble_fr, treble_target = self.parse_json(json.load(fh))
fr = FrequencyResponse(
name=item.true_name,
frequency=np.concatenate(
[bass_fr.frequency, mid_fr.frequency,
treble_fr.frequency]),
raw=np.concatenate([bass_fr.raw, mid_fr.raw, treble_fr.raw]))
target = FrequencyResponse(name=item.true_name,
frequency=np.concatenate([
bass_target.frequency,
mid_target.frequency,
treble_target.frequency
]),
raw=np.concatenate([
bass_target.raw, mid_target.raw,
treble_target.raw
]))
fr.interpolate()
if np.std(fr.raw) == 0:
# Frequency response data has non-zero target response, use that
target.interpolate()
target = target
print(f'Using target for {fr.name}')
elif item.form == 'inear':
# Using in-ear target response
target = INEAR_TARGET
else:
# Using on-ear or earbud target response
target = ONEAR_TARGET
target.center()
fr.raw += target.raw
fr.center()
# Inspection
dir_path = os.path.join(DIR_PATH, 'inspection')
os.makedirs(dir_path, exist_ok=True)
file_path = os.path.join(dir_path, f'{fr.name}.png')
fig, ax = fr.plot_graph(file_path=file_path, show=False)
plt.close(fig)
# Write to file
dir_path = os.path.join(DIR_PATH, 'data', item.form, fr.name)
os.makedirs(dir_path, exist_ok=True)
file_path = os.path.join(dir_path, fr.name + '.csv')
fr.write_to_csv(file_path)
print(f'Saved "{fr.name}" to "{file_path}"')