def to_main(self):
self.to_cut()
self.extract_hash()
entry = get_entry_by_id(self.options.entry_id)
source_url = self.source.parsed_url
file_ref = next(
(fr
for fr in entry.files if fr.reference == self.source.reference),
None)
purpose = file_ref.purpose if file_ref is not None else FilePurpose.unknown
if entry.metadata is not None and hasattr(
entry.metadata,
'taken_ts') and entry.metadata.taken_ts is not None:
taken_ts = entry.metadata.taken_ts[:10]
else:
taken_ts = datetime.datetime.fromtimestamp(
os.path.getmtime(self.cut_path)).strftime('%Y-%m-%d')
main_root = resolve_path(self.system.main_storage.root_path)
filename = os.path.basename(self.options.source_url)
parts = [
main_root, entry.type.value, purpose.value, taken_ts, filename
]
main_path = os.path.join(*parts)
logging.info("Source: %s", str(source_url))
logging.info("Main path: %s", str(main_path))
filecopy = FileCopy(
source=self.cut_path,
destination=main_path,
link=True,
remove_source=True,
)
filecopy.run()
@retry()
def push():
entry = get_entry_by_id(self.options.entry_id)
new_file_ref = FileReference(
purpose=file_ref.purpose,
version=file_ref.version,
reference=self.reference,
mime_type=file_ref.mime_type,
)
new_file = File(
reference=self.reference,
url=self.system.main_storage.get_file_url(main_path),
mime_type=self.source.mime_type,
status=FileStatus.managed,
)
create_file(new_file)
entry.files.append(new_file_ref)
update_entry_by_id(entry.id, entry)
push()
def run(self, job: Job):
logging.debug(job.to_json())
step = job.get_current_step()
system = current_system()
source = get_file_by_url(step.options.source_url)
source_url = source.parsed_url
source_path = source_url.path
cut = system.cut_storage
cut_filename = uuid.uuid4().hex
cut_path = os.path.join(resolve_path(cut.root_path), cut_filename)
if source_url.scheme == 'card':
if step.options.source_root_path is not None:
source_path = os.path.join(step.options.source_root_path,
source_path[1:])
else:
raise Exception("Only support card here")
filecopy = FileCopy(
source=source_path,
destination=cut_path,
link=False,
remove_source=False,
)
filecopy.run()
step.result = ToCutResult(path=cut_path)
step.status = StepStatus.done
def to_cut(self):
if self.cut_path is not None:
return
source = self.source
source_url = source.parsed_url
source_path = source_url.path
cut = self.system.cut_storage
cut_filename = uuid.uuid4().hex
cut_path = os.path.join(resolve_path(cut.root_path), cut_filename)
if source_url.scheme == 'card':
if self.options.source_root_path is not None:
source_path = os.path.join(self.options.source_root_path,
source_path[1:])
else:
raise Exception("Only support card here")
filecopy = FileCopy(
source=source_path,
destination=cut_path,
link=False,
remove_source=False,
)
filecopy.run()
self.cut_path = cut_path
def create_check(self, store, angle, mirror):
cut_target = self.full_original_file_path + '_check'
_create_check(
self.full_original_file_path,
cut_target,
angle=angle,
mirror=mirror,
size=self.system.config.get_job_settings('import').check_size,
)
reference = calculate_hash(cut_target)
main_root = resolve_path(self.system.main_storage.root_path)
parts = [
main_root, store, 'check', reference[:2], reference[2:] + '.jpg'
]
main_path = os.path.join(*parts)
FileCopy(
source=cut_target,
destination=main_path,
link=True,
remove_source=True,
).run()
f = File(
reference=reference,
url=self.system.main_storage.get_file_url(main_path),
mime_type='image/jpeg',
status=FileStatus.managed,
)
return f
def generate_rescaled(self, store, purpose, longest_edge, angle, mirror):
cut_target = self.full_original_file_path + '_' + purpose.value
_convert(
self.full_original_file_path,
cut_target,
longest_edge=longest_edge,
angle=angle,
mirror=mirror,
)
reference = calculate_hash(cut_target)
main_root = resolve_path(self.system.main_storage.root_path)
parts = [
main_root, store, purpose.value, reference[:2],
reference[2:] + '.jpg'
]
main_path = os.path.join(*parts)
FileCopy(
source=cut_target,
destination=main_path,
link=True,
remove_source=True,
).run()
f = File(
reference=reference,
url=self.system.main_storage.get_file_url(main_path),
mime_type='image/jpeg',
status=FileStatus.managed,
)
return f
def setup_filesystem(self):
self.server = next(
(x for x in self.config.servers if x.hostname == self.hostname),
None)
assert self.server is not None, 'Missing server config for %s' % self.hostname
self.main_storage = next(
(x for x in self.config.storages
if x.server == self.hostname and x.type == StorageType.main),
None)
self.cut_storage = next(
(x for x in self.config.storages
if x.server == self.hostname and x.type == StorageType.cut), None)
self.media_root = resolve_path(self.main_storage.root_path)
assert self.media_root is not None, 'Missing media root!'
os.makedirs(self.media_root, exist_ok=True)
logging.debug("Media root path: %s", self.media_root)
def setup_database(self):
db_config = next(
(x for x in self.config.databases if x.server == self.hostname),
None)
assert db_config is not None, 'Missing database config!'
db_root = resolve_path(db_config.path)
self.db = dict()
def get_taken_ts(o):
metadata = o.get('metadata')
if metadata is None: return None
return metadata.get('taken_ts')
def get_taken_ts_tuple(o):
t = get_taken_ts(o)
if t is None: return None
return tuple(int(x) for x in t[:10].split('-')) + (t[11:], ), None
def get_taken_date_tuple(o):
t = get_taken_ts(o)
if t is None: return None
return tuple(int(x) for x in t[:10].split('-')), None
def get_taken_date(o, get_value):
t = get_taken_ts(o)
if t is None: return None
return t[:10], get_value(o)
def get_source(o):
metadata = o.get('metadata')
if metadata is None: return None
source = metadata.get('source')
original_filename = metadata.get('original_filename')
if not all([source, original_filename]): return None
return (source, original_filename), None
def sum_per(field, values):
result = {}
for value in values:
v = value.get(field)
if v in result:
result[v] += 1
else:
result[v] = 1
result['total'] = len(values)
return result
def each_tag(value):
for subvalue in value.get('tags', []):
yield (subvalue, None)
def each_tag_with_taken_ts(value):
for subvalue in value.get('tags', []):
yield ((subvalue, get_taken_ts(value)), None)
def each_file_reference(value):
for file in value.get('files', []):
yield file.get('reference'), None
entry = jsondb.Database(os.path.join(db_root, 'entry'))
entry.define('by_taken_ts', get_taken_ts_tuple)
entry.define(
'state_by_date',
lambda o: get_taken_date(o, lambda oo: {'state': oo['state']}),
lambda keys, values, rereduce: sum_per('state', values))
entry.define('by_date', get_taken_date_tuple)
entry.define('by_state', lambda o: (o['state'], None))
entry.define('by_state_and_taken_ts', lambda o:
((o['state'], get_taken_ts(o)), None))
entry.define('by_source', get_source)
entry.define(
'by_tag',
each_tag,
lambda keys, values, rereduce: len(values),
)
entry.define('by_tag_and_taken_ts', each_tag_with_taken_ts)
entry.define('by_file_reference', each_file_reference)
self.db['entry'] = entry
file = jsondb.Database(os.path.join(db_root, 'file'))
file.define('by_reference', lambda o: (o['reference'], None))
self.db['file'] = file
date = jsondb.Database(os.path.join(db_root, 'date'))
date.define('by_date', lambda o: (o['_id'], None))
self.db['date'] = date
job = jsondb.Database(os.path.join(db_root, 'job'))
job.define(
'by_state',
lambda o: ((o['state'], o['release'], o['priority']), None),
)
job.define(
'by_updated',
lambda o: (10000000000 - int(o['updated']), None),
)
job.define(
'stats',
lambda o: (None, {
'state': o['state']
}),
lambda keys, values, rereduce: sum_per('state', values),
)
self.db['job'] = job
def run(self, job: Job):
logging.debug(job.to_json())
step = job.get_current_step()
entry = get_entry_by_id(step.options.entry_id)
source = get_file_by_url(step.options.source_url)
if step.options.path is not None:
source_path = step.options.path
else:
cut = job.get_step('to_cut')
source_path = cut.result.path
assert source_path, "Missing source path (forgot to_cut step?)"
if step.options.reference is not None:
reference = step.options.reference
else:
calculate_hash = job.get_step('calculate_hash')
reference = calculate_hash.result.calculated_hash
assert source_path, "Missing reference (forgot calculate_reference step?)"
metadata_step = job.get_step('read_metadata')
if metadata_step.result is not None and metadata_step.result.metadata is not None:
metadata = metadata_step.result.metadata
else:
metadata = entry.metadata
if metadata is not None and hasattr(
metadata, 'taken_ts') and metadata.taken_ts is not None:
taken_ts = metadata.taken_ts[:10]
else:
taken_ts = datetime.datetime.fromtimestamp(
os.path.getmtime(source_path)).strftime('%Y-%m-%d')
system = current_system()
file_ref = next(
(fr for fr in entry.files if fr.reference == source.reference),
None)
purpose = file_ref.purpose if file_ref is not None else FilePurpose.unknown
main_root = resolve_path(system.main_storage.root_path)
filename = os.path.basename(step.options.source_url)
parts = [
main_root, entry.type.value, purpose.value, taken_ts, filename
]
main_path = os.path.join(*parts)
logging.info("Source: %s", str(source.reference))
logging.info("Main path: %s", str(main_path))
filecopy = FileCopy(
source=source_path,
destination=main_path,
link=True,
remove_source=False,
)
filecopy.run()
@retry()
def push():
entry = get_entry_by_id(step.options.entry_id)
new_file_ref = FileReference(
purpose=file_ref.purpose,
version=file_ref.version,
reference=reference,
mime_type=file_ref.mime_type,
)
new_file = File(
reference=reference,
url=system.main_storage.get_file_url(main_path),
mime_type=source.mime_type,
status=FileStatus.managed,
)
create_file(new_file)
entry.files.append(new_file_ref)
update_entry_by_id(entry.id, entry)
push()
step.result = ToMainResult()
step.status = StepStatus.done