def ensure_downloaded(self):
pathed_container = self.container_path()
if os.path.exists(pathed_container):
return True
command = self.get_build_instructions_for(pathed_container,
self.container)
Logger.info("Couldn't find singularity container, building with: " +
" ".join(command))
try:
build_result = subprocess.check_output(command)
Logger.debug(build_result)
except subprocess.CalledProcessError as e:
Logger.log_ex(e)
def insert_or_update_many(self, els: List[T]):
if len(els) == 0:
return
queries: Dict[str, List[List[any]]] = {}
update_separator = ",\n"
tab = "\t"
idkeys = set(self.get_id_keys())
idkeys_ordered = list(idkeys)
pkeys_ordered = self.get_primary_keys()
existing_keys = set() # (*pkeys_ordered)
# get all primary keys
dbalias_map: Dict[str, DatabaseObjectField] = {
t.dbalias: t
for t in self._base.keymap()
}
updates, inserts = self.filter_updates(els)
def add_query(query, values):
if query in queries:
queries[query].append(values)
else:
queries[query] = [values]
for job in updates:
keys, values = job.prepare_insert()
# el_pkeys = [getattr(job, dbalias_map[_k]) for _k in idkeys_ordered]
keys_np, values_np = [], []
for k, v in zip(keys, values):
if k in idkeys:
continue
keys_np.append(k)
values_np.append(v)
# problem is we want to update matching on some fields when they are NULL, our WHERE statement
# should be something like:
# WHERE id1 = ? AND id2 = ? AND id3 is null AND id4 is null
id_keyvalues = {
pkey: prep_object_for_db(
getattr(job, dbalias_map[pkey].name),
encode=dbalias_map[pkey].encode,
)
for pkey in idkeys_ordered
}
id_withvalues_keyvalue_ordered = [
(idkey, idvalue) for idkey, idvalue in id_keyvalues.items()
if idvalue is not None
]
id_withvalues_updater_keys = [
f"{idkey} = ?" for idkey, _ in id_withvalues_keyvalue_ordered
]
id_withvalues_updater_values = [
idvalue for _, idvalue in id_withvalues_keyvalue_ordered
]
id_novalues_updater_keys = [
f"{idkey} is NULL" for idkey, idvalue in id_keyvalues.items()
if idvalue is None
]
prepared_statement = f"""
UPDATE {self._tablename}
SET {', '.join(f'{k} = ?' for k in keys_np)}
WHERE
{" AND ".join([*id_withvalues_updater_keys, *id_novalues_updater_keys])}
"""
vtuple = (
*values_np,
*id_withvalues_updater_values,
)
add_query(prepared_statement, vtuple)
for job in inserts:
keys, values = job.prepare_insert()
# el_pkeys = [getattr(job, dbalias_map[_k]) for _k in idkeys_ordered]
prepared_statement = f"""
INSERT INTO {self._tablename}
({', '.join(keys)})
VALUES
({', '.join(f'?' for _ in keys)});
"""
add_query(prepared_statement, values)
Logger.log(
f"DB {self._tablename}: Inserting {len(inserts)} and updating {len(updates)} rows"
)
with self.with_cursor() as cursor:
start = DateUtil.now()
if len(inserts) + len(updates) > 300:
Logger.warn(
f"DB '{self._tablename}' is inserting {len(inserts)} and updating {len(updates)} rows, this might take a while"
)
for query, vvalues in queries.items():
try:
Logger.log(
f"Running query: {query}\n\t: values: {vvalues}")
cursor.executemany(query, vvalues)
except OperationalError as e:
Logger.log_ex(e)
seconds = (DateUtil.now() - start).total_seconds()
if seconds > 2:
Logger.warn(
f"DB '{self._tablename}' took {second_formatter(seconds)} to insert {len(inserts)} and update {len(updates)} rows"
)
return True
