def ensure_downloaded(self): pathed_container = self.container_path() if os.path.exists(pathed_container): return True command = self.get_build_instructions_for(pathed_container, self.container) Logger.info("Couldn't find singularity container, building with: " + " ".join(command)) try: build_result = subprocess.check_output(command) Logger.debug(build_result) except subprocess.CalledProcessError as e: Logger.log_ex(e)
def insert_or_update_many(self, els: List[T]): if len(els) == 0: return queries: Dict[str, List[List[any]]] = {} update_separator = ",\n" tab = "\t" idkeys = set(self.get_id_keys()) idkeys_ordered = list(idkeys) pkeys_ordered = self.get_primary_keys() existing_keys = set() # (*pkeys_ordered) # get all primary keys dbalias_map: Dict[str, DatabaseObjectField] = { t.dbalias: t for t in self._base.keymap() } updates, inserts = self.filter_updates(els) def add_query(query, values): if query in queries: queries[query].append(values) else: queries[query] = [values] for job in updates: keys, values = job.prepare_insert() # el_pkeys = [getattr(job, dbalias_map[_k]) for _k in idkeys_ordered] keys_np, values_np = [], [] for k, v in zip(keys, values): if k in idkeys: continue keys_np.append(k) values_np.append(v) # problem is we want to update matching on some fields when they are NULL, our WHERE statement # should be something like: # WHERE id1 = ? AND id2 = ? AND id3 is null AND id4 is null id_keyvalues = { pkey: prep_object_for_db( getattr(job, dbalias_map[pkey].name), encode=dbalias_map[pkey].encode, ) for pkey in idkeys_ordered } id_withvalues_keyvalue_ordered = [ (idkey, idvalue) for idkey, idvalue in id_keyvalues.items() if idvalue is not None ] id_withvalues_updater_keys = [ f"{idkey} = ?" for idkey, _ in id_withvalues_keyvalue_ordered ] id_withvalues_updater_values = [ idvalue for _, idvalue in id_withvalues_keyvalue_ordered ] id_novalues_updater_keys = [ f"{idkey} is NULL" for idkey, idvalue in id_keyvalues.items() if idvalue is None ] prepared_statement = f""" UPDATE {self._tablename} SET {', '.join(f'{k} = ?' for k in keys_np)} WHERE {" AND ".join([*id_withvalues_updater_keys, *id_novalues_updater_keys])} """ vtuple = ( *values_np, *id_withvalues_updater_values, ) add_query(prepared_statement, vtuple) for job in inserts: keys, values = job.prepare_insert() # el_pkeys = [getattr(job, dbalias_map[_k]) for _k in idkeys_ordered] prepared_statement = f""" INSERT INTO {self._tablename} ({', '.join(keys)}) VALUES ({', '.join(f'?' for _ in keys)}); """ add_query(prepared_statement, values) Logger.log( f"DB {self._tablename}: Inserting {len(inserts)} and updating {len(updates)} rows" ) with self.with_cursor() as cursor: start = DateUtil.now() if len(inserts) + len(updates) > 300: Logger.warn( f"DB '{self._tablename}' is inserting {len(inserts)} and updating {len(updates)} rows, this might take a while" ) for query, vvalues in queries.items(): try: Logger.log( f"Running query: {query}\n\t: values: {vvalues}") cursor.executemany(query, vvalues) except OperationalError as e: Logger.log_ex(e) seconds = (DateUtil.now() - start).total_seconds() if seconds > 2: Logger.warn( f"DB '{self._tablename}' took {second_formatter(seconds)} to insert {len(inserts)} and update {len(updates)} rows" ) return True