def _repair_ts(): # fix any timestamps that are incorrect for md in db.session.query(Component).filter( Component.release_timestamp < 1980): fn = _get_absolute_path(md.fw) if not os.path.exists(fn): continue print(fn, md.release_timestamp) try: ufile = UploadedFile(is_strict=False) for cat in db.session.query(Category): ufile.category_map[cat.value] = cat.category_id for pro in db.session.query(Protocol): ufile.protocol_map[pro.value] = pro.protocol_id with open(fn, 'rb') as f: ufile.parse(os.path.basename(fn), f.read()) except MetadataInvalid as e: print('failed to parse file: {}'.format(str(e))) continue for md_local in ufile.fw.mds: if md_local.appstream_id == md.appstream_id: print('repairing timestamp from {} to {}'.format( md.release_timestamp, md_local.release_timestamp)) md.release_timestamp = md_local.release_timestamp md.fw.mark_dirty() # all done db.session.commit()
def _repair(): # fix any timestamps that are incorrect for md in db.session.query(Component).filter( Component.release_timestamp < 1980): fn = _get_absolute_path(md.fw) if not os.path.exists(fn): continue print(fn, md.release_timestamp) try: ufile = UploadedFile(is_strict=False) for cat in db.session.query(Category): ufile.category_map[cat.value] = cat.category_id for pro in db.session.query(Protocol): ufile.protocol_map[pro.value] = pro.protocol_id with open(fn, 'rb') as f: ufile.parse(os.path.basename(fn), f.read()) except MetadataInvalid as e: print('failed to parse file: {}'.format(str(e))) continue for md_local in ufile.fw.mds: if md_local.appstream_id == md.appstream_id: print('repairing timestamp from {} to {}'.format( md.release_timestamp, md_local.release_timestamp)) md.release_timestamp = md_local.release_timestamp md.fw.mark_dirty() # fix all the checksums and file sizes for fw in db.session.query(Firmware): try: with open(fw.filename_absolute, 'rb') as f: checksum_signed_sha1 = hashlib.sha1(f.read()).hexdigest() if checksum_signed_sha1 != fw.checksum_signed_sha1: print('repairing checksum from {} to {}'.format( fw.checksum_signed_sha1, checksum_signed_sha1)) fw.checksum_signed_sha1 = checksum_signed_sha1 fw.mark_dirty() checksum_signed_sha256 = hashlib.sha256(f.read()).hexdigest() if checksum_signed_sha256 != fw.checksum_signed_sha256: print('repairing checksum from {} to {}'.format( fw.checksum_signed_sha256, checksum_signed_sha256)) fw.checksum_signed_sha256 = checksum_signed_sha256 fw.mark_dirty() for md in fw.mds: sz = os.path.getsize(fw.filename_absolute) if sz != md.release_download_size: print('repairing size from {} to {}'.format( md.release_download_size, sz)) md.release_download_size = sz md.fw.mark_dirty() except FileNotFoundError as _: pass # ensure the test has been added for the firmware type if not fw.is_deleted: ploader.ensure_test_for_fw(fw) # all done db.session.commit()
def _fsck(): for firmware_id, in db.session.query(Firmware.firmware_id)\ .order_by(Firmware.firmware_id.asc()): fw = db.session.query(Firmware)\ .filter(Firmware.firmware_id == firmware_id)\ .one() fn = _get_absolute_path(fw) if not os.path.isfile(fn): print('firmware {} is missing, expected {}'.format(fw.firmware_id, fn))
def run_test_on_fw(self, test, fw): # get ClamAV version if self.get_setting_bool('clamav_use_daemon'): argv = ['clamdscan', '--version'] else: argv = ['clamscan', '--version'] try: ps = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if ps.wait() != 0: test.add_fail('Failed to scan', ps.stderr.read()) return stdout, _ = ps.communicate() except OSError as e: test.add_fail('Failed to scan', str(e)) return test.add_pass('Version', stdout) # scan cabinet archive fn = _get_absolute_path(fw) if self.get_setting_bool('clamav_use_daemon'): argv = ['clamdscan', '--fdpass', '--no-summary', fn] else: argv = [ 'clamscan', '--infected', '--scan-mail=no', '--phishing-scan-urls=no', '--phishing-sigs=no', '--scan-swf=no', '--nocerts', '--no-summary', fn ] if self.get_setting_bool('clamav_detect_pua'): argv.append('--detect-pua=yes') try: ps = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) rc = ps.wait() if rc == 2: test.add_fail('Failed to scan', ps.stderr.read()) return stdout, _ = ps.communicate() except OSError as e: test.add_fail('Failed to scan', str(e)) return # parse results if rc != 0: for ln in stdout.split('\n'): try: fn, status = ln.split(': ', 2) except ValueError as e: continue test.add_fail('Failed to scan', status)
def _repair_fn(): for firmware_id, in db.session.query(Firmware.firmware_id)\ .order_by(Firmware.firmware_id.asc()): fw = db.session.query(Firmware)\ .filter(Firmware.firmware_id == firmware_id)\ .one() filename = _get_sanitized_basename(fw.filename) if filename != fw.filename: print('moving {} to {}'.format(fw.filename, filename)) fn_old = _get_absolute_path(fw) fn_new = os.path.join(os.path.dirname(fn_old), filename) try: os.rename(fn_old, fn_new) fw.filename = filename fw.mark_dirty() except FileNotFoundError as _: pass # all done db.session.commit()
def _repair_csum(): # fix all the checksums and file sizes for firmware_id, in db.session.query(Firmware.firmware_id)\ .order_by(Firmware.firmware_id.asc()): fw = db.session.query(Firmware)\ .filter(Firmware.firmware_id == firmware_id)\ .one() try: print('checking {}…'.format(fw.filename_absolute)) fn = _get_absolute_path(fw) with open(fn, 'rb') as f: buf = f.read() checksum_signed_sha1 = hashlib.sha1(buf).hexdigest() if checksum_signed_sha1 != fw.checksum_signed_sha1: print('repairing checksum from {} to {}'.format( fw.checksum_signed_sha1, checksum_signed_sha1)) fw.checksum_signed_sha1 = checksum_signed_sha1 fw.mark_dirty() checksum_signed_sha256 = hashlib.sha256(buf).hexdigest() if checksum_signed_sha256 != fw.checksum_signed_sha256: print('repairing checksum from {} to {}'.format( fw.checksum_signed_sha256, checksum_signed_sha256)) fw.checksum_signed_sha256 = checksum_signed_sha256 fw.mark_dirty() sz = os.path.getsize(fn) for md in fw.mds: if sz != md.release_download_size: print('repairing size from {} to {}'.format( md.release_download_size, sz)) md.release_download_size = sz md.fw.mark_dirty() except FileNotFoundError as _: print('skipping {}'.format(fw.filename_absolute)) # all done db.session.commit()
def _fsck(): for fw in db.session.query(Firmware): fn = _get_absolute_path(fw) if not os.path.isfile(fn): print('firmware {} is missing, expected {}'.format( fw.firmware_id, fn))