class EncryptionMarkConfig(object): def __init__(self, logger, encryption_environment): self.logger = logger self.encryption_environment = encryption_environment self.command = None self.volume_type = None self.diskFormatQuery = None self.encryption_mark_config = ConfigUtil( self.encryption_environment.azure_crypt_request_queue_path, 'encryption_request_queue', self.logger) def get_volume_type(self): return self.encryption_mark_config.get_config( CommonVariables.EncryptionVolumeTypeKey) def get_current_command(self): return self.encryption_mark_config.get_config( CommonVariables.EncryptionEncryptionOperationKey) def get_encryption_disk_format_query(self): return self.encryption_mark_config.get_config( CommonVariables.EncryptionDiskFormatQueryKey) def config_file_exists(self): """ we should compare the timestamp of the file with the current system time if not match (in 30 minutes, then should skip the file) """ return self.encryption_mark_config.config_file_exists() def commit(self): key_value_pairs = [] command = ConfigKeyValuePair( CommonVariables.EncryptionEncryptionOperationKey, self.command) key_value_pairs.append(command) volume_type = ConfigKeyValuePair( CommonVariables.EncryptionVolumeTypeKey, self.volume_type) key_value_pairs.append(volume_type) disk_format_query = ConfigKeyValuePair( CommonVariables.EncryptionDiskFormatQueryKey, self.diskFormatQuery) key_value_pairs.append(disk_format_query) self.encryption_mark_config.save_configs(key_value_pairs) def clear_config(self): try: if os.path.exists(self.encryption_environment. azure_crypt_request_queue_path): os.remove( self.encryption_environment.azure_crypt_request_queue_path) return True except OSError as e: self.logger.log( "Failed to clear_queue with error: {0}, stack trace: {1}". format(e, traceback.format_exc())) return False
class DecryptionMarkConfig(object): def __init__(self, logger, encryption_environment): self.logger = logger self.encryption_environment = encryption_environment self.command = None self.volume_type = None self.decryption_mark_config = ConfigUtil(self.encryption_environment.azure_decrypt_request_queue_path, 'decryption_request_queue', self.logger) def get_current_command(self): return self.decryption_mark_config.get_config(CommonVariables.EncryptionEncryptionOperationKey) def config_file_exists(self): return self.decryption_mark_config.config_file_exists() def commit(self): key_value_pairs = [] command = ConfigKeyValuePair(CommonVariables.EncryptionEncryptionOperationKey, self.command) key_value_pairs.append(command) volume_type = ConfigKeyValuePair(CommonVariables.EncryptionVolumeTypeKey, self.volume_type) key_value_pairs.append(volume_type) self.decryption_mark_config.save_configs(key_value_pairs) def clear_config(self): try: if(os.path.exists(self.encryption_environment.azure_decrypt_request_queue_path)): os.remove(self.encryption_environment.azure_decrypt_request_queue_path) return True except OSError as e: self.logger.log("Failed to clear_queue with error: {0}, stack trace: {1}".format(e, traceback.format_exc())) return False
class EncryptionMarkConfig(object): def __init__(self, logger, encryption_environment): self.logger = logger self.encryption_environment = encryption_environment self.command = None self.volume_type = None self.diskFormatQuery = None self.encryption_mark_config = ConfigUtil(self.encryption_environment.azure_crypt_request_queue_path, 'encryption_request_queue', self.logger) def get_volume_type(self): return self.encryption_mark_config.get_config(CommonVariables.EncryptionVolumeTypeKey) def get_current_command(self): return self.encryption_mark_config.get_config(CommonVariables.EncryptionEncryptionOperationKey) def get_encryption_disk_format_query(self): return self.encryption_mark_config.get_config(CommonVariables.EncryptionDiskFormatQueryKey) def config_file_exists(self): """ we should compare the timestamp of the file with the current system time if not match (in 30 minutes, then should skip the file) """ return self.encryption_mark_config.config_file_exists() def commit(self): key_value_pairs = [] command = ConfigKeyValuePair(CommonVariables.EncryptionEncryptionOperationKey, self.command) key_value_pairs.append(command) volume_type = ConfigKeyValuePair(CommonVariables.EncryptionVolumeTypeKey, self.volume_type) key_value_pairs.append(volume_type) disk_format_query = ConfigKeyValuePair(CommonVariables.EncryptionDiskFormatQueryKey, self.diskFormatQuery) key_value_pairs.append(disk_format_query) self.encryption_mark_config.save_configs(key_value_pairs) def clear_config(self): try: if os.path.exists(self.encryption_environment.azure_crypt_request_queue_path): os.remove(self.encryption_environment.azure_crypt_request_queue_path) return True except OSError as e: self.logger.log("Failed to clear_queue with error: {0}, stack trace: {1}".format(e, traceback.format_exc())) return False
class DecryptionMarkConfig(object): def __init__(self, logger, encryption_environment): self.logger = logger self.encryption_environment = encryption_environment self.command = None self.volume_type = None self.decryption_mark_config = ConfigUtil( self.encryption_environment.azure_decrypt_request_queue_path, 'decryption_request_queue', self.logger) def get_current_command(self): return self.decryption_mark_config.get_config( CommonVariables.EncryptionEncryptionOperationKey) def config_file_exists(self): return self.decryption_mark_config.config_file_exists() def commit(self): key_value_pairs = [] command = ConfigKeyValuePair( CommonVariables.EncryptionEncryptionOperationKey, self.command) key_value_pairs.append(command) volume_type = ConfigKeyValuePair( CommonVariables.EncryptionVolumeTypeKey, self.volume_type) key_value_pairs.append(volume_type) self.decryption_mark_config.save_configs(key_value_pairs) def clear_config(self): try: if (os.path.exists(self.encryption_environment. azure_decrypt_request_queue_path)): os.remove(self.encryption_environment. azure_decrypt_request_queue_path) return True except OSError as e: self.logger.log( "Failed to clear_queue with error: {0}, stack trace: {1}". format(e, traceback.format_exc())) return False
match = re.search(r'cid=(\d+)', page.text) if match: cid = match.group(1) else: return 0 xml_path = "http://comment.bilibili.tv/{0}.xml".format(cid) try: save_xml(xml_path, av) except UnicodeDecodeError as e: raise e if __name__ == '__main__': nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 现在 p = ConfigUtil.get_config('from') print('{0} from av id {1}'.format(nowTime, p)) try: while p < 99999999: try: save_danmu(str(p)) except UnicodeDecodeError as e: print('{0} not saved for decoding error {1}'.format(p, e)) except Exception as e: print('{0} not saved for unknown error {1}'.format(p, e)) finally: p = p + 1 except SystemExit as ex: print('-----------------------exit raised-----------------------') print('code ' + ex.code) print(str(ex))
def clean(start, end): dir = os.getcwd() + '/resources/danmu/all/' for file in os.listdir(dir): # 避免非xml文件的干扰 if file.split('.')[1] == 'xml': name_id = int(file.split('-')[1].split('.')[0]) if start <= name_id <= end: os.remove(dir + file) print(file + " deleted") print('cleaned xml finished.') if __name__ == '__main__': xml_path = os.getcwd() + '/resources/danmu/all/' nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 现在 p = ConfigUtil.get_config('start') max_p = ConfigUtil.get_config('max') ConfigUtil.set_config('from', max_p + 1) print('{0} av id {1}-{2}'.format(nowTime, p, max_p)) # 追加 out_file = open(os.getcwd() + '/resources/config/out.txt', 'a') try: while p < max_p: try: out = parse(xml_path, 'all-{0}.xml'.format(p), p) if out is not None: out_file.write(out + '\n') except Exception as e: print('{0} not parsed error {1}'.format(p, e)) p = p + 1