class EncryptionMarkConfig(object):
def __init__(self, logger, encryption_environment):
self.logger = logger
self.encryption_environment = encryption_environment
self.command = None
self.volume_type = None
self.diskFormatQuery = None
self.encryption_mark_config = ConfigUtil(
self.encryption_environment.azure_crypt_request_queue_path,
'encryption_request_queue', self.logger)
def get_volume_type(self):
return self.encryption_mark_config.get_config(
CommonVariables.EncryptionVolumeTypeKey)
def get_current_command(self):
return self.encryption_mark_config.get_config(
CommonVariables.EncryptionEncryptionOperationKey)
def get_encryption_disk_format_query(self):
return self.encryption_mark_config.get_config(
CommonVariables.EncryptionDiskFormatQueryKey)
def config_file_exists(self):
"""
we should compare the timestamp of the file with the current system time
if not match (in 30 minutes, then should skip the file)
"""
return self.encryption_mark_config.config_file_exists()
def commit(self):
key_value_pairs = []
command = ConfigKeyValuePair(
CommonVariables.EncryptionEncryptionOperationKey, self.command)
key_value_pairs.append(command)
volume_type = ConfigKeyValuePair(
CommonVariables.EncryptionVolumeTypeKey, self.volume_type)
key_value_pairs.append(volume_type)
disk_format_query = ConfigKeyValuePair(
CommonVariables.EncryptionDiskFormatQueryKey, self.diskFormatQuery)
key_value_pairs.append(disk_format_query)
self.encryption_mark_config.save_configs(key_value_pairs)
def clear_config(self):
try:
if os.path.exists(self.encryption_environment.
azure_crypt_request_queue_path):
os.remove(
self.encryption_environment.azure_crypt_request_queue_path)
return True
except OSError as e:
self.logger.log(
"Failed to clear_queue with error: {0}, stack trace: {1}".
format(e, traceback.format_exc()))
return False
class DecryptionMarkConfig(object):
def __init__(self, logger, encryption_environment):
self.logger = logger
self.encryption_environment = encryption_environment
self.command = None
self.volume_type = None
self.decryption_mark_config = ConfigUtil(self.encryption_environment.azure_decrypt_request_queue_path,
'decryption_request_queue',
self.logger)
def get_current_command(self):
return self.decryption_mark_config.get_config(CommonVariables.EncryptionEncryptionOperationKey)
def config_file_exists(self):
return self.decryption_mark_config.config_file_exists()
def commit(self):
key_value_pairs = []
command = ConfigKeyValuePair(CommonVariables.EncryptionEncryptionOperationKey, self.command)
key_value_pairs.append(command)
volume_type = ConfigKeyValuePair(CommonVariables.EncryptionVolumeTypeKey, self.volume_type)
key_value_pairs.append(volume_type)
self.decryption_mark_config.save_configs(key_value_pairs)
def clear_config(self):
try:
if(os.path.exists(self.encryption_environment.azure_decrypt_request_queue_path)):
os.remove(self.encryption_environment.azure_decrypt_request_queue_path)
return True
except OSError as e:
self.logger.log("Failed to clear_queue with error: {0}, stack trace: {1}".format(e, traceback.format_exc()))
return False
class EncryptionMarkConfig(object):
def __init__(self, logger, encryption_environment):
self.logger = logger
self.encryption_environment = encryption_environment
self.command = None
self.volume_type = None
self.diskFormatQuery = None
self.encryption_mark_config = ConfigUtil(self.encryption_environment.azure_crypt_request_queue_path,
'encryption_request_queue',
self.logger)
def get_volume_type(self):
return self.encryption_mark_config.get_config(CommonVariables.EncryptionVolumeTypeKey)
def get_current_command(self):
return self.encryption_mark_config.get_config(CommonVariables.EncryptionEncryptionOperationKey)
def get_encryption_disk_format_query(self):
return self.encryption_mark_config.get_config(CommonVariables.EncryptionDiskFormatQueryKey)
def config_file_exists(self):
"""
we should compare the timestamp of the file with the current system time
if not match (in 30 minutes, then should skip the file)
"""
return self.encryption_mark_config.config_file_exists()
def commit(self):
key_value_pairs = []
command = ConfigKeyValuePair(CommonVariables.EncryptionEncryptionOperationKey, self.command)
key_value_pairs.append(command)
volume_type = ConfigKeyValuePair(CommonVariables.EncryptionVolumeTypeKey, self.volume_type)
key_value_pairs.append(volume_type)
disk_format_query = ConfigKeyValuePair(CommonVariables.EncryptionDiskFormatQueryKey, self.diskFormatQuery)
key_value_pairs.append(disk_format_query)
self.encryption_mark_config.save_configs(key_value_pairs)
def clear_config(self):
try:
if os.path.exists(self.encryption_environment.azure_crypt_request_queue_path):
os.remove(self.encryption_environment.azure_crypt_request_queue_path)
return True
except OSError as e:
self.logger.log("Failed to clear_queue with error: {0}, stack trace: {1}".format(e, traceback.format_exc()))
return False
class DecryptionMarkConfig(object):
def __init__(self, logger, encryption_environment):
self.logger = logger
self.encryption_environment = encryption_environment
self.command = None
self.volume_type = None
self.decryption_mark_config = ConfigUtil(
self.encryption_environment.azure_decrypt_request_queue_path,
'decryption_request_queue', self.logger)
def get_current_command(self):
return self.decryption_mark_config.get_config(
CommonVariables.EncryptionEncryptionOperationKey)
def config_file_exists(self):
return self.decryption_mark_config.config_file_exists()
def commit(self):
key_value_pairs = []
command = ConfigKeyValuePair(
CommonVariables.EncryptionEncryptionOperationKey, self.command)
key_value_pairs.append(command)
volume_type = ConfigKeyValuePair(
CommonVariables.EncryptionVolumeTypeKey, self.volume_type)
key_value_pairs.append(volume_type)
self.decryption_mark_config.save_configs(key_value_pairs)
def clear_config(self):
try:
if (os.path.exists(self.encryption_environment.
azure_decrypt_request_queue_path)):
os.remove(self.encryption_environment.
azure_decrypt_request_queue_path)
return True
except OSError as e:
self.logger.log(
"Failed to clear_queue with error: {0}, stack trace: {1}".
format(e, traceback.format_exc()))
return False
match = re.search(r'cid=(\d+)', page.text)
if match:
cid = match.group(1)
else:
return 0
xml_path = "http://comment.bilibili.tv/{0}.xml".format(cid)
try:
save_xml(xml_path, av)
except UnicodeDecodeError as e:
raise e
if __name__ == '__main__':
nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 现在
p = ConfigUtil.get_config('from')
print('{0} from av id {1}'.format(nowTime, p))
try:
while p < 99999999:
try:
save_danmu(str(p))
except UnicodeDecodeError as e:
print('{0} not saved for decoding error {1}'.format(p, e))
except Exception as e:
print('{0} not saved for unknown error {1}'.format(p, e))
finally:
p = p + 1
except SystemExit as ex:
print('-----------------------exit raised-----------------------')
print('code ' + ex.code)
print(str(ex))
def clean(start, end):
dir = os.getcwd() + '/resources/danmu/all/'
for file in os.listdir(dir):
# 避免非xml文件的干扰
if file.split('.')[1] == 'xml':
name_id = int(file.split('-')[1].split('.')[0])
if start <= name_id <= end:
os.remove(dir + file)
print(file + " deleted")
print('cleaned xml finished.')
if __name__ == '__main__':
xml_path = os.getcwd() + '/resources/danmu/all/'
nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 现在
p = ConfigUtil.get_config('start')
max_p = ConfigUtil.get_config('max')
ConfigUtil.set_config('from', max_p + 1)
print('{0} av id {1}-{2}'.format(nowTime, p, max_p))
# 追加
out_file = open(os.getcwd() + '/resources/config/out.txt', 'a')
try:
while p < max_p:
try:
out = parse(xml_path, 'all-{0}.xml'.format(p), p)
if out is not None:
out_file.write(out + '\n')
except Exception as e:
print('{0} not parsed error {1}'.format(p, e))
p = p + 1
