def test_27(self):
""" Data on volume persists even after the instance is deleted """
instance_id = self.create_instance(config.image_name, config.flavor, config.key_name,
config.network, config.security_group, config.instance_name)
volume_id = self.create_non_bootable_volume(config.non_bootable_volume_name, config.non_bootable_volume_size)
device = self.attach_volume(instance_id, volume_id)
instance_ip = utils.floating_ip_associate(self.tenant_id, self.auth_token, "public", instance_id)
time.sleep(30)
value = utils.create_filesystem(instance_ip, device, "/mnt/temp")
self.assertEquals("Filesystem Created", value)
value = utils.mount_volume(instance_ip, device,"/mnt/temp")
self.assertEquals("Successfully mounted", value)
size = config.volume_block_size
count = config.count
file_size = size*count
utils.write_data_on_volume(instance_ip, size, count, "/mnt/temp")
time.sleep(30)
value = utils.is_file_exists(instance_ip, "/mnt/temp", file_size)
self.assertEquals(True ,value, "File not found")
value = utils.instance_delete(self.tenant_id, self.auth_token, instance_id)
self.assertNotEqual(True, value, "Instance not deleted")
instance_id = self.create_instance(config.image_name, config.flavor, config.key_name,
config.network, config.security_group, config.instance_name)
instance_ip = utils.floating_ip_associate(self.tenant_id, self.auth_token, "public", instance_id)
time.sleep(30)
device = self.attach_volume(instance_id, volume_id)
value = utils.mount_volume(instance_ip, device, "/mnt/temp")
self.assertEquals("Successfully mounted", value)
value = utils.is_file_exists(instance_ip, "/mnt/temp", file_size)
self.assertEquals(True, value, "File not found")
def test_35(self):
""" Backup a volume with data when restored and attached to another instance data exists """
instance_id = self.create_instance(config.image_name, config.flavor, config.key_name,
config.network, config.security_group, config.instance_name)
volume_id = self.create_non_bootable_volume(config.non_bootable_volume_name, config.non_bootable_volume_size)
device = self.attach_volume(instance_id, volume_id)
instance_ip = utils.floating_ip_associate(self.tenant_id, self.auth_token, "public",instance_id)
time.sleep(30)
value = utils.create_filesystem(instance_ip, device, "/mnt/temp")
self.assertEquals("Filesystem Created", value, "File system not created")
value = utils.mount_volume(instance_ip, device, "/mnt/temp")
self.assertEquals("Successfully mounted", value, "Volume is not mounted")
size = config.volume_block_size
count = config.count
file_size = size*count
utils.write_data_on_volume(instance_ip, size,count, "/mnt/temp")
time.sleep(30)
value = utils.is_file_exists(instance_ip,"/mnt/temp",file_size)
self.assertEquals(True, value, "File not found")
value = utils.unmount_volume(instance_ip, "/mnt/temp")
self.assertEquals("Successfully unmounted", value, "Volume not mounted")
value = utils.instance_delete(self.tenant_id, self.auth_token, instance_id)
self.assertNotEqual(True, value, "Instance not deleted")
backup_id = self.create_backup(volume_id, config.backup_name)
volume_id = self.restore_backup(backup_id)
instance_id = self.create_instance(config.image_name, config.flavor,
config.key_name, config.network, config.security_group, config.instance_name)
instance_ip = utils.floating_ip_associate(self.tenant_id, self.auth_token, "public", instance_id)
time.sleep(30)
device = self.attach_volume(instance_id, volume_id)
value = utils.mount_volume(instance_ip, device, "/mnt/temp")
self.assertEquals("Successfully mounted", value, "Volume not mounted")
value = utils.is_file_exists(instance_ip, "/mnt/temp", file_size)
self.assertEquals(True, value, "File doesnot exists")
def __init__(self,
has_tag=None,
bam="",
samtools="samtools",
ctg_list=[],
ctg_range=None,
mapq_filter=10):
if not is_file_exists(bam):
print(
"... Error: Bam file does not exist, please check the input file path ..."
)
sys.exit(1)
if not is_file_exists(bam + ".bai"):
print(
"... Bam file is not indexed yet, the index file will now be generated ..."
)
bam_index = subprocess_popen(
shlex.split(f"{samtools} index {bam}"))
ret_code = bam_index.wait()
if ret_code != 0:
print(
"... Error: An error occured during indexing bam file, please check if the bam file is valid ..."
)
sys.exit(1)
queue = Manager().Queue()
process_list = []
subprocess_list = []
for contig in ctg_list:
subprocess = subprocess_popen(
shlex.split(
f'{samtools} view -q {mapq_filter} {bam} {contig}{f":{ctg_range}" if ctg_range else ""}'
))
subprocess_list.append(subprocess)
process = Process(target=gather_result,
args=(queue, subprocess.stdout, has_tag))
process_list.append(process)
for process in process_list:
process.start()
for process in process_list:
process.join()
for subprocess in subprocess_list:
subprocess.stdout.close()
self.reads = []
while not queue.empty():
self.reads.append(queue.get())
if len(self.reads) % 100000 == 0:
print("... %d reads processed ..." % len(self.reads))
print("... Finished reading bam, extracted %d reads ..." %
len(self.reads))
def __init__(self, api_key = "", output_dir = "", text_list_file = "", *args, **kwargs):
super(GetTextsSpider, self).__init__(*args, **kwargs)
utils.create_dir(output_dir)
if len(api_key) == 0:
raise scrapy.exceptions.CloseSpider("expected 'api_key' to be a string")
if len(output_dir) == 0:
raise scrapy.exceptions.CloseSpider("expected 'output_dir' to be a string")
if len(text_list_file) == 0:
raise scrapy.exceptions.CloseSpider("expected 'text_list_file' to be a string")
self.api_key = api_key
self.output_dir = output_dir
with open(text_list_file, "r") as f:
self.all_texts = json.load(f)['texts']
if utils.is_file_exists(output_dir+"/master_record.json"):
with open(output_dir+"/master_record.json", "r") as f:
json_file = json.load(f)
self.master_record = json_file["master_record"]
self.last_unallocated_number = json_file["last_unallocated_number"]
else:
self.master_record = {text["gid"]: "" for text in self.all_texts}
self.last_unallocated_number = 1
self.text_name_prefix = "Guardian"
self.text_name_length = int(math.floor(math.log(len(self.all_texts))/math.log(10)) + 1)
def any(message):
log.info(message)
group = Group.parse_group(message.text)
if group is None:
bot.send_message(message.chat.id, text=replies.WRONG_GROUP)
return
bot.send_message(message.chat.id, text='Уже ищу твое расписание...')
path_to_ics = make_path(config['VAULT_PATH'], group.get(), '.ics')
if group.has_type():
if not is_file_exists(path_to_ics):
try:
download_schedule(group, config['BMSTU_SCHEDULE_API'],
config['VAULT_PATH'])
except SystemExit:
bot.send_message(message.chat.id,
text=replies.GROUP_NOT_FOUND.format(
group.get()))
return
else:
available_types = get_available_types(config['VAULT_PATH'],
group.get())
if len(available_types) == 0:
bot.send_message(message.chat.id,
text=replies.GROUP_IS_NOT_DEFINED.format(
group.get()))
return
if len(available_types) == 1:
group.group_type = available_types[0]
else:
if is_file_exists(path_to_ics):
send_also_available_groups_msg(bot, message.chat.id, group,
available_types)
else:
send_choose_group_type_error_msg(bot, message.chat.id,
group, available_types)
return
send_schedule(bot, message.chat.id, group.get(), config['VAULT_PATH'])
def afk(message):
log.info(message)
path_to_afk = make_path(config['VAULT_PATH'], "afk.jpg")
if is_file_exists(path_to_afk):
with open(path_to_afk, 'rb') as file:
bot.send_photo(message.chat.id, file)
else:
bot.send_message(message.chat.id, text=replies.AFK)
def doctors(message):
log.info(message)
path_to_doctors = make_path(config['VAULT_PATH'], "doctors.jpeg")
if is_file_exists(path_to_doctors):
with open(path_to_doctors, 'rb') as file:
bot.send_photo(message.chat.id, file)
bot.send_message(message.chat.id, text=replies.DOCTORS[0])
else:
bot.send_message(message.chat.id, text=replies.DOCTORS[1])
def create_wallet_dir(self,
wallet_dir,
public_ip,
private_key,
delete_before=False):
if delete_before:
utils.execute_command(self.connection,
'rm -rf {}'.format(wallet_dir))
exists = utils.is_directory_exists(self.connection, wallet_dir)
if not exists:
utils.execute_command(self.connection,
'mkdir -p {}'.format(wallet_dir))
polis_conf = wallet_dir + 'polis.conf'
exists = utils.is_file_exists(self.connection, polis_conf)
if not exists:
# Transfer the inflated to file to the target
dir_path = os.path.dirname(os.path.realpath(__file__))
polis_conf_tpl = dir_path + '/polis.conf'
utils.send_file(self.connection, polis_conf_tpl, wallet_dir)
# Setup the config file
rpcuser = ''.join(
secrets.choice(string.ascii_lowercase +
string.ascii_uppercase + string.digits)
for _ in range(50))
utils.execute_command(
self.connection,
'sed -i \'s/<RPCUSER>/{}/g\' {}'.format(rpcuser, polis_conf))
rpcpassword = ''.join(
secrets.choice(string.ascii_lowercase +
string.ascii_uppercase + string.digits)
for _ in range(50))
utils.execute_command(
self.connection, 'sed -i \'s/<RPCPASSWORD>/{}/g\' {}'.format(
rpcpassword, polis_conf))
utils.execute_command(
self.connection, 'sed -i \'s/<PUBLICIP>/{}/g\' {}'.format(
public_ip, polis_conf))
utils.execute_command(
self.connection, 'sed -i \'s/<PRIVATEKEY>/{}/g\' {}'.format(
private_key, polis_conf))
return exists
def __init__(self,
api_key="",
output_dir="",
text_list_file="",
*args,
**kwargs):
super(GetTextsSpider, self).__init__(*args, **kwargs)
utils.create_dir(output_dir)
if len(api_key) == 0:
raise scrapy.exceptions.CloseSpider(
"expected 'api_key' to be a string")
if len(output_dir) == 0:
raise scrapy.exceptions.CloseSpider(
"expected 'output_dir' to be a string")
if len(text_list_file) == 0:
raise scrapy.exceptions.CloseSpider(
"expected 'text_list_file' to be a string")
self.api_key = api_key
self.output_dir = output_dir
with open(text_list_file, "r") as f:
self.all_texts = json.load(f)['texts']
if utils.is_file_exists(output_dir + "/master_record.json"):
with open(output_dir + "/master_record.json", "r") as f:
json_file = json.load(f)
self.master_record = json_file["master_record"]
self.last_unallocated_number = json_file[
"last_unallocated_number"]
else:
self.master_record = {text["gid"]: "" for text in self.all_texts}
self.last_unallocated_number = 1
self.text_name_prefix = "Guardian"
self.text_name_length = int(
math.floor(math.log(len(self.all_texts)) / math.log(10)) + 1)
def load_content(request_dict: Dict) -> Union[None, bytes]:
"""
加载批量响应对象
:param request_dict: 批量请求对象
:return: 批量响应对象
"""
path = '/tmp/star_crawler'
create_folder(path)
token = None
try:
name = request_dict["url"].split('&r')[0]
if request_dict["method"] == POST:
data = request_dict["data"]
token = generate_token(name, data)
elif request_dict["method"] == GET:
token = generate_token(name)
if not is_file_exists('{0}/{1}'.format(path, token)):
return None
with open('{0}/{1}'.format(path, token), mode='rb') as f:
buffer = f.read()
content = loads_content(buffer)
return content
except Exception:
return None
def test_13(self):
""" Write data on bootable volume """
image_id = utils.image_id(self.tenant_id, self.auth_token, config.image_name)
content = utils.volume_create(self.tenant_id, self.auth_token, config.bootable_volume_name,
config.bootable_volume_size_for_data, image_id)
volume_id = content["volume"]["id"]
is_available = utils.is_volume_available(self.tenant_id, self.auth_token, volume_id)
self.assertEquals(True, is_available)
content = utils.volume_boot_attach_while_creating_instance(self.tenant_id, self.auth_token,
volume_id,config.key_name, config.instance_name, config.flavor, config.network,config.security_group)
instance_id = utils.instance_id(self.tenant_id, self.auth_token, config.instance_name)
is_active = utils.is_instance_active(self.tenant_id, self.auth_token, instance_id)
self.assertEquals(True, is_active)
volume_details = utils.volume_details(self.tenant_id, self.auth_token, volume_id)
self.assertEquals(instance_id, volume_details["volume"]["attachments"][0]["server_id"],"notequal")
instance_ip = utils.floating_ip_associate(self.tenant_id, self.auth_token, "public", instance_id)
time.sleep(30)
size = config.volume_block_size
count = config.count
file_size = size*count
utils.write_data_on_volume(instance_ip, size, count, "/")
time.sleep(30)
value = utils.is_file_exists(instance_ip,"/", file_size)
self.assertEquals(True, value, "File not found")
def is_polis_installed(connection, dir):
if not dir.endswith('/'):
return is_file_exists(connection, "{}/{}".format(dir, 'polisd'))
else:
return is_file_exists(connection, "{}{}".format(dir, 'polisd'))
for url in urls:
site_name = re.sub('http(s)?://', '', url).replace('/', '').rstrip()
current_file_path = os.path.join(absolute_path,
'image/' + site_name + '/current')
previous_file_path = os.path.join(absolute_path,
'image/' + site_name + '/previous')
current_file_name = current_file_path + '/screen.png'
previous_file_name = previous_file_path + '/screen.png'
utils.mkdir(current_file_path)
utils.mkdir(previous_file_path)
if utils.is_file_exists(current_file_name):
utils.move_file(current_file_name, previous_file_name)
selenium_crawler = Crawler()
selenium_crawler.get_screenshot(url, current_file_name)
if utils.is_file_exists(current_file_name) and utils.is_file_exists(
previous_file_name):
if not utils.diff_image(current_file_name, previous_file_name):
message = MIMEMultipart()
message['Subject'] = '差分検知[' + site_name + ']'
message['From'] = from_address
message['To'] = to_address
message['Date'] = formatdate()