def download(self, path, url):
try:
cookie = None
anonymous = (self.user == '' or self.password == '')
code, result = client.request(url, output='response', error=True)
if code == '429' and anonymous is True:
control.dialog.ok(str('xsubs.tv'), str(result), str(''))
return
elif anonymous is False:
cookie = self.cookie()
result, headers, content, cookie = client.request(
url, cookie=cookie, output='extended')
subtitle = content['Content-Disposition']
subtitle = re.findall('"(.+?)"', subtitle)[0]
try:
subtitle = subtitle.decode('utf-8')
except Exception:
pass
subtitle = control.join(path, subtitle)
if not subtitle.endswith('.srt'):
raise Exception()
with open(subtitle, 'wb') as subFile:
subFile.write(result)
fileparts = os_split(subtitle)[1].split('.')
result = control.join(
os_split(subtitle)[0],
'subtitles.' + fileparts[len(fileparts) - 1])
rename(subtitle, result)
return result
except Exception as e:
_, __, tb = sys.exc_info()
print(traceback.print_tb(tb))
log_debug(
'Xsubstv subtitle download failed for the following reason: ' +
str(e))
return
def multichoice(filenames):
if filenames is None or len(filenames) == 0:
return
elif len(filenames) >= 1:
if len(filenames) == 1:
return filenames[0]
choices = [os_split(i)[1] for i in filenames]
choices.insert(0, control.lang(32215))
_choice = control.selectDialog(heading=control.lang(32214),
list=choices)
if _choice == 0:
filename = choice(filenames)
elif _choice != -1 and _choice <= len(filenames) + 1:
filename = filenames[_choice - 1]
else:
filename = choice(filenames)
return filename
else:
return
def compile(self):
filename = self.path
if filename[-5:] != '.rail':
filename += '.rail'
alias = (os_split(filename)[-1][:-5] if self.name is None
else self.name)
return interpreter.Import(filename=filename, alias=alias)
def _get_model_path_and_name(self):
model_path = self.config.get('model_path', '')
if not model_path:
raise ValueError('Model path should not be empty')
model_name, extension = os_splitext(os_split(model_path)[1])
if extension!= '.h5':
raise ValueError("This macro requires a .h5 file, {} given".format(extension))
return model_path, model_name
def find_parh_to_dit(target_dir_name, path=None):
from os import getcwd
from os.path import split as os_split, exists, join as os_join
path = path or getcwd()
old_path = ''
while target_dir_name not in path:
if exists(os_join(path, target_dir_name)):
path = os_join(path, target_dir_name)
break
now_dir = os_split(path)[1]
path = os_split(path)[0]
if old_path == path:
break
old_path = path
print(path, now_dir)
else:
all_path, end_dir = os_split(path)
while end_dir != target_dir_name:
now_dir = path[1]
path = os_split(path[0])[0] if not bool(path[-1]) else path[0]
return path
def get_inputs(self):
self.input_folder = Folder(
get_input_names_for_role("input_folder_id")[0])
output_folder_id = get_output_names_for_role("output_folder_id")[0]
self.output_folder = Folder(output_folder_id)
self.output_file_path = get_recipe_config()['output_model_path']
self.batch_size = int(get_recipe_config()['batch_size'])
if not get_recipe_config()['show_batch_size']:
self.batch_size = -1
self.overwrite_output_model = get_recipe_config(
)['overwrite_output_model']
self.model_path = get_recipe_config()['model_path']
self.model_name = os_splitext(os_split(self.model_path)[1])[0]
self.float_32 = get_recipe_config()["float_32"]
def download_img_and_link_to_place(place, img_url):
response_img = requests.get(img_url)
response_img.raise_for_status()
img_url_path = urlsplit(img_url)[2]
filename = os_split(img_url_path)[1]
with open(f'media/{filename}', 'wb') as f:
f.write(response_img.content)
PlaceImage.objects.filter(
place=place,
image=filename,
).get_or_create(
place=place,
image=filename,
)
def pre_process(self):
"""Pre-process the dataset."""
# First, load imposter images.
self._datasets = []
# TODO: need to refactor this somehow to allow varying subjects with caching.
# # If a training set exists, don't try to start again from scratch.
# if check_file_exists(self._output_filename):
# self._logger.info("File exists, so finish preprocessing early.")
# return
# TODO: go through preprocessing dataset into h5 file (all in one).
with h5py.File(self._output_filename, 'w') as hf:
for label in self._labels:
self._logger.info("Start looking at label %s in dataset." %
label)
label_images = []
# Go through each h5 file
glob_arg = self._filename + '/%s/**/*.hdf5' % label
print(glob_arg)
for vid_filename in glob.iglob(glob_arg, recursive=True):
print(vid_filename)
_, filename_only = os_split(vid_filename)
# Now we're going to extract the subject from the filename.
subject = get_subject_from_filename(filename_only)
# Skip any subjects TODO: rewrite this loop, because this is very bad practice.
if self.subjects is not None and subject not in self.subjects:
continue
# Now open h5py file for reading.
with h5py.File(vid_filename, "r") as mad_file:
file_images = mad_file['Color_Data']
for img in file_images[0:150:]:
# First, we need to transpose and reorder for Opencv.
img = img.transpose(2, 1, 0)
label_images.append(img)
# save to the dataset.
dataset = hf.create_dataset(label, data=label_images)
self._logger.info("Dataset created")
self._datasets.append(dataset)
self._logger.info("Dataset preprocessing completed.")
def multichoice(filenames, allow_random=False):
from random import choice
from os.path import split as os_split
if filenames is None or len(filenames) == 0:
return
elif len(filenames) >= 1:
if allow_random:
length = len(filenames) + 1
else:
length = len(filenames)
if len(filenames) == 1:
return filenames[0]
choices = [os_split(i)[1] for i in filenames]
if allow_random:
choices.insert(0, control.lang(32215))
_choice = control.selectDialog(heading=control.lang(32214), list=choices)
if _choice == 0:
if allow_random:
filename = choice(filenames)
else:
filename = filenames[0]
elif _choice != -1 and _choice <= length:
if allow_random:
filename = filenames[_choice - 1]
else:
filename = filenames[_choice]
else:
if allow_random:
filename = choice(filenames)
else:
return
return filename
else:
return
def main():
script_path = abspath(__file__)
script_dir = os_split(script_path)[0]
system('cd ' + script_dir)
dot_file_path = (script_dir + "\\" + json_extract('paths')['dot.exe'])
destination_file = json_extract('paths')['dot_file']
print(argv.__len__())
args = ""
if argv.__len__() < 2:
args = '-Tpng -o my_classes.png'
else:
for arg in argv[1:]:
args += arg
print(arg)
ar = [dot_file_path] + [destination_file] + split(args)
try:
sub_call(ar)
print('uml done')
except PermissionError:
print("you don't Hve valid permissions to create this file")
except FileNotFoundError:
print("cannot find file at " + ar)
def run(self, url, source):
log_debug('Source selected: {0}'.format(source))
path = control.join(control.dataPath, 'temp')
try:
path = path.decode('utf-8')
except Exception:
pass
control.deleteDir(control.join(path, ''), force=True)
control.makeFile(control.dataPath)
control.makeFile(path)
if control.setting('keep_subs') == 'true' or control.setting(
'keep_zips') == 'true':
if control.setting('output_folder').startswith('special://'):
output_path = control.transPath(
control.setting('output_folder'))
else:
output_path = control.setting('output_folder')
control.makeFile(output_path)
if source == 'subtitlesgr':
subtitle = subtitlesgr.Subtitlesgr().download(path, url)
elif source == 'xsubstv':
subtitle = xsubstv.Xsubstv().download(path, url)
elif source == 'podnapisi':
subtitle = podnapisi.Podnapisi().download(path, url)
elif source == 'vipsubs':
subtitle = vipsubs.Vipsubs().download(path, url)
else:
subtitle = None
if subtitle is not None:
if control.setting('keep_subs') == 'true':
# noinspection PyUnboundLocalVariable
copy(subtitle, control.join(output_path,
os_split(subtitle)[1]))
control.infoDialog(control.lang(30008))
item = control.item(label=subtitle)
control.addItem(handle=self.syshandle,
url=subtitle,
listitem=item,
isFolder=False)
control.directory(self.syshandle)
def download(self, path, url):
try:
url = re.findall(r'/(\d+)/', url + '/', re.I)[-1]
url = ''.join([self.download_link, '/getp.php?id={0}'.format(url)])
url = client.request(url,
output='geturl',
timeout=control.setting('timeout'))
req = Request(url)
req.add_header('User-Agent', randomagent())
opener = urlopen(req)
data = opener.read()
zip_file = zipfile.ZipFile(BytesIO(data))
opener.close()
files = zip_file.namelist()
files = [i for i in files if i.startswith('subs/')]
srt = [i for i in files if i.endswith(('.srt', '.sub'))]
archive = [i for i in files if i.endswith(('.rar', '.zip'))]
if len(srt) > 0:
if len(srt) > 1:
srt = multichoice(srt)
else:
srt = srt[0]
result = zip_file.open(srt).read()
subtitle = basename(srt)
try:
subtitle = control.join(path, subtitle.decode('utf-8'))
except Exception:
subtitle = control.join(path, subtitle)
with open(subtitle, 'wb') as subFile:
subFile.write(result)
return subtitle
elif len(archive) > 0:
if len(archive) > 1:
archive = multichoice(archive)
else:
archive = archive[0]
result = zip_file.open(archive).read()
f = control.join(path, os_split(url)[1])
with open(f, 'wb') as subFile:
subFile.write(result)
dirs, files = control.listDir(path)
if len(files) == 0:
return
if zipfile.is_zipfile(f):
zipped = zipfile.ZipFile(f)
zipped.extractall(path)
if not zipfile.is_zipfile(f):
if control.infoLabel('System.Platform.Windows'):
uri = "rar://{0}/".format(quote(f))
else:
uri = "rar://{0}/".format(quote_plus(f))
dirs, files = control.listDir(uri)
else:
dirs, files = control.listDir(path)
if dirs and not zipfile.is_zipfile(f):
for dir in dirs:
_dirs, _files = control.listDir(control.join(uri, dir))
[files.append(control.join(dir, i)) for i in _files]
if _dirs:
for _dir in _dirs:
_dir = control.join(_dir, dir)
__dirs, __files = control.listDir(
control.join(uri, _dir))
[
files.append(control.join(_dir, i))
for i in __files
]
filenames = [i for i in files if i.endswith(('.srt', '.sub'))]
filename = multichoice(filenames)
try:
filename = filename.decode('utf-8')
except Exception:
pass
if not control.exists(control.join(
path,
os_split(filename)[0])) and not zipfile.is_zipfile(f):
control.makeFiles(control.join(path,
os_split(filename)[0]))
subtitle = control.join(path, filename)
if not zipfile.is_zipfile(f):
with closing(control.openFile(uri + filename)) as fn:
try:
output = bytes(fn.readBytes())
except Exception:
output = bytes(fn.read())
content = output.decode('utf-16')
with closing(control.openFile(subtitle, 'w')) as subFile:
subFile.write(bytearray(content.encode('utf-8')))
fileparts = os_split(subtitle)[1].split('.')
# noinspection PyTypeChecker
result = control.join(
os_split(subtitle)[0],
'subtitles.' + fileparts[len(fileparts) - 1])
control.rename(subtitle, result)
return result
except Exception as e:
_, __, tb = sys.exc_info()
print(traceback.print_tb(tb))
log_debug(
'Subtitles.gr subtitle download failed for the following reason: '
+ str(e))
return
def download(path, url):
try:
result = client.request(url)
f = control.join(path, os_split(url)[1])
with open(f, 'wb') as subFile:
subFile.write(result)
dirs, files = control.listDir(path)
if len(files) == 0:
return
if not f.lower().endswith('.rar'):
try:
zipped = zipfile.ZipFile(f)
zipped.extractall(path)
except Exception:
control.execute('Extract("{0}","{0}")'.format(f, path))
if f.lower().endswith('.rar'):
if control.infoLabel('System.Platform.Windows'):
uri = "rar://{0}/".format(quote(f))
else:
uri = "rar://{0}/".format(quote_plus(f))
dirs, files = control.listDir(uri)
else:
dirs, files = control.listDir(path)
if dirs and f.lower().endswith('.rar'):
for dir in dirs:
_dirs, _files = control.listDir(control.join(uri, dir))
[files.append(control.join(dir, i)) for i in _files]
if _dirs:
for _dir in _dirs:
_dir = control.join(_dir, dir)
__dirs, __files = control.listDir(
control.join(uri, _dir))
[
files.append(control.join(_dir, i))
for i in __files
]
filenames = [i for i in files if i.endswith(('.srt', '.sub'))]
if len(filenames) == 1:
filename = filenames[0]
else:
filename = multichoice(filenames)
try:
filename = filename.decode('utf-8')
except Exception:
pass
if not control.exists(control.join(
path,
os_split(filename)[0])) and f.lower().endswith('.rar'):
control.makeFiles(control.join(path, os_split(filename)[0]))
subtitle = control.join(path, filename)
if f.lower().endswith('.rar'):
content = openFile(uri + filename).read()
with open(subtitle, 'wb') as subFile:
subFile.write(content)
fileparts = os_split(subtitle)[1].split('.')
result = control.join(
os_split(subtitle)[0],
'subtitles.' + fileparts[len(fileparts) - 1])
rename(subtitle, result)
return result
except Exception as e:
log.log(
'Subzxyz subtitle download failed for the following reason: ' +
str(e))
return
def run(self, url, source):
log_debug('Source selected: {0}'.format(source))
path = control.join(control.dataPath, 'temp')
try:
path = path.decode('utf-8')
except Exception:
pass
control.deleteDir(control.join(path, ''), force=True)
control.makeFile(control.dataPath)
control.makeFile(path)
if control.setting('keep_subs') == 'true' or control.setting(
'keep_zips') == 'true':
if not control.get_info_label('ListItem.Path').startswith(
'plugin://') and control.setting('destination') == '0':
output_path = control.get_info_label('Container.FolderPath')
elif control.setting('output_folder').startswith('special://'):
output_path = control.transPath(
control.setting('output_folder'))
else:
output_path = control.setting('output_folder')
if not exists(output_path):
control.makeFile(output_path)
if source == 'subtitlesgr':
subtitle = subtitlesgr.Subtitlesgr().download(path, url)
elif source == 'xsubstv':
subtitle = xsubstv.Xsubstv().download(path, url)
elif source == 'podnapisi':
subtitle = podnapisi.Podnapisi().download(path, url)
elif source == 'vipsubs':
subtitle = vipsubs.Vipsubs().download(path, url)
else:
subtitle = None
if subtitle is not None:
if control.setting('keep_subs') == 'true':
# noinspection PyUnboundLocalVariable
try:
if control.setting('destination') in ['0', '2']:
if control.infoLabel('{0}.Title'.format(
infolabel_prefix)).startswith('plugin://'):
copy(
subtitle,
control.join(output_path,
os_split(subtitle)[1]))
log_debug(
'Item currently selected is not a local file, cannot save subtitle next to it'
)
else:
output_filename = control.join(
output_path, ''.join([
splitext(
control.infoLabel('ListItem.FileName'))
[0],
splitext(os_split(subtitle)[1])[1]
]))
if exists(output_filename):
yesno = control.yesnoDialog(
control.lang(30015))
if yesno:
copy(subtitle, output_filename)
else:
copy(subtitle, output_filename)
if control.setting('destination') == '2':
if control.setting('output_folder').startswith(
'special://'):
output_path = control.transPath(
control.setting('output_folder'))
else:
output_path = control.setting(
'output_folder')
copy(
subtitle,
control.join(output_path,
os_split(subtitle)[1]))
else:
copy(subtitle,
control.join(output_path,
os_split(subtitle)[1]))
control.infoDialog(control.lang(30008))
except Exception:
control.infoDialog(control.lang(30013))
item = control.item(label=subtitle)
control.addItem(handle=self.syshandle,
url=subtitle,
listitem=item,
isFolder=False)
control.directory(self.syshandle)
from argparse import ArgumentParser
try: # python 3
from pickle import load
except ImportError: # python 2
from cPickle import load
from numpy import asarray
try:
from meta_waffle.plots import plot_waffle
except ImportError: # meta-waffle is not installed.. but it's still ok!!!
from os.path import join as os_join, split as os_split
import sys
sys.path.insert(0,
os_join(os_split(os_split(__file__)[0])[0], 'meta_waffle'))
from plots import plot_waffle
def main():
opts = get_options()
waffle_file = opts.infile
output = opts.outfile
title = opts.title
do_loop = opts.do_loop
try:
waffle = load(open(waffle_file, 'rb'))
except: # it's probably not a pickle
if count > max_deep_in_last:
break
continue
years.append(year)
# print(years)
return years
my_dir = "data_analyze"
if __name__ == '__main__':
a = globalGetChanges(2018, section="РАЗДЕЛ F СТРОИТЕЛЬСТВО")
print(a)
else:
from os import getcwd
from os.path import split as os_split, exists, join as os_join
path = getcwd()
while my_dir not in path:
if exists(os_join(path, my_dir)):
path = os_join(path, my_dir)
break
now_dir = os_split(path)[1]
path = os_split(path)[0]
print(path, now_dir)
else:
all_path, end_dir = os_split(path)
while end_dir != my_dir:
now_dir = path[1]
path = os_split(path[0])[0] if not bool(path[-1]) else path[0]
print('end', path)
from collections import defaultdict
from random import getrandbits
try: # python 3
from pickle import _Unpickler as Unpickler
except ImportError: # python 2
from pickle import Unpickler
from pytadbit.utils.file_handling import mkdir
from meta_waffle.utils import chromosome_from_bam
from meta_waffle import parse_peaks, generate_pairs
from meta_waffle import submatrix_coordinates, interactions_at_intersection
from meta_waffle.waffle_io import write_big_matrix, sort_BAMtsv
TEST_PATH = os_join(os_split(os_split(os_split(__file__)[0])[0])[0], 'test')
RESOLUTION = 10000
WINDOWS_SPAN = 4
COORD_CONV = ''
window_size = (WINDOWS_SPAN * 2) + 1
(SECTION_POS, CHROM_SIZES, BINS, PEAK_COORD1, PEAK_COORD2, ITER_PAIRS,
PAIR_PEAKS) = Unpickler(open(os_join(TEST_PATH, 'test_data.pickle'),
'rb')).load()
GROUPS = Unpickler(open(os_join(TEST_PATH, 'test_result.pickle'), 'rb')).load()
# Popen('python {}/Simulate_HiC.py'.format(TEST_PATH), shell=True).communicate()
class TestWaffle(unittest.TestCase):
"""
except ImportError:
from fractions import gcd
from subprocess import Popen
from functools import reduce
from pickle import dump
from os.path import join as os_join
from os.path import split as os_split
import numpy as np
from numpy.random import negative_binomial
from matplotlib import pyplot as plt
TEST_PATH = os_split(__file__)[0]
if not TEST_PATH:
TEST_PATH = '.'
QUICK = False
def load_genome(chroms):
sections = {}
total = 0
for c in chroms:
sections[c] = total
total += chroms[c]
bins = {}
for c in chroms:
chains_mps['send'].put(("end_work", []))
chains_mps['listen'].put(("end_work", []))
chains_mps['proc'][0].put(("end_work", []))
chains_mps['db'][0].put(("end_work", []))
print('finish ended_work')
def create_post_git_pull_file():
from os import environ, system
hostname = environ.get('HOST_NAME', "")
wsgi_module = environ.get('WSGI_MODULE', None)
# print(find_parh_to_dit('hooks'))
# print(2, hostname, "pythonanywhere" in str(hostname), wsgi_module)
if "pythonanywhere" in str(hostname) and wsgi_module:
git_path = find_parh_to_dit('.git')
print(git_path)
with open(join(git_path, "hooks", "post-merge.sample"), "w", encoding="utf-8") as file:
run_file_path = f"""/var/www/{wsgi_module}.py"""
text = [
"!/bin/sh",
f"if [[ -f {join(getcwd(), FILE_COUNTER_NAME)} ]]",
f"then\nrm {join(getcwd(), FILE_COUNTER_NAME)}\nfi",
# "killall uwsgi\n",
"touch {run_file_path}"
]
print("\n".join(text), file=file)
system(f"chmod +x {run_file_path}")
HOME_DIR = os_split(os_split(getsourcefile(callback_func))[0])[0]
from os.path import dirname, abspath, join, exists
import sys
from configparser import ConfigParser
if __name__ == '__main__':
from os import getcwd, chdir
from os.path import split as os_split
path = os_split(getcwd())
path = os_split(path[0])[0] if not bool(path[-1]) else path[0]
chdir(path)
base_path = dirname(abspath(__file__))
# dirname(dirname(__file__))
file_name = "settings.ini"
example_settings_filename = f"{base_path}/example_{file_name}"
config_path = join(base_path, file_name)
if not exists(example_settings_filename):
print(f"Config file ({config_path}) not found! Exiting!")
sys.exit(0)
if not exists(config_path):
example_cfg = ConfigParser(
allow_no_value=True,
converters={'list': lambda x: [i.strip() for i in x.split(',')]})
example_cfg.read(example_settings_filename)
user_input_tag = example_cfg.get("settings_ini_file", "user_input_tag")
print("Config file not found!")
print(f"I am trying to create {config_path}...")
print(
f"I am coping {example_settings_filename} and rename this to {config_path}"
)
def download(self, path, url):
# try:
url = re.findall('/(\d+)/', url + '/', re.I)[-1]
url = 'http://www.greeksubtitles.info/getp.php?id={0}'.format(url)
url = client.request(url, output='geturl')
data = urlopen(url, timeout=20).read()
zip_file = zipfile.ZipFile(StringIO(data))
files = zip_file.namelist()
files = [i for i in files if i.startswith('subs/')]
srt = [i for i in files if i.endswith(('.srt', '.sub'))]
archive = [i for i in files if i.endswith(('.rar', '.zip'))]
if len(srt) > 0:
if len(srt) > 1:
srt = multichoice(srt)
else:
srt = srt[0]
result = zip_file.open(srt).read()
subtitle = basename(srt)
try:
subtitle = control.join(path, subtitle.decode('utf-8'))
except Exception:
subtitle = control.join(path, subtitle)
with open(subtitle, 'wb') as subFile:
subFile.write(result)
return subtitle
elif len(archive) > 0:
if len(archive) > 1:
archive = multichoice(archive)
else:
archive = archive[0]
result = zip_file.open(archive).read()
f = control.join(path, os_split(url)[1])
with open(f, 'wb') as subFile:
subFile.write(result)
dirs, files = control.listDir(path)
if len(files) == 0:
return
if zipfile.is_zipfile(f):
# try:
# zipped = zipfile.ZipFile(f)
# zipped.extractall(path)
# except Exception:
control.execute('Extract("{0}","{0}")'.format(f, path))
if not zipfile.is_zipfile(f):
if control.infoLabel('System.Platform.Windows'):
uri = "rar://{0}/".format(quote(f))
else:
uri = "rar://{0}/".format(quote_plus(f))
dirs, files = control.listDir(uri)
else:
dirs, files = control.listDir(path)
if dirs and not zipfile.is_zipfile(f):
for dir in dirs:
_dirs, _files = control.listDir(control.join(uri, dir))
[files.append(control.join(dir, i)) for i in _files]
if _dirs:
for _dir in _dirs:
_dir = control.join(_dir, dir)
__dirs, __files = control.listDir(
control.join(uri, _dir))
[
files.append(control.join(_dir, i))
for i in __files
]
filenames = [i for i in files if i.endswith(('.srt', '.sub'))]
filename = multichoice(filenames)
try:
filename = filename.decode('utf-8')
except Exception:
pass
if not control.exists(control.join(
path,
os_split(filename)[0])) and not zipfile.is_zipfile(f):
control.makeFiles(control.join(path, os_split(filename)[0]))
subtitle = control.join(path, filename)
if not zipfile.is_zipfile(f):
content = openFile(uri + filename).read()
with open(subtitle, 'wb') as subFile:
subFile.write(content)
fileparts = os_split(subtitle)[1].split('.')
result = control.join(
os_split(subtitle)[0],
'subtitles.' + fileparts[len(fileparts) - 1])
rename(subtitle, result)
return result
