def backup_db(ctx, dbname="zam"):
create_directory(ctx, "/var/backups/zam", owner="postgres")
backup_filename = f"/var/backups/zam/postgres-dump-{timestamp()}.sql"
run_as_postgres(
ctx,
f"pg_dump --dbname={dbname} --create --encoding=UTF8 --file={backup_filename}",
)
def __init__(self):
self.lfsbuilder_src_directory = os.path.dirname(
os.path.realpath(__file__))
self.temporal_folder = os.path.join(self.lfsbuilder_src_directory,
"tmp")
self.basic_parser = None
self.build_parser = None
self.download_parser = None
self.xml_parser = None
self.cli = cli.Cli()
self.actual_owner_uid = pwd.getpwuid(os.stat(__file__).st_uid).pw_name
self.actual_owner_gid = grp.getgrgid(os.stat(__file__).st_gid).gr_name
# Parse sys.args and use dispatcher pattern to
# invoke method named as command
self.basic_parser = self.cli.configure_basic_parser()
self.all_args = self.basic_parser.parse_args()
self.build_args = None
self.download_args = None
self.xml_args = None
# Set boolean configuration flags
self.set_config_option(self.all_args)
if not hasattr(self, self.all_args.command[0]):
printer.warning(
"Unknown command '{c}'".format(c=self.all_args.command[0]))
self.basic_parser.print_help()
sys.exit(1)
# .- Create 'tmp' directory
tools.create_directory(self.temporal_folder)
# Run command
getattr(self, self.all_args.command[0])()
def copy_to_temp_directory(path_folder: str) -> str:
"""copy directory to OS temp directory
Arguments:
path_folder {str} -- directory path
Returns:
str -- temp directory path
"""
random_number = gen_random_bit_number(256)
# generate hash for temp folder
hash_folder = get_hash(random_number)
# get os temp directory
temp_dir = tools.get_temp_directory()
# create temp directory
temp_dir = os.path.join(temp_dir, hash_folder)
try:
# create directory
tools.create_directory(temp_dir)
# copy directory to OS temp directory
tools.copy_directory(path_folder, temp_dir)
except IOError as error:
print(error)
return False
return temp_dir
def copy_to_organized_directory(dir_src: str, dir_dst: str) -> bool:
"""copy source directory to organized directory
Get content from source directory and copy to a new directory
into new folders with the extentions name
Arguments:
dir_src {str} -- source directory
dir_dst {str} -- destination directory
Raises:
IOError: path dir_src not found
IOError: path dir_dst not found
Returns:
bool -- if successful return True
"""
if not checks.check_exists(dir_src):
raise IOError("Directory '{0}' not found.".format(dir_src))
if not checks.check_exists(dir_dst):
raise IOError("Directory '{0}' not found.".format(dir_dst))
# copy files temp to new directory
list_files = get_files(dir_src)
for file in list_files:
basename = os.path.basename(file)
search = re.search(r"\.([A-Za-z0-9]+)$", basename)
if search:
extension = search.group(1)
# check = [ext for ext in extensions if search.group(1) == ext]
directory_extension = os.path.join(dir_dst, extension)
if not checks.check_exists(directory_extension):
tools.create_directory(directory_extension)
# build target directory
target_directory = os.path.join(directory_extension, basename)
# copy file
tools.copy_file(file, target_directory)
return True
def build(self):
"""
'build' command.
"""
# 'build' command. It requires 'sudo' privileges to mount/umount directories
# and run admin commands such as 'chroot', 'loosetup' and others.
if os.getuid() != 0:
msg = "'build' command requires root privileges. Please try again using 'sudo'"
printer.error(msg)
# Parse command line arguments
self.build_parser = self.cli.configure_build_parser()
self.build_args = self.build_parser.parse_args(
self.all_args.command[1:])
# Set boolean configuration flags once arguments get actually parsed.
self.set_config_option(self.build_args)
# .- Check boot manager and meson builder combination
if tools.is_element_present(self.build_args.builders_list, "system") is True and \
tools.check_meson_builder_combination(m=config.INCLUDE_MESON_BUILDER,
sv=config.SYSV,
sd=config.SYSTEMD) is False:
printer.error(
"You can not use that combination of 'boot_manager' and 'meson builder'"
)
# Create 'config.BASE_DIRECTORY' if necessary
tools.create_directory(config.BASE_DIRECTORY)
# Create and build 'builders_list'
for builder in self.build_args.builders_list:
os.chdir(self.lfsbuilder_src_directory)
# Generate builder object from BuilderGenerator
bg = builders.BuilderGenerator(builder)
o = bg.get_builder_reference()
del bg
# Run the real builder
o.set_attributes()
o.build()
o.clean_workspace()
del o
# Set 'lfsbuilder_src_directory' permission back to the original 'uid' and 'gid'
tools.set_recursive_owner_and_group(self.lfsbuilder_src_directory,
self.actual_owner_uid,
self.actual_owner_gid)
def ProcessHandler(stg):
# setting up the path and grabbing the files
analysis_path = tools.create_directory(stg['DP'])
task = tools.Multiprocesser(data_dir=stg['DP'], pattern_a='frame*.jpg')
# finding background and reflections in the images
if stg['BR'] == 'on':
print('preprocessing:')
bg_a, bg_b = task.find_background(50, 10, 6)
reflection = tools.mark_reflection(
150, task.files_a, os.path.join(stg['DP'], 'reflection.tif'))
else:
bg_a, bg_b, reflection = None, None, None
# start processing data
print('main process:\nprocessing images...')
task.n_files = 4
main_process = partial(ProcessPIV,
bga=bg_a,
bgb=bg_b,
reflection=reflection,
stg=stg)
task.run(func=main_process, n_cpus=6)
print('- done processing')
'''
fig, ax = plt.subplots(2,2)
img = tools.imread(task.files_b[0])
bg = bg_b
ax[0,0].imshow(img, cmap='gray')
ax[0,1].imshow(bg, cmap='gray')
ax[1,0].imshow(reflection, cmap='gray')
img = img - bg
img[reflection==255] = 0
ax[1,1].imshow(img, cmap='gray')
plt.show()
img = tools.imread(task.files_b[0])
plt.imshow(img, cmap='gray')
plt.show()
'''
return bg_a
def deploy_repondeur(
ctx,
branch="master",
message="",
session_secret="",
auth_secret="",
wipe=False,
dbname="zam",
dbuser="******",
dbpassword="******",
):
if not session_secret:
session_secret = retrieve_secret_from_config(ctx, "session_secret")
if not session_secret:
session_secret = uuid4()
print(f"Initializing session_secret to {session_secret}")
if not auth_secret:
auth_secret = retrieve_secret_from_config(ctx, "auth_secret")
if not auth_secret:
auth_secret = uuid4()
print(f"Initializing auth_secret to {auth_secret}")
hostname = ctx.run("hostname").stdout.strip()
environment = hostname.split(".", 1)[0]
menu_badge_label = environment[4:] if environment.startswith("zam-") else ""
menu_badge_color = BADGE_COLORS.get(menu_badge_label, "#999999")
deploy_id = rollbar_deploy_start(
ctx, branch, environment, comment=f"[{branch}] {message}"
)
try:
install_locale(ctx, "fr_FR.utf8")
create_user(ctx, name=user, home_dir="/srv/repondeur")
clone_repo(
ctx,
repo="https://github.com/betagouv/zam.git",
branch=branch,
path="/srv/repondeur/src",
user=user,
)
# Stop workers (if running) to free up some system resources during deployment
stop_worker_service(ctx, warn=True)
create_virtualenv(ctx, venv_dir=venv_dir, user=user)
install_requirements(ctx, app_dir=app_dir, venv_dir=venv_dir, user=user)
create_directory(ctx, "/var/cache/zam/http", owner=user)
setup_config(
ctx,
app_dir=app_dir,
user=user,
context={
"db_url": f"postgres://{dbuser}:{dbpassword}@localhost:5432/{dbname}",
"environment": environment,
"branch": branch,
"session_secret": session_secret,
"auth_secret": auth_secret,
"rollbar_token": ctx.config["rollbar_token"],
"menu_badge_label": menu_badge_label,
"menu_badge_color": menu_badge_color,
},
)
# Also stop webapp (if running) to release any locks on the DB
stop_webapp_service(ctx, warn=True)
if wipe:
wipe_db(ctx, dbname=dbname)
setup_db(ctx, dbname=dbname, dbuser=dbuser, dbpassword=dbpassword)
migrate_db(ctx, app_dir=app_dir, venv_dir=venv_dir, user=user)
# Initialize email whitelist
if not whitelist_list(ctx):
whitelist_add(
ctx,
pattern=DEFAULT_EMAIL_WHITELIST_PATTERN,
comment="Default allowed email pattern",
)
setup_webapp_service(ctx)
setup_worker_service(ctx)
reset_data_locks(ctx, app_dir=app_dir, venv_dir=venv_dir, user=user)
# Load data into Redis cache
load_data(ctx, app_dir=app_dir, venv_dir=venv_dir, user=user)
# Update dossiers
update_dossiers(ctx)
# Start webapp and workers again
start_webapp_service(ctx)
start_worker_service(ctx)
except Exception as exc:
rollbar_deploy_update(ctx.config["rollbar_token"], deploy_id, status="failed")
else:
rollbar_deploy_update(
ctx.config["rollbar_token"], deploy_id, status="succeeded"
)
import tools
import string
from libs import cloudflare
from tm_libs import dom_parser
import cookielib
import json
from StringIO import StringIO
import gzip
import main_scrape
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/NineMovies")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'NineMovies/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
timeout = int(kodi.get_setting('scraper_timeout'))
def __enum(**enums):
return type('Enum', (), enums)
MAX_RESPONSE = 1024 * 1024 * 2
from tm_libs import dom_parser
from libs import log_utils
import tools
from libs import cloudflare
from libs import log_utils
from tm_libs import dom_parser
import cookielib
from StringIO import StringIO
import gzip
import main_scrape
import base64
addon_id = kodi.addon_id
timeout = int(kodi.get_setting('scraper_timeout'))
tools.create_directory(tools.AOPATH, "All_Cookies/Putlocker")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'Putlocker/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
def __enum(**enums):
return type('Enum', (), enums)
MAX_RESPONSE = 1024 * 1024 * 2
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low',
def extract_source_code(self):
"""
Find and extract source tarball for the component.
"""
# We look for a tar file
pattern = "{n}*.tar.*".format(n=self.component_data_dict["package_name"])
# Use 'package_version' in pattern if it is not None
if self.component_data_dict["version"] is not None:
pattern = "{n}*{v}*.tar.*".format(n=self.component_data_dict["package_name"],
v=self.component_data_dict["version"])
source_code_filename = tools.find_file(self.component_data_dict["sources_directory"],
pattern)
# Try a second run if 'source_code_filename' is None using only name as pattern.
if source_code_filename is None:
pattern = "{n}*.tar.*".format(n=self.component_data_dict["package_name"])
source_code_filename = tools.find_file(self.component_data_dict["sources_directory"],
pattern)
# Try to find a zip file in case the tar file was not found
if source_code_filename is None:
pattern = "{n}*.zip*".format(n=self.component_data_dict["package_name"])
# Use 'package_version' in pattern if it is not None
if self.component_data_dict["version"] is not None:
pattern = "{n}*{v}*.zip*".format(n=self.component_data_dict["package_name"],
v=self.component_data_dict["version"])
source_code_filename = tools.find_file(self.component_data_dict["sources_directory"],
pattern)
# Try a second run if 'source_code_filename' is None using only name as pattern.
if source_code_filename is None:
pattern = "{n}*.zip*".format(n=self.component_data_dict["package_name"])
source_code_filename = tools.find_file(
self.component_data_dict["sources_directory"],
pattern
)
# Give error if None
if source_code_filename is None:
msg = "Can't find source code file for '{n}' with pattern: '{p}'"
msg = msg.format(n=self.component_data_dict["name"], p=pattern)
printer.error(msg)
# Extract
tools.extract(source_code_filename)
# We get the name of the extracted directory
pattern = "{n}*".format(n=self.component_data_dict["package_name"])
# Use 'package_version' in pattern if it is not None
if self.component_data_dict["version"] is not None:
pattern = "{n}*{v}*".format(n=self.component_data_dict["package_name"],
v=self.component_data_dict["version"])
# Find directory using pattern
self.component_data_dict["extracted_directory"] = tools.find_directory(
self.component_data_dict["sources_directory"],
pattern)
# Try a second run if 'extracted_directory' is None using only name as pattern.
# If found, get the realpath
if self.component_data_dict["extracted_directory"] is None:
pattern = "{n}*".format(n=self.component_data_dict["package_name"])
self.component_data_dict["extracted_directory"] = tools.find_directory(
self.component_data_dict["sources_directory"],
pattern
)
else:
self.component_data_dict["extracted_directory"] = os.path.realpath(
self.component_data_dict["extracted_directory"]
)
# Fail if not found 'extracted_directroy'
if self.component_data_dict["extracted_directory"] is None:
msg = "Can't find extracted directory for '{n}' with pattern: '{p}'"
msg = msg.format(n=self.component_data_dict["name"], p=pattern)
printer.error(msg)
# Generate build_dir if necessary.
if self.component_data_dict["require_build_dir"] is True:
# Generate and save 'build_directory' path
value = os.path.realpath(
os.path.join(
self.component_data_dict["extracted_directory"],
self.build_directory_name
)
)
tools.add_to_dictionary(self.component_data_dict,
"build_directory_path",
value,
concat=False)
# Create directory
tools.create_directory(self.component_data_dict["build_directory_path"])
else:
# If not, build component into the extracted directory
tools.add_to_dictionary(self.component_data_dict,
"build_directory_path",
self.component_data_dict["extracted_directory"],
concat=False)
# Set directory owner if we are building the 'toolchain'
if self.component_data_dict["builder_name"] == "toolchain":
tools.set_recursive_owner_and_group(self.component_data_dict["extracted_directory"],
self.component_data_dict["run_as_username"])
import tools
from libs import cloudflare
from libs import log_utils
from tm_libs import dom_parser
import cookielib
from StringIO import StringIO
import gzip
import main_scrape
import base64
addon_id = kodi.addon_id
timeout = int(kodi.get_setting('scraper_timeout'))
tools.create_directory(tools.AOPATH, "All_Cookies/Putlocker")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','Putlocker/'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
def __enum(**enums):
return type('Enum', (), enums)
MAX_RESPONSE = 1024 * 1024 * 2
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low', MEDIUM='Medium', HIGH='High', HD720='HD720', HD1080='HD1080')
import tools
from libs import kodi
from tm_libs import dom_parser
from libs.trans_utils import i18n
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/MerDb")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'MerDb/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
base_url = kodi.get_setting('merdb_base_url')
def LogNotify(title, message, times, icon):
xbmc.executebuiltin("XBMC.Notification(" + title + "," + message + "," +
times + "," + icon + ")")
def run(path_folder, create_copy=True):
"""main script run
Arguments:
path_folder {str} -- folder to organize
Keyword Arguments:
create_copy {bool} -- create a folder copy (default: {True})
Raises:
IOError: if folder doesn't exists
"""
print(f"\n[Junkfile] - Parameters: ")
print(f"[Junkfile] - path_folder: {path_folder}")
print(f"[Junkfile] - create_copy: {create_copy}\n")
print("[Junkfile] - executing...")
# check if path exits
if not checks.check_exists(path_folder):
raise Exception("Please choose a directory.")
# check if is not inthe blacklist directory
if path_folder in get_blacklist_directories():
raise Exception("Directory may not be selected. Try another directory")
# remove last slash
path_folder = sanitize_path(path=path_folder)
'''directory contextmanager
check if path has permission to change, set as current directory and
change to old current directory at the end
'''
with current_directory_handler(path_folder):
# copy folder to to OS temp directory
temp_dir = copy_to_temp_directory(path_folder=path_folder)
# make path
new_directory = build_directory_path_with_date(path_folder=path_folder)
# create a new folder
if not tools.create_directory(new_directory):
raise Exception(f"new_directory could not be created. new_directory: '{new_directory}'")
# copy to new folder
copy_to_organized_directory(dir_src=temp_dir, dir_dst=new_directory)
# delete folders
if checks.check_exists(temp_dir):
tools.delete_directory(temp_dir)
# check if is a copy
if not create_copy:
tools.delete_directory(path_folder)
tools.move_directory(new_directory, path_folder)
if create_copy:
print(f"[Junkfile] - Was created new directory: {new_directory}")
print("[Junkfile] - executed!")
return True
default=False,
action='store_true')
args = parser.parse_args()
debug.info("***********************************")
# ---------------------------------------------
# -- check input
# ---------------------------------------------
if args.verbose == True:
debug.set_level(6);
if args.input == None \
or args.input == "" :
debug.error("must set an input directory")
tools.create_directory(args.output)
debug.info("==================================================================================================")
debug.info("== Preprocess corpus data: " + args.input + " to " + args.output)
debug.info("==================================================================================================")
debug.info("Get list of corpus files:")
audio_corpus_element = tools.get_list_of_file_in_path(args.input, ["*.json"], recursive=True)
debug.info("Corpus count " + str(len(audio_corpus_element)) + " element(s)")
elem_id = 0
for elem in audio_corpus_element:
debug.info("---------------------------[ " + str(elem_id) + " / " + str(len(audio_corpus_element)) + " ]---------------------------------------")
elem_id += 1
debug.info("Element: " + elem)
with open(elem) as file:
toolsDirectory = cosmoDir + 'tools/'
sys.path.extend( [ toolsDirectory, modulesDir ] )
# from load_data_enzo_old import load_snapshot_enzo, load_snapshot_enzo_yt
from generate_ics_particles_functions import *
from generate_ics_grid_functions import *
from domain_decomposition import get_domain_block, get_domain_parent
from tools import create_directory
# dataDir = '/home/bruno/Desktop/data/'
# dataDir = '/raid/bruno/data/'
# dataDir = '/gpfs/alpine/proj-shared/ast149/'
dataDir = '/data/groups/comp-astro/bruno/'
enzoDir = dataDir + 'cosmo_sims/enzo/512_hydro_50Mpc/'
inDir = enzoDir
outputDir = dataDir + 'cosmo_sims/512_hydro_50Mpc/ics_16/'
create_directory( outputDir )
nSnap = 0
# Load Enzo File
snapKey = '{0:03}'.format(nSnap)
inFileName = 'DD0{0}/data0{0}'.format( snapKey)
ds = yt.load( inDir + inFileName )
data = ds.all_data()
h = ds.hubble_constant
current_z = np.float(ds.current_redshift)
current_a = 1./(current_z + 1)
# Set Domain parameters
Lbox = 50000.
def run_post_steps(component_data_dict, parent_function):
parent_function()
# Get required paths
ssh_filename = os.path.join(
component_data_dict["lfsbuilder_src_directory"], "recipes",
"components", "openssh", "files",
component_data_dict["openssh_public_key_filename"])
# .- get $HOME directory path
if component_data_dict["openssh_username"] == "root":
# It can be dangerous!
printer.warning("WARNING: will configure SSH access for 'root'")
home_directory = os.path.join(config.BASE_DIRECTORY, "root")
elif component_data_dict[
"openssh_username"] == "config.NON_PRIVILEGED_USERNAME":
# Update dictionary value
tools.add_to_dictionary(component_data_dict,
"openssh_username",
config.NON_PRIVILEGED_USERNAME,
concat=False)
# Set home directory
home_directory = os.path.join(config.BASE_DIRECTORY, "home",
config.NON_PRIVILEGED_USERNAME)
else:
home_directory = os.path.join(config.BASE_DIRECTORY, "home",
component_data_dict["openssh_username"])
# .- '$HOME/.ssh' path
ssh_config_path = os.path.join(home_directory, ".ssh")
# .- destination file path
ssh_destination_filename = os.path.join(
ssh_config_path, component_data_dict["openssh_public_key_filename"])
# .- authorized_keys
authorized_keys = os.path.join(ssh_config_path, "authorized_keys")
if os.path.exists(ssh_filename) is False:
# Do not configure. SSH public key do not exists.
msg = """WARNING: SSH access will not be configured because \
the provided public key file '{k}' do not exists."""
msg = msg.format(k=component_data_dict["openssh_public_key_filename"])
printer.warning(msg)
elif tools.check_chroot_user_exists(
component_data_dict["openssh_username"]) is False:
# Do not configure. SSH username do not exists.
msg = """WARNING: SSH access will not be configured because \
the provided username '{u}' do not exists."""
msg = msg.format(u=component_data_dict["openssh_username"])
printer.warning(msg)
elif os.path.exists(home_directory) is False:
# Do not configure. SSH username's home directory do not exists.
msg = """WARNING: SSH access will not be configured because \
the home directory '{h}' do not exists."""
msg = msg.format(h=home_directory)
printer.warning(msg)
else:
msg = "Installing provided SSH public key '{k}' for username '{u}'"
msg = msg.format(k=component_data_dict["openssh_public_key_filename"],
u=component_data_dict["openssh_username"])
printer.substep_info(msg)
# .- create 'ssh_config_path' directory
tools.create_directory(ssh_config_path)
# .- copy public key file
tools.copy_file(ssh_filename, ssh_destination_filename)
# .- add to authorized keys
header = "# --- {f} ---".format(
f=component_data_dict["openssh_public_key_filename"])
tools.add_text_to_file(authorized_keys, header)
tools.add_text_to_file(authorized_keys, tools.read_file(ssh_filename))
# .- get 'UID' and 'GID' values to set permission
etc_passwd_values = tools.get_uid_gid_chroot_username(
component_data_dict["openssh_username"])
# .- set 'ssh_config_path' permission
tools.set_numeric_recursive_owner_and_group(ssh_config_path,
etc_passwd_values["uid"],
etc_passwd_values["gid"])
def execute_pipeline(parameters):
# Retrieve workspace
tools.create_directory(parameters["workspace"])
# Merge files
parameters["merge"]["merged_file"] = os.path.join(parameters["workspace"], parameters["merge"]["merged_file"])
if parameters["merge"]["do"]:
command = MERGING_COMMAND%parameters["merge"]
print "Issuing >> ", command
os.system(command)
if parameters["merge"]["delete_parts"]:
os.system("rm %s*"%(os.path.join(parameters["merge"]["working_directory"], parameters["merge"]["trajectory_prefix"])))
if parameters["merge"]["compress_logs"]:
os.system("tar -zcvf %(folder)s/logs.tar.gz %(folder)s/log_* --remove-files"%({"folder":os.path.join(parameters["merge"]["working_directory"])}))
# Perform all analyses
if parameters["analyze"]["do"]:
args = ["-i", parameters["merge"]["merged_file"]]
if "sasa_rgyr" in parameters["analyze"]:
if parameters["analyze"]["sasa_rgyr"]["sasa"]:
args.append("--sasa-vmd")
if parameters["analyze"]["sasa_rgyr"]["rgyr"]:
args.append("--rgyr-vmd")
if parameters["analyze"]["sasa_rgyr"]["selection"] != "":
args.append("--vmd-sel")
args.append('"%s"'%parameters["analyze"]["sasa_rgyr"]["selection"])
if parameters["analyze"]["rmsf"]:
args.append("--rmsf")
if "acceptance" in parameters["analyze"]:
args.append("--report")
args.append(parameters["analyze"]["acceptance"]["report_glob"])
args.append("--report-dir")
args.append(parameters["analyze"]["acceptance"]["working_directory"])
command = "python %s %s"%(parameters["analyze"]["script_location"],
" ".join(args))
print "Issuing >> ", command
os.system(command)
if parameters["compare"]["do"]:
try:
selection = parameters["analyze"]["sasa_rgyr"]["selection"]
except:
selection = "all"
base_path_for_sasa_rgyr = "%s.%s"%(parameters["merge"]["merged_file"],
selection.replace(" ","_"))
if "sasa" in parameters["compare"]:
parameters["compare"]["sasa"]["input"] = "%s.sasa"%(base_path_for_sasa_rgyr)
command = "python %(script_location)s %(reference)s %(input)s save"%(parameters["compare"]["sasa"])
print "Issuing >> ", command
os.system(command)
if "rgyr" in parameters["compare"]:
parameters["compare"]["rgyr"]["input"] = "%s.rgyr"%(base_path_for_sasa_rgyr)
command = "python %(script_location)s %(reference)s %(input)s save"%(parameters["compare"]["rgyr"])
print "Issuing >> ", command
os.system(command)
if "rmsf" in parameters["compare"]:
parameters["compare"]["rmsf"]["input"] = "%s.rmsf"%(parameters["merge"]["merged_file"])
command = "python %(script_location)s %(reference)s %(input)s save"%(parameters["compare"]["rmsf"])
print "Issuing >> ", command
os.system(command)
if "energy" in parameters["compare"]:
parameters["compare"]["energy"]["input"] = "%s.ener"%(parameters["merge"]["merged_file"])
command = "python %(script_location)s %(input)s %(skip)d"%(parameters["compare"]["energy"])
print "Issuing >> ", command
os.system(command)
from libs import kodi
from tm_libs import dom_parser
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import tools
import string
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/Afdah")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'Afdah/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
def __enum(**enums):
return type('Enum', (), enums)
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low',
MEDIUM='Medium',
from tm_libs import dom_parser
from libs.trans_utils import i18n
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import urlresolver
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id=kodi.addon_id
addon = Addon(addon_id, sys.argv)
ADDON = xbmcaddon.Addon(id=kodi.addon_id)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/PrimeWire")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','PrimeWire/'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
base_url = kodi.get_setting('primewire_base_url')
#base_url = 'http://www.primewire.ag/'
def LogNotify(title,message,times,icon):
xbmc.executebuiltin("XBMC.Notification("+title+","+message+","+times+","+icon+")")
def OPEN_URL(url):
req=urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Linux; U; Android 4.2.2; en-us; AFTB Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30')
def Network_config(class_num=4,
epoch=200,
initial_epoch=0,
batch_size=32,
train_data=None,
train_label=None,
test_data=None,
test_label=None,
fold=0):
adam = Adam(lr=0.005, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.000)
sgd = SGD(lr=0.001, momentum=0.9, decay=0.0, nesterov=False)
K.set_learning_phase(1)
base_model = VGG16(input_tensor=Input(shape=(224, 224, 3)),
weights='imagenet',
include_top=False)
x = base_model.output
x = Flatten()(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
predictions = Dense(class_num, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
for layer in (base_model.layers):
layer.trainable = False
if layer.name.startswith('bn'):
layer.call(layer.input, training=False)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=[keras.metrics.categorical_accuracy])
tools.create_directory('./tmpvgg/')
weights_file = './tmpvgg/' + str(
fold
) + '-weights.{epoch:02d}-{categorical_accuracy:.4f}-{val_loss:.4f}-{val_categorical_accuracy:.4f}.h5'
csv_file = './tmpvgg/record.csv'
lr_reducer = ReduceLROnPlateau(monitor='categorical_accuracy',
factor=0.5,
cooldown=0,
patience=5,
min_lr=0.5e-6)
early_stopper = EarlyStopping(monitor='val_categorical_accuracy',
min_delta=1e-4,
patience=50)
model_checkpoint = ModelCheckpoint(weights_file,
monitor='val_categorical_accuracy',
save_best_only=True,
verbose=2,
save_weights_only=True,
mode='max')
tensorboard = TensorBoard(log_dir='./logs/',
histogram_freq=0,
batch_size=8,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
CSV_record = CSVLogger(csv_file, separator=',', append=True)
callbacks = [
lr_reducer, early_stopper, model_checkpoint, tensorboard, CSV_record
]
gc.disable()
model.fit_generator(
generator=tools.batch_generator(np.array(train_data),
np.array(train_label), batch_size,
True, class_num),
steps_per_epoch=int(len(train_label) / batch_size) - 1,
max_q_size=50,
initial_epoch=initial_epoch,
epochs=epoch,
verbose=1,
callbacks=callbacks,
validation_data=tools.batch_generator(np.array(test_data),
np.array(test_label), batch_size,
True, class_num),
validation_steps=int(len(test_label) / batch_size) - 1,
class_weight='auto')
all_y_pred = []
all_y_true = []
for test_data_batch, test_label_batch in tools.batch_generator_confusion_matrix(
np.array(test_data), np.array(test_label), batch_size, True,
class_num):
y_pred = model.predict(test_data_batch, batch_size)
y_true = test_label_batch
for y_p in y_pred:
all_y_pred.append(np.where(y_p == max(y_p))[0][0])
for y_t in y_true:
all_y_true.append(np.where(y_t == max(y_t))[0][0])
confusion = confusion_matrix(y_true=all_y_true, y_pred=all_y_pred)
print(confusion)
f = open('confusion_matrix.txt', 'a+')
f.write(str(all_y_true) + "\n")
f.write(str(all_y_pred) + "\n")
f.write(str(confusion) + '\n')
f.close()
gc.enable()
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
n_procs = comm.Get_size()
else:
rank = 0
n_procs = 1
nPoints = 1024
dataDir = '/data/groups/comp-astro/bruno/'
inDir = dataDir + 'cosmo_sims/{0}_hydro_50Mpc/output_files_pchw18/'.format(
nPoints)
stats_dir = inDir + 'statistics/'
outDir = dataDir + 'cosmo_sims/{0}_hydro_50Mpc/snapshots_prepared/'.format(
nPoints)
if rank == 0: create_directory(outDir)
data_type = 'hydro'
# data_type = 'particles'
# Load Statistics
statistics = h5.File(stats_dir + 'stats_{0}.h5'.format(data_type), 'r')
fields = ['density']
precision = np.float32
Lbox = 5000 #kpc/h
if nPoints == 1024: proc_grid = [4, 2, 2]
if nPoints == 2048: proc_grid = [8, 8, 8]
box_size = [Lbox, Lbox, Lbox]
grid_size = [nPoints, nPoints, nPoints] #Size of the simulation grid
import tools
from libs import kodi
from tm_libs import dom_parser
from libs.trans_utils import i18n
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/IWatchOnline")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'IWatchOnline'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
base_url = 'http://www.merdb.link/'
def LogNotify(title, message, times, icon):
xbmc.executebuiltin("XBMC.Notification(" + title + "," + message + "," +
times + "," + icon + ")")
import tools
from libs import kodi
from tm_libs import dom_parser
from libs.trans_utils import i18n
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id=kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/MerDb")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','MerDb/'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
base_url = kodi.get_setting('merdb_base_url')
def LogNotify(title,message,times,icon):
xbmc.executebuiltin("XBMC.Notification("+title+","+message+","+times+","+icon+")")
def OPEN_URL(url):
req=urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Linux; U; Android 4.2.2; en-us; AFTB Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30')
response=urllib2.urlopen(req)
link=response.read()
from tm_libs import dom_parser
from libs.trans_utils import i18n
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import urlresolver
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
ADDON = xbmcaddon.Addon(id=kodi.addon_id)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/PrimeWire")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'PrimeWire/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
base_url = kodi.get_setting('primewire_base_url')
#base_url = 'http://www.primewire.ag/'
def LogNotify(title, message, times, icon):
xbmc.executebuiltin("XBMC.Notification(" + title + "," + message + "," +
times + "," + icon + ")")
addon_id = kodi.addon_id
timeout = int(kodi.get_setting('scraper_timeout'))
def __enum(**enums):
return type('Enum', (), enums)
VIDEO_TYPES = __enum(TVSHOW='TV Show',
MOVIE='Movie',
EPISODE='Episode',
SEASON='Season')
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/IceFilms")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'IceFilms/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
MAX_RESPONSE = 1024 * 1024 * 2
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low',
MEDIUM='Medium',
HIGH='High',
HD720='HD720',
HD1080='HD1080')
import HTMLParser
addon_id = kodi.addon_id
timeout = int(kodi.get_setting('scraper_timeout'))
def __enum(**enums):
return type('Enum', (), enums)
VIDEO_TYPES = __enum(TVSHOW='TV Show', MOVIE='Movie', EPISODE='Episode', SEASON='Season')
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/IceFilms")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','IceFilms/'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
MAX_RESPONSE = 1024 * 1024 * 2
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low', MEDIUM='Medium', HIGH='High', HD720='HD720', HD1080='HD1080')
XHR = {'X-Requested-With': 'XMLHttpRequest'}
USER_AGENT = "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
BR_VERS = [
from tm_libs import dom_parser
from libs import log_utils
import tools
from libs import cloudflare
from libs import log_utils
from tm_libs import dom_parser
import cookielib
from StringIO import StringIO
import gzip
import main_scrape
import base64
addon_id = kodi.addon_id
timeout = int(kodi.get_setting('scraper_timeout'))
tools.create_directory(tools.AOPATH, "All_Cookies/SantaSeries")
cookiepath = xbmc.translatePath(
os.path.join('special://home', 'addons', addon_id, 'All_Cookies',
'SantaSeries/'))
cookiejar = os.path.join(cookiepath, 'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath, 'cookies.lwp')
def __enum(**enums):
return type('Enum', (), enums)
MAX_RESPONSE = 1024 * 1024 * 2
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low',
def migrate_db(ctx, app_dir, venv_dir, user):
create_directory(ctx, "/var/lib/zam", owner=user)
cmd = f"{venv_dir}/bin/alembic -c production.ini upgrade head"
ctx.sudo(f'bash -c "cd {app_dir} && {cmd}"', user=user)
change_dir(name)
elif command == '1':
name = input('Enter file name: ')
if name == '':
print('File name is missing')
save_info('Error - File name is missing')
else:
create_file(name)
save_info(f'File {name} is created')
elif command == '2':
name = input('Enter folder name: ')
if name == '':
print('Folder name is missing')
save_info('Error - Folder name is missing')
else:
create_directory(name)
save_info(f'Folder {name} is created')
elif command == '3':
name = input('Enter file or folder name: ')
if name == '':
print('File or folder name is missing')
save_info('Error delete - File or folder name is missing')
else:
delete_file(name)
save_info(f'Deleted {name}')
elif command == '4':
name = input('Enter file or folder to copy: ')
new_name = input('Enter the new destination: ')
if name == '':
print('File or folder name is missing')
save_info('Error copy - File or folder name is missing')
def Network_config(class_num=4,
epoch=200,
initial_epoch=0,
batch_size=32,
train_data=None,
train_label=None,
test_data=None,
test_label=None,
fold=0):
adam = Adam(lr=0.005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0009)
sgd = SGD(lr=0.001, momentum=0.9, decay=0.0, nesterov=False)
input_tensor = Input(shape=(224, 224, 3))
#backbone
base_model = VGG16(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
base_output = base_model.output
#self-attention
x = non_local.non_local_block(base_output,
intermediate_dim=None,
compression=2,
mode='embedded',
add_residual=False)
x = BatchNormalization()(x)
#channel-attention
y = channel_attention.squeeze_excitation_layer(base_output,
512,
ratio=4,
concate=False)
y = BatchNormalization()(y)
#concat
x = concatenate([base_output, x], axis=3)
x = concatenate([x, y], axis=3)
# spp
gap = GlobalAveragePooling2D()(x)
x = Flatten()(x)
x = concatenate([gap, x])
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
predict = Dense(class_num, activation='softmax')(x)
model = Model(inputs=input_tensor, outputs=predict)
for layer in (base_model.layers):
layer.trainable = False
for l in model.layers:
print(l.name)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=[keras.metrics.categorical_accuracy])
model.summary()
tools.create_directory('./final/')
weights_file = './final/' + str(
fold
) + '-weights.{epoch:02d}-{categorical_accuracy:.4f}-{val_loss:.4f}-{val_categorical_accuracy:.4f}.h5'
csv_file = './final/record.csv'
lr_reducer = ReduceLROnPlateau(monitor='categorical_accuracy',
factor=0.2,
cooldown=0,
patience=2,
min_lr=0.5e-6)
early_stopper = EarlyStopping(monitor='val_categorical_accuracy',
min_delta=1e-4,
patience=30)
model_checkpoint = ModelCheckpoint(weights_file,
monitor='val_categorical_accuracy',
save_best_only=True,
verbose=1,
save_weights_only=True,
mode='max')
tensorboard = TensorBoard(log_dir='./logs/',
histogram_freq=0,
batch_size=8,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
CSV_record = CSVLogger(csv_file, separator=',', append=True)
callbacks = [
lr_reducer, early_stopper, model_checkpoint, tensorboard, CSV_record
]
gc.disable()
model.fit_generator(
generator=tools.batch_generator(np.array(train_data),
np.array(train_label), batch_size,
True, class_num, True),
steps_per_epoch=int(len(train_label) / batch_size) - 1,
max_q_size=20,
initial_epoch=initial_epoch,
epochs=epoch,
verbose=1,
callbacks=callbacks,
validation_data=tools.batch_generator(np.array(test_data),
np.array(test_label), batch_size,
True, class_num, False),
validation_steps=int(len(test_label) / batch_size) - 1,
class_weight='auto')
#confusion matrix
all_y_pred = []
all_y_true = []
for test_data_batch, test_label_batch in tools.batch_generator_confusion_matrix(
np.array(test_data), np.array(test_label), batch_size, True,
class_num):
y_pred = model.predict(test_data_batch, batch_size)
y_true = test_label_batch
for y_p in y_pred:
all_y_pred.append(np.where(y_p == max(y_p))[0][0])
for y_t in y_true:
all_y_true.append(np.where(y_t == max(y_t))[0][0])
confusion = confusion_matrix(y_true=all_y_true, y_pred=all_y_pred)
print(confusion)
f = open('confusion_matrix.txt', 'a+')
f.write(str(all_y_true) + "\n")
f.write(str(all_y_pred) + "\n")
f.write(str(confusion) + '\n')
f.close()
gc.enable()
from libs import kodi
from tm_libs import dom_parser
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import tools
import string
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/Afdah")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','Afdah/'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
def __enum(**enums):
return type('Enum', (), enums)
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
QUALITIES = __enum(LOW='Low', MEDIUM='Medium', HIGH='High', HD720='HD720', HD1080='HD1080')
def LogNotify(title,message,times,icon):
xbmc.executebuiltin("XBMC.Notification("+title+","+message+","+times+","+icon+")")
import string
from libs import cloudflare
from tm_libs import dom_parser
import cookielib
import json
from StringIO import StringIO
import gzip
import main_scrape
net = Net()
addon_id = kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/NineMovies")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','NineMovies/'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
timeout = int(kodi.get_setting('scraper_timeout'))
def __enum(**enums):
return type('Enum', (), enums)
MAX_RESPONSE = 1024 * 1024 * 2
FORCE_NO_MATCH = '***FORCE_NO_MATCH***'
import tools
from libs import kodi
from tm_libs import dom_parser
from libs.trans_utils import i18n
from libs import log_utils
from t0mm0.common.net import Net
from t0mm0.common.addon import Addon
import main_scrape
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
net = Net()
addon_id=kodi.addon_id
addon = Addon(addon_id, sys.argv)
#COOKIE STUFF
tools.create_directory(tools.AOPATH, "All_Cookies/IWatchOnline")
cookiepath = xbmc.translatePath(os.path.join('special://home','addons',addon_id,'All_Cookies','IWatchOnline'))
cookiejar = os.path.join(cookiepath,'cookies.lwp')
cj = cookielib.LWPCookieJar()
cookie_file = os.path.join(cookiepath,'cookies.lwp')
base_url = 'http://www.merdb.link/'
def LogNotify(title,message,times,icon):
xbmc.executebuiltin("XBMC.Notification("+title+","+message+","+times+","+icon+")")
def OPEN_URL(url):
req=urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Linux; U; Android 4.2.2; en-us; AFTB Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30')
response=urllib2.urlopen(req)
link=response.read()