def download(rqs, furl, path, level=3):
if furl[0:4] != 'http':
furl = "{}/{}".format(D2L_BASEURL, furl)
file = rqs.get(furl, stream=True, headers={
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0"
})
if file.status_code == 302: # D2L, you don't f*****g redirect a 404/403 error.
logger.error("Requested file is Not Found or Forbidden")
if not os.path.isdir(safeFilePath(path)):
# logger.info("Directory does not exist.")
logger.debug(safeFilePath(path))
mkdir_recursive(safeFilePath(path))
try:
name = furl.split('?')[0].split('/')[-1]
if name == "DirectFileTopicDownload":
name = file.headers['Content-Disposition'].split(';')[-1].split('=')[-1][1:-1]
path += "/" + safeFilePath(name)
with open(unquote(path), 'wb') as f:
for chunk in tqdm.tqdm(file.iter_content(chunk_size=1024), desc="Downloading {}".format(name),
position=level, unit="kb"): # is it kb or b?
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
except Exception as e:
logger.exception("Exception caught during file download. {}", str(e))
def on_task_output(self, task, config):
config = self.prepare_config(config)
# don't add when learning
if task.options.learn:
return
if not config['enabled']:
return
# Do not run if there is nothing to do
if not task.accepted:
return
if self.client is None:
self.client = self.create_rpc_client(config)
if self.client:
logger.debug('Successfully connected to transmission.')
else:
raise plugin.PluginError("Couldn't connect to transmission.")
session_torrents = self.client.get_torrents()
for entry in task.accepted:
if task.options.test:
logger.info('Would {} {} in transmission.', config['action'],
entry['title'])
continue
# Compile user options into appropriate dict
options = self._make_torrent_options_dict(config, entry)
torrent_info = None
for t in session_torrents:
if t.hashString.lower() == entry.get(
'torrent_info_hash',
'').lower() or t.id == entry.get('transmission_id'):
torrent_info = t
logger.debug(
'Found {} already loaded in transmission as {}',
entry['title'],
torrent_info.name,
)
break
if not torrent_info:
if config['action'] != 'add':
logger.warning(
'Cannot {} {} because it is not loaded in transmission.',
config['action'],
entry['title'],
)
continue
downloaded = not entry['url'].startswith('magnet:')
# Check that file is downloaded
if downloaded and 'file' not in entry:
entry.fail('`file` field missing?')
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry['file']):
tmp_path = os.path.join(task.manager.config_base, 'temp')
logger.debug('entry: {}', entry)
logger.debug('temp: {}', ', '.join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" %
entry['file'])
continue
try:
if downloaded:
with open(entry['file'], 'rb') as f:
filedump = base64.b64encode(
f.read()).decode('utf-8')
torrent_info = self.client.add_torrent(
filedump, 30, **options['add'])
else:
if options['post'].get('magnetization_timeout', 0) > 0:
options['add']['paused'] = False
torrent_info = self.client.add_torrent(
entry['url'], timeout=30, **options['add'])
except TransmissionError as e:
logger.opt(exception=True).debug('TransmissionError')
logger.debug('Failed options dict: {}', options['add'])
msg = 'Error adding {} to transmission. TransmissionError: {}'.format(
entry['title'], e.message or 'N/A')
logger.error(msg)
entry.fail(msg)
continue
logger.info('"{}" torrent added to transmission',
entry['title'])
# The info returned by the add call is incomplete, refresh it
torrent_info = self.client.get_torrent(torrent_info.id)
else:
# Torrent already loaded in transmission
if options['add'].get('download_dir'):
logger.verbose('Moving {} to "{}"', torrent_info.name,
options['add']['download_dir'])
# Move data even if current reported torrent location matches new location
# as transmission may fail to automatically move completed file to final
# location but continue reporting final location instead of real location.
# In such case this will kick transmission to really move data.
# If data is already located at new location then transmission just ignore
# this command.
self.client.move_torrent_data(
torrent_info.id, options['add']['download_dir'], 120)
try:
total_size = torrent_info.totalSize
main_id = None
find_main_file = (options['post'].get('main_file_only')
or 'content_filename' in options['post'])
skip_files = options['post'].get('skip_files')
# We need to index the files if any of the following are defined
if find_main_file or skip_files:
file_list = self.client.get_files(
torrent_info.id)[torrent_info.id]
if options['post'].get('magnetization_timeout',
0) > 0 and not file_list:
logger.debug(
'Waiting {} seconds for "{}" to magnetize',
options['post']['magnetization_timeout'],
entry['title'],
)
for _ in range(
options['post']['magnetization_timeout']):
sleep(1)
file_list = self.client.get_files(
torrent_info.id)[torrent_info.id]
if file_list:
total_size = self.client.get_torrent(
torrent_info.id,
['id', 'totalSize']).totalSize
break
else:
logger.warning(
'"{}" did not magnetize before the timeout elapsed, file list unavailable for processing.',
entry['title'],
)
# Find files based on config
dl_list = []
skip_list = []
main_list = []
ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass']
main_ratio = config['main_file_ratio']
if 'main_file_ratio' in options['post']:
main_ratio = options['post']['main_file_ratio']
for f in file_list:
# No need to set main_id if we're not going to need it
if find_main_file and file_list[f][
'size'] > total_size * main_ratio:
main_id = f
if 'include_files' in options['post']:
if any(
fnmatch(file_list[f]['name'], mask) for
mask in options['post']['include_files']):
dl_list.append(f)
elif options['post'].get('include_subs') and any(
fnmatch(file_list[f]['name'], mask)
for mask in ext_list):
dl_list.append(f)
if skip_files:
if any(
fnmatch(file_list[f]['name'], mask)
for mask in skip_files):
skip_list.append(f)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if options['post'].get('rename_like_files'):
for f in file_list:
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(file_list[f]['name'])
if fs[0] == os.path.splitext(
file_list[main_id]['name'])[0]:
main_list.append(f)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
elif find_main_file:
logger.warning(
'No files in "{}" are > {:.0f}% of content size, no files renamed.',
entry['title'],
main_ratio * 100,
)
# If we have a main file and want to rename it and associated files
if 'content_filename' in options[
'post'] and main_id is not None:
if 'download_dir' not in options['add']:
download_dir = self.client.get_session(
).download_dir
else:
download_dir = options['add']['download_dir']
# Get new filename without ext
file_ext = os.path.splitext(
file_list[main_id]['name'])[1]
file_path = os.path.dirname(
os.path.join(download_dir,
file_list[main_id]['name']))
filename = options['post']['content_filename']
if config['host'] == 'localhost' or config[
'host'] == '127.0.0.1':
counter = 1
while os.path.exists(
os.path.join(file_path,
filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = '%s(%s)' % (
options['post']['content_filename'],
counter,
)
counter += 1
else:
logger.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote transmission daemon.')
for index in main_list:
file_ext = os.path.splitext(
file_list[index]['name'])[1]
logger.debug(
'File {} renamed to {}',
file_list[index]['name'],
filename + file_ext,
)
# change to below when set_files will allow setting name, more efficient to have one call
# fl[index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
try:
self.client.rename_torrent_path(
torrent_info.id,
file_list[index]['name'],
os.path.basename(
str(pathscrub(filename + file_ext))),
)
except TransmissionError:
logger.error(
'content_filename only supported with transmission 2.8+'
)
if options['post'].get(
'main_file_only') and main_id is not None:
# Set Unwanted Files
options['change']['files_unwanted'] = [
x for x in file_list if x not in dl_list
]
options['change']['files_wanted'] = dl_list
logger.debug(
'Downloading {} of {} files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
elif (not options['post'].get('main_file_only')
or main_id is None) and skip_files:
# If no main file and we want to skip files
if len(skip_list) >= len(file_list):
logger.debug(
'skip_files filter would cause no files to be downloaded; '
'including all files in torrent.')
else:
options['change']['files_unwanted'] = skip_list
options['change']['files_wanted'] = [
x for x in file_list if x not in skip_list
]
logger.debug(
'Downloading {} of {} files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
# Set any changed file properties
if list(options['change'].keys()):
self.client.change_torrent(torrent_info.id, 30,
**options['change'])
if config['action'] == 'add':
# if add_paused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
start_paused = (
options['post']['paused']
if 'paused' in options['post'] else
not self.client.get_session().start_added_torrents)
if start_paused:
self.client.stop_torrent(torrent_info.id)
else:
self.client.start_torrent(torrent_info.id)
elif config['action'] in ('remove', 'purge'):
self.client.remove_torrent(
[torrent_info.id],
delete_data=config['action'] == 'purge')
logger.info('{}d {} from transmission', config['action'],
torrent_info.name)
elif config['action'] == 'pause':
self.client.stop_torrent([torrent_info.id])
logger.info('paused {} in transmission', torrent_info.name)
elif config['action'] == 'resume':
self.client.start_torrent([torrent_info.id])
logger.info('resumed {} in transmission',
torrent_info.name)
except TransmissionError as e:
logger.opt(exception=True).debug('TransmissionError')
logger.debug('Failed options dict: {}', options)
msg = 'Error trying to {} {}, TransmissionError: {}'.format(
config['action'], entry['title'], e.message or 'N/A')
logger.error(msg)
continue
def _check_passwd(self) -> bool:
if self.user.password != self.password:
logger.error(f"{self.user.email} 登录密码错误")
abort(403, message="密码错误")
return True
def run(self):
from ..autonlp import AutoNLP
logger.info(f"Uploading files for project: {self._name}")
client = AutoNLP()
try:
project = client.get_project(name=self._name)
except ValueError:
logger.error(f"Project {self._name} not found! You can create it using the create_project command.")
sys.exit(1)
splits = self._col_mapping.split(",")
col_maps = {}
for s in splits:
k, v = s.split(":")
col_maps[k] = v
logger.info(f"Mapping: {col_maps}")
files = self._files.split(",")
try:
project.upload(filepaths=files, split=self._split, col_mapping=col_maps, path_to_audio=self._path_to_audio)
print(
"🎉 Yupee! Your files have been uploaded.\n"
f"Once you're done, starting a training here: {RED}autonlp train --project {project.name}{RST}"
)
except ValueError as err:
logger.error("❌ Something went wrong!")
logger.error("Details:")
logger.error(str(err))
except FileNotFoundError as err:
logger.error("❌ One path you provided is invalid!")
logger.error("Details:")
logger.error(str(err))
except InvalidFileError as err:
logger.error("❌ Sorry, AutoNLP is not able to process the files you want to upload")
logger.error("Details:")
for line in str(err).splitlines():
logger.error(line)
except InvalidColMappingError as err:
logger.error("❌ The column mapping you provided is incorrect!")
logger.error("Details:")
for line in str(err).splitlines():
logger.error(line)
def Main():
default_config = os.path.join(CCF_Etc, 'cchost.toml')
parser = argparse.ArgumentParser(
description='Script to enable the CCF network')
parser.add_argument(
'--logfile',
help='Name of the log file, __screen__ for standard output',
type=str)
parser.add_argument('--loglevel',
help='Logging level',
default='WARNING',
type=str)
parser.add_argument('--ccf-config',
help='Name of the CCF configuration file',
default=default_config,
type=str)
parser.add_argument('--member-name',
help="Name of the member adding the user",
default="memberccf",
type=str)
parser.add_argument('--user-name',
help="Name of the user being added",
default="userccf",
type=str)
parser.add_argument('--add-node',
help="Add a new node to existing CCF network",
action="store_true")
parser.add_argument('--node-id',
help="id of the node to be added to the ccf network",
type=int)
options = parser.parse_args()
# -----------------------------------------------------------------
LOG.remove()
if options.logfile == '__screen__':
LOG.add(sys.stderr, level=options.loglevel)
else:
LOG.add(options.logfile)
# -----------------------------------------------------------------
try:
config = toml.load(options.ccf_config)
except:
LOG.error('unable to load ccf configuration file {0}'.format(
options.ccf_config))
pass
member_cert = os.path.join(CCF_Keys,
"{}_cert.pem".format(options.member_name))
member_key = os.path.join(CCF_Keys,
"{}_privk.pem".format(options.member_name))
network_cert = config["start"]["network-cert-file"]
(host, port) = config["rpc-address"].split(':')
try:
member_client = CCFClient(host=host,
port=port,
cert=member_cert,
key=member_key,
ca=network_cert,
format='json',
prefix='gov',
description="none",
version="2.0",
connection_timeout=3,
request_timeout=3)
except:
LOG.error('failed to connect to CCF service')
sys.exit(-1)
#Temporary fix to skip checking CCF host certificate. Version 0.11.7 CCF certificate expiration was hardcoded to end of 2021
member_client.client_impl.session.mount("https://", HTTPAdapter())
member_client.client_impl.session.verify = False
open_network_script(member_client, options, config)
add_user_script(member_client, options, config)
LOG.info('CCF network ready for use')
sys.exit(0)
def save_url_image_to_path(self, state, data_url, path, state_config={}):
"""Saves URL image from data_url to the specified path.
Parameters
----------
state : str
Two-letter abbreviation of the state or territory. Used for special-casing sizes, etc.
data_url : str
URL of data site to save
path : str
Local path to which to save .png screenshot of data_url
state_config : dict
If exists, this is a dict used for denoting phantomJScloud special casing or file type
"""
# if we need to just download the file, don't use phantomjscloud
if state_config and state_config.get('file'):
if self.dry_run:
logger.warning(f'Dry run: Downloading file from {data_url}')
return
logger.info(f"Downloading file from {data_url}")
response = requests.get(data_url)
if response.status_code == 200:
with open(path, 'wb') as f:
f.write(response.content)
return
else:
logger.error(f'Response status code: {response.status_code}')
raise ValueError(f'Could not download data from URL: {data_url}')
logger.info(f"Retrieving {data_url}")
data = {
'url': data_url,
'renderType': 'png',
}
if state_config:
# update data with state_config minus message
state_config_copy = state_config.copy()
message = state_config_copy.pop('message', None)
for field in ['url', 'name']: # we don't want to override the URL in case it's dynamic
state_config_copy.pop(field)
if message:
logger.info(message)
data.update(state_config_copy)
# set maxWait if unset
if 'requestSettings' in data:
if 'maxWait' not in data['requestSettings']:
data['requestSettings']['maxWait'] = 60000
else:
data['requestSettings'] = {'maxWait': 60000}
if self.dry_run:
logger.warning(f'Dry run: PhantomJsCloud request for {state} from {data_url}: {data}')
return
logger.info('Posting request %s...' % data)
response = requests.post(self.phantomjs_url, json.dumps(data))
logger.info('Done.')
if response.status_code == 200:
with open(path, 'wb') as f:
f.write(response.content)
else:
logger.error(f'Response status code: {response.status_code}')
if 'meta' in response.json():
response_metadata = response.json()['meta']
raise ValueError(f'Could not retrieve URL {data_url}, got response metadata {response_metadata}')
else:
raise ValueError(
'Could not retrieve URL %s and response has no metadata. Full response: %s' % (
data_url, response.json()))
def run(self):
# ---- Subscribe ----
with self.neuron:
# ---- Weights ----
self.row = self.neuron.metagraph.row
# --- Run state ---
self.global_step = 0
self.best_train_loss = math.inf
# --- Loop forever ---
for self.epoch in range(self.config.miner.n_epochs):
try:
# ---- Serve ----
self.neuron.axon.serve(self.model)
# ---- Train Model ----
self.train()
self.scheduler.step()
# If model has borked for some reason, we need to make sure it doesn't emit weights
# Instead, reload into previous version of model
if torch.any(
torch.isnan(
torch.cat([
param.view(-1)
for param in self.model.parameters()
]))):
self.model, self.optimizer = self.model_toolbox.load_model(
self.config)
continue
# ---- Emit row-weights ----
self.neuron.metagraph.set_weights(
self.row, wait_for_inclusion=True
) # Sets my row-weights on the chain.
# ---- Sync metagraph ----
self.neuron.metagraph.sync(
) # Pulls the latest metagraph state (with my update.)
self.row = self.neuron.metagraph.row
# --- Epoch logs ----
print(self.neuron.axon.__full_str__())
print(self.neuron.dendrite.__full_str__())
print(self.neuron.metagraph)
# ---- Update Tensorboard ----
self.neuron.dendrite.__to_tensorboard__(
self.tensorboard, self.global_step)
self.neuron.metagraph.__to_tensorboard__(
self.tensorboard, self.global_step)
self.neuron.axon.__to_tensorboard__(
self.tensorboard, self.global_step)
# ---- Save best loss and model ----
if self.training_loss and self.epoch % 10 == 0:
if self.training_loss < self.best_train_loss:
self.best_train_loss = self.training_loss # update best train loss
self.model_toolbox.save_model(
self.config.miner.full_path, {
'epoch':
self.epoch,
'model_state_dict':
self.model.state_dict(),
'loss':
self.best_train_loss,
'optimizer_state_dict':
self.optimizer.state_dict(),
})
self.tensorboard.add_scalar(
'Neuron/Train_loss', self.training_loss,
self.global_step)
# --- Catch Errors ----
except Exception as e:
logger.error('Exception in training script with error: {}',
e)
logger.info(traceback.print_exc())
logger.info('Continuing to train.')
time.sleep(1)
def error(self, msg: str, exception: Exception = None):
self.has_error = True
logger.error(msg)
if exception != None:
logger.exception(exception)
self.messages.append(("ERROR", msg, exception))
async def query_arm(ctx: Context, args: argparse.Namespace) -> None:
logger.info(f"Starting enumeration for ARM - {ctx.cloud['ARM']}")
async with SubscriptionClient(ctx.cred_async,
base_url=ctx.cloud["ARM"]) as sub_client:
async for tenant in sub_client.tenants.list():
tenant_dict = tenant.as_dict()
tenant_dict["subscriptions"] = []
logger.info(
f"Enumerating subscription and resource groups for tenant {tenant.tenant_id}"
)
# GET LIST OF SUBS.
sub_list = []
async for subscription in sub_client.subscriptions.list():
if args.subs:
if not subscription.subscription_id in args.subs:
continue
if args.nosubs:
if subscription.subscription_id in args.nosubs:
continue
sub_list.append(subscription)
if not sub_list:
logger.error(f"No subscriptions found for {tenant.tenant_id}")
continue
# ENUMERATE MANAGEMENT CERTS
if ctx.cloud["MGMT"]:
certsTasks = [
asyncio.create_task(_query_management_certs(ctx, sub))
for sub in sub_list
]
certs_output = OUTPUT_FOLDER / f"certs.sqlite"
for cert in asyncio.as_completed(*[certsTasks]):
if await cert:
await sqlite_writer(certs_output, cert)
# ENUMERATE RBAC
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=len(sub_list))
rbacTasks = {
executor.submit(_query_rbac, ctx, sub)
for sub in sub_list
}
backfills = {
"User": set(),
"Group": set(),
"ServicePrincipal": set(),
"Application": set(),
} # Dict of object IDs to hold for AAD enumeration
rbac_output = OUTPUT_FOLDER / f"rbac.sqlite"
for rbac in concurrent.futures.as_completed(*[rbacTasks]):
if rbac.result():
for role in rbac.result():
await sqlite_writer(rbac_output, role)
if args.backfill:
backfills[role["principal_type"]].add(
role["principal_id"])
# Only do backfill if azure argument is true (meaning specified on command line)
if args.azure and args.backfill:
await rbac_backfill(ctx, args, backfills)
# ENUMERATE TENANT DATA
subTasks = [
asyncio.create_task(_query_subscription(ctx, sub))
for sub in sub_list
]
for result in asyncio.as_completed(*[subTasks]):
tenant_dict["subscriptions"].append(await result)
tenant_output = OUTPUT_FOLDER / f"tenant.sqlite"
await sqlite_writer(tenant_output, tenant_dict)
def fetch_schema(client, prefix, file_prefix=None):
if file_prefix is None:
file_prefix = prefix
api_response = client.get(f"/{prefix}/api")
check(api_response,
error=lambda status, msg: status == http.HTTPStatus.OK.value)
response_body = api_response.body.json()
paths = response_body["paths"]
all_methods.extend(paths.keys())
fetched_version = response_body["info"]["version"]
formatted_schema = json.dumps(response_body, indent=2)
openapi_target_file = os.path.join(args.schema_dir,
f"{file_prefix}_openapi.json")
try:
old_schema.remove(openapi_target_file)
except KeyError:
pass
with open(openapi_target_file, "a+", encoding="utf-8") as f:
f.seek(0)
previous = f.read()
if previous != formatted_schema:
file_version = "0.0.0"
try:
from_file = json.loads(previous)
file_version = from_file["info"]["version"]
except (json.JSONDecodeError, KeyError):
pass
if version.parse(fetched_version) > version.parse(
file_version):
LOG.debug(
f"Writing schema to {openapi_target_file} - overwriting {file_version} with {fetched_version}"
)
f.truncate(0)
f.seek(0)
f.write(formatted_schema)
else:
LOG.error(
f"Found differences in {openapi_target_file}, but not overwriting as retrieved version is not newer ({fetched_version} <= {file_version})"
)
alt_file = os.path.join(
args.schema_dir,
f"{file_prefix}_{fetched_version}_openapi.json")
LOG.error(f"Writing to {alt_file} for comparison")
with open(alt_file, "w", encoding="utf-8") as f2:
f2.write(formatted_schema)
try:
old_schema.remove(alt_file)
except KeyError:
pass
changed_files.append(openapi_target_file)
else:
LOG.debug("Schema matches in {}".format(openapi_target_file))
try:
openapi_spec_validator.validate_spec(response_body)
except Exception as e:
LOG.error(f"Validation of {prefix} schema failed")
LOG.error(e)
return False
return True
def run(args):
os.makedirs(args.schema_dir, exist_ok=True)
changed_files = []
old_schema = set(
os.path.join(dir_path, filename)
for dir_path, _, filenames in os.walk(args.schema_dir)
for filename in filenames)
documents_valid = True
all_methods = []
def fetch_schema(client, prefix, file_prefix=None):
if file_prefix is None:
file_prefix = prefix
api_response = client.get(f"/{prefix}/api")
check(api_response,
error=lambda status, msg: status == http.HTTPStatus.OK.value)
response_body = api_response.body.json()
paths = response_body["paths"]
all_methods.extend(paths.keys())
fetched_version = response_body["info"]["version"]
formatted_schema = json.dumps(response_body, indent=2)
openapi_target_file = os.path.join(args.schema_dir,
f"{file_prefix}_openapi.json")
try:
old_schema.remove(openapi_target_file)
except KeyError:
pass
with open(openapi_target_file, "a+", encoding="utf-8") as f:
f.seek(0)
previous = f.read()
if previous != formatted_schema:
file_version = "0.0.0"
try:
from_file = json.loads(previous)
file_version = from_file["info"]["version"]
except (json.JSONDecodeError, KeyError):
pass
if version.parse(fetched_version) > version.parse(
file_version):
LOG.debug(
f"Writing schema to {openapi_target_file} - overwriting {file_version} with {fetched_version}"
)
f.truncate(0)
f.seek(0)
f.write(formatted_schema)
else:
LOG.error(
f"Found differences in {openapi_target_file}, but not overwriting as retrieved version is not newer ({fetched_version} <= {file_version})"
)
alt_file = os.path.join(
args.schema_dir,
f"{file_prefix}_{fetched_version}_openapi.json")
LOG.error(f"Writing to {alt_file} for comparison")
with open(alt_file, "w", encoding="utf-8") as f2:
f2.write(formatted_schema)
try:
old_schema.remove(alt_file)
except KeyError:
pass
changed_files.append(openapi_target_file)
else:
LOG.debug("Schema matches in {}".format(openapi_target_file))
try:
openapi_spec_validator.validate_spec(response_body)
except Exception as e:
LOG.error(f"Validation of {prefix} schema failed")
LOG.error(e)
return False
return True
with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes,
args.perf_nodes) as network:
network.start_and_open(args)
primary, _ = network.find_primary()
check = infra.checker.Checker()
with primary.client("user0") as user_client:
LOG.info("user frontend")
if not fetch_schema(user_client, "app"):
documents_valid = False
with primary.client() as node_client:
LOG.info("node frontend")
if not fetch_schema(node_client, "node"):
documents_valid = False
with primary.client("member0") as member_client:
LOG.info("member frontend")
if not fetch_schema(member_client, "gov"):
documents_valid = False
made_changes = False
if len(old_schema) > 0:
LOG.error(
"Removing old files which are no longer reported by the service:")
for f in old_schema:
LOG.error(" " + f)
os.remove(f)
f_dir = os.path.dirname(f)
# Remove empty directories too
while not os.listdir(f_dir):
os.rmdir(f_dir)
f_dir = os.path.dirname(f_dir)
made_changes = True
if len(changed_files) > 0:
LOG.error("Found problems with the following schema files:")
for f in changed_files:
LOG.error(" " + f)
made_changes = True
if args.list_all:
LOG.info("Discovered methods:")
for method in sorted(set(all_methods)):
LOG.info(f" {method}")
if made_changes or not documents_valid:
assert False
def main(
exp_cfg,
show=False,
demo_output_folder='demo_output',
pause=-1,
focal_length=5000,
sensor_width=36,
save_vis=True,
save_params=False,
save_mesh=False,
degrees=[],
):
device = torch.device('cuda')
if not torch.cuda.is_available():
logger.error('CUDA is not available!')
sys.exit(3)
logger.remove()
logger.add(lambda x: tqdm.write(x, end=''),
level=exp_cfg.logger_level.upper(),
colorize=True)
demo_output_folder = osp.expanduser(osp.expandvars(demo_output_folder))
logger.info(f'Saving results to: {demo_output_folder}')
os.makedirs(demo_output_folder, exist_ok=True)
model = SMPLXNet(exp_cfg)
try:
model = model.to(device=device)
except RuntimeError:
# Re-submit in case of a device error
sys.exit(3)
checkpoint_folder = osp.join(exp_cfg.output_folder,
exp_cfg.checkpoint_folder)
checkpointer = Checkpointer(model,
save_dir=checkpoint_folder,
pretrained=exp_cfg.pretrained)
arguments = {'iteration': 0, 'epoch_number': 0}
extra_checkpoint_data = checkpointer.load_checkpoint()
for key in arguments:
if key in extra_checkpoint_data:
arguments[key] = extra_checkpoint_data[key]
model = model.eval()
means = np.array(exp_cfg.datasets.body.transforms.mean)
std = np.array(exp_cfg.datasets.body.transforms.std)
render = save_vis or show
body_crop_size = exp_cfg.get('datasets', {}).get('body',
{}).get('transforms').get(
'crop_size', 256)
if render:
hd_renderer = HDRenderer(img_size=body_crop_size)
dataloaders = make_all_data_loaders(exp_cfg, split='test')
body_dloader = dataloaders['body'][0]
total_time = 0
cnt = 0
for bidx, batch in enumerate(tqdm(body_dloader, dynamic_ncols=True)):
full_imgs_list, body_imgs, body_targets = batch
if full_imgs_list is None:
continue
full_imgs = to_image_list(full_imgs_list)
body_imgs = body_imgs.to(device=device)
body_targets = [target.to(device) for target in body_targets]
full_imgs = full_imgs.to(device=device)
torch.cuda.synchronize()
start = time.perf_counter()
model_output = model(body_imgs,
body_targets,
full_imgs=full_imgs,
device=device)
torch.cuda.synchronize()
elapsed = time.perf_counter() - start
cnt += 1
total_time += elapsed
hd_imgs = full_imgs.images.detach().cpu().numpy().squeeze()
body_imgs = body_imgs.detach().cpu().numpy()
body_output = model_output.get('body')
_, _, H, W = full_imgs.shape
# logger.info(f'{H}, {W}')
# H, W, _ = hd_imgs.shape
if render:
hd_imgs = np.transpose(undo_img_normalization(hd_imgs, means, std),
[0, 2, 3, 1])
hd_imgs = np.clip(hd_imgs, 0, 1.0)
right_hand_crops = body_output.get('right_hand_crops')
left_hand_crops = torch.flip(body_output.get('left_hand_crops'),
dims=[-1])
head_crops = body_output.get('head_crops')
bg_imgs = undo_img_normalization(body_imgs, means, std)
right_hand_crops = undo_img_normalization(right_hand_crops, means,
std)
left_hand_crops = undo_img_normalization(left_hand_crops, means,
std)
head_crops = undo_img_normalization(head_crops, means, std)
body_output = model_output.get('body', {})
num_stages = body_output.get('num_stages', 3)
stage_n_out = body_output.get(f'stage_{num_stages - 1:02d}', {})
model_vertices = stage_n_out.get('vertices', None)
if stage_n_out is not None:
model_vertices = stage_n_out.get('vertices', None)
faces = stage_n_out['faces']
if model_vertices is not None:
model_vertices = model_vertices.detach().cpu().numpy()
camera_parameters = body_output.get('camera_parameters', {})
camera_scale = camera_parameters['scale'].detach()
camera_transl = camera_parameters['translation'].detach()
out_img = OrderedDict()
final_model_vertices = None
stage_n_out = model_output.get('body', {}).get('final', {})
if stage_n_out is not None:
final_model_vertices = stage_n_out.get('vertices', None)
if final_model_vertices is not None:
final_model_vertices = final_model_vertices.detach().cpu().numpy()
camera_parameters = model_output.get('body', {}).get(
'camera_parameters', {})
camera_scale = camera_parameters['scale'].detach()
camera_transl = camera_parameters['translation'].detach()
hd_params = weak_persp_to_blender(
body_targets,
camera_scale=camera_scale,
camera_transl=camera_transl,
H=H,
W=W,
sensor_width=sensor_width,
focal_length=focal_length,
)
if save_vis:
bg_hd_imgs = np.transpose(hd_imgs, [0, 3, 1, 2])
out_img['hd_imgs'] = bg_hd_imgs
if render:
# Render the initial predictions on the original image resolution
hd_orig_overlays = hd_renderer(
model_vertices,
faces,
focal_length=hd_params['focal_length_in_px'],
camera_translation=hd_params['transl'],
camera_center=hd_params['center'],
bg_imgs=bg_hd_imgs,
return_with_alpha=True,
)
out_img['hd_orig_overlay'] = hd_orig_overlays
# Render the overlays of the final prediction
if render:
hd_overlays = hd_renderer(
final_model_vertices,
faces,
focal_length=hd_params['focal_length_in_px'],
camera_translation=hd_params['transl'],
camera_center=hd_params['center'],
bg_imgs=bg_hd_imgs,
return_with_alpha=True,
body_color=[0.4, 0.4, 0.7])
out_img['hd_overlay'] = hd_overlays
for deg in degrees:
hd_overlays = hd_renderer(
final_model_vertices,
faces,
focal_length=hd_params['focal_length_in_px'],
camera_translation=hd_params['transl'],
camera_center=hd_params['center'],
bg_imgs=bg_hd_imgs,
return_with_alpha=True,
render_bg=False,
body_color=[0.4, 0.4, 0.7],
deg=deg,
)
out_img[f'hd_rendering_{deg:03.0f}'] = hd_overlays
if save_vis:
for key in out_img.keys():
out_img[key] = np.clip(
np.transpose(out_img[key], [0, 2, 3, 1]) * 255, 0,
255).astype(np.uint8)
for idx in tqdm(range(len(body_targets)), 'Saving ...'):
fname = body_targets[idx].get_field('fname')
curr_out_path = osp.join(demo_output_folder, fname)
os.makedirs(curr_out_path, exist_ok=True)
if save_vis:
for name, curr_img in out_img.items():
pil_img.fromarray(curr_img[idx]).save(
osp.join(curr_out_path, f'{name}.png'))
if save_mesh:
# Store the mesh predicted by the body-crop network
naive_mesh = o3d.geometry.TriangleMesh()
naive_mesh.vertices = Vec3d(model_vertices[idx] +
hd_params['transl'][idx])
naive_mesh.triangles = Vec3i(faces)
mesh_fname = osp.join(curr_out_path, f'body_{fname}.ply')
o3d.io.write_triangle_mesh(mesh_fname, naive_mesh)
# Store the final mesh
expose_mesh = o3d.geometry.TriangleMesh()
expose_mesh.vertices = Vec3d(final_model_vertices[idx] +
hd_params['transl'][idx])
expose_mesh.triangles = Vec3i(faces)
mesh_fname = osp.join(curr_out_path, f'{fname}.ply')
o3d.io.write_triangle_mesh(mesh_fname, expose_mesh)
if save_params:
params_fname = osp.join(curr_out_path, f'{fname}_params.npz')
out_params = dict(fname=fname)
for key, val in stage_n_out.items():
if torch.is_tensor(val):
val = val.detach().cpu().numpy()[idx]
out_params[key] = val
for key, val in hd_params.items():
if torch.is_tensor(val):
val = val.detach().cpu().numpy()
if np.isscalar(val[idx]):
out_params[key] = val[idx].item()
else:
out_params[key] = val[idx]
np.savez_compressed(params_fname, **out_params)
if show:
nrows = 1
ncols = 4 + len(degrees)
fig, axes = plt.subplots(ncols=ncols,
nrows=nrows,
num=0,
gridspec_kw={
'wspace': 0,
'hspace': 0
})
axes = axes.reshape(nrows, ncols)
for ax in axes.flatten():
ax.clear()
ax.set_axis_off()
axes[0, 0].imshow(hd_imgs[idx])
axes[0, 1].imshow(out_img['rgb'][idx])
axes[0, 2].imshow(out_img['hd_orig_overlay'][idx])
axes[0, 3].imshow(out_img['hd_overlay'][idx])
start = 4
for deg in degrees:
axes[0, start].imshow(
out_img[f'hd_rendering_{deg:03.0f}'][idx])
start += 1
plt.draw()
if pause > 0:
plt.pause(pause)
else:
plt.show()
logger.info(f'Average inference time: {total_time / cnt}')
def run(get_command, args):
if args.fixed_seed:
seed(getpass.getuser())
hosts = args.nodes
if not hosts:
hosts = ["localhost"] * number_of_local_nodes(args)
LOG.info("Starting nodes on {}".format(hosts))
with infra.ccf.network(
hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
) as network:
network.start_and_join(args)
primary, backups = network.find_nodes()
command_args = get_command_args(args, get_command)
if args.network_only:
run_client(args, primary, command_args)
else:
nodes = filter_nodes(primary, backups, args.send_tx_to)
clients = []
client_hosts = args.client_nodes or ["localhost"]
for client_id, client_host in enumerate(client_hosts):
node = nodes[client_id % len(nodes)]
remote_client = configure_remote_client(
args, client_id, client_host, node, command_args
)
clients.append(remote_client)
for remote_client in clients:
remote_client.start()
hard_stop_timeout = 90
try:
with cimetrics.upload.metrics() as metrics:
tx_rates = infra.rates.TxRates(primary)
start_time = time.time()
while True:
stop_waiting = True
for i, remote_client in enumerate(clients):
done = remote_client.check_done()
# all the clients need to be done
LOG.info(
f"Client {i} has {'completed' if done else 'not completed'} running"
)
stop_waiting = stop_waiting and done
if stop_waiting:
break
if time.time() > start_time + hard_stop_timeout:
raise TimeoutError(
f"Client still running after {hard_stop_timeout}s"
)
time.sleep(1)
tx_rates.get_metrics()
for remote_client in clients:
remote_client.print_and_upload_result(args.label, metrics)
remote_client.stop()
LOG.info(f"Rates:\n{tx_rates}")
tx_rates.save_results(args.metrics_file)
except Exception:
LOG.error("Stopping clients due to exception")
for remote_client in clients:
remote_client.stop()
raise
def handle(update, context):
util.log_chat("quotes", update)
try:
# Extract query...
query = update.message.text
query = query.split(" ")
command = None
try:
command = query[1].lower()
except:
command = "random"
if command == "add":
if update.message.reply_to_message.text:
response = add_quote(update)
update.message.reply_text(text=response,
parse_mode=ParseMode.MARKDOWN)
elif command == "remove":
if util.is_admin(update.message.from_user["id"]):
try:
id_to_remove = query[2]
except:
update.message.reply_text(
text="Please include the Quote ID you want to remove!",
parse_mode=ParseMode.MARKDOWN,
)
return
response = remove_quote(id_to_remove)
else:
response = "Chal kat re bsdk!"
update.message.reply_text(text=response,
parse_mode=ParseMode.MARKDOWN)
elif command == "get":
response = ""
try:
quote_id = query[2]
except:
update.message.reply_text(
text="Please include the Quote ID you want to get!",
parse_mode=ParseMode.MARKDOWN,
)
return
quote = get_quote_by_id(quote_id)
if quote is None:
group_id = util.get_group_id_from_update(update)
if group_id is None:
update.message.reply_text(
text="Can't run this command here!")
return
# Return a random quote
random_quote = get_random_quote(group_id)
if random_quote is None:
logger.info("[quotes] No quotes found! - group_id={}",
group_id)
update.message.reply_text(
text=
"No quotes found for this Group! You can add quotes with `/quotes add`",
parse_mode=ParseMode.MARKDOWN,
)
return
response = "Couldn't find quote with ID `{}`... but here's a random one - \n".format(
quote_id)
pretty_quote = generate_pretty_quote(quote)
response = response + pretty_quote
update.message.reply_text(text=response,
parse_mode=ParseMode.MARKDOWN)
elif command == "join":
response = ""
try:
quote_id = query[2]
except:
update.message.reply_text(
text="Please include the Quote ID you want to join!",
parse_mode=ParseMode.MARKDOWN,
)
return
quote = get_quote_by_id(quote_id)
if quote is None:
update.message.reply_text(
text="Couldn't find quote with ID `{}`!".format(quote_id),
parse_mode=ParseMode.MARKDOWN,
)
return
response = join_quotes(quote, update)
update.message.reply_text(text=response,
parse_mode=ParseMode.MARKDOWN)
else:
# Enforce rate limiting on getting random quotes
bakchod = dao.get_bakchod_by_id(update.message.from_user.id)
history = bakchod.history
if history is None:
history = {}
two_min_ago = datetime.datetime.now() - datetime.timedelta(
minutes=2)
if "random_quote_get" in history:
last_time_get = ciso8601.parse_datetime(
history["random_quote_get"])
if last_time_get > two_min_ago:
logger.info(
"[quotes] request random quote too soon... skipping")
update.message.reply_text(
"Quotes ki dukan band hai... come back later!")
return
history["random_quote_get"] = datetime.datetime.now()
bakchod.history = history
dao.insert_bakchod(bakchod)
group_id = util.get_group_id_from_update(update)
if group_id is None:
update.message.reply_text(text="Can't run this command here!")
return
# Return a random quote
random_quote = get_random_quote(group_id)
if random_quote is None:
logger.info("[quotes] No quotes found! - group_id={}",
group_id)
update.message.reply_text(
text=
"No quotes found for this Group! You can add quotes with `/quotes add`",
parse_mode=ParseMode.MARKDOWN,
)
return
logger.info("[quotes] Got a random quote '{}", random_quote)
pretty_quote = generate_pretty_quote(random_quote)
try:
quote_pics.generate_quote_pic(random_quote, update,
pretty_quote)
except Exception as e:
logger.error(
"Caught error in generate_quote_pic - {} \n {}",
e,
traceback.format_exc(),
)
update.message.reply_text(text=pretty_quote,
parse_mode=ParseMode.MARKDOWN)
return
except Exception as e:
logger.error(
"Caught Error in quotes.handle - {} \n {}",
e,
traceback.format_exc(),
)
def mkdir_recursive(path):
try:
os.makedirs(path, exist_ok=True)
except Exception as e:
logger.error("Exception: {}".format(e))
exit(1)
def forward(ctx, caller: Receptor, dummy: torch.Tensor, inputs: torch.Tensor, mode: bittensor.proto.Modality) -> Tuple[torch.Tensor, int]:
r""" Internal autograd-friendly Forward RPC call to a remote neuron (calls the Forward method on an Axon terminal.)
Args:
ctx: (:obj:`torch.autograd.ctx`, `required`):
Autograd context, saves state information between forward and backward calls. i.e. inputs for gradient computation.
caller: (:obj:`Receptor`, `required`):
Caller object the remote neuron containing the endpoint information, RPC channel etc.
dummy: (:obj:`torch.Tensor`, `required`):
Dummy torch tensor used to ensure that torch.backward computation is called on this function
regardless of the input types.
inputs (:obj:`List[torch.Tensor]` of shape :obj:`(shape)`, `required`):
Torch tensor to be sent to the caller associated endpoint neurons.
mode (:obj:`bittensor.proto.Modality` of shape :obj:`(1)`, `required`):
Bittensor forward modality type. Enum in [TEXT, IMAGE, TENSOR]
Returns:
output (:obj:`Tuple[torch.FloatTensor`, torch.LongTensor]`, `optional`):
Result from forward call. May be None in the case of failure.
code (:obj:`bittensor.proto.ReturnCode`, `required`):
Return code associated with forward call.
"""
# ---- Save for backward call ---
ctx.caller = caller
ctx.mode = mode
ctx.inputs = inputs
zeros = nill_response_for(inputs)
try:
# ---- Check inputs size ----
if torch.numel(inputs) == 0:
return zeros, torch.tensor(bittensor.proto.ReturnCode.EmptyRequest)
# ---- Inputs Serialization ----
try:
serializer = serialization.get_serializer( bittensor.proto.Serializer.MSGPACK )
serialized_inputs = serializer.serialize(inputs, modality = mode, from_type = bittensor.proto.TensorType.TORCH)
except Exception as e:
logger.warning('Serialization error with error {}', e)
return zeros, torch.tensor(bittensor.proto.ReturnCode.RequestSerializationException)
ctx.serialized_inputs = serialized_inputs
# ---- Build request ----
request = bittensor.proto.TensorMessage(
version = bittensor.__version__,
public_key = ctx.caller.wallet.hotkey.public_key,
nounce = ctx.caller.nounce,
signature = ctx.caller.signature,
tensors = [serialized_inputs])
# ---- Make RPC call ----
try:
start_time = time.time()
ctx.caller.stats.forward_qps.update(1)
ctx.caller.stats.forward_bytes_out.update(sys.getsizeof(request))
logger.debug('<white>Dendrite</white> <green>Forward Request</green> ---> <white>to</white>:{}, <white>inputs</white>:{}, <white>mode</white>:{}', caller.endpoint, inputs.shape, mode)
response = ctx.caller.stub.Forward(request, timeout=caller.config.receptor.timeout)
ctx.caller.stats.forward_bytes_in.update(sys.getsizeof(response))
ctx.caller.stats.forward_elapsed_time.update((time.time() - start_time))
# Get message
try:
response_message = response.message
except:
response_message = ''
# ---- Catch non-code ----
try:
bittensor_code = response.return_code
except:
logger.debug('<white>Dendrite</white> <green>Forward Response</> <--- <white>code</white>:<yellow>UnknownException</yellow>, <white>from</white>:{}, message:<red>{}</red>', caller.endpoint, inputs.shape, mode)
return zeros, torch.tensor(bittensor_code)
# ---- Catch bittensor errors ----
if bittensor_code == bittensor.proto.ReturnCode.UnknownException:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<yellow>UnknownException</yellow>, <white>from</white>:{}, message:<red>{}</red>', caller.endpoint, response_message)
return zeros, torch.tensor(bittensor_code)
elif bittensor_code != bittensor.proto.ReturnCode.Success:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<yellow>{}</yellow>, <white>from</white>:{}, message:<red>{}</red>', bittensor_code, caller.endpoint,response_message)
return zeros, torch.tensor(bittensor_code)
# ---- Catch GRPC Errors ----
except grpc.RpcError as rpc_error_call:
grpc_code = rpc_error_call.code()
if grpc_code == grpc.StatusCode.DEADLINE_EXCEEDED:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<yellow>Timeout</yellow>, <white>from</white>:{}', caller.endpoint )
return zeros, torch.tensor(bittensor.proto.ReturnCode.Timeout)
elif grpc_code == grpc.StatusCode.UNAVAILABLE:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<yellow>Unavailable</yellow>, <white>from</white>:{}', caller.endpoint )
return zeros, torch.tensor(bittensor.proto.ReturnCode.Unavailable)
else:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<red>UnknownException</red>, <white>from</white>:{} ', caller.endpoint )
return zeros, torch.tensor(bittensor.proto.ReturnCode.UnknownException)
# ---- Catch Unknown Errors ----
except Exception as e:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<red>UnknownException</red>, <white>from</white>:{}, <white>message</white>:<red>{}</red>', caller.endpoint, e)
return zeros, torch.tensor(bittensor.proto.ReturnCode.UnknownException)
# ---- Check tensor response length ----
if len(response.tensors) == 0:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<yellow>EmptyResponse</yellow>, <white>from</white>:{}', caller.endpoint )
return zeros, torch.tensor(bittensor.proto.ReturnCode.EmptyResponse)
# ---- Deserialize response ----
try:
outputs = response.tensors[0]
deserializer = serialization.get_serializer( outputs.serializer )
outputs = deserializer.deserialize( outputs, to_type = bittensor.proto.TensorType.TORCH )
except Exception as e:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<red>ResponseDeserializationException</red>, <white>from</white>:{}, message:<red>{}</red> ]', caller.endpoint, e)
return zeros, torch.tensor(bittensor.proto.ReturnCode.ResponseDeserializationException)
# ---- Check response shape ----
if outputs.size(0) != inputs.size(0) \
or outputs.size(1) != inputs.size(1) \
or outputs.size(2) != bittensor.__network_dim__:
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<red>ResponseShapeException</red>, <white>from</white>:{}, <white>shape</white>:{}, <white>expected</white>:{}', caller.endpoint, list(outputs.shape), [inputs.size(0), inputs.size(1), bittensor.__network_dim__])
return zeros, torch.tensor(bittensor.proto.ReturnCode.ResponseShapeException)
# ---- Safe catch NaNs and replace with 0.0 ----
outputs = torch.where(torch.isnan(outputs), torch.zeros_like(outputs), outputs)
# ---- Catch all ----
except Exception as e:
logger.error('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<red>UnknownException</red>, <white>from</white>:{}, <white>message</white>:<red>{}</red>', caller.endpoint, e)
return zeros, torch.tensor(bittensor.proto.ReturnCode.UnknownException)
# ---- Return ----
logger.debug('<white>Dendrite</white> <green>Forward Response</green> <--- <white>code</white>:<green>Success</green>, <white>from</white>:{}, <white>outputs</white>:{}', caller.endpoint, outputs.shape)
return outputs, torch.tensor(response.return_code)
def install_agent(request):
from knox.models import AuthToken
client_id = request.data["client"]
site_id = request.data["site"]
version = settings.LATEST_AGENT_VER
arch = request.data["arch"]
# response type is blob so we have to use
# status codes and render error message on the frontend
if arch == "64" and not os.path.exists(
os.path.join(settings.EXE_DIR, "meshagent.exe")):
return Response(status=status.HTTP_406_NOT_ACCEPTABLE)
if arch == "32" and not os.path.exists(
os.path.join(settings.EXE_DIR, "meshagent-x86.exe")):
return Response(status=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
inno = (f"winagent-v{version}.exe"
if arch == "64" else f"winagent-v{version}-x86.exe")
download_url = settings.DL_64 if arch == "64" else settings.DL_32
_, token = AuthToken.objects.create(
user=request.user, expiry=dt.timedelta(hours=request.data["expires"]))
if request.data["installMethod"] == "exe":
from tacticalrmm.utils import generate_winagent_exe
return generate_winagent_exe(
client=client_id,
site=site_id,
agent_type=request.data["agenttype"],
rdp=request.data["rdp"],
ping=request.data["ping"],
power=request.data["power"],
arch=arch,
token=token,
api=request.data["api"],
file_name=request.data["fileName"],
)
elif request.data["installMethod"] == "manual":
cmd = [
inno,
"/VERYSILENT",
"/SUPPRESSMSGBOXES",
"&&",
"ping",
"127.0.0.1",
"-n",
"5",
"&&",
r'"C:\Program Files\TacticalAgent\tacticalrmm.exe"',
"-m",
"install",
"--api",
request.data["api"],
"--client-id",
client_id,
"--site-id",
site_id,
"--agent-type",
request.data["agenttype"],
"--auth",
token,
]
if int(request.data["rdp"]):
cmd.append("--rdp")
if int(request.data["ping"]):
cmd.append("--ping")
if int(request.data["power"]):
cmd.append("--power")
resp = {
"cmd": " ".join(str(i) for i in cmd),
"url": download_url,
}
return Response(resp)
elif request.data["installMethod"] == "powershell":
ps = os.path.join(settings.BASE_DIR, "core/installer.ps1")
with open(ps, "r") as f:
text = f.read()
replace_dict = {
"innosetupchange": inno,
"clientchange": str(client_id),
"sitechange": str(site_id),
"apichange": request.data["api"],
"atypechange": request.data["agenttype"],
"powerchange": str(request.data["power"]),
"rdpchange": str(request.data["rdp"]),
"pingchange": str(request.data["ping"]),
"downloadchange": download_url,
"tokenchange": token,
}
for i, j in replace_dict.items():
text = text.replace(i, j)
file_name = "rmm-installer.ps1"
ps1 = os.path.join(settings.EXE_DIR, file_name)
if os.path.exists(ps1):
try:
os.remove(ps1)
except Exception as e:
logger.error(str(e))
with open(ps1, "w") as f:
f.write(text)
if settings.DEBUG:
with open(ps1, "r") as f:
response = HttpResponse(f.read(), content_type="text/plain")
response[
"Content-Disposition"] = f"inline; filename={file_name}"
return response
else:
response = HttpResponse()
response[
"Content-Disposition"] = f"attachment; filename={file_name}"
response["X-Accel-Redirect"] = f"/private/exe/{file_name}"
return response
def print(self):
if self.has_error:
logger.error("Could not complete run")
pass
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in (
'path',
'add_paused',
'honor_limits',
'bandwidth_priority',
'max_connections',
'max_up_speed',
'max_down_speed',
'ratio',
'main_file_only',
'main_file_ratio',
'magnetization_timeout',
'include_subs',
'content_filename',
'include_files',
'skip_files',
'rename_like_files',
'queue_position',
):
# Values do not merge config with task
# Task takes priority then config is used
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}, 'post': {}}
add = options['add']
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path']))
except RenderError as e:
logger.error('Error setting path for {}: {}', entry['title'],
e)
else:
# Transmission doesn't like it when paths end in a separator
path = path.rstrip('\\/')
add['download_dir'] = pathscrub(path)
# make sure we add it paused, will modify status after adding
add['paused'] = True
change = options['change']
if 'bandwidth_priority' in opt_dic:
change['bandwidthPriority'] = opt_dic['bandwidth_priority']
if 'honor_limits' in opt_dic and not opt_dic['honor_limits']:
change['honorsSessionLimits'] = False
if 'max_up_speed' in opt_dic:
change['uploadLimit'] = opt_dic['max_up_speed']
change['uploadLimited'] = True
if 'max_down_speed' in opt_dic:
change['downloadLimit'] = opt_dic['max_down_speed']
change['downloadLimited'] = True
if 'max_connections' in opt_dic:
change['peer_limit'] = opt_dic['max_connections']
if 'ratio' in opt_dic:
change['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change['seedRatioMode'] = 2
else:
change['seedRatioMode'] = 1
if 'queue_position' in opt_dic:
change['queuePosition'] = opt_dic['queue_position']
post = options['post']
# set to modify paused status after
if 'add_paused' in opt_dic:
post['paused'] = opt_dic['add_paused']
if 'main_file_only' in opt_dic:
post['main_file_only'] = opt_dic['main_file_only']
if 'main_file_ratio' in opt_dic:
post['main_file_ratio'] = opt_dic['main_file_ratio']
if 'magnetization_timeout' in opt_dic:
post['magnetization_timeout'] = opt_dic['magnetization_timeout']
if 'include_subs' in opt_dic:
post['include_subs'] = opt_dic['include_subs']
if 'content_filename' in opt_dic:
try:
post['content_filename'] = entry.render(
opt_dic['content_filename'])
except RenderError as e:
logger.error('Unable to render content_filename {}: {}',
entry['title'], e)
if 'skip_files' in opt_dic:
post['skip_files'] = opt_dic['skip_files']
if not isinstance(post['skip_files'], list):
post['skip_files'] = [post['skip_files']]
if 'include_files' in opt_dic:
post['include_files'] = opt_dic['include_files']
if not isinstance(post['include_files'], list):
post['include_files'] = [post['include_files']]
if 'rename_like_files' in opt_dic:
post['rename_like_files'] = opt_dic['rename_like_files']
return options
async def cog_command_error(self, ctx: commands.Context,
error: commands.CommandError) -> None:
logger.error(error)
logger.error(traceback.format_exc())
await ctx.send(f"{ctx.author.mention}, {error}")
load_dotenv()
environ["no_proxy"] = "*"
from auth_swust import Login
from loguru import logger
from src.gen_ics import generate_class_schedule
from src.parse_api import get_course_api
# 登录部分
username = _("c2i_username")
password = _("c2i_password")
path = _("c2i_path", "./")
if not username or not password:
raise ValueError("未设置用户名密码")
username_li = username.split(",")
password_li = password.split(",")
for u, p in zip(username_li, password_li):
login = Login(u, p)
res, _ = login.try_login()
if res:
sess = login.sess
api = get_course_api(sess)
generate_class_schedule(api, u, path)
else:
logger.error(f"{u} 生成日历出错！")
def error(update, context):
logger.warning('Update "%s" caused error "%s"', update, context.error)
logger.error(traceback.format_exc())
def search(self, task, entry, config=None):
"""
Search for name from piratebay.
"""
if not isinstance(config, dict):
config = {}
self.set_urls(config.get('url', URL))
sort = SORT.get(config.get('sort_by', 'seeds'))
if config.get('sort_reverse'):
sort += 1
if isinstance(config.get('category'), int):
category = config['category']
else:
category = CATEGORIES.get(config.get('category', 'all'))
filter_url = '/0/%d/%d' % (sort, category)
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
# TPB search doesn't like dashes or quotes
query = query.replace('-', ' ').replace("'", " ")
# urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
url = '%s/search/%s%s' % (self.url, quote(
query.encode('utf-8')), filter_url)
logger.debug('Using {} as piratebay search url', url)
page = task.requests.get(url).content
soup = get_soup(page)
for link in soup.find_all('a', attrs={'class': 'detLink'}):
entry = Entry()
entry['title'] = self.extract_title(link)
if not entry['title']:
logger.error(
'Malformed search result. No title or url found. Skipping.'
)
continue
href = link.get('href')
if href.startswith('/'): # relative link?
href = self.url + href
entry['url'] = href
row = link.parent.parent.parent
description = row.find_all('a', attrs={'class': 'detDesc'})
if description and description[0].contents[0] == "piratebay ":
logger.debug('Advertisement entry. Skipping.')
continue
tds = row.find_all('td')
entry['torrent_seeds'] = int(tds[-2].contents[0])
entry['torrent_leeches'] = int(tds[-1].contents[0])
entry['torrent_availability'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
# Parse content_size
size_text = link.find_next(attrs={
'class': 'detDesc'
}).get_text()
if size_text:
size = re.search(r'Size (\d+(\.\d+)?\xa0(?:[PTGMK])?i?B)',
size_text)
if size:
entry['content_size'] = parse_filesize(size.group(1))
else:
logger.error(
'Malformed search result? Title: "{}", No size? {}',
entry['title'],
size_text,
)
entries.add(entry)
return sorted(entries,
reverse=True,
key=lambda x: x.get('torrent_availability'))
def start_server_over_ssh(self):
try:
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(WarningPolicy())
client.connect(
self.ssh_host,
port=self.ssh_port,
username=self.ssh_user,
password=self.text,
)
transport = client.get_transport()
ip, _ = transport.getpeername()
if ip:
self.update_ip_linedt_signal.emit(ip)
logger.info(f"IP for {self.ssh_host} detected as {ip}.")
ws_name = self.run_config["workspace_name"]
server_port = self.run_config["server_port"]
# TODO Check if the server port is already in use
logger.info(
f"Checking if server port: {server_port} at ip: {ip} is already in use."
)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((ip, int(server_port)))
if result == 0:
logger.error(f"Port {server_port} is already open.")
self.button_message_signal.emit(
[f"Port: {server_port} at ip: {ip} is already in use!", "maroon", 3]
)
self.error_signal.emit()
sock.close()
client.close()
return
else:
logger.info(f"Port {server_port} is not open.")
sock.close()
cuda_command = "module load cuda/10.1\n"
command = (
"/dls_sw/apps/SuRVoS2/s2_conda/bin/python -u "
"/dls/science/groups/das/SuRVoS/s2/s2_dec/SuRVoS2/survos.py "
f"start_server {ws_name} {server_port} > {date.today()}_survos2.log &\n"
)
logger.info(f"Running command on remote machine: {command}")
session = transport.open_session()
session.setblocking(0) # Set to non-blocking mode
session.get_pty()
session.invoke_shell()
# Send commands
session.send(cuda_command)
session.send(command)
# Loop for 15 seconds
self.button_message_signal.emit(
[f"Starting server on {self.ssh_host}. Please Wait!", "navy", 14]
)
start = time.time()
while time.time() - start < 15:
if session.recv_ready():
data = session.recv(512)
print(data.decode(), flush=True)
time.sleep(1) # Yield CPU so we don't take up 100% usage...
self.finished.emit()
except AuthenticationException:
logger.error("SSH Authentication failed!")
self.button_message_signal.emit(["Incorrect Password!", "maroon", 3])
self.error_signal.emit()
async def is_sqlite(self, file: Path):
async with aiofiles.open(str(file), "rb") as f:
sixteen = await f.read(16)
return sixteen == b"SQLite format 3\000"
@logger.catch
async def process_sqlite(self, sql_file: Path):
logger.info(f"Processing {sql_file.stem} ")
try:
async with aiosqlite.connect(sql_file) as db:
async with db.execute("SELECT * from results") as cursor:
while row := await cursor.fetchone():
await self._process_json(row[1])
logger.info(f"Finished processing {sql_file.name} ")
except Exception as e:
logger.error(e)
async def process(self, upload: SpooledTemporaryFile, filename: str,
neo_user: str, neo_pass: str):
server = ("bolt://stormspotter-neo4j:7687"
if os.environ.get("DOCKER_STORMSPOTTER") else
"bolt://localhost:7687")
# TODO: Pass whole neo4j params from frontend or use .env for server
self.neo = Neo4j(server=server, user=neo_user, password=neo_pass)
if zipfile.is_zipfile(upload):
self.status = f"Unzipping {filename}"
tempdir = mkdtemp()
zipfile.ZipFile(upload).extractall(tempdir)
sqlite_files = [
f for f in Path(tempdir).glob("*") if await self.is_sqlite(f)
def main():
# Launching appium
parser = argparse.ArgumentParser(
description=
'instr = True to collect code coverage, algo = RL to use RL or random'
'to choose random approach')
parser.add_argument('--timesteps', type=int, default=3600)
parser.add_argument('--iterations', type=int, default=10)
parser.add_argument('--instr_jacoco', default=False, action='store_true')
parser.add_argument('--instr_emma', default=False, action='store_true')
parser.add_argument('--save_policy', default=False, action='store_true')
parser.add_argument('--real_device', default=False, action='store_true')
parser.add_argument('--rotation', default=False, action='store_true')
parser.add_argument('--internet', default=False, action='store_true')
parser.add_argument('--menu', default=False, action='store_true')
parser.add_argument('--algo',
choices=['TD3', 'SAC', 'random', 'Q', 'DDPG', 'test'],
type=str,
required=True)
parser.add_argument('--emu',
choices=['normal', 'headless'],
type=str,
required=False,
default='normal')
parser.add_argument('--appium_port', type=int, required=True)
parser.add_argument('--platform_name',
choices=['Android', 'iOS'],
type=str,
default='Android')
parser.add_argument('--platform_version', type=str, default='9.0')
parser.add_argument('--udid', type=str, default='emulator-5554')
parser.add_argument('--device_name', type=str, default='test0')
parser.add_argument('--android_port', type=str, default='5554')
parser.add_argument('--apps', type=str, required=True)
parser.add_argument('--timer', type=int, default=60)
parser.add_argument('--max_timesteps', type=int, default=250)
parser.add_argument('--pool_strings', type=str, default='strings.txt')
parser.add_argument('--trials_per_app', type=int, default=3)
args = parser.parse_args()
save_policy = args.save_policy
max_trials = args.trials_per_app
if max_trials <= 0:
raise Exception('max_trials must be > 0')
timesteps = args.timesteps
max_timesteps = args.max_timesteps
pool_strings = args.pool_strings
N = args.iterations
instr_jacoco = args.instr_jacoco
instr_emma = args.instr_emma
if instr_emma and instr_jacoco:
raise AssertionError
real_device = args.real_device
algo = args.algo
emu = args.emu
appium_port = args.appium_port
platform_name = args.platform_name
platform_version = args.platform_version
udid = args.udid
# Check this in case of name error
device_name = args.device_name.replace('_', ' ')
rotation = args.rotation
internet = args.internet
merdoso_button_menu = args.menu
android_port = args.android_port
apps = [p for p in args.apps.split(",")]
timer = args.timer
if emu == 'normal':
is_headless = False
else:
is_headless = True
# Put all APKs in folder apps
# path = os.path.join(os.getcwd(), app_path)
my_log = logger.add(
os.path.join('logs', 'logger.log'),
format="{time} {level} {message}",
filter=lambda record: record["level"].name == "INFO" or "ERROR")
appium = AppiumLauncher(appium_port)
if real_device:
emulator = None
else:
emulator = EmulatorLauncher(emu, device_name, android_port)
if len(apps) == 0:
raise Exception(f'The folder is empty or the path is wrong')
exit()
for application in apps:
app_name = os.path.basename(os.path.splitext(application)[0])
logger.info(f'now testing: {app_name}\n')
cycle = 0
trial = 0
coverage_dict_template = {}
try:
exported_activities, services, receivers, providers, string_activities, my_package = apk_analyzer.analyze(
application, coverage_dict_template)
ready = True
except Exception as e:
logger.error(f'{e} at app: {application}')
ready = False
if ready:
package = None
while cycle < N:
logger.info(f'app: {app_name}, test {cycle} of {N} starting')
# coverage dir
coverage_dir = ''
if instr_emma or instr_jacoco:
coverage_dir = os.path.join(os.getcwd(), 'coverage',
app_name, algo, str(cycle))
os.makedirs(coverage_dir, exist_ok=True)
# logs dir
log_dir = os.path.join(os.getcwd(), 'logs', app_name, algo,
str(cycle))
os.makedirs(log_dir, exist_ok=True)
policy_dir = os.path.join(os.getcwd(), 'policies', app_name,
algo)
os.makedirs(policy_dir, exist_ok=True)
# instantiating timer in minutes
coverage_dict = dict(coverage_dict_template)
widget_list = []
bug_set = set()
visited_activities = []
clicked_buttons = []
number_bugs = []
os.system(f'adb -s {udid} install -t -r {application}')
result = subprocess.run([
"adb", "shell", "su", "0", "find", "/data/data/", "-type",
"d", "-name", f'"{my_package}*"'
],
capture_output=True)
package = result.stdout.decode('utf-8').strip('\n').rsplit(
'/')[-1]
try:
app = RLApplicationEnv(
coverage_dict,
app_path=application,
list_activities=list(coverage_dict.keys()),
widget_list=widget_list,
bug_set=bug_set,
coverage_dir=coverage_dir,
log_dir=log_dir,
visited_activities=visited_activities,
clicked_buttons=clicked_buttons,
number_bugs=number_bugs,
string_activities=string_activities,
appium_port=appium_port,
internet=internet,
instr_emma=instr_emma,
instr_jacoco=instr_jacoco,
merdoso_button_menu=merdoso_button_menu,
rotation=rotation,
platform_name=platform_name,
platform_version=platform_version,
udid=udid,
pool_strings=pool_strings,
device_name=device_name,
max_episode_len=max_timesteps,
is_headless=is_headless,
appium=appium,
emulator=emulator,
package=package,
exported_activities=exported_activities,
services=services,
receivers=receivers)
if algo == 'TD3':
algorithm = TD3Algorithm()
elif algo == 'random':
algorithm = RandomAlgorithm()
elif algo == 'SAC':
algorithm = SACAlgorithm()
elif algo == 'Q':
algorithm = QLearnAlgorithm()
elif algo == 'DDPG':
algorithm = DDPGAlgorithm()
elif algo == 'test':
algorithm = TestApp()
flag = algorithm.explore(app, emulator, appium, timesteps,
timer, save_policy, policy_dir,
cycle)
if flag:
with open(f'logs{os.sep}success.log', 'a+') as f:
f.write(f'{app_name}\n')
else:
with open(f'logs{os.sep}error.log', 'a+') as f:
f.write(f'{app_name}\n')
except Exception as e:
logger.error(e)
flag = False
try:
# bye handler, it has been an honour
os.kill(app.bug_proc_pid, 9)
except Exception:
pass
if flag:
try:
app.reset()
app.driver.quit()
except InvalidSessionIdException:
pass
except WebDriverException:
pass
logger.remove(app.logger_id)
logger.remove(app.bug_logger_id)
# save_pickles(algo, app_name, cycle, clicked_buttons, visited_activities, number_bugs, bug_set)
logger.info(
f'app: {app_name}, test {cycle} of {N} ending\n')
cycle += 1
else:
trial += 1
try:
logger.remove(app.logger_id)
logger.remove(app.bug_logger_id)
except Exception:
pass
if trial == max_trials:
try:
app.driver.quit()
except Exception:
pass
logger.error(
f'Too Many Times tried, app: {app_name}, iteration: {cycle}'
)
break
# in order to avoid faulty behavior we uninstall the application
if package:
os.system(f'adb -s {udid} uninstall {package}')
if emulator is not None:
emulator.terminate()
appium.terminate()
return 0
def parse_configs(file_name):
with open(file_name, encoding='utf-8') as y:
y_content = yaml.load(y, Loader=yaml.Loader)
return y_content['addr'], y_content['port']
def get_msg(msg_in):
json_in = json.loads(msg_in)
return json_in
def send_msg(destination_host, msg):
g = json.dumps(msg)
destination_host.send(g.encode('utf-8'))
timestr = time.ctime(time.time()) + "\n"
logger.debug("get_addr_port(dedug)!")
logger.info("get_addr_port(dedug)!")
logger.error("get_addr_port(dedug)!")
logger.debug("parse_configs(dedug)!")
logger.info("parse_configs(dedug)!")
logger.error("parse_configs(dedug)!")
logger.debug("send_msg(dedug)!")
logger.info("send_msg(dedug)!")
logger.error("send_msg(dedug)!")
def getDefaultBrowserByUserName(self, userName):
output = self.volumeInfo
if "FAT" or "NTFS" in output.split(" ")[0]:
os.chdir("%s/%s/" % (self.mountDir, output.split(" ")[2]))
logger.info(
"Loading every user info!") # TODO:It should be per user!
try:
os.chdir("Users/")
except FileNotFoundError:
logger.error("Couldn't find Users folder!")
return None
for userDir in os.listdir("."):
if userName == userDir:
if os.access(
"{0}/AppData/Local/Microsoft/Windows/UsrClass.dat".
format(userDir), os.F_OK | os.R_OK):
registry = Registry.Registry(
"{0}/NTUSER.DAT".format(userDir))
else:
logger.warning("Couldn't find user registry on %s" %
userDir)
continue
try:
open1 = registry.open("http\\shell\\open\\command")
except Registry.RegistryKeyNotFoundException:
logger.error(
"Couldn't find UsrClass http registry on user {0}".
format(userDir))
continue
logger.info("Now showing %s default http open info!" %
userDir)
logger.debug("Default HTTP open handler : {0}".format(
open1.value("(default)").value()))
try:
open2 = registry.open("https\\shell\\open\\command")
except Registry.RegistryKeyNotFoundException:
logger.error(
"Couldn't find UsrClass https registry on user {0}"
.format(userDir))
continue
logger.info("Now showing %s default http open info!" %
userDir)
logger.debug("Default HTTPS open handler : {0}".format(
open2.value("(default)").value()))
try:
open3 = registry.open(".html")
except Registry.RegistryKeyNotFoundException:
logger.error(
"Couldn't find UsrClass http registry on user {0}".
format(userDir))
continue
html_value = open3.value("(default)").value()
try:
open4 = registry.open(html_value)
except Registry.RegistryKeyNotFoundException:
logger.error(
".html association on user {0} may be broken!".
format(userDir))
continue
logger.info("Now showing %s default .html open info!" %
userDir)
logger.debug("Default .html open handler : {0}".format(
open4.subkey("shell").subkey("open").subkey(
"command").value("(default)").value()))
def _add_node(
self,
node,
lib_name,
args,
target_node=None,
recovery=False,
ledger_dir=None,
copy_ledger_read_only=False,
read_only_ledger_dir=None,
from_snapshot=False,
snapshot_dir=None,
):
forwarded_args = {
arg: getattr(args, arg)
for arg in infra.network.Network.node_args_to_forward
}
# Contact primary if no target node is set
if target_node is None:
target_node, _ = self.find_primary(
timeout=args.ledger_recovery_timeout if recovery else 3
)
LOG.info(f"Joining from target node {target_node.node_id}")
# Only retrieve snapshot from target node if the snapshot directory is not
# specified
if from_snapshot and snapshot_dir is None:
snapshot_dir = self.get_committed_snapshots(target_node)
assert os.listdir(
snapshot_dir
), f"There are no snapshots to resume from in directory {snapshot_dir}"
committed_ledger_dir = None
current_ledger_dir = None
if snapshot_dir is not None:
LOG.info(f"Joining from snapshot directory: {snapshot_dir}")
# Only when joining from snapshot, retrieve ledger dirs from target node
# if the ledger directories are not specified. When joining without snapshot,
# the entire ledger will be retransmitted by primary node
current_ledger_dir = ledger_dir or None
committed_ledger_dir = read_only_ledger_dir or None
if copy_ledger_read_only and read_only_ledger_dir is None:
current_ledger_dir, committed_ledger_dir = target_node.get_ledger(
include_read_only_dirs=True
)
node.join(
lib_name=lib_name,
workspace=args.workspace,
label=args.label,
common_dir=self.common_dir,
target_rpc_address=f"{target_node.host}:{target_node.rpc_port}",
snapshot_dir=snapshot_dir,
ledger_dir=current_ledger_dir,
read_only_ledger_dir=committed_ledger_dir,
**forwarded_args,
)
# If the network is opening, node are trusted without consortium approval
if self.status == ServiceStatus.OPENING:
try:
node.wait_for_node_to_join(timeout=JOIN_TIMEOUT)
except TimeoutError:
LOG.error(f"New node {node.node_id} failed to join the network")
raise
node.network_state = infra.node.NodeNetworkState.joined
except Exception:
progress.clear()
logger.warning("Shibboleth rejected your username and/or password.")
exit()
rq = session.post(D2L_BASEURL + "/Shibboleth.sso/SAML2/POST", data=dta, allow_redirects=True)
session.get(D2L_BASEURL + "/d2l/lp/auth/login/ProcessLoginActions.d2l")
logger.info("Successfully logged into MyCourses")
with halo.Halo(text="Discovering Courses", spinner="dots") as progress:
# We need to get the XSRF.Token to move forward
bph = session.get("{}/d2l/le/manageCourses/search/6605".format(D2L_BASEURL))
if bph.status_code != 200:
logger.error("Course Query failed. Invalid response code! Expected 200, got {}", bph.status_code)
exit(1)
token = get_xfrs_token(bph.text)
now = datetime.datetime.now()
query_data = {
"gridPartialInfo$_type": "D2L.LP.Web.UI.Desktop.Controls.GridPartialArgs",
"gridPartialInfo$SortingInfo$SortField": "OrgUnitName",
"gridPartialInfo$SortingInfo$SortDirection": "0",
"gridPartialInfo$NumericPagingInfo$PageNumber": "1",
"gridPartialInfo$NumericPagingInfo$PageSize": "100",
"searchTerm": "",
"status": "-1",
"toStartDate$Year": str(now.year),
"toStartDate$Month": str(now.month),
"toStartDate$Day": str(now.day),
def start_and_join(self, args):
"""
Starts a CCF network.
:param args: command line arguments to configure the CCF nodes.
"""
self.common_dir = get_common_folder_name(args.workspace, args.label)
assert (
args.gov_script is not None
), "--gov-script argument must be provided to start a network"
self._setup_common_folder(args.gov_script)
mc = max(1, args.initial_member_count)
initial_members_info = []
for i in range(mc):
initial_members_info += [
(
i,
(i < args.initial_recovery_member_count),
{"is_operator": True}
if (i < args.initial_operator_count)
else None,
)
]
self.consortium = infra.consortium.Consortium(
self.common_dir,
self.key_generator,
self.share_script,
initial_members_info,
args.participants_curve,
)
initial_users = list(range(max(0, args.initial_user_count)))
self.create_users(initial_users, args.participants_curve)
primary = self._start_all_nodes(args)
self.wait_for_all_nodes_to_catch_up(primary)
LOG.success("All nodes joined network")
self.consortium.activate(primary)
if args.js_app_script:
LOG.error(
"--js-app-script is deprecated - update to --js-app-bundle instead"
)
infra.proc.ccall(
"cp", args.js_app_script, args.binary_dir
).check_returncode()
self.consortium.set_js_app(
remote_node=primary, app_script_path=args.js_app_script
)
if args.js_app_bundle:
self.consortium.deploy_js_app(
remote_node=primary, app_bundle_path=args.js_app_bundle
)
for path in args.jwt_issuer:
self.consortium.set_jwt_issuer(remote_node=primary, json_path=path)
self.consortium.add_users(primary, initial_users)
LOG.info(f"Initial set of users added: {len(initial_users)}")
self.consortium.open_network(remote_node=primary)
self.status = ServiceStatus.OPEN
LOG.success("***** Network is now open *****")
img_file = f'frame_{counter}.jpg'
cv2.imwrite(img_file, frame)
response = f'frame captured from {output["src"]} and saved to {img_file}'
else:
response = queue_img(data=output,
queue_url=output['queue_url'])
logger.info(response)
time.sleep(data['sleep'])
counter += 1
except AttributeError:
# re-establish video capture if 'NoneType' error occurs because the connection has been dropped.
logger.error(
f"re-establishing video connection with {data['src']}")
cap = make_capture_obj(data['src'])
except:
logger.error(traceback.format_exc())
continue
if __name__ == '__main__':
"""entry point"""
try:
data = json.loads(os.getenv('data'))
logger.info(f'received data: {data}')
except TypeError as e:
logger.error('no environmental variables available')
raise
stream_feed(data)
def forward(self, inputs: torch.Tensor, mode: bittensor.proto.Modality) -> Tuple[torch.Tensor, int]:
r""" Torch.nn.Module forward call: Triggers the grpc call to the remote neuron on the associated endpoint.
Call returns the output tensor and a bittensor.proto.ReturnCode.
Args:
inputs (:obj:`List[torch.Tensor]` of shape :obj:`(shape)`, `required`):
Single torch tensor to be sent to the remote neuron endpoint.
mode (:obj:`bittensor.proto.Modality` of shape :obj:`(1)`, `required`):
Bittensor forward modality type. Enum in [TEXT, IMAGE, TENSOR]
Returns:
output (:obj:`Tuple[torch.FloatTensor, torch.LongTensor]`, `required`):
Result tuple from the forward call.
"""
# ---- On Backoff: We dont make an RPC and return zeros instead ----
if self.config.receptor.do_backoff and self.backoff >= 1:
outputs = nill_response_for(inputs)
code = torch.tensor(bittensor.proto.ReturnCode.Backoff)
# ---- On Not-backoff: We make the Forward RPC ----
else:
try:
# Make and time the query.
outputs, code = _ReceptorCall.apply(self, DUMMY, inputs, mode)
# ---- On unknown failure: we return zeros and unknown code ----
except Exception as e:
logger.error('Uncaught error in forward call with error {}, {}'.format( e, traceback.format_exc()))
outputs = nill_response_for(inputs)
code = torch.tensor(bittensor.proto.ReturnCode.UnknownException)
# ---- On Success: set zero backoff and halve the next backoff ----
try:
self.stats.codes[code.item()] += 1
except Exception:
pass
if code.item() == bittensor.proto.ReturnCode.Success:
self.backoff = 0
self.next_backoff = max(1, self.next_backoff / 2)
elif code.item() == bittensor.proto.ReturnCode.EmptyRequest:
# This was a NO-OP
pass
# ---- On Backoff: Lower backoff value by 1 ----
elif code.item() == bittensor.proto.ReturnCode.Backoff:
# We slowly lower the backoff count until 0.
self.backoff -= 1
# ---- On failure: Increase backoff and double next_backoff towards max value ----
# Catch all non-success / non-backoff codes and trigger backoff increase. This catches
# serialization errors, timeouts, unavailable endpoints etc. Note, it can
# be triggered by invalid requests on this side of the query.
else:
# ---- Do backoff: incease backoff until max_backoff is reached ----
self.backoff = self.next_backoff
self.next_backoff = min(self.config.receptor.max_backoff, self.next_backoff * 2)
# ---- Finally return ----
return outputs, code