async def handle_register(req, websocket):
if 'register' == req['method']:
if 'params' in req and 'secret_key' in req[
'params'] and RTR_SECRET_KEY_SHA1 == get_sha1(
req['params']['secret_key']):
if 'view' in req['params']:
view_name = Cached.get_view_name(req['params']['view'])
else:
view_name = Cached.VIEW_DEFAULT
await Cached.add_client(websocket, req['id'], view_name)
data = await Cached.get_global()
torrents = await Cached.get_torrents()
result = {
'version': RTR_VERSION,
'global': data.__dict__,
'torrents': [t.__dict__ for t in torrents]
}
plugins_data = {}
for plugin in Cached.plugins:
plugin_output = await plugin.get(False)
if plugin_output is not None:
plugins_data[plugin.name()] = plugin_output
if plugins_data:
result['plugins'] = plugins_data
result = Cached.filter_by_view(result, view_name)
response_json = get_json_response(req['id'], result)
return response_json
def get_objects_on_disk(self):
"""
get_objects_on_disk(self)
Walk though local storage and build one giant dictionary of objects on disk
"""
objects_on_disk = {}
download_path = self.options['dest_sync']
if 'shelf' in self.options:
download_path = os.path.join(download_path, self.options['shelf'])
for (root, dirs, files) in os.walk(download_path):
for f in files:
obj = os.path.join(root, f)
object_name = utils.unix_path(
os.path.relpath(obj, self.options['dest_sync'])
)
# Return sha1 hash if checksum is enabled
if self.options['checksum']:
objects_on_disk.update({object_name: {
'sha1_hash': utils.get_sha1(obj),
}})
else:
objects_on_disk.update({object_name: {
'modified_time': utils.get_modified_time(obj),
'file_size': utils.get_file_size(obj)
}})
return objects_on_disk
def get_objects_on_disk(self):
"""
get_objects_on_disk(self)
Walk though local storage and build one giant dictionary of objects on disk
"""
objects_on_disk = {}
download_path = self.options['dest_sync']
if 'shelf' in self.options:
download_path = os.path.join(download_path, self.options['shelf'])
for (root, dirs, files) in os.walk(download_path):
for f in files:
obj = os.path.join(root, f)
object_name = utils.unix_path(
os.path.relpath(obj, self.options['dest_sync']))
# Return sha1 hash if checksum is enabled
if self.options['checksum']:
objects_on_disk.update(
{object_name: {
'sha1_hash': utils.get_sha1(obj),
}})
else:
objects_on_disk.update({
object_name: {
'modified_time': utils.get_modified_time(obj),
'file_size': utils.get_file_size(obj)
}
})
return objects_on_disk
def checking_inaccessibility(self, key, data_len=None):
""" Checking that data is inaccessible
"""
try:
result_data = self.read_data_sync(key).pop().data
except Exception as e:
print e.message
else:
print len(result_data), '/', data_len, 'bytes already accessible'
assert utils.get_sha1(result_data) != key
def Main():
(options, args) = parse_options()
assert len(args) == 1
name = args[0]
print 'Creating archive for %s' % name
if not name in os.listdir('.'):
print 'You must be standing directly below the directory you are uploading'
return 1
filename = utils.create_archive(name)
sha1 = utils.get_sha1(filename)
dest = os.path.join(GMSCORE_DEPS, sha1)
uploadFile(filename, dest)
sha1_file = '%s.sha1' % filename
with open(sha1_file, 'w') as output:
output.write(sha1)
print 'Sha (%s) written to: %s' % (sha1, sha1_file)
def Main():
(options, args) = parse_options()
assert len(args) == 1
sha1_file = args[0]
dest = sha1_file[:-5]
print 'Ensuring %s' % dest
with open(sha1_file, 'r') as input_sha:
sha1 = input_sha.readline()
if os.path.exists(dest) and utils.get_sha1(dest) == sha1:
print 'sha1 matches, not downloading'
dest_dir = extract_dir(dest)
if os.path.exists(dest_dir):
print 'destination directory exists, no extraction'
else:
unpack_archive(dest)
return
src = os.path.join(GMSCORE_DEPS, sha1)
if not os.path.exists(src):
print 'File (%s) does not exist on x20' % src
print 'Downloading %s to %s' % (src, dest)
shutil.copyfile(src, dest)
unpack_archive(dest)
def main():
# We need prodaccess to upload to x20
utils.check_prodacces()
working_dir = run_on_as_app.WORKING_DIR
print 'Removing directories that do not match checked out revision'
with utils.ChangedWorkingDirectory(working_dir):
for repo in run_on_as_app.APP_REPOSITORIES:
repo_dir = os.path.join(working_dir, repo.name)
if os.path.exists(repo_dir) \
and utils.get_HEAD_sha1_for_checkout(repo_dir) != repo.revision:
print 'Removing %s' % repo_dir
shutil.rmtree(repo_dir)
print 'Downloading all missing apps'
run_on_as_app.clone_repositories(quiet=False)
# Package all files as x20 dependency
parent_dir = os.path.dirname(working_dir)
with utils.ChangedWorkingDirectory(parent_dir):
print 'Creating archive for opensource_apps (this may take some time)'
working_dir_name = os.path.basename(working_dir)
repo_dirs = [working_dir_name + '/' + repo.name
for repo in run_on_as_app.APP_REPOSITORIES]
filename = utils.create_archive("opensource_apps", repo_dirs)
sha1 = utils.get_sha1(filename)
dest = os.path.join(upload_to_x20.GMSCORE_DEPS, sha1)
upload_to_x20.uploadFile(filename, dest)
sha1_file = '%s.sha1' % filename
with open(sha1_file, 'w') as output:
output.write(sha1)
shutil.move(sha1_file,
os.path.join(utils.THIRD_PARTY, 'opensource_apps.tar.gz.sha1'))
print 'To have apps benchmarked on Golem, the updated apps have to be ' \
'downloaded to the runners by ssh\'ing into each runner and do:\n' \
'cd ../golem\n' \
'update_dependencies.sh\n'
from cachetools import TTLCache, cached
from websockets import WebSocketException
from diffs import map_diff, map_get_multi_diff
from model import Client
from plugins import DiskUsage
from remote import Remote
from utils import Logger, get_sha1, getenv_path
# logging.getLogger("asyncio").setLevel(logging.INFO)
RTR_CERT_PATH = getenv_path('RTR_CERT_PATH', './cert/cert.pem')
RTR_RETR_INTERVAL = int(os.getenv('RTR_RETR_INTERVAL', 5))
RTR_SHORT_CACHE_TTL = int(os.getenv('RTR_SHORT_CACHE_TTL', 5))
RTR_LISTEN_HOST = os.getenv('RTR_LISTEN_HOST', '127.0.0.1')
RTR_LISTEN_PORT = int(os.getenv('RTR_LISTEN_PORT', 8765))
RTR_SECRET_KEY_SHA1 = getenv_path('RTR_SECRET_KEY_SHA1', get_sha1('abc123'))
SOCK_PATH = getenv_path('RTR_SCGI_SOCKET_PATH', './.rtorrent.sock')
RTR_PID_PATH = getenv_path('RTR_PID_PATH', './wss_server.pid')
RTR_PLUGINS_DISK_USAGE_PATHS = os.getenv('RTR_PLUGINS_DISK_USAGE_PATHS', '/')
logger = Logger.get_logger()
RTR_VERSION = '__RTR_VERSION_PLACEHOLDER__'
class Cached:
VIEW_DEFAULT = 'main'
VIEW_NAME = 'name'
VIEWS = {
VIEW_DEFAULT, VIEW_NAME, 'started', 'stopped', 'complete',
'incomplete', 'hashing', 'seeding', 'leeching', 'active'
}
global_data = None
# Skip weekends
dt = datetime.utcnow()
isWeekend = (datetime.now(timezone('US/Eastern')).isoweekday() >= 6)
if not isWeekend:
# DOWNLOAD THE FILE
try:
r = requests.get(url)
if r.ok:
with open(file_name, 'wb') as outfile:
outfile.write(r.content)
date_time_path = dt.strftime(
'%Y/%m/%d/%H/%Y-%m-%dT%H-%M-%S')
# 2021/02/17/02/2021-02-17T02-01-09
# SAVE NEW VERSIONS TO B2 STORAGE
sha1str = get_sha1(file_name)
if sha1str != db[
KEY.grab_file_hash]: # It's new! Save it to B2!
print("[{0}] SHA1: {1}".format(run_count, sha1str))
db[KEY.grab_file_hash] = sha1str
# Upload new version to b2
destination_path = 'TDTrackerMattZ/{0}_{1}'.format(
date_time_path, file_name)
try:
bucket.upload_file(Filename=file_name,
Key=destination_path)
except:
print(
'FAILED TO UPLOAD, OH WELL BETTER LUCK NEXT TIME!'
)
except:
def process_objects(self, expanded_objects=[]):
"""
process_objects(expanded_objects)
Given a list of objects, determines if uploadable (binary), and
then create a dictionary of:
sha1_hash
sha256_hash
modified_time
filesize
Sha1_hash is only determined on first upload or if modified time and
file size changed.
"""
objects_metadata = {}
for obj in expanded_objects:
# Process if object is uploadable
if self.uploadable_object(obj):
# Object name in metadata file. Replace \\ with / to remain consistent
# accoss platforms
object_name = utils.unix_path(
os.path.relpath(obj, self.paths['shelves']))
# Determine paths
object_path = os.path.abspath(obj)
object_metadata_file = '%s.pitem' % object_path
# Add object to gitignore
self.add_object_to_gitignore(obj)
object_mtime = utils.get_modified_time(obj)
object_file_size = utils.get_file_size(obj)
# Use cached checksum since checksum hashing is cpu intensive and
# file size and modified times are quicker. Checksums are force using
# cli flag --checksum.
if (not self.options['checksum']
and os.path.exists(object_metadata_file)):
with open(object_metadata_file) as json_file:
cached_metadata = json.load(json_file)
# Use cached hash if filesize and mtime are the same
if (object_file_size
== cached_metadata[object_name]['file_size']
and object_mtime
== cached_metadata[object_name]['modified_time']):
object_sha1_hash = cached_metadata[object_name][
'sha1_hash']
if 'sha26_hash' in cached_metadata[object_name]:
object_sha256_hash = cached_metadata[object_name][
'sha256_hash']
else:
object_sha256_hash = utils.get_sha256(obj)
else:
object_sha1_hash = utils.get_sha1(obj)
object_sha256_hash = utils.get_sha256(obj)
else:
# Genertate hash if cached_metadat is not present
object_sha1_hash = utils.get_sha1(obj)
object_sha256_hash = utils.get_sha256(obj)
# TODO remove sha1 check as its not needed.
# Add object to metadata dictionary
objects_metadata[object_name] = {
'sha1_hash': object_sha1_hash,
'sha256_hash': object_sha256_hash,
'modified_time': object_mtime,
'file_size': object_file_size,
}
return objects_metadata
def process_objects(self, expanded_objects=[]):
"""
process_objects(expanded_objects)
Given a list of objects, determines if uploadable (binary), and
then create a dictionary of:
sha1_hash
sha256_hash
modified_time
filesize
Sha1_hash is only determined on first upload or if modified time and
file size changed.
"""
objects_metadata = {}
for obj in expanded_objects:
# Process if object is uploadable
if self.uploadable_object(obj):
# Object name in metadata file. Replace \\ with / to remain consistent
# accoss platforms
object_name = utils.unix_path(
os.path.relpath(obj, self.paths['shelves'])
)
# Determine paths
object_path = os.path.abspath(obj)
object_metadata_file = '%s.pitem' % object_path
# Add object to gitignore
self.add_object_to_gitignore(obj)
object_mtime = utils.get_modified_time(obj)
object_file_size = utils.get_file_size(obj)
# Use cached checksum since checksum hashing is cpu intensive and
# file size and modified times are quicker. Checksums are force using
# cli flag --checksum.
if (
not self.options['checksum'] and
os.path.exists(object_metadata_file)
):
with open(object_metadata_file) as json_file:
cached_metadata = json.load(json_file)
# Use cached hash if filesize and mtime are the same
if (
object_file_size == cached_metadata[object_name]['file_size'] and
object_mtime == cached_metadata[object_name]['modified_time']
):
object_sha1_hash = cached_metadata[object_name]['sha1_hash']
if 'sha26_hash' in cached_metadata[object_name]:
object_sha256_hash = cached_metadata[object_name]['sha256_hash']
else:
object_sha256_hash = utils.get_sha256(obj)
else:
object_sha1_hash = utils.get_sha1(obj)
object_sha256_hash = utils.get_sha256(obj)
else:
# Genertate hash if cached_metadat is not present
object_sha1_hash = utils.get_sha1(obj)
object_sha256_hash = utils.get_sha256(obj)
# TODO remove sha1 check as its not needed.
# Add object to metadata dictionary
objects_metadata[object_name] = {
'sha1_hash': object_sha1_hash,
'sha256_hash': object_sha256_hash,
'modified_time': object_mtime,
'file_size': object_file_size,
}
return objects_metadata
f'https://api.codenation.dev/v1/challenge/dev-ps/submit-solution?token={key}'
}
r = requests.get(urls['generate_file'])
utils.write_json_content('answer.json', r.json())
# Second part
json_content = utils.read_json_content('answer.json')
json_dict = utils.to_dict(json_content)
encrypted_text = json_dict['cifrado']
key = json_dict['numero_casas']
decrypted_text = utils.decrypt(encrypted_text, key)
sha1_text = utils.get_sha1(decrypted_text)
json_dict['decifrado'] = decrypted_text
json_dict['resumo_criptografico'] = sha1_text
utils.write_json_content('answer.json', json_dict)
# Third Part
files = {'answer': open('answer.json', 'rb')}
r = requests.post(urls['submit_file'], files=files)
print(r.text)
