def test_storage_simple_save(self):
"""
Make sure that saving to remote locations actually works
"""
def task(name, local, remote, cache_key):
remote.save(name, local.open(name))
self.assertTrue(isinstance(local, FileSystemStorage))
self.assertTrue(isinstance(remote, FileSystemStorage))
remote.save(name, local.open(name))
def delay(*args, **kwargs):
task(*args, **kwargs)
task.delay = delay
storage = QueuedStorage(
local=FileSystemStorage(location=self.local_dir),
remote=FileSystemStorage(location=self.remote_dir), task=task)
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertTrue(path.isfile(path.join(self.local_dir, obj.file.name)))
self.assertTrue(path.isfile(path.join(self.remote_dir, obj.file.name)))
def load_data():
"""Loads movie_data, cust_data, and answers from pickles.
Returns:
The tuple (movie_data, cust_data, answers) with the objects loaded from
their pickles.
"""
# load movie data cache
if isfile(CACHE_LOC + MOVIE_PICKLE):
with open(CACHE_LOC + MOVIE_PICKLE, 'rb') as movie_file:
movie_data = load(movie_file)
else:
movie_data = loads(urlopen(CACHE_URL + MOVIE_PICKLE).read())
# load customer data cache
if isfile(CACHE_LOC + CUSTOMER_PICKLE):
with open(CACHE_LOC + CUSTOMER_PICKLE, 'rb') as cust_file:
cust_data = load(cust_file)
else:
cust_data = loads(urlopen(CACHE_URL + CUSTOMER_PICKLE).read())
# load answers
if isfile(CACHE_LOC + ANSWER_PICKLE):
with open(CACHE_LOC + ANSWER_PICKLE, 'rb') as answer_file:
answers = load(answer_file)
else:
answers = loads(urlopen(CACHE_URL + ANSWER_PICKLE).read())
return(movie_data, cust_data, answers)
def main(max_stations=0, folder='.'):
try:
makedirs(output_folder+'/'+folder)
except OSError:
pass
all_files = [ f for f in listdir(data_folder) if isfile(join(data_folder,f)) and f.endswith('.gz') ]
for ndf in all_files:
string = '_%dstations' % max_stations
new_name=ndf[:-7]+string+ndf[-7:]
rename(data_folder+'/'+ndf, data_folder+'/'+new_name)
all_files = [ f for f in listdir(data_folder) if isfile(join(data_folder,f)) and f.endswith('.gz') ]
for a_f in all_files:
move(data_folder+'/'+a_f, output_folder+'/'+folder+'/'+a_f)
print "Moved:", a_f[0:-3]
data_files = [ f for f in listdir(output_folder+'/'+folder) if isfile(join(output_folder+'/'+folder,f)) and f.endswith('.dat.gz') ]
print "\n"
for d_f in data_files:
fin = gzip.open(output_folder+'/'+folder+'/'+d_f, 'rb')
data = fin.read()
fin.close()
with open(output_folder+'/'+folder+'/'+d_f[0:-3],'w') as fout:
fout.write(data)
print "Unzipped:", d_f[0:-3]
def loadPlugin(package, filename, *args):
modName = "%s.%s" % (__name__, path.basename(filename))
# Search for full filename
fullPath = path.join(_ROOT, package, filename)
if not path.isfile(fullPath):
fullPath = fullPath + ".py"
if not path.isfile(fullPath):
raise Exception("Unable to find module: %s" % fullPath)
try:
# Load file
logging.debug("Loading module '%s' at: %s" % (modName, fullPath))
module = imp.load_source(__name__, fullPath)
if not module:
raise Exception('Error loading module source')
# Create instance using `create`
logging.debug("Creating instance of module '%s'" % modName)
inst = module.create(*args)
# Validate
if not inst:
raise Exception("Create did not return a valid instance")
if len(inst.__class__.__bases__) == 0:
logging.warning("Class '%s' does not inherit from base class", modName)
return inst
except Exception as e:
logging.error("Error loading module: %s" % e)
raise Exception("Error loading module: %s" % e)
def check_local_config_schema(build):
log.info("Verifying your configuration settings...")
# leave this import here: might not be on sys.path in some situations
import validictory
local_conf_filename = build.tool_config.get('general.local_config')
if local_conf_filename is not None:
# explicit conf file defined
if not path.isfile(local_conf_filename):
raise ConfigurationError("{file} does not exist!".format(file=local_conf_filename))
else:
local_conf_filename = 'local_config.json'
if not path.isfile(local_conf_filename):
log.warning("Local configuration file '{file}' does not exist!".format(file=local_conf_filename))
with open(local_conf_filename) as local_conf_file:
local_conf = json.load(local_conf_file)
from forge.remote import Remote
from forge import build_config
remote = Remote(build_config.load())
local_conf_schema = remote._api_get('platform/{platform_version}/local_config_schema'.format(
platform_version=build.config['platform_version']))
try:
validictory.validate(local_conf, local_conf_schema)
except validictory.validator.UnexpectedPropertyError as e:
log.warning('Unexpected setting: "{error}" in "{file}". This will be ignored.'.format(
file=local_conf_filename,
error=e)
)
log.info("Configuration settings check complete")
def test_delayed_storage(self):
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
delayed=True)
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertIsNone(getattr(obj.file.storage, 'result', None))
self.assertFalse(
path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file should not be transferred automatically.")
result = obj.file.storage.transfer(obj.file.name)
result.get()
self.assertTrue(
path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file is not available.")
def initializeInitD(ownerName):
if (os.path.isdir(initdDirName)):
fn = join(RANGER_USERSYNC_HOME, initdProgramName)
initdFn = join(initdDirName, initdProgramName)
shutil.copy(fn, initdFn)
if (ownerName != 'ranger'):
f = open(initdFn, 'r')
filedata = f.read()
f.close()
find_str = "LINUX_USER=ranger"
replace_str = "LINUX_USER="******"/etc/rc2.d", "/etc/rc3.d", "/etc/rc.d/rc2.d", "/etc/rc.d/rc3.d"]
for rcDir in rcDirList:
if (os.path.isdir(rcDir)):
for prefix in initPrefixList:
scriptFn = prefix + initdProgramName
scriptName = join(rcDir, scriptFn)
if isfile(scriptName) or os.path.islink(scriptName):
os.remove(scriptName)
os.symlink(initdFn, scriptName)
userSyncScriptName = "ranger-usersync-services.sh"
localScriptName = os.path.abspath(join(RANGER_USERSYNC_HOME, userSyncScriptName))
ubinScriptName = join("/usr/bin", initdProgramName)
if isfile(ubinScriptName) or os.path.islink(ubinScriptName):
os.remove(ubinScriptName)
os.symlink(localScriptName, ubinScriptName)
def download_properties(root_dir):
""" Get the download properties.
First tries to find the properties file in the given root directory,
and if not found there, tries in the Gerrit settings folder in the
user's home directory.
Returns a set of download properties, which may be empty.
"""
p = {}
local_prop = path.join(root_dir, LOCAL_PROPERTIES)
if not path.isfile(local_prop):
local_prop = path.join(GERRIT_HOME, LOCAL_PROPERTIES)
if path.isfile(local_prop):
try:
with open(local_prop) as fd:
for line in fd:
if line.startswith('download.'):
d = [e.strip() for e in line.split('=', 1)]
name, url = d[0], d[1]
p[name[len('download.'):]] = url
except OSError:
pass
return p
def test_transfer_and_delete(self):
"""
Make sure the TransferAndDelete task does what it says
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
task='queued_storage.tasks.TransferAndDelete')
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
obj.file.storage.result.get()
self.assertFalse(
path.isfile(path.join(self.local_dir, obj.file.name)),
"Local file is still available")
self.assertTrue(
path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file is not available.")
def get_module_source(modname):
"""Try to find the source code for a module.
Can return ('file', 'filename') in which case the source is in the given
file, or ('string', 'source') which which case the source is the string.
"""
if modname not in sys.modules:
try:
__import__(modname)
except Exception as err:
raise PycodeError('error importing %r' % modname, err)
mod = sys.modules[modname]
if hasattr(mod, '__loader__'):
try:
source = mod.__loader__.get_source(modname)
except Exception as err:
raise PycodeError('error getting source for %r' % modname, err)
return 'string', source
filename = getattr(mod, '__file__', None)
if filename is None:
raise PycodeError('no source found for module %r' % modname)
filename = path.normpath(path.abspath(filename))
lfilename = filename.lower()
if lfilename.endswith('.pyo') or lfilename.endswith('.pyc'):
filename = filename[:-1]
if not path.isfile(filename) and path.isfile(filename + 'w'):
filename += 'w'
elif not (lfilename.endswith('.py') or lfilename.endswith('.pyw')):
raise PycodeError('source is not a .py file: %r' % filename)
if not path.isfile(filename):
raise PycodeError('source file is not present: %r' % filename)
return 'file', filename
def delete_certificate(self, certificate_id, hackathon):
"""Delete certificate by azureKey.id and hackathon
Delete the hackathon-azureKey relationship first. If the auzreKey is not needed any more, delete the azureKey too
:type certificate_id: int
:param certificate_id: id of AzureKey
:type hackathon: Hackathon
:param hackathon: instance of Hackathon
"""
# delete all hackathon-azureKey relationships first
azure_key = AzureKey.objects(id=certificate_id).first()
# if no relations left, delete the azureKey itself
if azure_key in hackathon.azure_keys:
try:
if isfile(azure_key.cert_url):
os.remove(azure_key.cert_url)
else:
self.storage.delete(azure_key.cert_url)
if isfile(azure_key.pem_url):
os.remove(azure_key.pem_url)
else:
self.storage.delete(azure_key.pem_url)
except Exception as e:
self.log.error(e)
hackathon.azure_keys.remove(azure_key)
hackathon.save()
return ok(True)
def nb_renderer(full_path):
directory, base = split(full_path)
cache_file = join(directory, '.%s.html' % base)
if not current_app.config.get('DEBUG'):
try:
if isfile(cache_file) and getmtime(full_path) < getmtime(cache_file):
current_app.logger.debug('Using Cache File %s' % cache_file)
return raw_renderer(cache_file)
except:
current_app.logger.warn('There was an error reading from the cache file %s' % cache_file)
ex = HTMLExporter(extra_loaders=[current_app.jinja_env.loader],
template_file='wakari_notebook.html')
ex.environment.globals.update(current_app.jinja_env.globals)
current_app.update_template_context(ex.environment.globals)
ex.environment.globals.update(dirname=dirname(request.view_args['path']))
output, _ = ex.from_filename(full_path)
try:
with open(cache_file, 'w') as fd:
current_app.logger.debug('Writing Cache File %s' % cache_file)
fd.write(output.encode(errors='replace'))
except (OSError, IOError):
current_app.logger.warn('There was an error writing to the cache file %s' % cache_file)
try:
if isfile(cache_file): os.unlink(cache_file)
except OSError:
current_app.logger.warn('There was an error removing the cache file %s' % cache_file)
pass
return output
def get_recipes(filter):
with open('support.json') as fi:
supported = json.load(fi)
result = 'result.txt'
if isfile(result):
with open(result) as fi:
for line in fi:
line = line.split()
if(line[1] == "OK"):
OK.append(line[0])
for dn in sorted(os.listdir('..')):
if(dn not in OK):
recipe_dir = join('..', dn)
meta_path = join(recipe_dir, 'meta.yaml')
if not isfile(meta_path):
continue
with open(meta_path) as fi:
data = fi.read()
name = yaml.load(data)['package']['name']
if name not in supported:
continue
sl = supported[name]
if sl == 'all':
ALLOWED.append(name)
yield recipe_dir, name
else:
assert isinstance(sl, list)
if filter in sl:
ALLOWED.append(name)
yield recipe_dir, name
def _get_feature(self, typename):
fname_feature = 'features/feature_%s.npy' % typename
fname_names = 'features/names_%s.pkl' % typename
if path.isfile(fname_feature) and path.isfile(fname_names):
feature = np.load(fname_feature)
names = pickle.load(open(fname_names, 'rb'))
return feature, names
def test_create_structure(tmpfolder):
struct = {"my_file": "Some content",
"my_folder": {
"my_dir_file": "Some other content",
"empty_file": "",
"file_not_created": None
},
"empty_folder": {}}
expected = {"my_file": "Some content",
"my_folder": {
"my_dir_file": "Some other content",
"empty_file": ""
},
"empty_folder": {}}
changed, _ = structure.create_structure(struct, {})
assert changed == expected
assert isdir("my_folder")
assert isdir("empty_folder")
assert isfile("my_folder/my_dir_file")
assert isfile("my_folder/empty_file")
assert not isfile("my_folder/file_not_created")
assert isfile("my_file")
assert open("my_file").read() == "Some content"
assert open("my_folder/my_dir_file").read() == "Some other content"
assert open("my_folder/empty_file").read() == ""
def install():
"""Install the Python CGI"""
# Update the Package Index
local("sudo apt-get update")
# Check pip
try:
pip_version = check_output("pip -V", shell=True)
except CalledProcessError:
# Install pip
downloads_path = expanduser("~/Downloads")
local("cd %s; wget https://bootstrap.pypa.io/get-pip.py" %
downloads_path)
local("cd %s; sudo python get-pip.py" % downloads_path)
pip_path = join(downloads_path, "get-pip.py")
if exists(pip_path) and isfile(pip_path):
remove(pip_path)
# Installing Apache 2
local("echo \"Y\" | sudo apt-get install apache2")
# Disable multithreading processes
local("sudo a2dismod mpm_event")
# Give Apache explicit permission to run scripts
local("sudo a2enmod mpm_prefork cgi")
# Configuration
config_file_abs_path = "/etc/apache2/sites-enabled/000-default.conf"
if exists(config_file_abs_path) and isfile(config_file_abs_path):
try:
with open(config_file_abs_path, 'r') as f:
config_file_lines = f.readlines()
except Exception as exception: # Python3 PermissionError
error_code = exception.errno
if error_code == EACCES: # 13
print(messages["_error_NoRoot"])
exit(1)
else:
print(messages["_error_Oops"] % strerror(error_code))
exit(1)
try:
with open(config_file_abs_path, 'w') as f:
for line in config_file_lines:
f.write(line)
line_strip = line.strip()
if line_strip == "<VirtualHost *:80>":
f.write("\t<Directory /var/www/html>\n")
f.write("\t\tOptions +ExecCGI\n")
f.write("\t\tDirectoryIndex index.html\n")
f.write("\t</Directory>\n")
f.write("\tAddHandler cgi-script .py\n")
except Exception as exception: # Python3 PermissionError
error_code = exception.errno
if error_code == EACCES: # 13
print(messages["_error_NoRoot"])
exit(1)
else:
print(messages["_error_Oops"] % strerror(error_code))
exit(1)
# Restart Apache
local("sudo service apache2 restart")
# Output
print(messages["_installed"])
def get_npz(name):
fname = 'npz_data/%s.npz' % name
if self.use_saved_npz and path.isfile(fname):
all_data = np.load(fname)
# Each work contains many parts. Loop through each one.
return [all_data[i] for i in all_data.files]
music_file = self._get_path('data/', name + '.krn')
if not path.isfile(music_file):
music_file = music_file[:-3] + 'xml'
if not path.isfile(music_file):
raise Exception("Cannot find score for %s" % music_file[:-4])
score = music21.converter.parse(music_file)
all_arr = []
for part in score.parts:
arr = []
for note in part.flat:
if isinstance(note, music21.note.Note):
elem = (note.ps, note.quarterLength)
elif isinstance(note, music21.note.Rest):
elem = (0.0, note.quarterLength)
else:
continue
arr.append(elem)
all_arr.append(np.array(arr))
if self.save_data:
np.savez(fname, *all_arr)
return all_arr
def test_render_report():
"""Test rendering -*.fif files for mne report.
"""
report = Report(info_fname=raw_fname)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.parse_folder(data_path=base_dir)
assert_true(len(w) == 1)
# Check correct paths and filenames
assert_true(raw_fname in report.fnames)
assert_true(event_name in report.fnames)
assert_true(report.data_path == base_dir)
# Check if all files were rendered in the report
fnames = glob.glob(op.join(base_dir, '*.fif'))
bad_name = 'test_ctf_comp_raw-eve.fif'
decrement = any(fname.endswith(bad_name) for fname in fnames)
fnames = [fname for fname in fnames if
fname.endswith(('-eve.fif', '-ave.fif', '-cov.fif',
'-sol.fif', '-fwd.fif', '-inv.fif',
'-src.fif', '-trans.fif', 'raw.fif',
'sss.fif', '-epo.fif')) and
not fname.endswith(bad_name)]
# last file above gets created by another test, and it shouldn't be there
for fname in fnames:
assert_true(''.join(report.html).find(op.basename(fname)) != -1)
assert_equal(len(report.fnames), len(fnames))
assert_equal(len(report.html), len(report.fnames))
evoked1 = read_evokeds(evoked1_fname)
evoked2 = read_evokeds(evoked2_fname)
assert_equal(len(report.fnames) + len(evoked1) + len(evoked2) - 2,
report.initial_id - decrement)
# Check saving functionality
report.data_path = tempdir
report.save(fname=op.join(tempdir, 'report.html'), open_browser=False)
assert_true(op.isfile(op.join(tempdir, 'report.html')))
# Check add_section functionality
fig = evoked1[0].plot(show=False)
report.add_section(figs=fig, # test non-list input
captions=['evoked response'])
assert_equal(len(report.html), len(fnames) + 1)
assert_equal(len(report.html), len(report.fnames))
assert_raises(ValueError, report.add_section, figs=[fig, fig],
captions='H')
# Check saving same report to new filename
report.save(fname=op.join(tempdir, 'report2.html'), open_browser=False)
assert_true(op.isfile(op.join(tempdir, 'report2.html')))
# Check overwriting file
report.save(fname=op.join(tempdir, 'report.html'), open_browser=False,
overwrite=True)
assert_true(op.isfile(op.join(tempdir, 'report.html')))
def main_loop(argv=None):
if argv is None:
argv = sys.argv
args = docopt.docopt(get_updated_docstring(), argv=argv[1:],
version=".".join(map(str, __version__)))
if not args["--silent"]:
logcfg.set_loglevel(log, "INFO")
for h in log.handlers:
logcfg.set_loglevel(h, "INFO")
elif args["--verbose"] > 0:
logcfg.make_verbose()
log.debug(pf(args))
ext = args["--extension"]
recursive = args["--recursive"]
files_and_folders = []
files_and_folders.extend(args["<file_or_folder>"])
for faf in files_and_folders:
if osp.isfile(faf):
parse_file(faf, args)
elif osp.isdir(faf):
for entry in os.listdir(faf):
path = osp.join(faf, entry)
valid_file = osp.isfile(path)\
and osp.splitext(path)[-1] == ext\
and osp.basename(osp.splitext(path)[0]) != "cfg"
valid_folder = recursive and osp.isdir(path)
if valid_file or valid_folder:
files_and_folders.append(path)
def build_chm_doc(libname):
"""Return CHM documentation file (on Windows only), which is copied under
{PythonInstallDir}\Doc, hence allowing Spyder to add an entry for opening
package documentation in "Help" menu. This has no effect on a source
distribution."""
args = ''.join(sys.argv)
if os.name == 'nt' and ('bdist' in args or 'build' in args):
try:
import sphinx # analysis:ignore
except ImportError:
print('Warning: `sphinx` is required to build documentation',
file=sys.stderr)
return
hhc_base = r'C:\Program Files%s\HTML Help Workshop\hhc.exe'
for hhc_exe in (hhc_base % '', hhc_base % ' (x86)'):
if osp.isfile(hhc_exe):
break
else:
print('Warning: `HTML Help Workshop` is required to build CHM '\
'documentation file', file=sys.stderr)
return
doctmp_dir = 'doctmp'
subprocess.call('sphinx-build -b htmlhelp doc %s' % doctmp_dir,
shell=True)
atexit.register(shutil.rmtree, osp.abspath(doctmp_dir))
fname = osp.abspath(osp.join(doctmp_dir, '%s.chm' % libname))
subprocess.call('"%s" %s' % (hhc_exe, fname), shell=True)
if osp.isfile(fname):
return fname
else:
print('Warning: CHM building process failed', file=sys.stderr)
def crop(self, dimensions=None, links_to_create=None):
if path.isfile(settings.USER_FILES_ROOT + self.full_filename()):
return # already done, skip
if not path.isfile(settings.UPLOADED_FILES_ROOT + self.full_filename()):
return # source image doesn't exist, can't crop it
if not links_to_create:
links_to_create = []
x = y = w = h = 0
if dimensions:
x = dimensions['x']
y = dimensions['y']
w = dimensions['w']
h = dimensions['h']
# Queue a job for the cropping/resizing gearman worker
gm_client = libgearman.Client()
for server in settings.GEARMAN_SERVERS:
gm_client.add_server(server)
workload = {'file_hash': self.filename, 'format': self.format,
'x': x, 'y': y, 'w': w, 'h': h, 'links': links_to_create}
gm_client.do_background('cropresize', json.dumps(workload))
def build_specific(self, filenames):
# type: (List[unicode]) -> None
"""Only rebuild as much as needed for changes in the *filenames*."""
# bring the filenames to the canonical format, that is,
# relative to the source directory and without source_suffix.
dirlen = len(self.srcdir) + 1
to_write = []
suffixes = None # type: Tuple[unicode]
suffixes = tuple(self.config.source_suffix) # type: ignore
for filename in filenames:
filename = path.normpath(path.abspath(filename))
if not filename.startswith(self.srcdir):
logger.warning(__('file %r given on command line is not under the '
'source directory, ignoring'), filename)
continue
if not (path.isfile(filename) or
any(path.isfile(filename + suffix) for suffix in suffixes)):
logger.warning(__('file %r given on command line does not exist, '
'ignoring'), filename)
continue
filename = filename[dirlen:]
for suffix in suffixes:
if filename.endswith(suffix):
filename = filename[:-len(suffix)]
break
filename = filename.replace(path.sep, SEP)
to_write.append(filename)
self.build(to_write, method='specific',
summary=__('%d source files given on command line') % len(to_write))
def process(filename, size=-1):
file = out_dir + filename
if path.isfile(file) and stat(file).st_size == size:
print 'Skipping: ' + filename
return
print 'Processing: ' + filename
handle = urlopen(base_url + filename)
headers = handle.info()
content_length = int(headers.getheader('Content-Length'))
last_modified = mktime(strptime(headers.getheader('Last-Modified'), '%a, %d %b %Y %H:%M:%S %Z'))
if rfind(filename, '/') > 0:
dir = out_dir + filename[:rfind(filename, '/')]
else:
dir = out_dir
if not path.isdir(dir):
print 'Creating ' + dir
makedirs(dir)
if not path.isfile(file):
download(filename, last_modified)
else:
file_stat = stat(file)
if file_stat.st_mtime != last_modified or file_stat.st_size != content_length:
download(filename, last_modified)
else:
print 'Skipping: ' + filename
def status(request, task_id):
if request.method == 'GET':
status_file_path = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, 'status.json')
status = {'status': 'unknown'}
if path.isfile(status_file_path):
with open(status_file_path, 'rb') as f:
statusdata = json.load(f)
if statusdata['status'] == 'pending' and settings.USE_CACHE:
tlist = cache.get('task_list_cache', [])
num_preceding = -1;
if tlist:
for index, tuple in enumerate(tlist):
if task_id in tuple:
num_preceding = index
break
statusdata['num_preceding'] = num_preceding
elif statusdata['status'] == 'running':
asn_path = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, (task_id+'.asn'))
if path.isfile(asn_path):
with open(asn_path, 'r') as asn_f:
astr = asn_f.read()
processed_seq_count = astr.count('title \"')
statusdata['processed'] = processed_seq_count
else:
statusdata['processed'] = 0
return HttpResponse(json.dumps(statusdata))
return HttpResponse(json.dumps(status))
else:
return HttpResponse('Invalid Post')
def is_fresh(self, key, depends):
if self.force:
return
mtime_cache = self.mtime_cache
if self.prereq:
output = join(self.base_dir, key)
if not isfile(output):
self.prereq_data.pop(key, None)
return
for dep in depends:
if newer(dep, output, mtime_cache):
self.prereq_data.pop(key, None)
return
if newer(self.config_path, output, mtime_cache):
self.prereq_data.pop(key, None)
return
return 1
paths = self.output_data.get(key)
if not paths:
return
output_dir = self.output_dir
for output in paths:
output = join(output_dir, output)
if not isfile(output):
self.output_data.pop(key)
return
output = join(output_dir, list(paths).pop())
for dep in depends:
if newer(dep, output, mtime_cache):
self.output_data.pop(key)
return
if newer(self.config_path, output, mtime_cache):
self.output_data.pop(key)
return
return 1
def test_demo_html_normalized(self):
"""
Ensure a demo.html in zip file is normalized to index.html when
unpacked
"""
fout = StringIO()
zf = zipfile.ZipFile(fout, "w")
zf.writestr("demo.html", """<html></html""")
zf.writestr("css/main.css", "h1 { color: red }")
zf.writestr("js/main.js", 'alert("HELLO WORLD");')
zf.close()
s = Submission(
title="Hello world", slug="hello-world", description="This is a hello world demo", creator=self.user
)
s.demo_package.save("play_demo.zip", ContentFile(fout.getvalue()))
s.demo_package.close()
s.clean()
s.save()
s.process_demo_package()
path = s.demo_package.path.replace(".zip", "")
ok_(isdir(path))
ok_(isfile("%s/index.html" % path))
ok_(isfile("%s/css/main.css" % path))
ok_(isfile("%s/js/main.js" % path))
rmtree(path)
def test_demo_unicode_filenames(self):
"""Bug 741660: Demo package containing filenames with non-ASCII
characters works"""
fout = StringIO()
zf = zipfile.ZipFile(fout, "w")
zf.writestr("demo.html", """<html></html""")
zf.writestr("css/예제.css", "h1 { color: red }")
zf.writestr("js/示例.js", 'alert("HELLO WORLD");')
zf.close()
s = Submission(
title="Hello world", slug="hello-world", description="This is a hello world demo", creator=self.user
)
s.demo_package.save("play_demo.zip", ContentFile(fout.getvalue()))
s.demo_package.close()
s.clean()
s.save()
s.process_demo_package()
path = s.demo_package.path.replace(".zip", "")
ok_(isdir(path))
ok_(isfile((u"%s/index.html" % path).encode("utf-8")))
ok_(isfile((u"%s/css/예제.css" % path).encode("utf-8")))
ok_(isfile((u"%s/js/示例.js" % path).encode("utf-8")))
rmtree(path)
def dump_db(dumpfile="pootle_DB_backup.sql"):
"""Dumps the DB as a SQL script and downloads it"""
require('environment', provided_by=[production, staging])
if ((isfile(dumpfile) and confirm('\n%s already exists locally. Do you '
'want to overwrite it?' % dumpfile, default=False))
or not isfile(dumpfile)):
remote_filename = '%s/%s' % (env['project_path'], dumpfile)
if ((exists(remote_filename) and confirm('\n%s already exists. Do you '
'want to overwrite it?' % remote_filename, default=False))
or not exists(remote_filename)):
print('\nDumping DB...')
with settings(hide('stderr')):
sudo('mysqldump -u %s -p %s > %s' % (env['db_user'],
env['db_name'],
remote_filename))
get(remote_filename, '.')
else:
print('\nAborting.')
else:
print('\nAborting.')
def mainFunc():
parser = argparse.ArgumentParser(description='Run Elastix registration protocol for all images in the directory')
parser.add_argument('--refDir', '-r', dest='refDir', required = True, \
help='The directory containing the reference images.')
parser.add_argument('--floatFile', '-f', dest='floatFile', required = True, \
help='Path to the floating image.')
parser.add_argument('--outDir', '-o', dest='outDir', required = False, \
help='Path to store the output images/parameters (default: current dir)', default=os.getcwd())
parser.add_argument('--atlas', '-a', dest='atlas', required = False, \
help='Path to the atlas segmentation file which will be resampled with the CPP file from the registration.', default=None)
args = parser.parse_args()
refImgs = [join(args.refDir, File) for File in listdir(args.refDir)]
refImgs = [img for img in refImgs if isfile(img) and img.endswith('.nii')]
if not refImgs:
print('Couldn\'t find any reference images')
return
if not path.isfile(args.floatFile):
print('Coudln\'t find the float image')
refImgs.sort(key=str.lower)
refFloatPairs = [[refImg, args.floatFile] for refImg in refImgs]
f3dParStr = paramListToShortString(f3d_params)
aladinParStr = paramListToShortString(aladin_params)
for rfPair in refFloatPairs:
baseName = basename(rfPair[0])[:-4]+'_'+basename(rfPair[1])[:-4]
currOutDir = join(args.outDir,baseName)
mkdir(currOutDir)
elastixLogPath = join(currOutDir,basename+'_LOG.txt')
elastixCommand = elastixExec+' -f '+rfPair[0]+' -m '+rfPair[1]+' -p '.join(elastixParams)+' -o '+currOutDir
elastixLog = ''
try:
elastixLog = ccall(elastixCommand, shell=True, stderr=STDOUT)
except CalledProcessError as err:
writeAndDie(err.output, elastixLogPath)
with open(elastixLogPath, 'w') as f:
f.write(elastixLog)
transformParameterFiles = ['TransformParameters.0.txt', 'TransformParameters.1.txt']
transformParameterFiles = [join(currOutDir,tpFile) for tpFile in transformParameterFiles]
for tpFilePath in transformParameterFiles:
with open(tpFilePath,'r') as tpFile:
tpCont = tpFile.read()
tpCont = tpCont.replace('(FinalBSplineInterpolationOrder 3)', '(FinalBSplineInterpolationOrder 1)')
with open(tpFilePath,'w') as tpFile:
tpCont = tpFile.write(tpCont)
if args.atlas is not None:
atlasOutDir = join(currOutDir,'atlas')
mkdir(atlasOutDir)
trfxCmd = trfxExec+' -in '+args.atlas+' -out '+atlasOutDir+' tp '+transformParameterFiles[-1]
try:
resampleLog = ccall(trfxCmd, shell=True, stderr=STDOUT)
except CalledProcessError as err:
writeAndDie(err.output, join(atlasOutDir,'ERR.txt'))
def MODULE_LIST(force_compile=False):
"""Load scripts from scripts directory and return list of modules."""
modules = []
for search_path in [search_path for search_path in SCRIPT_SEARCH_PATHS if exists(search_path)]:
to_compile = [file for file in os.listdir(search_path)
if file[-7:] == ".script" and file[0] != "_"
and ((not isfile(join(search_path, file[:-7] + '.py'))) or
(isfile(join(search_path, file[:-7] + '.py')) and
(getmtime(join(search_path, file[:-7] + '.py')) <
getmtime(join(search_path, file))))
or force_compile)
]
for script in to_compile:
script_name = '.'.join(script.split('.')[:-1])
compile_script(join(search_path, script_name))
files = [file for file in os.listdir(search_path)
if file[-3:] == ".py" and file[0] != "_"
and '#retriever' in open(join(search_path, file), 'r').readline().lower()]
for script in files:
script_name = '.'.join(script.split('.')[:-1])
file, pathname, desc = imp.find_module(script_name, [search_path])
try:
new_module = imp.load_module(script_name, file, pathname, desc)
new_module.SCRIPT.download
modules.append(new_module)
except:
sys.stderr.write("Failed to load script: %s (%s)" % (script_name, search_path))
return modules
else '07_{}_joint_angles.npy'.format(test_sequence_2)
cmu_data = np.load(DATA_PATH + 'cmu_mocap/' + np_file)
# Subsample test sequence.
test_data_2 = cmu_data[::frame_subsamples]
# Normalise data to zero mean and unit variance.
y_test_2 = scaler.transform(test_data_2)
# Define file path for results.
dataset_str = 'cmu_subject7_training_seq_{}_test_seqs_{}_{}'.format(
training_sequence, test_sequence_1, test_sequence_2)
dp_gp_lvm_results_file = RESULTS_FILE_NAME.format(model='dp_gp_lvm', dataset=dataset_str)
# Define instance of necessary model.
if not isfile(dp_gp_lvm_results_file):
# Reset default graph before building new model graph. This speeds up script.
tf.reset_default_graph()
np.random.seed(1) # Random seed.
# Define instance of DP-GP-LVM.
model = dp_gp_lvm(y_train=y_train,
num_inducing_points=num_inducing_points,
num_latent_dims=num_latent_dimensions,
truncation_level=truncation_level,
mask_size=1) # Treat each observed dimension as independent.
model_training_objective = model.objective
predict_lower_bound_1, x_mean_test_1, x_covar_test_1, test_log_likelihood_1 = \
model.predict_new_latent_variables(y_test=y_test_1)
model_test_objective_1 = tf.negative(predict_lower_bound_1)
predict_lower_bound_2, x_mean_test_2, x_covar_test_2, test_log_likelihood_2 = \
str += "</{}>\n".format(value)
else:
str += " " * tabCount
str += "<{}> {} </{}>\n".format(type, value, type)
return str
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: parser.py path/file.jack")
print(" parser.py path")
exit(0)
jackFilePath = sys.argv[1]
if path.isfile(jackFilePath):
fullPath = os.path.abspath(jackFilePath)
filePath, fileName = os.path.split(fullPath)
filePre, fileExt = os.path.splitext(fileName)
if fileName.endswith(".jack"):
output = main(fullPath)
xmlPath = os.path.join(fullPath, "_" + filePre + ".xml")
with open(xmlPath, 'w') as xmlFile:
xmlFile.write(output)
elif path.isdir(jackFilePath):
fullPath = os.path.abspath(jackFilePath)
fileList = os.listdir(fullPath)
for fileName in fileList:
filePre, fileExt = os.path.splitext(fileName)
"handlers": ["mail_admins"],
"level": "ERROR",
"propagate": True,
},
"django.request.tastypie": {"level": "ERROR"},
"administration": {"level": "DEBUG"},
"common": {"level": "DEBUG"},
"locations": {"level": "DEBUG"},
"sword2": {"level": "INFO"},
"boto3": {"level": "INFO"},
"botocore": {"level": "INFO"},
},
"root": {"handlers": ["console"], "level": "WARNING"},
}
if isfile(LOGGING_CONFIG_FILE):
with open(LOGGING_CONFIG_FILE, "rt") as f:
LOGGING = logging.config.dictConfig(json.load(f))
else:
logging.config.dictConfig(LOGGING)
# ######## END LOGGING CONFIGURATION
# ######## SESSION CONFIGURATION
# So the cookies don't conflict with archivematica cookies
SESSION_COOKIE_NAME = "storageapi_sessionid"
# ######## END SESSION CONFIGURATION
# ######## WSGI CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
#++++++++++++++++++++++ MAIN ++++++++++++++++++++++++++++++++++++++++++++++
#++++++Sampling Frequency Assignment
if channel == 'Koerperschall':
fs = 1000.0
elif channel == 'Drehmoment':
fs = 1000.0
elif channel == 'AE_Signal':
fs = 1000000.0
else:
print('Error fs assignment')
dt = 1.0/fs
#++++++List of Files in Directory
filenames = [f for f in os.listdir(mypath) if isfile(join(mypath, f))]
features = [
# 'KURT_WFM_0',
# 'RMS_WFM_0',
# 'LP6_WFM_0',
# 'LP7_WFM_0',
# 'LP16_WFM_0',
# 'LP17_WFM_0',
# 'LP21_WFM_0',
# 'LP24_WFM_0',
# 'LP16_FFT_0',
# 'LP17_FFT_0',
# 'LP21_FFT_0',
# 'LP24_FFT_0',
'NBU_WFM_0',
else:
tk.messagebox.showinfo(title='提示', message='下载失败！')
bar.quit()
WorkThread(action=copyAnrTool).start()
bar.start('更新软件','正在下载......')
HEIGHT = 600
WIDTH = 800
if __name__ == '__main__':
window = tk.Tk()
window.title('Anr 工具')
window.resizable(0, 0)
ico = sep.join(['res',"anr.ico"])
if isfile(ico):
window.iconbitmap(ico)
sw = window.winfo_screenwidth()
sh = window.winfo_screenheight()
ww = WIDTH
wh = HEIGHT
x = (sw - ww) / 2
y = (sh - wh) / 2
window.geometry("%dx%d+%d+%d" % (ww, wh, x, y))
bg = sep.join(['res','window_bg.png'])
if isfile(bg):
canvas = tk.Canvas(window, height=HEIGHT, width=WIDTH)
image_file = tk.PhotoImage(file=bg)
image = canvas.create_image(WIDTH / 2, HEIGHT / 2, anchor='center', image=image_file) # n 北方，s 南方， w西方，e东方，center中间
canvas.pack(side='top')
def fmda_advance_region(cycle, cfg, rtma, wksp_path, lookback_length, meso_token):
"""
Advance the fuel moisture estimates in the region specified by the configuration.
The function assumes that the fuel moisture model has not been advanced to this
cycle yet and will overwrite any previous computations.
Control flow:
1) read in RTMA variables
2) check if there is a stored FM model for previous cycle
2a) yes -> load it, advance one time-step, perform DA
2b) no -> compute equilibrium, use background covariance to do DA
3) store model
:param cycle: the datetime indicating the processed cycle in UTC
:param cfg: the configuration dictionary specifying the region
:param rtma: the RTMA object that can be used to retrieve variables for this cycle
:param wksp_path: the workspace path for the cycler
:param lookback_length: number of cycles to search before we find a computed cycle
:param meso_token: the mesowest API access token
:return: the model advanced and assimilated at the current cycle
"""
logging.info("rtma_cycler.fmda_advance_region: %s" % str(cycle))
model = None
prev_cycle = cycle - timedelta(hours=1)
prev_model_path = compute_model_path(prev_cycle, cfg.code, wksp_path)
if not osp.exists(prev_model_path):
logging.info('CYCLER cannot find model from previous cycle %s' % str(prev_cycle))
if lookback_length > 0:
model = fmda_advance_region(cycle - timedelta(hours=1), cfg, rtma, wksp_path, lookback_length - 1, meso_token)
else:
logging.info('CYCLER found previous model for cycle %s.' % str(prev_cycle))
model = FuelMoistureModel.from_netcdf(prev_model_path)
# retrieve the variables and make sure they are available (we should not be here if they are not)
try:
dont_have_vars, have_vars = rtma.retrieve_rtma(cycle)
except ValueError as e:
logging.error(e)
sys.exit(1)
assert not dont_have_vars
logging.info('CYCLER loading RTMA data for cycle %s.' % str(cycle))
TD, T2, RH, precipa, hgt, lats, lons = load_rtma_data(have_vars, cfg.bbox)
Ed, Ew = compute_equilibria(T2, RH)
rain = precipa[:,:] + 0
# remove rain that is too small to make any difference
rain[rain < 0.01] = 0
# remove bogus rain that is too large
rain[rain > 1e10] = 0
dom_shape = T2.shape
# store the lons/lats for this domain
geo_path = osp.join(wksp_path, '%s-geo.nc' % cfg.code)
if not osp.isfile(geo_path):
logging.info('CYCLER initializing new file %s.' % (geo_path))
d = netCDF4.Dataset(geo_path, 'w', format='NETCDF4')
d.createDimension('south_north', dom_shape[0])
d.createDimension('west_east', dom_shape[1])
xlat = d.createVariable('XLAT', 'f4', ('south_north', 'west_east'))
xlat[:,:] = lats
xlong = d.createVariable('XLONG', 'f4', ('south_north', 'west_east'))
xlong[:,:] = lons
d.close()
else:
logging.info('CYCLER file already exists: %s.' % (geo_path))
# the process noise matrix
Q = np.diag([1e-4,5e-5,1e-5,1e-6,1e-6])
# background covariance
P0 = np.diag([0.01,0.01,0.01,0.001,0.001])
# check if we must start from equilibrium
if model is None:
logging.info('CYCLER initializing from equilibrium for cycle %s.' % (str(cycle)))
# setup model parameters
Nk = 3
Tk = np.array([1.0, 10.0, 100.0])
m0 = np.expand_dims(0.5 * (Ed + Ew), axis=2)
model = FuelMoistureModel(m0[:,:,[0,0,0]], Tk, P0)
else:
logging.info('CYCLER advancing model one hour to cycle %s.' % (str(cycle)))
dt = 3600 # always 1 hr step in RTMA
model.advance_model(Ed, Ew, rain, dt, Q)
logging.info('CYCLER retrieving fm-10 observations for cycle %s.' % (str(cycle)))
# perform assimilation with mesowest observations
tm_start = cycle - timedelta(minutes=30)
tm_end = cycle + timedelta(minutes=30)
fm10 = retrieve_mesowest_observations(meso_token, tm_start, tm_end, lats, lons)
fm10v = []
for fm10_obs in fm10.values():
for obs in fm10_obs:
fm10v.append(obs.get_value())
logging.info('CYCLER retrieved %d valid observations, min/mean/max [%g/%g/%g].' %
(len(fm10),np.amin(fm10v),np.mean(fm10v),np.amax(fm10v)))
# run the data assimilation step
covs = [np.ones(dom_shape), hgt / 2000.0]
covs_names = ['const','hgt/2000']
if np.any(rain > 0.01):
covs.append(rain)
covs_names.append('rain')
execute_da_step(model, cycle, covs, covs_names, fm10)
# make geogrid files for WPS; datasets and lines to add to GEOGRID.TBL
geo_path = compute_model_path(cycle, cfg.code, wksp_path,ext="geo")
index = rtma.geogrid_index()
print('index',index)
model.to_geogrid(geo_path,index,lats,lons)
# store the new model
model_path = compute_model_path(cycle, cfg.code, wksp_path)
logging.info('CYCLER writing model variables to: %s.' % model_path)
model.to_netcdf(ensure_dir(model_path),
{'EQUILd FM':Ed,'EQUILw FM':Ew,'TD':TD,'T2':T2,'RH':RH,'PRECIPA':precipa,'PRECIP':rain,'HGT':hgt})
return model
def __init__(self,
fieldset,
ptype,
pyfunc=None,
funcname=None,
funccode=None,
py_ast=None,
funcvars=None,
c_include=""):
self.fieldset = fieldset
self.ptype = ptype
self._lib = None
# Derive meta information from pyfunc, if not given
self.funcname = funcname or pyfunc.__name__
if pyfunc is AdvectionRK4_3D:
warning = False
if isinstance(fieldset.W, Field) and fieldset.W.creation_log != 'from_nemo' and \
fieldset.W._scaling_factor is not None and fieldset.W._scaling_factor > 0:
warning = True
if type(fieldset.W) in [SummedField, NestedField]:
for f in fieldset.W:
if f.creation_log != 'from_nemo' and f._scaling_factor is not None and f._scaling_factor > 0:
warning = True
if warning:
logger.warning_once(
'Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n'
' If z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)'
)
if funcvars is not None:
self.funcvars = funcvars
elif hasattr(pyfunc, '__code__'):
self.funcvars = list(pyfunc.__code__.co_varnames)
else:
self.funcvars = None
self.funccode = funccode or inspect.getsource(pyfunc.__code__)
# Parse AST if it is not provided explicitly
self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0]
if pyfunc is None:
# Extract user context by inspecting the call stack
stack = inspect.stack()
try:
user_ctx = stack[-1][0].f_globals
user_ctx['math'] = globals()['math']
user_ctx['random'] = globals()['random']
user_ctx['ErrorCode'] = globals()['ErrorCode']
except:
logger.warning(
"Could not access user context when merging kernels")
user_ctx = globals()
finally:
del stack # Remove cyclic references
# Compile and generate Python function from AST
py_mod = Module(body=[self.py_ast])
exec(compile(py_mod, "<ast>", "exec"), user_ctx)
self.pyfunc = user_ctx[self.funcname]
else:
self.pyfunc = pyfunc
assert len(inspect.getargspec(self.pyfunc).args) == 3, \
'Since Parcels v2.0, kernels do only take 3 arguments: particle, fieldset, time !! AND !! Argument order in field interpolation is time, depth, lat, lon.'
self.name = "%s%s" % (ptype.name, self.funcname)
# Generate the kernel function and add the outer loop
if self.ptype.uses_jit:
kernelgen = KernelGenerator(fieldset, ptype)
kernel_ccode = kernelgen.generate(deepcopy(self.py_ast),
self.funcvars)
self.field_args = kernelgen.field_args
self.vector_field_args = kernelgen.vector_field_args
fieldset = self.fieldset
for fname in self.vector_field_args:
f = getattr(fieldset, fname)
Wname = f.W.name if f.W else 'not_defined'
for sF in [f.U.name, f.V.name, Wname]:
if sF not in self.field_args:
try:
self.field_args[sF] = getattr(fieldset, sF)
except:
continue
self.const_args = kernelgen.const_args
loopgen = LoopGenerator(fieldset, ptype)
if path.isfile(c_include):
with open(c_include, 'r') as f:
c_include_str = f.read()
else:
c_include_str = c_include
self.ccode = loopgen.generate(self.funcname, self.field_args,
self.const_args, kernel_ccode,
c_include_str)
basename = path.join(get_cache_dir(), self._cache_key)
self.src_file = "%s.c" % basename
self.lib_file = "%s.%s" % (basename,
'dll' if platform == 'win32' else 'so')
self.log_file = "%s.log" % basename
def main(argv):
"""
Main entry point for gcexport.py
"""
args = parse_arguments(argv)
if args.version:
print(argv[0] + ", version " + SCRIPT_VERSION)
exit(0)
print('Welcome to Garmin Connect Exporter!')
# Create directory for data files.
if isdir(args.directory):
print(
'Warning: Output directory already exists. Will skip already-downloaded files and \
append to the CSV file.')
login_to_garmin_connect(args)
# We should be logged in now.
if not isdir(args.directory):
mkdir(args.directory)
csv_filename = args.directory + '/activities.csv'
csv_existed = isfile(csv_filename)
csv_file = open(csv_filename, 'a')
# Write header to CSV file
if not csv_existed:
csv_file.write('Activity name,\
Description,\
Begin timestamp,\
Duration (h:m:s),\
Moving duration (h:m:s),\
Distance (km),\
Average speed (km/h or min/km),\
Average moving speed (km/h or min/km),\
Max. speed (km/h or min/km),\
Elevation loss uncorrected (m),\
Elevation gain uncorrected (m),\
Elevation min. uncorrected (m),\
Elevation max. uncorrected (m),\
Min. heart rate (bpm),\
Max. heart rate (bpm),\
Average heart rate (bpm),\
Calories,\
Avg. cadence (rpm),\
Max. cadence (rpm),\
Strokes,\
Avg. temp (°C),\
Min. temp (°C),\
Max. temp (°C),\
Map,\
End timestamp,\
Begin timestamp (ms),\
End timestamp (ms),\
Device,\
Activity type,\
Event type,\
Time zone,\
Begin latitude (°DD),\
Begin longitude (°DD),\
End latitude (°DD),\
End longitude (°DD),\
Elevation gain corrected (m),\
Elevation loss corrected (m),\
Elevation max. corrected (m),\
Elevation min. corrected (m),\
Sample count\n')
if args.count == 'all':
# If the user wants to download all activities, first download one,
# then the result of that request will tell us how many are available
# so we will modify the variables then.
print(
"Making result summary request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
print(URL_GC_SEARCH)
result = http_req(URL_GC_SEARCH)
print(
"Finished result summary request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
# Persist JSON
write_to_file(args.directory + '/activities-summary.json', result, 'a')
# Modify total_to_download based on how many activities the server reports.
json_results = json.loads(
result) # TODO: Catch possible exceptions here.
total_to_download = int(json_results['results']['totalFound'])
else:
total_to_download = int(args.count)
total_downloaded = 0
device_dict = dict()
# load some dictionaries with lookup data from REST services
activity_type_props = http_req(URL_GC_ACT_PROPS)
# write_to_file(args.directory + '/activity_types.properties', activity_type_props, 'a')
activity_type_name = load_properties(activity_type_props)
event_type_props = http_req(URL_GC_EVT_PROPS)
# write_to_file(args.directory + '/event_types.properties', event_type_props, 'a')
event_type_name = load_properties(event_type_props)
# This while loop will download data from the server in multiple chunks, if necessary.
while total_downloaded < total_to_download:
# Maximum chunk size 'LIMIT_MAXIMUM' ... 400 return status if over maximum. So download
# maximum or whatever remains if less than maximum.
# As of 2018-03-06 I get return status 500 if over maximum
if total_to_download - total_downloaded > LIMIT_MAXIMUM:
num_to_download = LIMIT_MAXIMUM
else:
num_to_download = total_to_download - total_downloaded
search_params = {'start': total_downloaded, 'limit': num_to_download}
# Query Garmin Connect
print("Making activity request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
print(URL_GC_LIST + urlencode(search_params))
result = http_req(URL_GC_LIST + urlencode(search_params))
print(
"Finished activity request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
# Persist JSON
write_to_file(args.directory + '/activities.json', result, 'a')
json_results = json.loads(
result) # TODO: Catch possible exceptions here.
# search = json_results['results']['search']
# Pull out just the list of activities.
activities = json_results
# Process each activity.
for a in activities:
# Display which entry we're working on.
print('Garmin Connect activity: [' + str(a['activityId']) + '] ',
end='')
print(a['activityName'])
# Retrieve also the detail data from the activity (the one displayed on
# the https://connect.garmin.com/modern/activity/xxx page), because some
# data are missing from 'a' (or are even different, e.g. for my activities
# 86497297 or 86516281)
activity_details = None
details = None
tries = MAX_TRIES
while tries > 0:
activity_details = http_req(URL_GC_ACTIVITY +
str(a['activityId']))
details = json.loads(activity_details)
# I observed a failure to get a complete JSON detail in about 5-10 calls out of 1000
# retrying then statistically gets a better JSON ;-)
if len(details['summaryDTO']) > 0:
tries = 0
else:
print('retrying for ' + str(a['activityId']))
tries -= 1
if tries == 0:
raise Exception('Didn\'t get "summaryDTO" after ' +
str(MAX_TRIES) + ' tries for ' +
str(a['activityId']))
parent_type_id = 4 if absent_or_null(
'activityType', a) else a['activityType']['parentTypeId']
type_id = 4 if absent_or_null('activityType',
a) else a['activityType']['typeId']
start_time_with_offset = offset_date_time(a['startTimeLocal'],
a['startTimeGMT'])
elapsed_duration = details['summaryDTO'][
'elapsedDuration'] if 'summaryDTO' in details and 'elapsedDuration' in details[
'summaryDTO'] else None
duration = elapsed_duration if elapsed_duration else a['duration']
duration_seconds = int(round(duration))
end_time_with_offset = start_time_with_offset + timedelta(
seconds=duration_seconds) if duration else None
# get some values from detail if present, from a otherwise
start_latitude = from_activities_or_detail('startLatitude', a,
details, 'summaryDTO')
start_longitude = from_activities_or_detail(
'startLongitude', a, details, 'summaryDTO')
end_latitude = from_activities_or_detail('endLatitude', a, details,
'summaryDTO')
end_longitude = from_activities_or_detail('endLongitude', a,
details, 'summaryDTO')
print('\t' + start_time_with_offset.isoformat() + ', ', end='')
if 'duration' in a:
print(hhmmss_from_seconds(a['duration']) + ', ', end='')
else:
print('??:??:??, ', end='')
if 'distance' in a and isinstance(a['distance'], (float)):
print("{0:.3f}".format(a['distance'] / 1000) + 'km')
else:
print('0.000 km')
# try to get the device details (and cache them, as they're used for multiple activities)
device = None
device_app_inst_id = None if absent_or_null(
'metadataDTO', details
) else details['metadataDTO']['deviceApplicationInstallationId']
if device_app_inst_id:
if not device_dict.has_key(device_app_inst_id):
# print '\tGetting device details ' + str(device_app_inst_id)
device_details = http_req(URL_GC_DEVICE +
str(device_app_inst_id))
write_to_file(
args.directory + '/device_' + str(device_app_inst_id) +
'.json', device_details, 'a')
device_dict[
device_app_inst_id] = None if not device_details else json.loads(
device_details)
device = device_dict[device_app_inst_id]
# Write stats to CSV.
empty_record = '"",'
csv_record = ''
csv_record += empty_record if absent_or_null(
'activityName',
a) else '"' + a['activityName'].replace('"', '""') + '",'
csv_record += empty_record if absent_or_null(
'description',
a) else '"' + a['description'].replace('"', '""') + '",'
csv_record += '"' + start_time_with_offset.strftime(
ALMOST_RFC_1123) + '",'
# csv_record += '"' + startTimeWithOffset.isoformat() + '",'
csv_record += empty_record if not duration else hhmmss_from_seconds(
round(duration)) + ','
csv_record += empty_record if absent_or_null(
'movingDuration',
details['summaryDTO']) else hhmmss_from_seconds(
details['summaryDTO']['movingDuration']) + ','
csv_record += empty_record if absent_or_null(
'distance',
a) else '"' + "{0:.5f}".format(a['distance'] / 1000) + '",'
csv_record += empty_record if absent_or_null(
'averageSpeed', a) else '"' + trunc6(
pace_or_speed_raw(type_id, parent_type_id,
a['averageSpeed'])) + '",'
csv_record += empty_record if absent_or_null(
'averageMovingSpeed', details['summaryDTO']) else '"' + trunc6(
pace_or_speed_raw(
type_id, parent_type_id,
details['summaryDTO']['averageMovingSpeed'])) + '",'
csv_record += empty_record if absent_or_null(
'maxSpeed', details['summaryDTO']) else '"' + trunc6(
pace_or_speed_raw(
type_id, parent_type_id,
details['summaryDTO']['maxSpeed'])) + '",'
csv_record += empty_record if a[
'elevationCorrected'] or absent_or_null(
'elevationLoss', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['elevationLoss'],
2)) + '",'
csv_record += empty_record if a[
'elevationCorrected'] or absent_or_null(
'elevationGain', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['elevationGain'],
2)) + '",'
csv_record += empty_record if a[
'elevationCorrected'] or absent_or_null(
'minElevation', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['minElevation'], 2)) + '",'
csv_record += empty_record if a[
'elevationCorrected'] or absent_or_null(
'maxElevation', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['maxElevation'], 2)) + '",'
csv_record += empty_record # no minimum heart rate in JSON
csv_record += empty_record if absent_or_null(
'maxHR', a) else '"' + "{0:.0f}".format(a['maxHR']) + '",'
csv_record += empty_record if absent_or_null(
'averageHR',
a) else '"' + "{0:.0f}".format(a['averageHR']) + '",'
csv_record += empty_record if absent_or_null(
'calories', details['summaryDTO']) else '"' + str(
details['summaryDTO']['calories']) + '",'
csv_record += empty_record if absent_or_null(
'averageBikingCadenceInRevPerMinute', a) else '"' + str(
a['averageBikingCadenceInRevPerMinute']) + '",'
csv_record += empty_record if absent_or_null(
'maxBikingCadenceInRevPerMinute',
a) else '"' + str(a['maxBikingCadenceInRevPerMinute']) + '",'
csv_record += empty_record if absent_or_null(
'strokes', a) else '"' + str(a['strokes']) + '",'
csv_record += empty_record if absent_or_null(
'averageTemperature', details['summaryDTO']) else '"' + str(
details['summaryDTO']['averageTemperature']) + '",'
csv_record += empty_record if absent_or_null(
'minTemperature', details['summaryDTO']) else '"' + str(
details['summaryDTO']['minTemperature']) + '",'
csv_record += empty_record if absent_or_null(
'maxTemperature', details['summaryDTO']) else '"' + str(
details['summaryDTO']['maxTemperature']) + '",'
csv_record += '"https://connect.garmin.com/modern/activity/' + str(
a['activityId']) + '",'
csv_record += empty_record if not end_time_with_offset else '"' + end_time_with_offset.strftime(
ALMOST_RFC_1123) + '",'
csv_record += empty_record if not start_time_with_offset else '"' + start_time_with_offset.isoformat(
) + '",'
# csv_record += empty_record if absent_or_null('beginTimestamp', a) else '"' + str(a['beginTimestamp']) + '",'
csv_record += empty_record if not end_time_with_offset else '"' + end_time_with_offset.isoformat(
) + '",'
# csv_record += empty_record if absent_or_null('beginTimestamp', a) else '"' + str(a['beginTimestamp']+durationSeconds*1000) + '",'
csv_record += empty_record if absent_or_null(
'productDisplayName',
device) else '"' + device['productDisplayName'].replace(
'"', '""') + ' ' + device['versionString'] + '",'
csv_record += empty_record if absent_or_null(
'activityType', a) else '"' + value_if_found_else_key(
activity_type_name, 'activity_type_' +
a['activityType']['typeKey']) + '",'
csv_record += empty_record if absent_or_null(
'eventType', a) else '"' + value_if_found_else_key(
event_type_name, a['eventType']['typeKey']) + '",'
csv_record += '"' + start_time_with_offset.isoformat()[-6:] + '",'
csv_record += empty_record if not start_latitude else '"' + trunc6(
start_latitude) + '",'
csv_record += empty_record if not start_longitude else '"' + trunc6(
start_longitude) + '",'
csv_record += empty_record if not end_latitude else '"' + trunc6(
end_latitude) + '",'
csv_record += empty_record if not end_longitude else '"' + trunc6(
end_longitude) + '",'
csv_record += empty_record if not a[
'elevationCorrected'] or absent_or_null(
'elevationGain', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['elevationGain'],
2)) + '",'
csv_record += empty_record if not a[
'elevationCorrected'] or absent_or_null(
'elevationLoss', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['elevationLoss'],
2)) + '",'
csv_record += empty_record if not a[
'elevationCorrected'] or absent_or_null(
'maxElevation', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['maxElevation'], 2)) + '",'
csv_record += empty_record if not a[
'elevationCorrected'] or absent_or_null(
'minElevation', details['summaryDTO']) else '"' + str(
round(details['summaryDTO']['minElevation'], 2)) + '",'
csv_record += '""' # no Sample Count in JSON
csv_record += '\n'
csv_file.write(csv_record.encode('utf8'))
export_data_file(str(a['activityId']), activity_details, args)
total_downloaded += num_to_download
# End while loop for multiple chunks.
csv_file.close()
print('Open CSV output.')
print(csv_filename)
# open CSV file. Comment this line out if you don't want this behavior
# call(["/usr/bin/libreoffice6.0", "--calc", csv_filename])
print('Done!')
def add_mim_extension():
"""Add extra files that are required to support MIM into the package.
These files will be added by creating a symlink to the originals if the
package is installed in `editable` mode (e.g. pip install -e .), or by
copying from the originals otherwise.
"""
# parse installment mode
if 'develop' in sys.argv:
# installed by `pip install -e .`
if platform.system() == 'Windows':
# set `copy` mode here since symlink fails on Windows.
mode = 'copy'
else:
mode = 'symlink'
elif 'sdist' in sys.argv or 'bdist_wheel' in sys.argv or \
platform.system() == 'Windows':
# installed by `pip install .`
# or create source distribution by `python setup.py sdist`
# set `copy` mode here since symlink fails with WinError on Windows.
mode = 'copy'
else:
return
filenames = ['tools', 'configs', 'model-index.yml']
repo_path = osp.dirname(__file__)
mim_path = osp.join(repo_path, 'mmseg', '.mim')
os.makedirs(mim_path, exist_ok=True)
for filename in filenames:
if osp.exists(filename):
src_path = osp.join(repo_path, filename)
tar_path = osp.join(mim_path, filename)
if osp.isfile(tar_path) or osp.islink(tar_path):
os.remove(tar_path)
elif osp.isdir(tar_path):
shutil.rmtree(tar_path)
if mode == 'symlink':
src_relpath = osp.relpath(src_path, osp.dirname(tar_path))
try:
os.symlink(src_relpath, tar_path)
except OSError:
# Creating a symbolic link on windows may raise an
# `OSError: [WinError 1314]` due to privilege. If
# the error happens, the src file will be copied
mode = 'copy'
warnings.warn(
f'Failed to create a symbolic link for {src_relpath}, '
f'and it will be copied to {tar_path}')
else:
continue
if mode == 'copy':
if osp.isfile(src_path):
shutil.copyfile(src_path, tar_path)
elif osp.isdir(src_path):
shutil.copytree(src_path, tar_path)
else:
warnings.warn(f'Cannot copy file {src_path}.')
else:
raise ValueError(f'Invalid mode {mode}')
def export_data_file(activity_id, activity_details, args):
if args.format == 'gpx':
data_filename = args.directory + '/activity_' + activity_id + '.gpx'
download_url = URL_GC_GPX_ACTIVITY + activity_id + '?full=true'
# download_url = URL_GC_GPX_ACTIVITY + activity_id + '?full=true' + '&original=true'
print(download_url)
file_mode = 'w'
elif args.format == 'tcx':
data_filename = args.directory + '/activity_' + activity_id + '.tcx'
download_url = URL_GC_TCX_ACTIVITY + activity_id + '?full=true'
file_mode = 'w'
elif args.format == 'original':
data_filename = args.directory + '/activity_' + activity_id + '.zip'
fit_filename = args.directory + '/' + activity_id + '.fit'
download_url = URL_GC_ORIGINAL_ACTIVITY + activity_id
file_mode = 'wb'
elif args.format == 'json':
data_filename = args.directory + '/activity_' + activity_id + '.json'
file_mode = 'w'
else:
raise Exception('Unrecognized format.')
if isfile(data_filename):
print('\tData file already exists; skipping...')
return
# Regardless of unzip setting, don't redownload if the ZIP or FIT file exists.
if args.format == 'original' and isfile(fit_filename):
print('\tFIT data file already exists; skipping...')
return
if args.format != 'json':
# Download the data file from Garmin Connect. If the download fails (e.g., due to timeout),
# this script will die, but nothing will have been written to disk about this activity, so
# just running it again should pick up where it left off.
print('\tDownloading file...')
try:
data = http_req(download_url)
except urllib2.HTTPError as e:
# Handle expected (though unfortunate) error codes; die on unexpected ones.
if e.code == 500 and args.format == 'tcx':
# Garmin will give an internal server error (HTTP 500) when downloading TCX files
# if the original was a manual GPX upload. Writing an empty file prevents this file
# from being redownloaded, similar to the way GPX files are saved even when there
# are no tracks. One could be generated here, but that's a bit much. Use the GPX
# format if you want actual data in every file, as I believe Garmin provides a GPX
# file for every activity.
print(
'Writing empty file since Garmin did not generate a TCX file for this \
activity...')
data = ''
elif e.code == 404 and args.format == 'original':
# For manual activities (i.e., entered in online without a file upload), there is
# no original file. # Write an empty file to prevent redownloading it.
print(
'Writing empty file since there was no original activity data...'
)
data = ''
else:
raise Exception('Failed. Got an unexpected HTTP error (' +
str(e.code) + download_url + ').')
else:
data = activity_details
# Persist file
write_to_file(data_filename, data, file_mode)
if args.format == 'gpx' and data:
# Validate GPX data. If we have an activity without GPS data (e.g., running on a
# treadmill). Garmin Connect still kicks out a GPX (sometimes), but there is only activity
# information, no GPS data. N.B. You can omit the XML parse (and the associated log
# messages) to speed things up.
gpx = parseString(data)
gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0
if gpx_data_exists:
print('Done. GPX data saved.')
else:
print('Done. No track points found.')
elif args.format == 'original':
# Even manual upload of a GPX file is zipped, but we'll validate the extension.
if args.unzip and data_filename[-3:].lower() == 'zip':
print("Unzipping and removing original files...")
print('Filesize is: ' + str(stat(data_filename).st_size))
if stat(data_filename).st_size > 0:
zip_file = open(data_filename, 'rb')
z = zipfile.ZipFile(zip_file)
for name in z.namelist():
z.extract(name, args.directory)
zip_file.close()
else:
print('Skipping 0Kb zip file.')
remove(data_filename)
print('Done.')
elif args.format == 'json':
# print nothing here
pass
else:
# TODO: Consider validating other formats.
print('Done.')
def make_tagged_corpus(perashki_dir=settings.PERASHKI_UNTAGGED_DIR, perashki_tagged_dir=settings.PERASHKI_TAGGED_DIR):
perashki = [(join(perashki_dir, f), join(perashki_tagged_dir, f)) for f in listdir(perashki_dir) if isfile(join(perashki_dir, f))]
morph = pymorphy2.MorphAnalyzer()
for perashok_file, perashok_tagged_file in perashki:
with open(perashok_file, encoding='utf-8') as input_stream, open(perashok_tagged_file, 'w', encoding='utf-8') as output_stream:
for line in input_stream.readlines()[2:-1]:
words = line.split()
words_tagged = []
for word in words:
words_tagged.append('{}:{}:{}'.format(word, get_lemma(word, morph), get_pos(word, morph)))
print(*words_tagged, file=output_stream)
from keras import models
from keras.utils import np_utils
from keras.datasets import mnist
import keras as keras
import keras.backend as K
import numpy as np
from os import listdir
from os.path import isfile, join
import sys
path = sys.argv[1]
X_train = []
Y_train = []
onlyfiles = [f for f in listdir(path) if isfile(join(path, f))]
print("Loading data...")
for file in onlyfiles:
file = join(path, file)
f = open(file, "r", encoding='latin-1')
line = f.readline()
while line:
try:
splitted = line.replace("(", "").replace(")", "") \
.replace("[", "").replace("]", "") \
.replace('\n', "").replace(" ", "").split(",")
def _exists(self) -> bool:
return isfile(self._filepath)
folder = 'dataset/' new_folder = 'processed_dataset/' index = 5 TEST_DATA = [] def squash_arrays(files, path): arrays = [np.load(f) for f in files] res = np.concatenate(arrays) np.random.shuffle(res) TEST_DATA.extend(res[-10:]) res = res[:-10] np.save(path, res) del arrays del res if __name__ == '__main__': if not isdir(new_folder): makedirs(new_folder) npy_files = [join(folder, f) for f in listdir(folder) if isfile(join(folder, f))] while index <= len(npy_files) or index == 5: files = npy_files[index - 5: index] path = join(new_folder, str(index // 5)) squash_arrays(files, path) index += 5 np.save(join(new_folder, 'test.npy'), TEST_DATA)
Searches for all files matching a particular glob pattern, extracts the given slice as an
integer, and makes sure it is in the list of allowed numbers. If not, the file is deleted.
"""
from glob import iglob
from os import unlink
from os.path import basename, join, isfile
files = ((f, basename(f)[match_slice]) for f in iglob(join(d, pattern))
if isfile(f))
for f in (f for f, x in files if x.isdigit() and int(x) not in allowed):
try:
unlink(f)
except Exception, e:
pass
files = ((f, basename(f)[match_slice])
for f in iglob(join(d, '.' + pattern)) if isfile(f))
for f in (f for f, x in files if x.isdigit() and int(x) not in allowed):
try:
unlink(f)
except Exception, e:
pass
def get_terminal_width():
"""Gets the width of the terminal if there is a terminal, in which case 80 is returned."""
from os import environ
from sys import platform
if platform == "win32":
# Windows
from ctypes import windll, c_short, c_ushort, c_int, c_uint, c_void_p, byref, Structure
def initGui(self):
# Setup menu
self.quickosm_menu = QMenu('Quick OSM')
self.quickosm_menu.setIcon(QIcon(':/plugins/QuickOSM/icon.png'))
self.dock_menu = QMenu(tr('QuickOSM', u'Dock'))
self.vector_menu = self.iface.vectorMenu()
self.vector_menu.addMenu(self.quickosm_menu)
# Main window
self.mainWindowAction = QAction(
QIcon(':/plugins/QuickOSM/icon.png'),
u'QuickOSM',
self.iface.mainWindow())
# noinspection PyUnresolvedReferences
self.mainWindowAction.triggered.connect(self.openMainWindow)
self.toolbar.addAction(self.mainWindowAction)
self.iface.QuickOSM_mainWindowDialog = MainWindowDialog()
# OSM File
self.osmFileAction = QAction(
QIcon(':/plugins/QuickOSM/resources/open.png'),
tr('ui_osm_file', 'OSM File'),
self.iface.mainWindow())
# noinspection PyUnresolvedReferences
self.osmFileAction.triggered.connect(self.openOsmFileDockWidget)
self.osmFileDockWidget = OsmFileDockWidget()
self.iface.addDockWidget(
Qt.RightDockWidgetArea, self.osmFileDockWidget)
self.osmFileDockWidget.hide()
self.osmFileDockWidget.setObjectName('osmFileWidget')
# My queries
self.myQueriesAction = QAction(
QIcon(':/plugins/QuickOSM/resources/favorites.png'),
tr('ui_my_queries', 'My queries'),
self.iface.mainWindow())
# noinspection PyUnresolvedReferences
self.myQueriesAction.triggered.connect(self.openMyQueriesDockWidget)
self.myQueriesDockWidget = MyQueriesDockWidget()
self.iface.addDockWidget(
Qt.RightDockWidgetArea, self.myQueriesDockWidget)
self.myQueriesDockWidget.hide()
self.myQueriesDockWidget.setObjectName('myQueriesWidget')
# Query
self.queryAction = QAction(
QIcon(':/plugins/QuickOSM/resources/edit.png'),
tr('ui_query', 'Query'),
self.iface.mainWindow())
# noinspection PyUnresolvedReferences
self.queryAction.triggered.connect(self.openQueryDockWidget)
self.queryDockWidget = QueryDockWidget()
self.iface.addDockWidget(Qt.RightDockWidgetArea, self.queryDockWidget)
self.queryDockWidget.hide()
self.queryDockWidget.setObjectName('queryWidget')
# Quick query
self.quickQueryAction = QAction(
QIcon(':/plugins/QuickOSM/resources/quick.png'),
tr('ui_quick_query', 'Quick query'),
self.iface.mainWindow())
# noinspection PyUnresolvedReferences
self.quickQueryAction.triggered.connect(self.openQuickQueryDockWidget)
self.quickQueryDockWidget = QuickQueryDockWidget()
self.iface.addDockWidget(
Qt.RightDockWidgetArea, self.quickQueryDockWidget)
self.quickQueryDockWidget.hide()
self.quickQueryDockWidget.setObjectName('quickQueryWidget')
# Insert in the good order
self.quickosm_menu.addAction(self.mainWindowAction)
self.quickosm_menu.addMenu(self.dock_menu)
self.dock_menu.addAction(self.quickQueryAction)
self.dock_menu.addAction(self.queryAction)
self.dock_menu.addAction(self.myQueriesAction)
self.dock_menu.addAction(self.osmFileAction)
# Connect signals and slots from dock
self.queryDockWidget.signal_new_query_successful.connect(
self.iface.QuickOSM_mainWindowDialog.refresh_my_queries_tree)
self.queryDockWidget.signal_new_query_successful.connect(
self.myQueriesDockWidget.refresh_my_queries_tree)
self.myQueriesDockWidget.signal_delete_query_successful.connect(
self.myQueriesDockWidget.refresh_my_queries_tree)
self.myQueriesDockWidget.signal_delete_query_successful.connect(
self.iface.QuickOSM_mainWindowDialog.refresh_my_queries_tree)
# Connect signals and slots from mainWindow
self.iface.QuickOSM_mainWindowDialog.signal_new_query_successful.\
connect(self.myQueriesDockWidget.refresh_my_queries_tree)
self.iface.QuickOSM_mainWindowDialog.signal_new_query_successful.\
connect(
self.iface.QuickOSM_mainWindowDialog.refresh_my_queries_tree)
self.iface.QuickOSM_mainWindowDialog.signal_delete_query_successful.\
connect(self.myQueriesDockWidget.refresh_my_queries_tree)
self.iface.QuickOSM_mainWindowDialog.signal_delete_query_successful.\
connect(
self.iface.QuickOSM_mainWindowDialog.refresh_my_queries_tree)
# Read the config file
json_file_config = join(dirname(abspath(__file__)), 'config.json')
if isfile(json_file_config):
config_json = load(open(json_file_config))
for server in config_json['overpass_servers']:
self.iface.QuickOSM_mainWindowDialog.comboBox_default_OAPI.\
addItem(server)
# Check previous version and if new queries are available
version = get_setting('version')
current_version = get_current_version()
if version != current_version:
if new_queries_available():
message = 'New queries are available in the plugin. Would ' \
'like to install them ? This will overwrite the ' \
'current default queries.'
title = 'QuickOSM'
message = tr('QuickOSM', message)
widget = self.iface.messageBar().createMessage(title, message)
button_ok = QPushButton(widget)
button_ok.setText(
tr('QuickOSM', 'Install'))
button_ok.pressed.connect(self.restoreDefaultQueries)
widget.layout().addWidget(button_ok)
self.iface.messageBar().pushWidget(
widget, QgsMessageBar.INFO, 0)
set_setting('version', current_version)
def create_summary_evaluationmetrics(destdir):
'''Creates a new HDF file storing some metrics for all precidtions
(HDF files) found inside `destdir` and subfolders
:param destdir: a destination directory **whose FILE SUBTREE STRUCTURE
MUST HAVE BEEN CREATED BY `run_evaluation` or (if calling from scrippt file)
`evaluate.py`: a list of scikit model files with associated directories
(the model name without the extension '.sklmodel') storing each
prediction run on HDF datasets.
`'''
print('Computing summary evaluation metrics from '
'predictions data frames (HDF file)')
eval_df_path = join(destdir, 'evaluationmetrics.hdf')
dfr, already_processed_tuples = None, set()
if isfile(eval_df_path):
dfr = pd.read_hdf(eval_df_path)
already_processed_tuples = set(dfr._key)
def build_key(clfdir, testname):
return join(basename(clfdir), testname)
# a set is faster than a dataframe for searching already processed
# couples of (clf, testset_hdf):
newrows = []
clfs_prediction_paths = []
for clfname in [] if not isdir(destdir) else listdir(destdir):
clfdir, ext = splitext(clfname)
if ext != '.sklmodel':
continue
clfdir = join(destdir, clfdir)
for testname in [] if not isdir(clfdir) else listdir(clfdir):
if build_key(clfdir, testname) in already_processed_tuples:
continue
clfs_prediction_paths.append((clfdir, testname))
print('%d new prediction(s) found' % len(clfs_prediction_paths))
if clfs_prediction_paths:
errors = []
pool = Pool(processes=int(cpu_count()))
with click.progressbar(length=len(clfs_prediction_paths),
fill_char='o', empty_char='.') as pbar:
for clfdir, testname, dic in \
pool.imap_unordered(_get_summary_evaluationmetrics_mp,
clfs_prediction_paths):
pbar.update(1)
if isinstance(dic, Exception):
errors.append(dic)
else:
dic['_key'] = build_key(clfdir, testname)
newrows.append(dic)
if newrows:
new_df = pd.DataFrame(data=newrows)
if dfr is None:
dfr = new_df
else:
dfr = pd.concat([dfr, new_df], axis=0, copy=True,
sort=False, ignore_index=True)
# dfr = dfr.append(new_df,
# ignore_index=True,
# verify_integrity=False,
# sort=False).reset_index(drop=True)
save_df(dfr, eval_df_path)
if errors:
print('%d prediction(s) discarded due to error' % len(errors))
print('(possible cause: only one class found in the prediction)')
def file_checker(top, itp, suff):
"""This function checks for .itp (not containing 'posre' or '/') included in .top and try to
look for them in the current directory or in the directory name of the .top file given by -p.\n
Check if .itp/.top files have an [ atoms ] block to modifiy, else remove it from input files.\n
Finally, builds output names for the topologies"""
if top != []:
print(f'Working on following .top file: {top}\n')
regxitp = re.compile(r'(?<=#include ")(?!.*(\b\/\b|posre)).*(?=")')
with open(top[0], 'r') as intop:
for line in intop:
if regxitp.search(line):
if regxitp.search(line).group() not in itp:
itp.append(regxitp.search(line).group())
# Check if those .itp are in current directory or in the dirname of .top given by -p
for i in itp:
if not path.isfile(i):
dirname_of_top = path.dirname(top[0])
print(
f"{i} included in .top not found in './', "
f"let's look in dirname of the -p argument (i.e. {dirname_of_top})"
)
guess_itp_path = f'{dirname_of_top}/{i}'
if path.isfile(guess_itp_path):
print(f'{i} found in {dirname_of_top}')
itp[itp.index(i)] = guess_itp_path
else:
print(
f"{i} included in .top also not present in {dirname_of_top}:"
f" exiting script")
raise SystemExit()
topols = top + itp
regx_atmblock = re.compile(r'^\[ atoms \]\n')
topol2rm = []
for i in topols:
keep = 0
with open(i, 'r') as topfile:
for line in topfile:
if regx_atmblock.search(line):
keep = 1
if keep == 0:
print(f'No [ atoms ] block in {i}, this file will be ignored')
topol2rm.append(i)
topols = [topfile for topfile in topols if topfile not in topol2rm]
print(
f'\nThe following .itp and/or .top will be used for generating heavy H topologies:'
f'\n{topols}\n')
topolout = [
re.sub(r'(\.itp|\.top)', '', path.basename(name)) + suff + '.' +
name.split('.')[-1] for name in topols
]
topolin_topolout = {topols[i]: topolout[i] for i in range(len(topols))}
return topolin_topolout
from os.path import isfile, isdir
from tqdm import tqdm
import problem_unittests as tests
import tarfile
cifar10_dataset_folder_path = 'cifar-10-batches-py'
class DLProgress(tqdm):
last_block = 0
def hook(self, block_num=1, block_size=1, total_size=None):
self.total = total_size
self.update((block_num - self.last_block) * block_size)
self.last_block = block_num
if not isfile('cifar-10-python.tar.gz'):
with DLProgress(unit='B', unit_scale=True, miniters=1, desc='CIFAR-10 Dataset') as pbar:
urlretrieve(
'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz',
'cifar-10-python.tar.gz',
pbar.hook)
if not isdir(cifar10_dataset_folder_path):
with tarfile.open('cifar-10-python.tar.gz') as tar:
tar.extractall()
tar.close()
tests.test_folder_path(cifar10_dataset_folder_path)
from pkgutil import extend_path
__extended_path = "/home/callum/buggy_ws/src/robot_upstart-jade-devel/src".split(
";")
for p in reversed(__extended_path):
sys_path.insert(0, p)
del p
del sys_path
__path__ = extend_path(__path__, __name__)
del extend_path
__execfiles = []
for p in __extended_path:
src_init_file = os_path.join(p, __name__ + '.py')
if os_path.isfile(src_init_file):
__execfiles.append(src_init_file)
else:
src_init_file = os_path.join(p, __name__, '__init__.py')
if os_path.isfile(src_init_file):
__execfiles.append(src_init_file)
del src_init_file
del p
del os_path
del __extended_path
for __execfile in __execfiles:
with open(__execfile, 'r') as __fh:
exec(__fh.read())
del __fh
del __execfile
def runTraining(opts):
print('' * 41)
print('~' * 50)
print('~~~~~~~~~~~~~~~~~ PARAMETERS ~~~~~~~~~~~~~~~~')
print('~' * 50)
print(' - Number of image modalities: {}'.format(opts.numModal))
print(' - Number of classes: {}'.format(opts.numClasses))
print(' - Directory to load images: {}'.format(opts.root_dir))
for i in range(len(opts.modality_dirs)):
print(' - Modality {}: {}'.format(i + 1, opts.modality_dirs[i]))
print(' - Directory to save results: {}'.format(opts.save_dir))
print(' - To model will be saved as : {}'.format(opts.modelName))
print('-' * 41)
print(' - Number of epochs: {}'.format(opts.numClasses))
print(' - Batch size: {}'.format(opts.batchSize))
print(' - Number of samples per epoch: {}'.format(opts.numSamplesEpoch))
print(' - Learning rate: {}'.format(opts.l_rate))
print('' * 41)
print('-' * 41)
print('~~~~~~~~ Starting the training... ~~~~~~')
print('-' * 41)
print('' * 40)
a = 64
b = 64
samplesPerEpoch = opts.numSamplesEpoch
batch_size = opts.batchSize
lr = opts.l_rate
epoch = opts.numEpochs
root_dir = opts.root_dir
model_name = opts.modelName
if not (len(opts.modality_dirs) == opts.numModal): raise AssertionError
moda_1 = root_dir + 'Training/' + opts.modality_dirs[0]
moda_2 = root_dir + 'Training/' + opts.modality_dirs[1]
if (opts.numModal == 3):
moda_3 = root_dir + 'Training/' + opts.modality_dirs[2]
moda_g = root_dir + 'Training/GT'
print(' --- Getting image names.....')
print(' - Training Set: -')
if os.path.exists(moda_1):
imageNames_tr = [
f for f in os.listdir(moda_1) if isfile(join(moda_1, f))
]
np.random.seed(1)
np.random.shuffle(imageNames_tr)
imageNames_val = imageNames_tr[0:40]
imageNames_train = list(set(imageNames_tr) - set(imageNames_val))
print(' ------- Images found ------')
for i in range(len(imageNames_train)):
print(' - {}'.format(imageNames_train[i]))
else:
raise Exception(' - {} does not exist'.format(moda_1))
moda_1_val = root_dir + 'Training/' + opts.modality_dirs[0]
moda_2_val = root_dir + 'Training/' + opts.modality_dirs[1]
if (opts.numModal == 3):
moda_3_val = root_dir + 'Training/' + opts.modality_dirs[2]
moda_g_val = root_dir + 'Training/GT'
print(' --------------------')
print(' - Validation Set: -')
if os.path.exists(moda_1):
# imageNames_val = [f for f in os.listdir(moda_1_val) if isfile(join(moda_1_val, f))]
# imageNames_val.sort()
print(' ------- Images found ------')
for i in range(len(imageNames_val)):
print(' - {}'.format(imageNames_val[i]))
else:
raise Exception(' - {} does not exist'.format(moda_1_val))
print("~~~~~~~~~~~ Creating the model ~~~~~~~~~~")
num_classes = opts.numClasses
# Define UNet_FEE_2mod
# To-Do. Get as input the config settings to create different networks
if (opts.numModal == 2):
hdNet = UNet_FEE(in_dim=2, out_dim=1, num_filters=4)
#
'''try:
hdNet = torch.load(os.path.join(model_name, "Best_" + model_name + ".pkl"))
print("--------model restored--------")
except:
print("--------model not restored--------")
pass'''
#softMax = nn.Softmax()
softMax = nn.Sigmoid()
CE_loss = DicexCELoss()
if torch.cuda.is_available():
hdNet.cuda()
softMax.cuda()
CE_loss.cuda()
# To-DO: Check that optimizer is the same (and same values) as the Theano implementation
optimizer = torch.optim.Adam(hdNet.parameters(), lr=lr, betas=(0.9, 0.999))
#optimizer = torch.optim.SGD(hdNet.parameters(), lr=lr, momentum = 0.6)
print(" ~~~~~~~~~~~ Starting the training ~~~~~~~~~~")
dscAll = []
accall = []
train_eval = []
dsc_eval = []
acc_eval = []
d1 = 0
if (opts.numModal == 2):
imgPaths = [moda_1, moda_2]
if (opts.numModal == 3):
imgPaths = [moda_1, moda_2, moda_3]
val_epochs = [0, 21, 41, 61, 81, 101, 121, 141]
x_train, y_train, img_shape = load_data_trainG(imgPaths, moda_g,
imageNames_train,
samplesPerEpoch,
opts.numModal)
print(x_train.shape)
numBatches = int(x_train.shape[0] / batch_size)
idx = np.arange(x_train.shape[0])
for e_i in range(epoch):
hdNet.train()
lossEpoch = []
np.random.shuffle(idx)
x_train = x_train[idx]
y_train = y_train[idx]
for b_i in range(numBatches):
optimizer.zero_grad()
hdNet.zero_grad()
MRIs = numpy_to_var(x_train[b_i * batch_size:b_i * batch_size +
batch_size, :, :, :, :])
Segmentation = numpy_to_var(
y_train[b_i * batch_size:b_i * batch_size +
batch_size, :, :, :])
segmentation_prediction = hdNet(MRIs)
#print("segmentation_prediction : ", segmentation_prediction.shape)
#predClass_y = softMax(segmentation_prediction)
segmentation_prediction = softMax(segmentation_prediction)
# To adapt CE to 3D
# LOGITS:
segmentation_prediction = segmentation_prediction.permute(
0, 2, 3, 4, 1).contiguous()
segmentation_prediction = segmentation_prediction.reshape(
segmentation_prediction.numel() // num_classes, num_classes)
if e_i == 0 and b_i == 0:
print("MRIS : ", MRIs.shape)
print("Segmentation : ", Segmentation.shape)
print("segmentation_prediction : ",
segmentation_prediction.shape)
CE_loss_batch = CE_loss(
segmentation_prediction,
Segmentation.reshape(-1).type(torch.cuda.FloatTensor))
loss = CE_loss_batch
loss.backward()
optimizer.step()
lossEpoch.append(CE_loss_batch.cpu().data.numpy())
printProgressBar(b_i + 1,
numBatches,
prefix="[Training] Epoch: {} ".format(e_i),
length=15)
del MRIs
del Segmentation
del segmentation_prediction
# del predClass_y
if not os.path.exists(model_name):
os.makedirs(model_name)
np.save(os.path.join(model_name, model_name + '_loss.npy'), dscAll)
print(' Epoch: {}, loss: {}'.format(e_i, np.mean(lossEpoch)))
if (e_i % 5) == 0:
if (opts.numModal == 2):
moda_n = [moda_1_val, moda_2_val]
if (opts.numModal == 3):
moda_n = [moda_1_val, moda_2_val, moda_3_val]
dsct, acct = inferencee(hdNet, x_train, y_train, imageNames_train,
e_i, opts.save_dir, opts.numModal)
dsc_eval.append(dsct)
acc_eval.append(acct)
print(' Metrics: The mean of train Accuracy is : {} '.format(acct))
print(' Metrics: The mean of train DSC is : {} '.format(dsct))
np.save(os.path.join(model_name, model_name + 'train_DSCs.npy'),
dsc_eval)
np.save(os.path.join(model_name, model_name + 'train_ACC.npy'),
acc_eval)
dsc, acc = inference(hdNet, moda_n, moda_g_val, imageNames_val,
e_i, opts.save_dir, opts.numModal)
dscAll.append(dsc)
accall.append(acc)
print(' Metrics: The mean of Accuracy is : {} '.format(
np.mean(acc)))
print(' Metrics: The mean of DSC is : {} '.format(np.mean(dsc)))
if not os.path.exists(model_name):
os.makedirs(model_name)
np.save(os.path.join(model_name, model_name + '_DSCs.npy'), dscAll)
np.save(os.path.join(model_name, model_name + '_ACC.npy'), accall)
if np.mean(dsc) > 0.60:
if not os.path.exists(model_name):
os.makedirs(model_name)
torch.save(
hdNet,
os.path.join(model_name,
"Best2_" + model_name + str(e_i) + ".pkl"))
"""
def run_evaluation(training_param, test_param, destdir):
'''Runs the model evaluation
'''
if not isdir(destdir):
raise NotADirectoryError(''"%s"'' % destdir)
print('Running Evaluator. All files will be stored in:\n%s' %
destdir)
print('Step 1 of 2: Training (creating models)')
newly_created_models = 0
print('Reading Training file (HDF)')
# returns iterable of classifier TO BE CREATED (already existing are not
# yielded):
iterargs = list(training_param.iterargs(destdir))
if iterargs:
pool = Pool(processes=int(cpu_count()))
with click.progressbar(length=len(iterargs),
fill_char='o', empty_char='.') as pbar:
try:
print('Building classifiers from parameters and training file')
for clf, destpath in \
pool.imap_unordered(_classifier_mp, iterargs):
# save on the main process (here):
if clf is not None:
save_clf(clf, destpath)
newly_created_models += 1
pbar.update(1)
# absolutely call these methods
# although in impa and imap unordered
# do make sense?
pool.close()
pool.join()
except Exception as exc:
_kill_pool(pool, str(exc))
raise exc
classifier_paths = [
_ for _ in training_param.classifier_paths(destdir) if isfile(_)
]
print("%d of %d models created (already existing were not overwritten)" %
(newly_created_models, len(classifier_paths)))
print('Step 2 of 2: Testing (creating prediction data frames)')
pred_filepaths = []
test_param.set_classifiers_paths(classifier_paths)
# with set_classifiers_paths above, we internally stored classifiers
# who do NOT have a prediction dataframe. `num_iterations` below accounts
# for thius, thus being zero for several reasons, among which also
# if all classifiers already have a relative prediction dataframe:
num_iterations = test_param.num_iterations
if num_iterations:
pool = Pool(processes=int(cpu_count()))
with click.progressbar(length=num_iterations,
fill_char='o', empty_char='.') as pbar:
try:
for test_df_chunk in test_param.read_testset():
iterargs = test_param.iterargs(test_df_chunk)
for pred_df, destpath in \
pool.imap_unordered(_predict_mp, iterargs):
if pred_df is not None:
# save on the main process (here):
kwargs = {'append': True}
if test_param.min_itemsize:
kwargs['min_itemsize'] = test_param.min_itemsize
save_df(pred_df, destpath, **kwargs)
if destpath not in pred_filepaths:
pred_filepaths.append(destpath)
pbar.update(1)
# absolutely call these methods,
# although in imap and imap unordered
# do make sense?
pool.close()
pool.join()
except Exception as exc:
_kill_pool(pool, str(exc))
raise exc
print("%d of %d prediction HDF file(s) created "
"(already existing were not overwritten)" %
(len(pred_filepaths), len(classifier_paths)))
print()
print('Creating evaluation metrics')
create_summary_evaluationmetrics(destdir)
print('DONE')
#!/usr/bin/python
import os
from os import listdir
from os.path import isfile, join
from subprocess import call
fileList = [ f for f in listdir(".") if isfile(join(".",f)) and f.endswith('.zip') ]
for f in fileList:
name = f.strip('.zip')
os.mkdir(name)
os.chdir(name)
call(["unzip","../"+f])
os.chdir('..')
os.rename('twinnings','twinings')
def initialize_options(self):
tf_gcc = check_tf_cuda_compatibility()
self.gcc = tf_gcc if isfile(tf_gcc) else 'gcc'
self.nvcc = '/usr/local/cuda/bin/nvcc' if isfile('/usr/local/cuda/bin/nvcc') else 'nvcc'
self.cuda_lib = '/usr/local/cuda/lib64/'
def read_doc(self, docname, app=None):
"""Parse a file and add/update inventory entries for the doctree."""
self.temp_data['docname'] = docname
# defaults to the global default, but can be re-set in a document
self.temp_data['default_domain'] = \
self.domains.get(self.config.primary_domain)
self.settings['input_encoding'] = self.config.source_encoding
self.settings['trim_footnote_reference_space'] = \
self.config.trim_footnote_reference_space
self.settings['gettext_compact'] = self.config.gettext_compact
docutilsconf = path.join(self.srcdir, 'docutils.conf')
# read docutils.conf from source dir, not from current dir
OptionParser.standard_config_files[1] = docutilsconf
if path.isfile(docutilsconf):
self.note_dependency(docutilsconf)
with sphinx_domains(self):
if self.config.default_role:
role_fn, messages = roles.role(self.config.default_role, english,
0, dummy_reporter)
if role_fn:
roles._roles[''] = role_fn
else:
self.warn(docname, 'default role %s not found' %
self.config.default_role)
codecs.register_error('sphinx', self.warn_and_replace)
# publish manually
reader = SphinxStandaloneReader(self.app, parsers=self.config.source_parsers)
pub = Publisher(reader=reader,
writer=SphinxDummyWriter(),
destination_class=NullOutput)
pub.set_components(None, 'restructuredtext', None)
pub.process_programmatic_settings(None, self.settings, None)
src_path = self.doc2path(docname)
source = SphinxFileInput(app, self, source=None, source_path=src_path,
encoding=self.config.source_encoding)
pub.source = source
pub.settings._source = src_path
pub.set_destination(None, None)
pub.publish()
doctree = pub.document
# post-processing
self.process_dependencies(docname, doctree)
self.process_images(docname, doctree)
self.process_downloads(docname, doctree)
self.process_metadata(docname, doctree)
self.create_title_from(docname, doctree)
for manager in itervalues(self.managers):
manager.process_doc(docname, doctree)
for domain in itervalues(self.domains):
domain.process_doc(self, docname, doctree)
# allow extension-specific post-processing
if app:
app.emit('doctree-read', doctree)
# store time of reading, for outdated files detection
# (Some filesystems have coarse timestamp resolution;
# therefore time.time() can be older than filesystem's timestamp.
# For example, FAT32 has 2sec timestamp resolution.)
self.all_docs[docname] = max(
time.time(), path.getmtime(self.doc2path(docname)))
if self.versioning_condition:
old_doctree = None
if self.versioning_compare:
# get old doctree
try:
with open(self.doc2path(docname,
self.doctreedir, '.doctree'), 'rb') as f:
old_doctree = pickle.load(f)
except EnvironmentError:
pass
# add uids for versioning
if not self.versioning_compare or old_doctree is None:
list(add_uids(doctree, self.versioning_condition))
else:
list(merge_doctrees(
old_doctree, doctree, self.versioning_condition))
# make it picklable
doctree.reporter = None
doctree.transformer = None
doctree.settings.warning_stream = None
doctree.settings.env = None
doctree.settings.record_dependencies = None
# cleanup
self.temp_data.clear()
self.ref_context.clear()
roles._roles.pop('', None) # if a document has set a local default role
# save the parsed doctree
doctree_filename = self.doc2path(docname, self.doctreedir,
'.doctree')
ensuredir(path.dirname(doctree_filename))
with open(doctree_filename, 'wb') as f:
pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
def test_CreatePythonEntryPointAction_noarch_python(self):
target_python_version = '%d.%d' % sys.version_info[:2]
transaction_context = {
'target_python_version': target_python_version,
}
package_info = AttrDict(package_metadata=AttrDict(noarch=AttrDict(
type=NoarchType.python,
entry_points=(
'command1=some.module:main',
'command2=another.somewhere:go',
),
)))
axns = CreatePythonEntryPointAction.create_actions(transaction_context, package_info,
self.prefix, LinkType.hardlink)
grouped_axns = groupby(lambda ax: isinstance(ax, LinkPathAction), axns)
windows_exe_axns = grouped_axns.get(True, ())
assert len(windows_exe_axns) == (2 if on_win else 0)
py_ep_axns = grouped_axns.get(False, ())
assert len(py_ep_axns) == 2
py_ep_axn = py_ep_axns[0]
command, module, func = parse_entry_point_def('command1=some.module:main')
assert command == 'command1'
if on_win:
target_short_path = "%s\\%s-script.py" % (get_bin_directory_short_path(), command)
else:
target_short_path = "%s/%s" % (get_bin_directory_short_path(), command)
assert py_ep_axn.target_full_path == join(self.prefix, target_short_path)
assert py_ep_axn.module == module == 'some.module'
assert py_ep_axn.func == func == 'main'
mkdir_p(dirname(py_ep_axn.target_full_path))
py_ep_axn.execute()
assert isfile(py_ep_axn.target_full_path)
if not on_win:
assert is_executable(py_ep_axn.target_full_path)
with open(py_ep_axn.target_full_path) as fh:
lines = fh.readlines()
first_line = lines[0].strip()
last_line = lines[-1].strip()
if not on_win:
python_full_path = join(self.prefix, get_python_short_path(target_python_version))
assert first_line == "#!%s" % python_full_path
assert last_line == "exit(%s())" % func
py_ep_axn.reverse()
assert not isfile(py_ep_axn.target_full_path)
if on_win:
windows_exe_axn = windows_exe_axns[0]
target_short_path = "%s\\%s.exe" % (get_bin_directory_short_path(), command)
assert windows_exe_axn.target_full_path == join(self.prefix, target_short_path)
mkdir_p(dirname(windows_exe_axn.target_full_path))
windows_exe_axn.verify()
windows_exe_axn.execute()
assert isfile(windows_exe_axn.target_full_path)
assert is_executable(windows_exe_axn.target_full_path)
src = compute_md5sum(join(context.conda_prefix, 'Scripts/conda.exe'))
assert src == compute_md5sum(windows_exe_axn.target_full_path)
windows_exe_axn.reverse()
assert not isfile(windows_exe_axn.target_full_path)
async def google_drive(gdrive):
reply = ''
""" - Parsing all google drive function - """
value = gdrive.pattern_match.group(1)
file_path = None
uri = None
if not value and not gdrive.reply_to_msg_id:
return
elif value and gdrive.reply_to_msg_id:
return await gdrive.edit(
"`[UNKNOWN - ERROR]`\n\n"
"`Status` : **failed**\n"
"`Reason` : Confused to upload file or the replied message/media."
)
service = await create_app(gdrive)
if service is False:
return
if isfile(value):
file_path = value
if file_path.endswith(".torrent"):
uri = [file_path]
file_path = None
elif isdir(value):
folder_path = value
global parent_Id
folder_name = await get_raw_name(folder_path)
folder = await create_dir(service, folder_name)
parent_Id = folder.get('id')
webViewURL = "https://drive.google.com/drive/folders/" + parent_Id
try:
await task_directory(gdrive, service, folder_path)
except CancelProcess:
await gdrive.respond(
"`[FOLDER - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
await reset_parentId()
return await gdrive.delete()
except Exception as e:
await gdrive.edit(
"`[FOLDER - UPLOAD]`\n\n"
f"`{folder_name}`\n"
"`Status` : **BAD**\n"
f"`Reason` : {str(e)}"
)
return await reset_parentId()
else:
await gdrive.edit(
"`[FOLDER - UPLOAD]`\n\n"
f"[{folder_name}]({webViewURL})\n"
"`Status` : **OK** - Successfully uploaded.\n\n"
)
return await reset_parentId()
elif not value and gdrive.reply_to_msg_id:
reply += await download(gdrive, service)
await gdrive.respond(reply)
return await gdrive.delete()
else:
if re.findall(r'\bhttps?://drive\.google\.com\S+', value):
""" - Link is google drive fallback to download - """
value = re.findall(r'\bhttps?://drive\.google\.com\S+', value)
for uri in value:
try:
reply += await download_gdrive(gdrive, service, uri)
except CancelProcess:
reply += (
"`[FILE - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
break
except Exception as e:
reply += (
"`[FILE - ERROR]`\n\n"
"`Status` : **BAD**\n"
f"`Reason` : {str(e)}\n\n"
)
continue
if reply:
await gdrive.respond(reply, link_preview=False)
return await gdrive.delete()
else:
return
elif re.findall(r'\bhttps?://.*\.\S+', value) or "magnet:?" in value:
uri = value.split()
else:
for fileId in value.split():
if any(map(str.isdigit, fileId)):
one = True
else:
one = False
if "-" in fileId or "_" in fileId:
two = True
else:
two = False
if True in [one or two]:
try:
reply += await download_gdrive(gdrive, service, fileId)
except CancelProcess:
reply += (
"`[FILE - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
break
except Exception as e:
reply += (
"`[FILE - ERROR]`\n\n"
"`Status` : **BAD**\n"
f"`Reason` : {str(e)}\n\n"
)
continue
if reply:
await gdrive.respond(reply, link_preview=False)
return await gdrive.delete()
else:
return
if not uri and not gdrive.reply_to_msg_id:
return await gdrive.edit(
"`[VALUE - ERROR]`\n\n"
"`Status` : **BAD**\n"
"`Reason` : given value is not URL nor file/folder path. "
"If you think this is wrong, maybe you use .gd with multiple "
"value of files/folders, e.g `.gd <filename1> <filename2>` "
"for upload from files/folders path this doesn't support it."
)
if uri and not gdrive.reply_to_msg_id:
for dl in uri:
try:
reply += await download(gdrive, service, dl)
except Exception as e:
if " not found" in str(e) or "'file'" in str(e):
reply += (
"`[FILE - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
await asyncio.sleep(2.5)
break
else:
""" - if something bad happened, continue to next uri - """
reply += (
"`[UNKNOWN - ERROR]`\n\n"
"`Status` : **BAD**\n"
f"`Reason` : `{dl}` | `{str(e)}`\n\n"
)
continue
await gdrive.respond(reply, link_preview=False)
return await gdrive.delete()
mimeType = await get_mimeType(file_path)
file_name = await get_raw_name(file_path)
try:
result = await upload(gdrive, service,
file_path, file_name, mimeType)
except CancelProcess:
gdrive.respond(
"`[FILE - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
if result:
await gdrive.respond(
"`[FILE - UPLOAD]`\n\n"
f"`Name :` `{file_name}`\n"
f"`Size :` `{humanbytes(result[0])}`\n"
f"`Link :` [{file_name}]({result[1]})\n"
"`Status :` **OK** - Successfully uploaded.\n",
link_preview=False
)
await gdrive.delete()
return
def _run_interface(self, runtime):
raw_filename = self.inputs.raw_filename
cov_fname_in = self.inputs.cov_fname_in
is_epoched = self.inputs.is_epoched
is_evoked = self.inputs.is_evoked
events_id = self.inputs.events_id
t_min = self.inputs.t_min
t_max = self.inputs.t_max
data_path, basename, ext = split_f(raw_filename)
self.cov_fname_out = op.join(data_path, '%s-cov.fif' % basename)
if not op.isfile(cov_fname_in):
if is_epoched and is_evoked:
raw = read_raw_fif(raw_filename)
events = find_events(raw)
if not op.isfile(self.cov_fname_out):
print(('\n*** COMPUTE COV FROM EPOCHS ***\n' +
self.cov_fname_out))
reject = create_reject_dict(raw.info)
picks = pick_types(raw.info,
meg=True,
ref_meg=False,
exclude='bads')
epochs = Epochs(raw,
events,
events_id,
t_min,
t_max,
picks=picks,
baseline=(None, 0),
reject=reject)
# TODO method='auto'? too long!!!
noise_cov = compute_covariance(epochs,
tmax=0,
method='diagonal_fixed')
write_cov(self.cov_fname_out, noise_cov)
else:
print(('\n *** NOISE cov file %s exists!!! \n' %
self.cov_fname_out))
else:
'\n *** RAW DATA \n'
for er_fname in glob.glob(op.join(data_path, cov_fname_in)):
print(('\n found file name %s \n' % er_fname))
try:
if er_fname.rfind('cov.fif') > -1:
print(('\n *** NOISE cov file %s exists!! \n' %
er_fname))
self.cov_fname_out = er_fname
else:
if er_fname.rfind('.fif') > -1:
er_raw = read_raw_fif(er_fname)
er_fname = er_fname.replace('.fif', '-raw-cov.fif')
elif er_fname.rfind('.ds') > -1:
er_raw = read_raw_ctf(er_fname)
er_fname = er_fname.replace('.ds', '-raw-cov.fif')
self.cov_fname_out = op.join(data_path, er_fname)
if not op.isfile(self.cov_fname_out):
reject = create_reject_dict(er_raw.info)
picks = pick_types(er_raw.info,
meg=True,
ref_meg=False,
exclude='bads')
noise_cov = compute_raw_covariance(er_raw,
picks=picks,
reject=reject)
write_cov(self.cov_fname_out, noise_cov)
else:
print(('\n *** NOISE cov file %s exists!!! \n' %
self.cov_fname_out))
except NameError:
sys.exit("No covariance matrix as input!")
# TODO creare una matrice diagonale?
else:
print(('\n *** NOISE cov file %s exists!!! \n' % cov_fname_in))
self.cov_fname_out = cov_fname_in
return runtime
async def download(gdrive, service, uri=None):
reply = ''
""" - Download files to local then upload - """
if not isdir(TEMP_DOWNLOAD_DIRECTORY):
os.makedirs(TEMP_DOWNLOAD_DIRECTORY)
required_file_name = None
if uri:
full_path = os.getcwd() + TEMP_DOWNLOAD_DIRECTORY.strip('.')
if isfile(uri) and uri.endswith(".torrent"):
downloads = aria2.add_torrent(
uri,
uris=None,
options={'dir': full_path},
position=None)
else:
uri = [uri]
downloads = aria2.add_uris(
uri,
options={'dir': full_path},
position=None)
gid = downloads.gid
await check_progress_for_dl(gdrive, gid, previous=None)
file = aria2.get_download(gid)
filename = file.name
if file.followed_by_ids:
new_gid = await check_metadata(gid)
await check_progress_for_dl(gdrive, new_gid, previous=None)
try:
required_file_name = TEMP_DOWNLOAD_DIRECTORY + filenames
except Exception:
required_file_name = TEMP_DOWNLOAD_DIRECTORY + filename
else:
try:
current_time = time.time()
downloaded_file_name = await gdrive.client.download_media(
await gdrive.get_reply_message(),
TEMP_DOWNLOAD_DIRECTORY,
progress_callback=lambda d, t: asyncio.get_event_loop(
).create_task(progress(d, t, gdrive, current_time,
"[FILE - DOWNLOAD]")))
except Exception as e:
await gdrive.edit(str(e))
else:
required_file_name = downloaded_file_name
try:
file_name = await get_raw_name(required_file_name)
except AttributeError:
reply += (
"`[ENTRY - ERROR]`\n\n"
"`Status` : **BAD**\n"
)
return reply
mimeType = await get_mimeType(required_file_name)
try:
status = "[FILE - UPLOAD]"
if isfile(required_file_name):
try:
result = await upload(
gdrive, service, required_file_name, file_name, mimeType)
except CancelProcess:
reply += (
"`[FILE - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
return reply
else:
reply += (
f"`{status}`\n\n"
f"`Name :` `{file_name}`\n"
f"`Size :` `{humanbytes(result[0])}`\n"
f"`Link :` [{file_name}]({result[1]})\n"
"`Status :` **OK** - Successfully uploaded.\n\n"
)
return reply
else:
status = status.replace("[FILE", "[FOLDER")
global parent_Id
folder = await create_dir(service, file_name)
parent_Id = folder.get('id')
webViewURL = (
"https://drive.google.com/drive/folders/"
+ parent_Id
)
try:
await task_directory(gdrive, service, required_file_name)
except CancelProcess:
reply += (
"`[FOLDER - CANCELLED]`\n\n"
"`Status` : **OK** - received signal cancelled."
)
await reset_parentId()
return reply
except Exception:
await reset_parentId()
else:
reply += (
f"`{status}`\n\n"
f"[{file_name}]({webViewURL})\n"
"`Status` : **OK** - Successfully uploaded.\n\n"
)
await reset_parentId()
return reply
except Exception as e:
status = status.replace("DOWNLOAD]", "ERROR]")
reply += (
f"`{status}`\n\n"
"`Status` : **failed**\n"
f"`Reason` : `{str(e)}`\n\n"
)
return reply
return
from lib.router import Router
router = Router()
print '>> Walk, Don`t Run'
# --------------------------------------------------------------------------
#STEP: modify version?
configVersion = config['version']
config['version'] = round(
float(configVersion) +
.1, 1) if config['options']['increment-version'] == True else configVersion
#STEP: load
sourcePath = router.getRoute(
config['source']['route']) + config['source']['dir']
sourceFilesNames = [
f for f in listdir(sourcePath) if isfile(join(sourcePath, f))
]
sourceFilesNames.remove('links.json')
'''
Create data frame with: participantName, date, clicks, in, time, dayWeek
- replace characters
-> list by particpant with: {in, clicks, date, time}, sort by date
'''
# STEP: create registry of participants
fileIdx = 0
rawObj = open(sourcePath + sourceFilesNames[fileIdx], 'r')
fileJsonContent = json.load(rawObj)
sys.exit(0)
