def fetch_env_configs(self):
fp = FilePath()
get_structure = fp.get_filepaths(
'{lib_path}/{lib_repo}/config/environments'.format(
lib_path=LIB_PATH, lib_repo=suite_runner_lib_repo()))
return get_structure
def check_exists_recursive(self, filepath_tree):
filepath = FilePath()
for location in filepath_tree.locations:
filepath.add(location)
if not self.check_is_folder(filepath):
return filepath
return True
def get_user_input():
print("Choose process: (delete, copy, move)")
process = input()
print("Enter regex for input files:")
regex = input()
print("Enter origin directory:")
origin = FilePath.from_string(input())
file_manager = FileManager()
origin_files = file_manager.get_folder_contents(origin)
if process in ["copy", "move"]:
print("Enter destination directory:")
destination = FilePath.from_string(input())
else:
destination = None
file_rule = create_file_rule(regex)
process = create_process(file_manager, process)
rule = Rule(file_rule, process, destination)
filtered = rule.apply_files_rule(origin_files)
for p in rule.apply_process(filtered):
input("Enter to continue...")
def main(argv):
minecraft_dir = FilePath(argv[1])
assert minecraft_dir.isdir(), minecraft_dir
servers_filename = minecraft_dir.child(SERVERS_DAT)
servers = get_servers(servers_filename)
for server in servers:
status = server.server.status()
show_server_players(server, status)
return 0
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def gather_elements(self, loc, test_type, template_name):
locator = {}
fp = FilePath()
structure = fp.get_filepaths('{lib_path}/{lib_repo}/elements/{test_type}/{template}'.format(lib_path=LIB_PATH,
test_type=test_type,
template=template_name,
lib_repo=suite_runner_lib_repo()))
for element_file in structure:
with open(element_file) as elements_map:
if bool(re.search(loc, element_file)) is True:
elements = json.load(elements_map)
for element_tag, element_value in elements.iteritems():
locator[element_tag] = element_value
return locator
def mapCallings(client, data_dir='data', template_root='templates'):
data_fp = FilePath(data_dir)
output_root = data_fp.child('output')
if not output_root.exists():
output_root.makedirs()
jenv = Environment(loader=FileSystemLoader(template_root))
jenv.filters['abbr'] = abbreviateCalling
jenv.globals['math'] = math
template = jenv.get_template('callingmap.html')
#members = client.getRawValue('member_list')
callings = client.getRawValue('members_with_callings')
no_calling = client.getRawValue('members_without_callings')
no_calling = [x for x in no_calling if x['age'] >= 12]
# get the groups and subgroups organized into dicts
groups = OrderedDict()
by_suborg = {}
for line in prefOrder:
if line.startswith('#'):
# heading
groups[line[1:].strip()] = OrderedDict()
elif line.strip():
subgroup_key = line.strip()
groups[groups.keys()
[-1]][subgroup_key] = by_suborg[subgroup_key] = []
# put each calling into the right subgroup
# also count the number of callings per person
calling_counts = defaultdict(lambda: 0)
for calling in callings:
suborg = calling['subOrgType'] or calling['organization']
by_suborg[suborg].append(calling)
calling_counts[calling['id']] += 1
fp = output_root.child('callingmap.html')
fp.setContent(
template.render(orgs=groups,
calling_counts=calling_counts,
no_calling=no_calling).encode('utf-8'))
print 'wrote', fp.path
def mapCallings(client, data_dir='data', template_root='templates'):
data_fp = FilePath(data_dir)
output_root = data_fp.child('output')
if not output_root.exists():
output_root.makedirs()
jenv = Environment(loader=FileSystemLoader(template_root))
jenv.filters['abbr'] = abbreviateCalling
jenv.globals['math'] = math
template = jenv.get_template('callingmap.html')
#members = client.getRawValue('member_list')
callings = client.getRawValue('members_with_callings')
no_calling = client.getRawValue('members_without_callings')
no_calling = [x for x in no_calling if x['age'] >= 12]
# get the groups and subgroups organized into dicts
groups = OrderedDict()
by_suborg = {}
for line in prefOrder:
if line.startswith('#'):
# heading
groups[line[1:].strip()] = OrderedDict()
elif line.strip():
subgroup_key = line.strip()
groups[groups.keys()[-1]][subgroup_key] = by_suborg[subgroup_key] = []
# put each calling into the right subgroup
# also count the number of callings per person
calling_counts = defaultdict(lambda:0)
for calling in callings:
suborg = calling['subOrgType'] or calling['organization']
by_suborg[suborg].append(calling)
calling_counts[calling['id']] += 1
fp = output_root.child('callingmap.html')
fp.setContent(template.render(
orgs=groups,
calling_counts=calling_counts,
no_calling=no_calling).encode('utf-8'))
print 'wrote', fp.path
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def stats(albumdir, options):
"""
Prints out statistics.
"""
import statistics
stats = statistics.Stats()
if options.recursive:
for root, dirs, files in albumdir.walk():
if len(dirs) > 0:
for dir in dirs:
fp = FilePath(root, dir)
if isIgnored(options, fp):
continue
try:
stats = statistics.dirstat(fp, stats, options.verbose)
except NamingMuseException, strerr:
print strerr
def readconfig(options):
home = os.getenv("HOME")
homeconfdir = FilePath(home, ".namingmuse")
configfile = homeconfdir + "config"
cp = ConfigParser(defaultconfig)
if os.access(str(configfile), os.R_OK):
cp.read([str(configfile)])
defitems = cp.items("DEFAULT")
for key, value in dict(defitems).items():
if str(value).lower() == "true":
value = True
elif str(value).lower() == "false":
value = False
options.ensure_value(key, value)
if options.sysencoding == 'terminal':
options.sysencoding = sys.stdout.encoding
if options.tagencoding == 'terminal':
options.tagencoding = sys.stdout.encoding
if isinstance(options.ignore, basestring):
options.ignore = [options.ignore]
class CachingDownloader(object):
cache_dir = FilePath('.cache')
def getURL(self, url):
"""
Fetch a web page either from the local cache or else
from the Internet.
"""
global cache_dir
if not self.cache_dir.exists():
self.cache_dir.makedirs()
cache_key = hashlib.sha1(url).hexdigest()
cache_file = self.cache_dir.child(cache_key)
if not cache_file.exists():
log('GET', url)
r = requests.get(url)
cache_file.setContent(r.content)
cache_file.chmod(0664)
else:
log('[CACHE] GET', url)
return cache_file.getContent()
def namefix(albumdir, options):
"""
Prettifies filenames, see doc in the namefix module.
"""
from namefix import namefix
from terminal import colorize
filelist = getFilelist(albumdir)
renamesign = "->"
if options.dryrun:
renamesign = "-dry->"
for filepath in filelist:
tofile = albumdir + namefix(filepath.getName())
print filepath
print "\t", colorize(renamesign), tofile
if not options.dryrun:
os.rename(str(filepath), str(tofile))
todir = namefix(albumdir.getName())
print "\n", albumdir.getName()
print "\t", colorize(renamesign), todir
if not options.dryrun:
os.rename(str(albumdir), str(FilePath(albumdir.getParent(), todir)))
def cli():
op = makeOptionParser()
options, args = op.parse_args()
readconfig(options)
if options.doc:
print getDoc()
exit()
if len(args) == 0:
op.print_help()
exit(0)
if not options.cmd and len(args) >= 1:
options.cmd = "discmatch"
if options.cmd and len(args) == 0:
op.print_help()
exit("error: <albumdir> not specified")
if options.words:
options.cmd = "search"
if options.cddb:
options.cmd = 'cddb'
if options.releaseid:
options.cmd = 'releaseid'
albumdir = FilePath(args[0])
try:
os.listdir(str(albumdir))
except OSError, (errno, strerror):
exit(strerror + ": " + str(albumdir))
class LDSClient(object):
vals = ValueGetters()
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def assertOk(self, response):
if not response.ok:
sys.stderr.write('not okay: %r\n' % (response, ))
sys.stderr.write(repr(response.text)[:200] + '\n')
raise Exception('response not okay', response)
def log(self, message):
sys.stderr.write(message + '\n')
def authenticate(self):
if self._session:
return self._session
self.log('Signing in...')
s = self._session = requests.session()
r = s.get('https://ident.lds.org/sso/UI/Login')
r = s.post('https://ident.lds.org/sso/UI/Login',
params={
'IDToken1': self.username,
'IDToken2': self.password,
'IDButton': 'Log In',
})
self.assertOk(r)
return self._session
def storeRawValue(self, filename, value):
self.raw_root.child(filename).setContent(json.dumps(value))
def hasRawValue(self, filename):
fp = self.raw_root.child(filename)
if fp.exists():
return fp
return None
def getRawValue(self, filename, default_value=None):
fp = self.hasRawValue(filename)
if fp:
return json.loads(fp.getContent())
else:
return default_value
def updateRawData(self):
for name, func in self.vals.raws.items():
if self.hasRawValue(name):
# already has a value; do not recompute
self.log('[%s] data already present' % (name, ))
continue
self.log('[%s] fetching...' % (name, ))
data = func(self)
self.storeRawValue(name, data)
@vals.raw('unit_number')
def get_unitNumber(self):
s = self.authenticate()
r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng')
self.assertOk(r)
# this is probably pretty fragile...
re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';")
m = re_unit_number.search(r.text)
return m.groups()[0]
@vals.raw('member_list')
def get_memberList(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/member-list',
params={
'lang': 'eng',
'unitNumber': unit_number,
})
self.assertOk(r)
return r.json()
@vals.raw('members_with_callings')
def get_membersWithCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get(
'https://www.lds.org/mls/mbr/services/report/members-with-callings',
params={
'lang': 'eng',
'unitNumber': unit_number,
},
headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
@vals.raw('members_without_callings')
def get_membersWithoutCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get(
'https://www.lds.org/mls/mbr/services/orgs/members-without-callings',
params={
'lang': 'eng',
'unitNumber': unit_number,
},
headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
#----------------------------
# photos
#----------------------------
def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'):
"""
Valid size options are:
- large
- medium
- original
- thumbnail
"""
return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext))
def _memberIDsWithNoPhoto(self, size='large'):
members = self.getRawValue('member_list')
for member in members:
member_id = member['id']
photo_fp = self._memberPhotoFilePath(member_id, size)
if photo_fp.exists():
continue
yield member_id
def updatePhotos(self, size='large'):
s = self.authenticate()
self.log('Getting photos...')
for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19):
if not member_ids:
continue
try:
r = s.get(
'https://www.lds.org/directory/services/ludrs/photo/url/' +
','.join(map(str, member_ids)) + '/individual')
data = r.json()
except ValueError:
print 'Error on', member_ids
raise
for member_id, result in zip(member_ids, data):
fp = self._memberPhotoFilePath(member_id, size)
uri = result[size + 'Uri']
if uri:
print 'fetching photo for', member_id
uri = 'https://www.lds.org' + uri
image_data = s.get(uri)
content_type = image_data.headers['content-type']
if content_type != 'image/jpeg':
print 'NON-JPEG: ', content_type, member_id
continue
fp.setContent(image_data.content)
else:
print 'no photo for', member_id
time.sleep(0.5)
def exists(self):
"""
Returns true if the underlying archive exists.
"""
return FilePath(self.zipfile.filename).exists()
def tagfiles(albumdir, album, options):
'''Rename and tag files using freedb information for
the specified album.'''
# XXX: doesn't really belong here
missing = album.validate()
if len(missing) > 0:
if options.strict:
amiss = ",".join(missing)
raise TagIncompleteWarning(amiss)
else:
for miss in missing:
print TagIncompleteWarning(miss)
print
album.ignoreMissing(True)
localalbum = LocalAlbumInfo(albumdir)
filelist = localalbum.getfilelist(options.sysencoding)
a = len(album.tracks)
b = len(filelist)
if a != b:
raise NamingMuseError('there are %d files, but %d metainfo tracks' % (b,a))
namebinder = get_namebinder(options, filelist)
tracks = namebinder(filelist, album, options.sysencoding)
if not sortedcmp(tracks, album.tracks):
options.dryrun = True
print NamingMuseError("binding was not exact, forcing dry run")
print "Tagging album: %s, %s - %s, %s.\n" % \
(album.year, album.artist, album.title, album.genre)
# Process files
renamealbum = True
renameTempDir = FilePath(albumdir, 'rename-tmp')
if renameTempDir.exists():
raise NamingMuseError('%s exists!' % renameTempDir)
renameTempDir.mkdir()
# a list of tuples (from, to)
# we use temporary renames to avoid filename collision on filename swaps
# this holds the list of final renames to be executed
finalrenames = []
# a list of tuples (from, to)
# used to rollback renames in case of error
rollback = []
renamesign = "->"
rollbacksign = '<-'
if options.tagonly:
renamesign = "-tag->"
if options.dryrun:
renamesign = "-dry->"
try:
for i in range(0, len(filelist)):
fpath = filelist[i]
# XXX: move bug check to freedbalbuminfo parser
#if album.isVarious:
# if not "/" in title and "-" in title:
# # workaround: this is a bug in the freedb entry
# # (according to submission guidelines)
# trackartist, title = title.split("-")
# print NamingMuseWarning("bugged database entry with - instead of /")
# else:
# trackartist, title = title.split("/")
# trackartist, title = trackartist.strip(), title.strip()
#else:
# trackartist = albumartist
track = tracks[i]
tofile = policy.genfilename(filelist[i], album, track)
tofile = tofile.encode(options.sysencoding)
totmpfile = FilePath(renameTempDir, tofile, encoding=options.sysencoding)
tofinalfile = FilePath(albumdir, tofile, encoding=options.sysencoding)
# Tag and rename file
print fpath.getName()
print "\t", colorize(renamesign), tofinalfile.getName()
if not options.dryrun:
# Tag file
#preserve stat
fd = tempfile.NamedTemporaryFile()
tmpfilename = fd.name
shutil.copystat(str(fpath), tmpfilename)
# tag the file
tagfile(fpath, album, track, options)
# restore filestat
shutil.copystat(tmpfilename, str(fpath))
# deletes tempfile
fd.close()
# Rename file to temporary name
if not options.tagonly:
if totmpfile.exists():
raise NamingMuseError('tried to rename file over existing: %s' % str(totmpfile))
if fpath != totmpfile:
fpath.rename(totmpfile)
rollback.append((fpath, totmpfile))
finalrenames.append((totmpfile, tofinalfile))
except Exception, e:
print
print colorize('Error: an error occurred. rolling back %d renames.' % len(rollback))
for frompath, topath in reversed(rollback):
print frompath.getName()
print "\t", colorize(rollbacksign), topath.getName()
topath.rename(frompath)
renameTempDir.rmdir()
raise
# Get new albumdir name
newalbum = policy.genalbumdirname(albumdir, album)
newalbum = newalbum.encode(options.sysencoding)
artistdir = ""
# Check if parent dir name matches fairly well with
# artistname. If it does, we ignore options.artistdir
parent = albumdir.getParent().getName()
artistdirdiff = distwrap(parent, album.artist.encode(options.sysencoding))
if DEBUG: print "Distance between %s and %s is : %s" \
%(parent, album.artist, artistdirdiff)
needartistdirmove = options.artistdir and \
(artistdirdiff > 0.25) #XXX sane value?
if needartistdirmove:
newalbumdir = FilePath(albumdir.getParent(),
album.artist.encode(options.sysencoding),
newalbum, encoding=options.sysencoding)
else:
newalbumdir = FilePath(albumdir.getParent(), newalbum, \
encoding=options.sysencoding)
# Make parent directory of albumdir if needed
parent = newalbumdir.getParent()
if not parent.isdir():
parent.mkdir()
# Rename album (if no "manual" mp3 files in that dir)
rollbacksign = '<-'
renamesign = "->"
if options.dryrun or options.tagonly:
renamesign = "-dry->"
# Django settings for code_scouts_platform project.
from filepath import FilePath
PROJECT_ROOT = FilePath(__file__).parent().parent()
DEBUG = True
TEMPLATE_DEBUG = DEBUG
ADMINS = (
('Kevin Turner', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'devdb.sqlite', # Or path to database file if using sqlite3.
'USER': '', # Not used with sqlite3.
'PASSWORD': '', # Not used with sqlite3.
'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
'PORT': '', # Set to empty string for default. Not used with sqlite3.
}
}
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Los_Angeles'
# Language code for this installation. All choices can be found here:
def command_comp(arguments, fcapture=None):
if not any(arguments[name]
for name in ('--full', '--quick', '--none', '--size', '--time',
'--extension', '--basename')):
arguments['--full'] = True
arguments['--extension'] = True
if not any(arguments[name] for name in ('--full', '--quick', '--none')):
arguments['--none'] = True
if not arguments['--none']:
arguments['--size'] = True
if (arguments['--lhs-update']
or not arguments['--lhs-db']) and arguments['--lhs-path']:
arguments['--lhs-path'] = os.path.realpath(arguments['--lhs-path'])
if (arguments['--rhs-update']
or not arguments['--rhs-db']) and arguments['--rhs-path']:
arguments['--rhs-path'] = os.path.realpath(arguments['--rhs-path'])
if arguments['--dry-run']:
arguments['--echo'] = True
haslhs = arguments['--lhs-path'] or arguments['--lhs-db']
hasrhs = arguments['--rhs-path'] or arguments['--rhs-db']
if (haslhs and hasrhs):
attach = True
else:
attach = False
def match(sel, lhs, rhs, complete=True):
if arguments['--size']:
sel = sel.where(lhs.c.size == rhs.c.size)
if arguments['--time']:
sel = sel.where(lhs.c.time == rhs.c.time)
if arguments['--extension']:
sel = sel.where(lhs.c.extension == rhs.c.extension)
if arguments['--basename']:
sel = sel.where(lhs.c.basename == rhs.c.basename)
if arguments['--skip-empty']:
sel = sel.where(lhs.c.size != 0)
if complete:
if arguments['--quick']:
sel = sel.where(
and_(lhs.c.hash_quick == rhs.c.hash_quick,
lhs.c.hash_quick != None))
if arguments['--full']:
sel = sel.where(
and_(lhs.c.hash_total == rhs.c.hash_total,
lhs.c.hash_total != None))
# Determine if we are on the same file system and ignore the same literal files
if not (arguments['--rhs-path'] or arguments['--rhs-db']) or \
(arguments['--rhs-update'] or not arguments['--rhs-db']) and arguments['--rhs-path']:
sel = sel.where(lhs.c.path != rhs.c.path)
return sel
#ToDo add the LHSPATH and RHSPATH options
argsSubstitutions = set(
chain(*(re.findall(
r'\{((?:(?:LHS|RHS|LHSONLY|RHSONLY|DUPE|UNIQUE)(GROUP)?)|LHSPATH|RHSPATH)(?:\:(dirpath|basename|ext|name|drive|dirpathnodrive|fullpath))?\}',
arg) for arg in arguments['COMMAND'])))
args = set(name for name, group, section in argsSubstitutions)
if 'LHSPATH' in args:
if not arguments['--lhs-path']:
raise DocoptExit('{LHSPATH} without --lhs--path')
args.remove('LHSPATH')
if 'RHSPATH' in args:
if not arguments['--rhs-path']:
raise DocoptExit('{RHSPATH} without --rhs--path')
args.remove('RHSPATH')
if haslhs and hasrhs and args == {'LHS'}:
def get_sel(lhs, rhs):
sel = select([lhs.c.path.label('LHS')])
sel = match(sel, lhs, rhs)
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHS', 'RHS'}:
def get_sel(lhs, rhs):
sel = select([lhs.c.path.label('LHS'), rhs.c.path.label('RHS')])
sel = match(sel, lhs, rhs)
sel = sel.order_by(lhs.c.path).order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHS', 'RHSGROUP'}:
def get_sel(lhs, rhs):
sel = select([
lhs.c.path.label('LHS'),
func.group_filepath(rhs.c.path).label('RHSGROUP')
])
sel = match(sel, lhs, rhs)
sel = sel.group_by(lhs.c.path)
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHSGROUP'}:
def get_sel(lhs, rhs):
sel = select([func.group_filepath(lhs.c.path).label('LHSGROUP')])
sel = match(sel, lhs, rhs)
sel = sel.group_by(rhs.c.path)
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHSGROUP', 'RHS'}:
def get_sel(lhs, rhs):
sel = select([
func.group_filepath(lhs.c.path).label('LHSGROUP'),
rhs.c.path.label('RHS')
])
sel = match(sel, lhs, rhs)
sel = sel.group_by(rhs.c.path)
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHSGROUP', 'RHSGROUP'}:
def get_sel(lhs, rhs):
sel = select([
func.group_filepath(lhs.c.path).label('LHSGROUP'),
func.group_filepath(rhs.c.path.distinct()).label('RHSGROUP')
])
sel = match(sel, lhs, rhs)
if arguments['--size']:
sel = sel.group_by(lhs.c.size).order_by(lhs.c.size)
if arguments['--time']:
sel = sel.group_by(lhs.c.time).order_by(lhs.c.time)
if arguments['--extension']:
sel = sel.group_by(lhs.c.extension).order_by(lhs.c.extension)
if arguments['--basename']:
sel = sel.group_by(lhs.c.basename).order_by(lhs.c.basename)
if arguments['--quick']:
sel = sel.group_by(lhs.c.hash_quick).order_by(lhs.c.hash_quick)
if arguments['--full']:
sel = sel.group_by(lhs.c.hash_total).order_by(lhs.c.hash_total)
sel = sel.order_by(lhs.c.path)
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHSONLY'}:
def get_sel(lhs, rhs):
sel = select([lhs.c.path.label('LHSONLY')])
sel = sel.where(~exists(match(select(['*']), lhs, rhs)))
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'LHSONLYGROUP'}:
def get_sel(lhs, rhs):
sel = select(
[func.group_filepath(lhs.c.path).label('LHSONLYGROUP')])
sel = sel.where(~exists(match(select(['*']), lhs, rhs)))
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'RHS'}:
def get_sel(lhs, rhs):
sel = select([rhs.c.path.label('RHS')])
sel = match(sel, lhs, rhs)
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'RHSGROUP'}:
def get_sel(lhs, rhs):
sel = select([func.group_filepath(rhs.c.path).label('RHSGROUP')])
sel = match(sel, lhs, rhs)
sel = sel.group_by(lhs.c.path)
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'RHSONLY'}:
def get_sel(lhs, rhs):
sel = select([rhs.c.path.label('RHSONLY')])
sel = sel.where(~exists(match(select(['*']), lhs, rhs)))
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and hasrhs and args == {'RHSONLYGROUP'}:
def get_sel(lhs, rhs):
sel = select(
[func.group_filepath(rhs.c.path).label('RHSONLYGROUP')])
sel = sel.where(~exists(match(select(['*']), lhs, rhs)))
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and not hasrhs and args == {'DUPE'}:
def get_sel(lhs, rhs):
sel = select([lhs.c.path.label('DUPE')])
sel = match(sel, lhs, rhs)
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and not hasrhs and args == {'DUPEGROUP'}:
def get_sel(lhs, rhs):
sel = select([
func.group_filepath(lhs.c.path.distinct()).label('DUPEGROUP')
])
sel = match(sel, lhs, rhs)
if arguments['--size']:
sel = sel.group_by(lhs.c.size).order_by(lhs.c.size)
if arguments['--time']:
sel = sel.group_by(lhs.c.time).order_by(lhs.c.time)
if arguments['--extension']:
sel = sel.group_by(lhs.c.extension).order_by(lhs.c.extension)
if arguments['--basename']:
sel = sel.group_by(lhs.c.basename).order_by(lhs.c.basename)
if arguments['--quick']:
sel = sel.group_by(lhs.c.hash_quick).order_by(lhs.c.hash_quick)
if arguments['--full']:
sel = sel.group_by(lhs.c.hash_total).order_by(lhs.c.hash_total)
sel = sel.order_by(lhs.c.path)
sel = sel.order_by(rhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and not hasrhs and args == {'UNIQUE'}:
def get_sel(lhs, rhs):
sel = select([lhs.c.path.label('UNIQUE')])
sel = sel.where(~exists(match(select(['*']), lhs, rhs)))
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
elif haslhs and not hasrhs and args == {'UNIQUEGROUP'}:
def get_sel(lhs, rhs):
sel = select(
[func.group_filepath(lhs.c.path).label('UNIQUEGROUP')])
sel = sel.where(~exists(match(select(['*']), lhs, rhs)))
sel = sel.order_by(lhs.c.path)
sel = sel.distinct()
return sel
else:
raise DocoptExit(
'COMMAND does not contain a valid combination of special arguments'
)
lhsrwFiles, lhsroFiles = None, None
rhsrwFiles, rhsroFiles = None, None
if attach:
engine = create(None, separator=arguments['--separator'])
else:
engine, lhsrwFiles, lhsroFiles, rhsroFiles = create_side(
arguments['--lhs-db'],
arguments['--lhs-update'],
arguments['--lhs-path'],
separator=arguments['--separator'])
with engine_dispose(engine):
if attach:
if haslhs:
lhsrwFiles, lhsroFiles = attach_side(engine, 'lhs',
arguments['--lhs-db'],
arguments['--lhs-update'],
arguments['--lhs-path'])
if hasrhs:
rhsrwFiles, rhsroFiles = attach_side(engine, 'rhs',
arguments['--rhs-db'],
arguments['--rhs-update'],
arguments['--rhs-path'])
if not arguments['--none'] and (lhsrwFiles != None
or rhsrwFiles != None):
# Do a preliminary comparison
lhssel = match(select([lhsroFiles.c.path]), lhsroFiles, rhsroFiles,
False)
rhssel = match(select([rhsroFiles.c.path]), lhsroFiles, rhsroFiles,
False)
if arguments['--quick']:
lhssel = lhssel.where(lhsroFiles.c.hash_quick == None)
rhssel = rhssel.where(rhsroFiles.c.hash_quick == None)
if arguments['--full']:
lhssel = lhssel.where(lhsroFiles.c.hash_total == None)
rhssel = rhssel.where(rhsroFiles.c.hash_total == None)
# Update rw table
conn = engine.connect()
def updaterw(ro, rw, sel):
if rw is None:
return
for result in conn.execute(sel):
file = conn.execute(rw.select().where(
rw.c.path == result.path)).fetchone()
try:
stat = os.stat(result.path, follow_symlinks=False)
except Exception:
hash_quick, hash_total = None, None
else:
hash_quick, hash_total = hashfile(
result.path, stat, arguments['--quick'])
if hash_quick != None or hash_total != None:
conn.execute(
rw.update().where(rw.c.path == result.path).values(
size=stat.st_size,
time=stat.st_mtime_ns,
hash_quick=hash_quick,
hash_total=hash_total))
else:
conn.execute(
rw.delete().where(rw.c.path == result.path))
try:
updaterw(lhsroFiles, lhsrwFiles, lhssel)
updaterw(rhsroFiles, rhsrwFiles, rhssel)
finally:
conn.close()
# Do the full comparison
sel = get_sel(lhsroFiles if lhsrwFiles is None else lhsrwFiles,
rhsroFiles if rhsrwFiles is None else rhsrwFiles)
conn = engine.connect()
try:
for result in conn.execute(sel):
result = dict(result.items())
resultNormalized = {}
for name, group, section in argsSubstitutions:
if section == '':
# simple
resultNormalized[name] = result[name]
elif group == '':
# single, sectioned
path = FilePath(result[name])
resultNormalized[name + ':' + section] = getattr(
path, section)
else:
# grouped, sectioned
paths = FilePath.splitpaths(result[name],
arguments['--separator'])
resultNormalized[name + ':' +
section] = FilePath.joinpaths([
getattr(p, section) for p in paths
], arguments['--separator'])
result = resultNormalized
cmd = [
re.sub(
r'\{(%s)\}' % '|'.join(
name + (':' + section if section != '' else '')
for name, group, section in argsSubstitutions),
(lambda match: result[match.group(1)]), arg)
for arg in arguments['COMMAND']
]
if arguments['--echo']:
print(' '.join(escape_for_shell(arg) for arg in cmd))
if fcapture != None:
fcapture(cmd)
elif not arguments['--dry-run']:
try:
subprocess.run(cmd, shell=True, check=True)
except subprocess.CalledProcessError as ex:
if arguments['--ignore-errors']:
print(ex, file=sys.stderr)
else:
raise ex
finally:
conn.close()
def tagfiles(albumdir, album, options):
'''Rename and tag files using freedb information for
the specified album.'''
# XXX: doesn't really belong here
missing = album.validate()
if len(missing) > 0:
if options.strict:
amiss = ",".join(missing)
raise TagIncompleteWarning(amiss)
else:
for miss in missing:
print TagIncompleteWarning(miss)
print
album.ignoreMissing(True)
localalbum = LocalAlbumInfo(albumdir)
filelist = localalbum.getfilelist(options.sysencoding)
a = len(album.tracks)
b = len(filelist)
if a != b:
raise NamingMuseError('there are %d files, but %d metainfo tracks' %
(b, a))
namebinder = get_namebinder(options, filelist)
tracks = namebinder(filelist, album, options.sysencoding)
if not sortedcmp(tracks, album.tracks):
options.dryrun = True
print NamingMuseError("binding was not exact, forcing dry run")
print "Tagging album: %s, %s - %s, %s.\n" % \
(album.year, album.artist, album.title, album.genre)
# Process files
renamealbum = True
renameTempDir = FilePath(albumdir, 'rename-tmp')
if renameTempDir.exists():
raise NamingMuseError('%s exists!' % renameTempDir)
renameTempDir.mkdir()
# a list of tuples (from, to)
# we use temporary renames to avoid filename collision on filename swaps
# this holds the list of final renames to be executed
finalrenames = []
# a list of tuples (from, to)
# used to rollback renames in case of error
rollback = []
renamesign = "->"
rollbacksign = '<-'
if options.tagonly:
renamesign = "-tag->"
if options.dryrun:
renamesign = "-dry->"
try:
for i in range(0, len(filelist)):
fpath = filelist[i]
# XXX: move bug check to freedbalbuminfo parser
#if album.isVarious:
# if not "/" in title and "-" in title:
# # workaround: this is a bug in the freedb entry
# # (according to submission guidelines)
# trackartist, title = title.split("-")
# print NamingMuseWarning("bugged database entry with - instead of /")
# else:
# trackartist, title = title.split("/")
# trackartist, title = trackartist.strip(), title.strip()
#else:
# trackartist = albumartist
track = tracks[i]
tofile = policy.genfilename(filelist[i], album, track)
tofile = tofile.encode(options.sysencoding)
totmpfile = FilePath(renameTempDir,
tofile,
encoding=options.sysencoding)
tofinalfile = FilePath(albumdir,
tofile,
encoding=options.sysencoding)
# Tag and rename file
print fpath.getName()
print "\t", colorize(renamesign), tofinalfile.getName()
if not options.dryrun:
# Tag file
#preserve stat
fd = tempfile.NamedTemporaryFile()
tmpfilename = fd.name
shutil.copystat(str(fpath), tmpfilename)
# tag the file
tagfile(fpath, album, track, options)
# restore filestat
shutil.copystat(tmpfilename, str(fpath))
# deletes tempfile
fd.close()
# Rename file to temporary name
if not options.tagonly:
if totmpfile.exists():
raise NamingMuseError(
'tried to rename file over existing: %s' %
str(totmpfile))
if fpath != totmpfile:
fpath.rename(totmpfile)
rollback.append((fpath, totmpfile))
finalrenames.append((totmpfile, tofinalfile))
except Exception, e:
print
print colorize('Error: an error occurred. rolling back %d renames.' %
len(rollback))
for frompath, topath in reversed(rollback):
print frompath.getName()
print "\t", colorize(rollbacksign), topath.getName()
topath.rename(frompath)
renameTempDir.rmdir()
raise
# Get new albumdir name
newalbum = policy.genalbumdirname(albumdir, album)
newalbum = newalbum.encode(options.sysencoding)
artistdir = ""
# Check if parent dir name matches fairly well with
# artistname. If it does, we ignore options.artistdir
parent = albumdir.getParent().getName()
artistdirdiff = distwrap(parent, album.artist.encode(options.sysencoding))
if DEBUG: print "Distance between %s and %s is : %s" \
%(parent, album.artist, artistdirdiff)
needartistdirmove = options.artistdir and \
(artistdirdiff > 0.25) #XXX sane value?
if needartistdirmove:
newalbumdir = FilePath(albumdir.getParent(),
album.artist.encode(options.sysencoding),
newalbum,
encoding=options.sysencoding)
else:
newalbumdir = FilePath(albumdir.getParent(), newalbum, \
encoding=options.sysencoding)
# Make parent directory of albumdir if needed
parent = newalbumdir.getParent()
if not parent.isdir():
parent.mkdir()
# Rename album (if no "manual" mp3 files in that dir)
rollbacksign = '<-'
renamesign = "->"
if options.dryrun or options.tagonly:
renamesign = "-dry->"
def getAccessTime(self):
"""
Return the archive file's last access time.
"""
return FilePath(self.zipfile.filename).getAccessTime()
class LDSClient(object):
vals = ValueGetters()
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def assertOk(self, response):
if not response.ok:
sys.stderr.write('not okay: %r\n' % (response,))
sys.stderr.write(repr(response.text)[:200] + '\n')
raise Exception('response not okay', response)
def log(self, message):
sys.stderr.write(message + '\n')
def authenticate(self):
if self._session:
return self._session
self.log('Signing in...')
s = self._session = requests.session()
r = s.get('https://ident.lds.org/sso/UI/Login')
r = s.post('https://ident.lds.org/sso/UI/Login', params={
'IDToken1': self.username,
'IDToken2': self.password,
'IDButton': 'Log In',
})
self.assertOk(r)
return self._session
def storeRawValue(self, filename, value):
self.raw_root.child(filename).setContent(json.dumps(value))
def hasRawValue(self, filename):
fp = self.raw_root.child(filename)
if fp.exists():
return fp
return None
def getRawValue(self, filename, default_value=None):
fp = self.hasRawValue(filename)
if fp:
return json.loads(fp.getContent())
else:
return default_value
def updateRawData(self):
for name, func in self.vals.raws.items():
if self.hasRawValue(name):
# already has a value; do not recompute
self.log('[%s] data already present' % (name,))
continue
self.log('[%s] fetching...' % (name,))
data = func(self)
self.storeRawValue(name, data)
@vals.raw('unit_number')
def get_unitNumber(self):
s = self.authenticate()
r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng')
self.assertOk(r)
# this is probably pretty fragile...
re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';")
m = re_unit_number.search(r.text)
return m.groups()[0]
@vals.raw('member_list')
def get_memberList(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/member-list', params={
'lang': 'eng',
'unitNumber': unit_number,
})
self.assertOk(r)
return r.json()
@vals.raw('members_with_callings')
def get_membersWithCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/members-with-callings', params={
'lang': 'eng',
'unitNumber': unit_number,
}, headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
@vals.raw('members_without_callings')
def get_membersWithoutCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/orgs/members-without-callings', params={
'lang': 'eng',
'unitNumber': unit_number,
}, headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
#----------------------------
# photos
#----------------------------
def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'):
"""
Valid size options are:
- large
- medium
- original
- thumbnail
"""
return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext))
def _memberIDsWithNoPhoto(self, size='large'):
members = self.getRawValue('member_list')
for member in members:
member_id = member['id']
photo_fp = self._memberPhotoFilePath(member_id, size)
if photo_fp.exists():
continue
yield member_id
def updatePhotos(self, size='large'):
s = self.authenticate()
self.log('Getting photos...')
for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19):
if not member_ids:
continue
try:
r = s.get('https://www.lds.org/directory/services/ludrs/photo/url/'+','.join(map(str, member_ids))+'/individual')
data = r.json()
except ValueError:
print 'Error on', member_ids
raise
for member_id, result in zip(member_ids, data):
fp = self._memberPhotoFilePath(member_id, size)
uri = result[size + 'Uri']
if uri:
print 'fetching photo for', member_id
uri = 'https://www.lds.org' + uri
image_data = s.get(uri)
content_type = image_data.headers['content-type']
if content_type != 'image/jpeg':
print 'NON-JPEG: ', content_type, member_id
continue
fp.setContent(image_data.content)
else:
print 'no photo for', member_id
time.sleep(0.5)
def getModificationTime(self):
"""
Return the archive file's modification time.
"""
return FilePath(self.zipfile.filename).getModificationTime()
def getStatusChangeTime(self):
"""
Return the archive file's status change time.
"""
return FilePath(self.zipfile.filename).getStatusChangeTime()
''''I assume this walk method is here (as opposed to using os.walk) because
we modify the directory tree by renaming directories?'''
try:
names = os.listdir(str(top))
except os.error:
return
try:
if not isIgnored(options, top):
doMusicbrainz(options, top)
except NoFilesException:
pass
except NamingMuseException,(errstr):
print errstr
for name in names:
name = FilePath(top, name)
if name.isdir():
walk2(name, options)
def walk(top, cddb, options):
''''I assume this walk method is here (as opposed to using os.walk) because
we modify the directory tree by renaming directories?'''
try:
names = os.listdir(str(top))
except os.error:
return
try:
if not isIgnored(options, top):
doDiscmatch(options, top, cddb)
except CDDBPException, err:
"""
from __future__ import unicode_literals
from yaml import safe_load
from testtools import skipUnless
from testtools.matchers import AfterPreprocessing, Equals
from lae_util.testtools import TestCase
from filepath import FilePath
from lae_automation import __file__ as lae_package_location
ROOT = FilePath(lae_package_location).parent().parent()
STAGING = ROOT.child("k8s").child("secrets.staging.enc.yaml")
PRODUCTION = ROOT.child("k8s").child("secrets.production.enc.yaml")
class SecretsFileTests(TestCase):
"""
Tests for the files in the repository which contain the S4 secrets.
"""
@skipUnless(
STAGING.exists() and PRODUCTION.exists(),
"Staging and production secrets missing.",
)
def test_same_staging_and_production_structure(self):
"""
The staging and production secrets files have the same overall structure
albumtag.tagfiles(albumdir, albuminfo, options)
elif options.cmd == 'musicbrainz2':
if options.recursive:
walk2(albumdir, options)
else:
doMusicbrainz(options, albumdir)
elif options.cmd == 'releaseid':
albuminfo = MusicBrainz2AlbumInfo(options.releaseid)
albumtag.tagfiles(albumdir, albuminfo, options)
elif options.cmd == "local":
if options.recursive:
for root, dirs, files in albumdir.walk():
if len(dirs) > 0:
for dir in dirs:
try:
doLocal(FilePath(root, dir), options)
except NoFilesException:
pass
except NamingMuseInfo, err:
print err
except NamingMuseWarning, strerr:
print strerr
#the recursive function doesn't run doLocal on the root
#so we have to do it either way
doLocal(albumdir, options)
elif options.cmd == "namefix":
namefix(albumdir, options)
elif options.cmd == "stats":
stats(albumdir, options)
else:
exit("error: no action option specified")
destination = None
file_rule = create_file_rule(regex)
process = create_process(file_manager, process)
rule = Rule(file_rule, process, destination)
filtered = rule.apply_files_rule(origin_files)
for p in rule.apply_process(filtered):
input("Enter to continue...")
if __name__ == "__main__":
file_manager = FileManager()
test_file_path = FilePath()
test_file_path.add("origin_folder")
test_file_path.add("test_file")
test_destination = FilePath("test_folder")
file_manager.copy_file(test_file_path, test_destination)
# # Standard file creation/deletion
# assert file_manager.check_exists(test_file_path) == False
# assert file_manager.create_file(test_file_path) == True
# assert file_manager.check_exists(test_file_path) == True
# assert file_manager.delete_file(test_file_path) == True
# assert file_manager.check_exists(test_file_path) == False
# # Standard folder create/deletion
# test_folder_path = FilePath("test_folder")