def detect_format(source):
fs = OSFS(source)
if fs.walkfiles('/', '*.morph'):
return 'baserock-morphologies'
if fs.walkfiles('/', '*.cida'):
return 'cida-definitions'
return None
def write_chunk_metafile(defs, chunk):
'''Writes a chunk .meta file to the baserock dir of the chunk
The split rules are used to divide up the installed files for the chunk
into artifacts in the 'products' list
'''
app.log(chunk['name'], 'splitting chunk')
rules, splits = compile_rules(defs, chunk)
install_dir = chunk['install']
fs = OSFS(install_dir)
files = fs.walkfiles('.', search='depth')
dirs = fs.walkdirs('.', search='depth')
for path in files:
for artifact, rule in rules:
if rule.match(path):
splits[artifact].append(path)
break
all_files = [a for x in splits.values() for a in x]
for path in dirs:
if not any(map(lambda y: y.startswith(path),
all_files)) and path != '':
for artifact, rule in rules:
if rule.match(path) or rule.match(path + '/'):
splits[artifact].append(path)
break
write_metafile(rules, splits, chunk)
def copy_wiki():
ws = OSFS('wiki-small')
for file in ws.walkfiles():
print(file)
copyfile('wiki-small%s' % file,
'htmls/%s' % file[file.rfind('/') + 1:])
ws.close()
def create_free_editorslf(name):
"""
Creates a BufferedSlfFS exclusively from the contents of the editor directory.
The python files inside the editor directory are responsible for creating the STI images and adding them to the SLF.
Each file should contain the function:
```
def add_to_free_editorslf(source_fs, target_fs):
pass
```
source_fs is the source OSFS (editor directory)
target_fs is the target BufferedSlfFS
"""
target_fs = BufferedSlfFS()
target_fs.library_name = name or "Free editor.slf"
target_fs.library_path = "editor\\"
target_fs.version = 0x0200 # 2.0
target_fs.sort = 0 # BufferedSlfFS does not guarantee that the entries are sorted
source_fs = OSFS('editor')
for path in source_fs.walkfiles():
if path.endswith(".py"):
# run python file inside the editor directory
name = ("editor" + path)[:-3].replace("/", ".")
spec = spec_from_file_location(name, source_fs.getsyspath(path))
module = module_from_spec(spec)
spec.loader.exec_module(module)
module.add_to_free_editorslf(source_fs, target_fs)
for path in sorted(target_fs.walkfiles()):
print(path)
return target_fs
def write_chunk_metafile(defs, chunk):
'''Writes a chunk .meta file to the baserock dir of the chunk
The split rules are used to divide up the installed files for the chunk
into artifacts in the 'products' list
'''
log(chunk['name'], 'Splitting', chunk.get('kind'))
rules, splits = compile_rules(defs, chunk)
install_dir = chunk['install']
fs = OSFS(install_dir)
files = fs.walkfiles('.', search='depth')
dirs = fs.walkdirs('.', search='depth')
for path in files:
for artifact, rule in rules:
if rule.match(path):
splits[artifact].append(path)
break
all_files = [a for x in splits.values() for a in x]
for path in dirs:
if not any(map(lambda y: y.startswith(path),
all_files)) and path != '':
for artifact, rule in rules:
if rule.match(path) or rule.match(path + '/'):
splits[artifact].append(path)
break
write_metafile(rules, splits, chunk)
def sanitiz():
base = 'band_to_tm'
metal_btm = OSFS(base)
for metal in metal_btm.walkfiles():
p1 = metal.find('/')
p2 = metal.find('.')
b = metal[p1 + 1:p2]
print(b)
with open('%s%s' % ('band_to_tm', metal), 'r') as min:
metal = json.load(min)
info = metal['info']
mem_point = []
for i in info:
for f, t in map(extract_binfo_date, i['active']):
mem_point.append({
"f": f.format('YYYYMMDDHHMMSS'),
"t": t.format('YYYYMMDDHHMMSS'),
"plays": i['plays'],
"member": i['member']
})
mtm = metal['tm']
metal_plot = {
"member_points": mem_point,
"mementos": mtm['mementos'],
"first": mtm['first'],
"last": mtm['last'],
"timemap": mtm['self'],
"timegate": mtm['timegate'],
"original": mtm['original'],
}
with open('plot/%s.json' % b, 'w') as mout:
json.dump(metal_plot, mout, indent=2)
print('----------------------------------------')
metal_btm.close()
def extract_dl_bmember_pages():
base = 'band_pages'
bandpages = OSFS(base)
c = 0
with requests.session() as session:
for page in bandpages.walkfiles():
path = '%s/%s' % (base, page)
band = page[page.find('/') + 1:page.rfind('/')]
page_type = page[page.rfind('/') + 1:page.find('.')]
if page_type == 'wiki':
with open(path, 'r') as win:
wsoup = BeautifulSoup(win.read(), 'lxml')
band_mem_page = wsoup.find_all(
'a',
href=True,
title=re.compile('list\sof\s.+\sband\smembers',
re.IGNORECASE))
if len(band_mem_page) != 0:
session.headers.update({'User-Agent': useragents[c]})
request = session.get('https://en.wikipedia.org%s' %
band_mem_page[0]['href'])
with open('bandmember_pages/%s.html' % band,
'w+') as page_out:
page_out.write(request.text)
c += 1
if c == 3:
c = 0
def pick_file_list():
ws = OSFS('wiki-small')
small_list = []
small_set = set()
for file in ws.walkfiles():
small_list.append('wiki-small%s' % file)
small_set.add(file[file.rfind('/') + 1:])
dump_pickle((small_list, small_set), 'pickled/wsmall.pickle')
ws.close()
def pick_file_list():
wl = OSFS('wiki-large')
large_list = []
large_set = set()
for file in wl.walkfiles():
large_list.append('wiki-large%s' % file)
large_set.add(file[file.rfind('/') + 1:])
dump_pickle((large_list, large_set), 'pickled/wiki-large2.pickle')
wl.close()
ws = OSFS('wiki-small')
small_list = []
small_set = set()
for file in ws.walkfiles():
small_list.append('wiki-small%s' % file)
small_set.add(file[file.rfind('/') + 1:])
dump_pickle((small_list, small_set), 'pickled/wiki-small2.pickle')
ws.close()
def list_photos(folder):
try:
photo_fs = OSFS('~/Pictures/Playground/' + folder)
photos = []
for photo in photo_fs.walkfiles():
if is_photo(photo):
photos.append(Photo(photo, photo_fs))
return jsonify(photos)
except ResourceNotFoundError as err:
return jsonify({'error': unicode(err)})
def do_combine():
with open('band_timeline.json', 'r') as bndin:
bands = json.load(bndin)
print(bands)
mtml = OSFS('timemaps/json')
for metal_tml in mtml.walkfiles():
p1 = metal_tml.find('/')
p2 = metal_tml.find('.')
b = metal_tml[p1 + 1:p2]
with open('timemaps/json%s' % metal_tml, 'r') as bndin:
band_tm = json.load(bndin)
with open('band_to_tm/%s.json' % b, 'w+') as btm:
json.dump({"info": bands[b], "tm": band_tm}, btm, indent=2)
print(band_tm)
mtml.close()
def rest_wiki():
skip = set()
populate_skip(skip, skip2=False)
base = 'band_pages'
bandpages = OSFS(base)
bands = {}
no_do = {'Sunn_O)))', 'Alice_Cooper', 'Hellbastard'}
for page in bandpages.walkfiles():
path = '%s/%s' % (base, page)
band = page[page.find('/') + 1:page.rfind('/')]
page_type = page[page.rfind('/') + 1:page.find('.')]
if page_type == 'wiki' and band not in skip:
bands[band] = {}
with open(path, 'r') as win:
wsoup = BeautifulSoup(win.read(), 'lxml')
if band in no_do:
if band == 'Alice_Cooper':
alice_cooper(wsoup, bands, band)
else:
find_members_dt(wsoup, bands, band)
find_members_span(wsoup, bands, band)
# for k,v in bands.items():
# if v == {}:
# print(k)
# print(len(list(bands.keys())))
with open('band_timeline_a2.json', 'w+') as out:
nbands = {}
for band, bdict in bands.items():
nbands[band] = []
print(band)
for member, infor in bdict.items():
print(infor)
nbands[band].append({
'member': member,
'plays': infor.get('plays', []),
'active': infor.get('active', [])
})
json.dump(nbands, out, indent=2)
def cmd_makeadminpreviews(self, *params, **options):
try:
iconset = params[0]
except IndexError:
iconset = ''
icon_fs = OSFS(settings.MEDIA_ROOT).opendir('iconsets')
if params:
icon_fs = icon_fs.opendir(params[0])
done_dirs = set()
for path in icon_fs.walkfiles(wildcard='*.png'):
dirpath = dirname(path)
png_path = icon_fs.getsyspath(path)
img = Image.open(png_path).convert('RGBA')
background_img = Image.new('RGB', img.size, (255, 255, 255))
background_img.paste(img, None, img)
new_path = os.path.splitext(png_path)[0] + '.jpg'
background_img.save(new_path)
if dirpath not in done_dirs:
print "Generating admin previews in %s/*" % dirpath
done_dirs.add(dirpath)
request = session.get(tm_link % metal['site'])
save_tm(metal['band'], request)
c += 1
if c == 3:
c = 0
print(metal['band'])
if __name__ == '__main__':
c = 0
bands = {}
with open('bands.csv', 'r') as metalIn:
for metal in DictReader(metalIn):
bands[metal['band']] = {'site': metal['site']}
mtml = OSFS('timemaps/link/')
for metal_tml in mtml.walkfiles():
p1 = metal_tml.find('/')
p2 = metal_tml.find('.')
b = metal_tml[p1 + 1:p2]
with open('timemaps/link%s' % metal_tml, 'r') as mtmlin:
metal_tm = Timemap(mtmlin.readlines())
print(metal_tm)
bands[b] = metal_tm
print('----------------------------------------')
mtml.close()
for band, tm in bands.items():
print(band, tm)
with open('timemaps/json/%s.json' % band, 'w+') as btm:
json.dump(tm, btm, default=lambda x: x.to_json(), indent=2)
# with open('band_to_tm.json','w+') as btm:
# json.dump(bands,btm,default=lambda x:x.to_json(),indent=2)
def extract_members_bmemberpages():
base = 'bandmember_pages'
bandpages = OSFS(base)
bandl = []
bands = {}
for page in bandpages.walkfiles():
path = '%s/%s' % (base, page)
band = page[page.find('/') + 1:page.rfind('.')]
bandl.append(band)
# if band != 'AC_DC':
# continue
with open(path, 'r') as win:
wsoup = BeautifulSoup(win.read(), 'lxml')
allm = wsoup.find_all('span', id=member_finder)
bands[band] = {}
if len(allm) == 0:
tables = wsoup.find_all('table')
extract_caption_table(tables, bands, band)
else:
for span in allm:
# print(span)
parent = span.find_parent() # type: bs4.element.Tag
# print(type(parent))
sibling = parent.find_next_sibling(
next_sibling_filter) # type: bs4.element.Tag
# print('next sibling is a ', sibling.name)
if sibling.name == 'table':
extract_table(sibling, bands, band)
elif sibling.name == 'dl':
siblings = [
dl for dl in parent.find_next_siblings('dl')
] # type: list(bs4.element.Tag)
extract_dl(siblings, bands, band)
else:
if band == 'Twisted_Sister' or band == 'Opeth':
for ul in [
ul
for ul in parent.find_next_siblings('ul')
]:
for li in ul.find_all('li'):
dash = li.text.find('-')
name = li.text[:dash]
if dash == -1:
dash = li.text.find('-')
name = li.text[:dash]
if dash == -1:
dash = li.text.find('–')
name = li.text[:dash]
if bands[band].get(name, None) is None:
bands[band][name] = {}
if bands[band][name].get('active',
None) is None:
bands[band][name]['active'] = []
if bands[band][name].get('plays',
None) is None:
bands[band][name]['plays'] = []
plays_in = li.text[dash + 2:]
plays = plays_in[:plays_in.find('(')]
ain = plays_in[plays_in.find('(') +
1:plays_in.rfind(')')]
for found in really.finditer(ain):
it = found.group(0)
ft = from_to_re.match(it)
if ft:
bands[band][name]['active'].append(
{
'from': ft.group('from'),
'to': ft.group('to')
})
else:
bands[band][name]['active'].append(
{
'from': it,
'to': it
})
if ',' in plays:
for play in plays.split(','):
bands[band][name]['plays'].append(
play)
else:
bands[band][name]['plays'].append(
plays.rstrip())
if bands[band] == {}:
tables = wsoup.find_all('table')
extract_caption_table(tables, bands, band)
for k, v in bands.items():
if v == {}:
print(k)
print(len(list(bands.keys())))
