def write(self):
dockerfile = os.path.join(self.image_dir, 'Dockerfile')
os.makedirs(self.image_dir, exist_ok=True)
if os.path.exists(dockerfile):
os.remove(dockerfile)
with open(dockerfile, 'x'):
pass
snippets = []
snippets.extend(self.snippets)
snippets.append(
Snippet(os.path.join(docker_path, 'snippets/workdir.in')))
snippets.append(
Snippet(os.path.join(docker_path, 'snippets/entrypoint.in')))
for snippet in snippets:
with open(dockerfile, 'a') as f:
try:
f.write(snippet.substitute(self.substitutions))
except Exception as e:
raise RuntimeError(
'Error creating Dockerfile for {}: {}'.format(
self.name, e))
for file in self.files:
shutil.copy(file, self.image_dir)
def parse_wrap(nd):
"""This parses one section with class="wrap" (it builds a package) """
# Title has package name and version
title = ''
a = nd.find('.//div/div/div/h2/a')
if a is None or a.tail is None:
return
# Parse the section title
title = a.tail.strip()
title = re.sub('\s+', ' ', title)
# Patch
title = title.replace('xml::parser', 'xml-parser')
# Extract build instructions
snippets = []
divs = nd.findall('./div')
with_pkg = False
sbu = sz = None
for div in divs:
# A section may have different (sub-)classes
if 'class' in div.attrib:
kl = div.attrib['class']
if kl == 'package':
sbu, sz = parse_package(div)
# print(f'{title}, {sbu} SBU, {sz} GB')
with_pkg = True
continue
# sect2 is used for the 8.4 grub section
if kl not in ['installation', 'configuration', 'sect2']:
continue
for pre in div.findall('.//pre'):
s = ''
if pre.text:
t = pre.text.strip()
s += t
if 'class' in pre.attrib:
kl = pre.attrib['class']
# Note: <pre class="root" has only been found in sect1 sections
if kl in ['userinput', 'root']:
# This is a code snippet
kbds = pre.findall('.//kbd[@class="command"]')
s += get_kbds_script(kbds)
if len(s) > 0:
snippets.append(Snippet('userinput', s))
elif kl == 'screen':
# This is some program output
s += get_em_code(pre)
if len(s) > 0:
snippets.append(Snippet('screen', s))
# Only create section object if there are instructions
if len(snippets) > 0:
return Section(title, snippets, with_pkg=with_pkg, sbu=sbu, sz=sz)
def on_cell_edited(self, cell, path, new_text):
if new_text != '':
piter = self.model.get_iter(path)
node = self.model.get_value(piter, self.SNIPPET_COLUMN)
if node:
if node == self.snippet.data:
s = self.snippet
else:
s = Snippet(node)
s['description'] = new_text
self.snippet_changed(piter)
self.select_iter(piter)
else:
# This is the `Add a new snippet...` item
# We create a new snippet
snippet = self.new_snippet({'description': new_text})
if snippet:
self.model.set_value(piter, self.SNIPPET_COLUMN,
snippet.data)
self.snippet_changed(piter)
self.snippet = snippet
self.selection_changed()
def mixGPAll(snippets, params):
logger.mixer("Starting mixGPAll mix method")
combined = Snippet("combined")
# copy a few things, assume first snippet has the defaults
combined.optSteps = snippets[0].optSteps
combined.learningRate = snippets[0].learningRate
combined.lossTolerance = snippets[0].lossTolerance
logger.mixer("Adding training data")
# add training data
for snippet in snippets:
for tdata in snippet.data:
combined.addData(tdata)
# train the new snippet
logger.mixer("Training new snippet")
combined.train()
# sample the stuff
logger.mixer("Sampling new snippet")
sampler = Rejection(combined, params["x0"], n=20)
sampler.start()
# this is threaded so we'll join then pull the results out
sampler.join()
return sampler.results
def new_snippet(self, properties=None):
if not self.language_path:
return None
snippet = Library().new_snippet(self.get_language(self.language_path),
properties)
return Snippet(snippet)
def __init__(self, name, snippets, files, substitutions):
self.name = name
self.snippets = [
Snippet(os.path.join(docker_path, 'snippets', snippet + '.in'))
for snippet in snippets
]
self.substitutions = substitutions
self.files = files
self.status = None
self.image_dir = os.path.join(images_dir, self.name)
self.image_name = 'gulinux/planetaryimager_build:{}'.format(self.name)
def addSnippet(self, name, paramInfo=None):
if name not in self.snippets:
self.snippets[name] = Snippet(name, paramInfo)
return DSStatus(code=0, message="Added Snippet: {0}".format(name))
else:
return DSStatus(
code=-1,
message="Failed to add Snippet with name {0}. Snippet already exists.".format(
name
),
)
def parse_sect1(nd):
# Title has package name and version
title = ''
a = nd.find('.//div/div/div/h2/a')
if a is None or a.tail is None:
return
# Parse the section title
title = a.tail.strip()
title = re.sub('\s+', ' ', title)
# Extract build instructions
snippets = []
for pre in nd.findall('.//pre'):
s = ''
if pre.text:
t = pre.text.strip()
s += t
if 'class' in pre.attrib:
kl = pre.attrib['class']
# Note: <pre class="root" has only been found in sect1 sections
if kl in ['userinput', 'root']:
# This is a code snippet
kbds = pre.findall('.//kbd[@class="command"]')
s += get_kbds_script(kbds)
if len(s) > 0:
snippets.append(Snippet('userinput', s))
elif kl == 'screen':
# This is some program output
s += get_em_code(pre)
if len(s) > 0:
snippets.append(Snippet('screen', s))
# Only create section object if there are instructions
if len(snippets) > 0:
return Section(title, snippets, with_pkg=False)
def snippet_changed(self, piter=None):
if piter:
node = self.model.get_value(piter, self.SNIPPET_COLUMN)
s = Snippet(node)
else:
s = self.snippet
piter = self.find_iter(self.model.get_iter(self.language_path), s)
if piter:
nm = s.display()
self.model.set_value(piter, self.NAME_COLUMN, nm)
self.model.set_value(piter, self.SORT_COLUMN, nm)
self.update_toolbar_buttons()
self.entry_tab_trigger_update_valid()
return piter
def parse_snippets(self, path: str) -> Type[HeaderFile]:
result = HeaderFile(path)
snippet = None
group = []
for line_num, line in enumerate(readlines(path), 1):
if line.startswith(BEGIN_TAG):
dprint(f'Process {line}')
key = line[len(BEGIN_TAG):].strip().strip('`')
if snippet is not None:
raise Exception(f'Unexpected open section '
f'({key}) {path}:{line_num} {line}')
snippet = Snippet(key,
enabled=key.startswith(ALWAYS_INCLUDE_TAG))
elif line.startswith(END_TAG):
dprint(f'Process {line}')
key = line[len(END_TAG):].strip().strip('`')
if snippet is None or snippet.tag != key:
raise Exception(f'Unexpected close section '
f'({key}) {path}:{line_num} {line}')
if key in self.snippets:
raise Exception(f'Duplicated key found '
f'({key}) {path}:{line_num}')
snippet.body = '\n'.join(group)
self.snippets[snippet.tag] = snippet
result.add_snippet(snippet)
snippet = None
group = []
elif line.startswith(REQUIRED_TAG) and snippet is not None:
for dependency in filter(
None,
map(str.strip, line[len(REQUIRED_TAG):].split(','))):
dprint(f'{snippet.tag} depends on `{dependency}`')
if dependency not in self.snippets:
raise Exception(f'Unknown dependency {dependency} '
f'for snippet {snippet.tag}'
f'({key}) {path}:{line_num}')
snippet.add_dependency(self.snippets[dependency])
elif snippet is not None:
group.append(line)
if snippet is not None:
raise Exception(f'Missed last closing tag {path} ({snippet.tag})')
return result
def on_tree_view_selection_changed(self, selection):
parent, piter, node = self.selected_snippet()
if self.snippet:
self.on_entry_tab_trigger_focus_out(self['entry_tab_trigger'],
None)
self.on_source_view_snippet_focus_out(self['source_view_snippet'],
None)
self.on_entry_drop_targets_focus_out(
self['combo_drop_targets'].get_child(), None)
self.update_language_path()
if node:
self.snippet = Snippet(node)
else:
self.snippet = None
self.selection_changed()
def find_snippet(filename: str = Constants.PYTHON_SNIPPETS_FILE) -> List[Snippet]:
with open(filename) as snippets_file:
snippets_db = []
snippet = Snippet()
for line in snippets_file.readlines():
if "snippet " in line:
if (snippet.name and snippet.prefix) is not None:
snippets_db.append(copy.deepcopy(snippet))
snippet.clean()
snippet.name = line.replace("snippet", "").lstrip().rstrip()
snippet.prefix = snippet.name.split()[0]
continue
if line.find("#") == 0:
continue
clean_line = line.replace("\t", "", 1).replace("\n", "")
if line:
snippet.body.append(clean_line)
return snippets_db
#!/usr/bin/python3
from snippet import Snippet
from utils import *
text = """
#!/usr/bin python3
print('hello world!')
"""
with open('example.html') as fp:
text = fp.read().strip()
text = text[:int(len(text) / 32)]
snippet = Snippet(text)
png_path = 'hello.png'
screen_width_px, screen_height_px = 1920, 1080
font_size_px = 16
snippet.to_image(png_path, font_size=font_size_px)
size = [1920, 1080]
scale_image(png_path, 'scaled.png', size)
def __init__(self,
snippet_limit: int = 3,
take_words: int = 3,
results_limit: int = 10,
data_dir: str = 'pages') -> None:
"""
This class is used for basic, no SQL searching.
:param take_words: How many words to display is snippets before/after the found word.
:param results_limit: How many rows to show when displaying top results with snippets.
:param data_dir: Directory in which the HTML sites are saved relative to the current directory.
"""
# nltk.download('punkt')
# nltk.download('stopwords')
self.time_needed_to_search = None
self.take_words = take_words
self.results_limit = results_limit
self.data_dir = os.path.join(os.getcwd(), data_dir)
self.path = data_dir
self.snippet_limit = snippet_limit
self.snip = Snippet(self.take_words, self.results_limit)
self.stop_words_slovene = set(stopwords.words("Slovene")).union({
"ter", "nov", "novo", "nova", "zato", "še", "zaradi", "a", "ali",
"april", "avgust", "b", "bi", "bil", "bila", "bile", "bili",
"bilo", "biti", "blizu", "bo", "bodo", "bojo", "bolj", "bom",
"bomo", "boste", "bova", "boš", "brez", "c", "cel", "cela", "celi",
"celo", "d", "da", "daleč", "dan", "danes", "datum", "december",
"deset", "deseta", "deseti", "deseto", "devet", "deveta", "deveti",
"deveto", "do", "dober", "dobra", "dobri", "dobro", "dokler",
"dol", "dolg", "dolga", "dolgi", "dovolj", "drug", "druga",
"drugi", "drugo", "dva", "dve", "e", "eden", "en", "ena", "ene",
"eni", "enkrat", "eno", "etc.", "f", "februar", "g", "g.", "ga",
"ga.", "gor", "gospa", "gospod", "h", "halo", "i", "idr.", "ii",
"iii", "in", "iv", "ix", "iz", "j", "januar", "jaz", "je", "ji",
"jih", "jim", "jo", "julij", "junij", "jutri", "k", "kadarkoli",
"kaj", "kajti", "kako", "kakor", "kamor", "kamorkoli", "kar",
"karkoli", "katerikoli", "kdaj", "kdo", "kdorkoli", "ker", "ki",
"kje", "kjer", "kjerkoli", "ko", "koder", "koderkoli", "koga",
"komu", "kot", "kratek", "kratka", "kratke", "kratki", "l",
"lahka", "lahke", "lahki", "lahko", "le", "lep", "lepa", "lepe",
"lepi", "lepo", "leto", "m", "maj", "majhen", "majhna", "majhni",
"malce", "malo", "manj", "marec", "me", "med", "medtem", "mene",
"mesec", "mi", "midva", "midve", "mnogo", "moj", "moja", "moje",
"mora", "morajo", "moram", "moramo", "morate", "moraš", "morem",
"mu", "n", "na", "nad", "naj", "najina", "najino", "najmanj",
"naju", "največ", "nam", "narobe", "nas", "nato", "nazaj", "naš",
"naša", "naše", "ne", "nedavno", "nedelja", "nek", "neka", "nekaj",
"nekatere", "nekateri", "nekatero", "nekdo", "neke", "nekega",
"neki", "nekje", "neko", "nekoga", "nekoč", "ni", "nikamor",
"nikdar", "nikjer", "nikoli", "nič", "nje", "njega", "njegov",
"njegova", "njegovo", "njej", "njemu", "njen", "njena", "njeno",
"nji", "njih", "njihov", "njihova", "njihovo", "njiju", "njim",
"njo", "njun", "njuna", "njuno", "no", "nocoj", "november", "npr.",
"o", "ob", "oba", "obe", "oboje", "od", "odprt", "odprta",
"odprti", "okoli", "oktober", "on", "onadva", "one", "oni",
"onidve", "osem", "osma", "osmi", "osmo", "oz.", "p", "pa", "pet",
"peta", "petek", "peti", "peto", "po", "pod", "pogosto", "poleg",
"poln", "polna", "polni", "polno", "ponavadi", "ponedeljek",
"ponovno", "potem", "povsod", "pozdravljen", "pozdravljeni",
"prav", "prava", "prave", "pravi", "pravo", "prazen", "prazna",
"prazno", "prbl.", "precej", "pred", "prej", "preko", "pri",
"pribl.", "približno", "primer", "pripravljen", "pripravljena",
"pripravljeni", "proti", "prva", "prvi", "prvo", "r", "ravno",
"redko", "res", "reč", "s", "saj", "sam", "sama", "same", "sami",
"samo", "se", "sebe", "sebi", "sedaj", "sedem", "sedma", "sedmi",
"sedmo", "sem", "september", "seveda", "si", "sicer", "skoraj",
"skozi", "slab", "smo", "so", "sobota", "spet", "sreda", "srednja",
"srednji", "sta", "ste", "stran", "stvar", "sva", "t", "ta", "tak",
"taka", "take", "taki", "tako", "takoj", "tam", "te", "tebe",
"tebi", "tega", "težak", "težka", "težki", "težko", "ti", "tista",
"tiste", "tisti", "tisto", "tj.", "tja", "to", "toda", "torek",
"tretja", "tretje", "tretji", "tri", "tu", "tudi", "tukaj", "tvoj",
"tvoja", "tvoje", "u", "v", "vaju", "vam", "vas", "vaš", "vaša",
"vaše", "ve", "vedno", "velik", "velika", "veliki", "veliko",
"vendar", "ves", "več", "vi", "vidva", "vii", "viii", "visok",
"visoka", "visoke", "visoki", "vsa", "vsaj", "vsak", "vsaka",
"vsakdo", "vsake", "vsaki", "vsakomur", "vse", "vsega", "vsi",
"vso", "včasih", "včeraj", "x", "z", "za", "zadaj", "zadnji",
"zakaj", "zaprta", "zaprti", "zaprto", "zdaj", "zelo", "zunaj",
"č", "če", "često", "četrta", "četrtek", "četrti", "četrto", "čez",
"čigav", "š", "šest", "šesta", "šesti", "šesto", "štiri", "ž",
"že", "svoj", "jesti", "imeti", "\u0161e", "iti", "kak", "www",
"km", "eur", "pač", "del", "kljub", "šele", "prek", "preko",
"znova", "morda", "kateri", "katero", "katera", "ampak", "lahek",
"lahka", "lahko", "morati", "torej", "gl", "xsd", "ipd", "om",
"gt", "lt", "d.o.o"
})
def musicThumbnailing(x,
Fs,
shortTermSize=1.0,
shortTermStep=0.5,
thumbnailSize=10.0,
Limit1=0,
Limit2=1):
# self-similarity matrix
x = audioBasicIO.stereo2mono(x)
stFeatures = aF.stFeatureExtraction(x, Fs, Fs * shortTermSize,
Fs * shortTermStep)
S = aS.selfSimilarityMatrix(stFeatures)
# moving filter:
M = int(round(thumbnailSize / shortTermStep))
B = np.eye(M, M)
S = scipy.signal.convolve2d(S, B, 'valid')
MIN = np.min(S)
# post-processing (remove main diagonal elements)
for i in range(S.shape[0]):
for j in range(S.shape[1]):
if abs(i - j) < 5.0 / shortTermStep or i > j:
S[i, j] = MIN
# find max position:
S[0:int(Limit1 * S.shape[0]), :] = MIN
S[:, 0:int(Limit1 * S.shape[0])] = MIN
S[int(Limit2 * S.shape[0])::, :] = MIN
S[:, int(Limit2 * S.shape[0])::] = MIN
matches = []
maxMax = maxVal = np.max(S)
i1 = i2 = j1 = j2 = 0
Sbak = np.copy(S)
while maxVal > maxMax / 3 * 2 > MIN: # currently arbitrary cutoff
[I, J] = np.unravel_index(S.argmax(), S.shape)
# expand:
i1 = I
i2 = I
j1 = J
j2 = J
while i2 - i1 < M:
if i1 <= 0 or j1 <= 0 or i2 >= S.shape[0] - 2 or j2 >= S.shape[
1] - 2:
break
if S[i1 - 1, j1 - 1] > S[i2 + 1, j2 + 1]:
i1 -= 1
j1 -= 1
else:
i2 += 1
j2 += 1
S[i1, j1] = S[i2, j2] = MIN
# only add to potential matches if we have enough overlap or new record
if i2 - i1 >= M:
matches.append(
Snippet(maxVal, shortTermStep * i1, shortTermStep * i2,
shortTermStep * j1, shortTermStep * j2))
S[I, J] = MIN
maxVal = np.max(S)
return (matches, Sbak)
def apply_directive(self,
directive,
bk,
code=None,
s=None,
repl_records=None):
"""Apply directives.
The value in directive[2] depends on the command:
'replace': the argument is the string to use as a replacement
'push': the argument is the filepath to save into
"""
snip_id = directive[0]
cmd = directive[1]
if cmd == 'push':
# Perform replacements on the directive argument if necessary
arg = directive[2]
if repl_records is not None:
for rec in repl_records:
# rec[0] = snippet id, rec[1] = array of replacement values
if rec[0] == snip_id:
arg = replace_values(arg, rec[1])
break
# Save the current (code, filepath) on the stack
bk.code_stack.append((code + s, arg))
code = s = ''
elif cmd == 'pop':
# Finalize code being generated
code += s
# Pop up one stack level and resume that level's code generation
prev_code, filepath = bk.code_stack.pop()
print(f'{self.id}: apply_directive: writing "{filepath}"')
# Write out to file
chrooted = self.id.startswith('6.') or self.id.startswith('9.')
with open(filepath, 'w') as f:
f.write(
file_header(filepath, bk, chrooted=chrooted) + code +
file_footer(filepath))
# Reset code generation for this stack level
code = prev_code
s = ''
elif cmd == 'add':
arg = directive[2]
t = Snippet('userinput', arg)
# Replace placeholders with actual values
if repl_records is not None:
for rec in repl_records:
# rec[0] = snippet id, rec[1] = array of replacement values
if rec[0] == snip_id:
t.text = t.replace_values(rec[1])
break
s += t.generate(f'{self.id}_{snip_id} [add]')
else:
print(f'Sect: {self.id}, snippet directive: {snip_id}' +
f', unknown command "{cmd}"')
return code, s
def __init__(self, snippet):
GObject.Object.__init__(self)
self._snippet = Snippet(snippet)
def apply_snippet(self, snippet, start=None, end=None, environ={}):
if not snippet.valid:
return False
# Set environmental variables
env = self.get_environment()
if environ:
for k in environ['utf8']:
env['utf8'][k] = environ['utf8'][k]
for k in environ['noenc']:
env['noenc'][k] = environ['noenc'][k]
buf = self.view.get_buffer()
s = Snippet(snippet, env)
if not start:
start = buf.get_iter_at_mark(buf.get_insert())
if not end:
end = buf.get_iter_at_mark(buf.get_selection_bound())
if start.equal(end) and self.uses_current_word(s):
# There is no tab trigger and no selection and the snippet uses
# the current word. Set start and end to the word boundary so that
# it will be removed
start, end = buffer_word_boundary(buf)
elif start.equal(end) and self.uses_current_line(s):
# There is no tab trigger and no selection and the snippet uses
# the current line. Set start and end to the line boundary so that
# it will be removed
start, end = buffer_line_boundary(buf)
# You know, we could be in an end placeholder
(current, next) = self.next_placeholder()
if current and current.__class__ == PlaceholderEnd:
self.goto_placeholder(current, None)
buf.begin_user_action()
# Remove the tag, selection or current word
buf.delete(start, end)
# Insert the snippet
holders = len(self.placeholders)
if len(self.active_snippets) == 0:
self.first_snippet_inserted()
sn = s.insert_into(self, start)
self.active_snippets.append(sn)
# Put cursor at first tab placeholder
keys = filter(lambda x: x > 0, sn.placeholders.keys())
if len(keys) == 0:
if 0 in sn.placeholders:
self.goto_placeholder(self.active_placeholder,
sn.placeholders[0])
else:
buf.place_cursor(sn.begin_iter())
else:
self.goto_placeholder(self.active_placeholder,
sn.placeholders[keys[0]])
if sn in self.active_snippets:
# Check if we can get end_iter in view without moving the
# current cursor position out of view
cur = buf.get_iter_at_mark(buf.get_insert())
last = sn.end_iter()
curloc = self.view.get_iter_location(cur)
lastloc = self.view.get_iter_location(last)
if (lastloc.y + lastloc.height) - curloc.y <= \
self.view.get_visible_rect().height:
self.view.scroll_mark_onscreen(sn.end_mark)
buf.end_user_action()
self.view.grab_focus()
return True
def fetch_snippets(num_snippets, start_time, end_time, extra_tags):
"""
Fetches snippets from StackOverflow by looking at 'python' questions
in the time-period specified, and retrieving `<pre><code>` code blocks
from these questions (and their answers, where applicable)
Note that currently the number of snippets returned is not the same as
the number requested: it depends on the number of retrieved questions with
a code block, as well as the number with an accepted answer with a code block
"""
filters = load_filter_file()
snippets = []
answer_ids = []
questions_retrieved = 0
page_num = 1
while questions_retrieved < num_snippets:
questions, has_more = fetch_recent_questions(
num_questions=(num_snippets - questions_retrieved),
from_time=start_time,
to_time=end_time,
tags=extra_tags,
page_num=page_num,
filter_name=filters.Questions)
current_time = datetime.utcnow()
for q in questions:
snippets += [
Snippet(snippet_id=q['question_id'],
code=block,
url=q['link'],
author='stack-overflow',
retrieved_at=current_time,
additional_url=None)
for block in get_snippets(q['body'])
if check_source_and_warn(block, q['link'])
]
answer_ids.append(q.get('accepted_answer_id', None))
questions_retrieved += len(questions)
page_num += 1
if not has_more:
warn('No more questions to fetch: Terminating')
break
# Filter out `None` accepted_answer_id's
answer_ids = [str(ans_id) for ans_id in answer_ids if ans_id]
log('Retrieving {} accepted answers for analysis...'.format(
len(answer_ids)))
for i in range(0, len(answer_ids), ANSWER_BATCH_SIZE):
batch = answer_ids[i:i + ANSWER_BATCH_SIZE]
answers = fetch_answers(batch, filters.Answers)
for a in answers:
snippets += [
Snippet(snippet_id=a['answer_id'],
code=block,
url=a['link'],
author='stack-overflow',
retrieved_at=current_time,
additional_url=None)
for block in get_snippets(a['body'])
if check_source_and_warn(block, q['link'])
]
success('Retrieved {} snippets'.format(len(snippets)))
return snippets