def handle(self, *args, **kwargs):
with open('%s/tutorials/deploy_colab.md' % settings.UI_ROOT, 'w') as readme_file:
readme_file.write('# Using Scripts As A Colab Notebook\n')
readme_file.write('\n')
readme_file.write('All StarThinker recipes and solutions can be run from [Google Collaboratory](https://colab.research.google.com/github/google/starthinker/blob/master). ')
readme_file.write('Also visit the [Solution Gallery](google.github.io/starthinker/) or click a link below to deploy a notebook.\n')
readme_file.write('\n')
readme_file.write('## List Of Notebooks\n')
for script in Script.get_scripts():
if script.get_open_source():
readme_file.write('* [%s](%s) - %s\n' % (script.get_name(), script.get_link_colab(), script.get_description()))
readme_file.write('---\n')
readme_file.write('© 2019 Google Inc. - Apache License, Version 2.0\n')
for script in Script.get_scripts():
if script.get_open_source():
print('Writing: %s/colabs/%s.ipynb' % (settings.UI_ROOT, script.get_tag()))
with open('%s/colabs/%s.ipynb' % (settings.UI_ROOT, script.get_tag()), 'w') as colab_file:
colab_file.write(script_to_colab(
script.get_name(),
script.get_description(),
script.get_instructions(),
script.get_tasks(),
))
def __init__(self, manual, recipe, account, *args, **kwargs):
post = args[0] if len(args) > 0 else None
# load scripts ( used when creating new recipe )
if 'scripts' in kwargs:
scripts = [s for s in kwargs.pop('scripts').split(',') if s]
else:
scripts = []
# fetch the instance and load initial data
self.instance = recipe or Recipe()
self.setup = SetupForm(manual, account, post, instance=self.instance)
super(ScriptForm, self).__init__(*args, **kwargs)
# create blank forms
self.blanks = []
for s in Script.get_scripts(account.email):
if s.is_manual() == manual:
self.blanks.append(
ScriptJsonForm('[BLANK]',
s, {},
prefix='%s_[BLANK]' % s.get_tag()))
# processing incoming form
self.forms = []
if post:
for prefix in post.getlist('form_prefix'):
tag, sequence = prefix.rsplit('_', 1)
sequence = int(sequence) - 1
s = Script(tag)
self.forms.append(
ScriptJsonForm(
sequence,
s,
self.instance.get_values()[sequence]['values']
if self.instance
and sequence < len(self.instance.get_values()) else {},
post,
prefix=prefix))
# loading from existing recipe
elif self.instance:
for sequence, script in enumerate(self.instance.get_values()):
s = Script(script['tag'])
self.forms.append(
ScriptJsonForm(sequence + 1,
s,
script['values'],
prefix='%s_%d' %
(s.get_tag(), sequence + 1)))
# starting a new recipe
else:
for sequence, script in enumerate(scripts):
s = Script(script)
self.forms.append(
ScriptJsonForm(sequence + 1,
s, [],
prefix='%s_%d' %
(s.get_tag(), sequence + 1)))
def solutions(request):
scripts = [s for s in Script.get_scripts() if s.is_solution()]
# if open source then request will be null
if not request:
scripts = [s for s in scripts if s.get_open_source()]
context = {
'scripts':
scripts,
'categories':
sorted(set(chain.from_iterable([s.get_categories()
for s in scripts]))),
'catalysts':
sorted(set(chain.from_iterable([s.get_catalysts() for s in scripts]))),
'requirements':
sorted(
set(chain.from_iterable([s.get_requirements() for s in scripts]))),
'external': (request is None)
}
# if open source then request will be null
if not request:
return render_to_string("website/solutions.html", context)
else:
return render(request, "website/solutions.html", context)
def handle(self, *args, **kwargs):
settings.GOOGLE_ANALYTICS = kwargs['analytics']
directory = '%s/docs' % settings.UI_ROOT
print('Writing:', directory)
with open('%s/index.html' % directory, 'w') as index_file:
index_file.write(
self.strip_trailing_whitespace(solutions(request=None)))
directory = '%s/docs/help' % settings.UI_ROOT
print('Writing:', directory)
os.makedirs(directory, exist_ok=True)
with open('%s/index.html' % directory, 'w') as index_file:
index_file.write(self.strip_trailing_whitespace(
help(request=None)))
directory = '%s/docs/solution' % settings.UI_ROOT
print('Writing:', directory)
os.makedirs(directory, exist_ok=True)
with open('%s/index.html' % directory, 'w') as index_file:
index_file.write(
self.strip_trailing_whitespace(solutions(request=None)))
for s in Script.get_scripts():
if s.get_open_source():
directory = '%s/docs/solution/%s' % (settings.UI_ROOT,
s.get_tag())
print('Writing:', directory)
os.makedirs(directory, exist_ok=True)
with open('%s/index.html' % directory, 'w') as solution_file:
solution_file.write(
self.strip_trailing_whitespace(
solution(request=None, tag=s.get_tag())))
def handle(self, *args, **kwargs):
directory = '%s/docs' % kwargs['path']
print 'Writing:', directory
with open('%s/index.html' % directory, 'w') as index_file:
index_file.write(solutions(request=None))
directory = '%s/docs/solution' % kwargs['path']
print 'Writing:', directory
with open('%s/index.html' % directory, 'w') as index_file:
index_file.write(solutions(request=None))
directory = '%s/docs/code' % kwargs['path']
print 'Writing:', directory
if not os.path.exists(directory): os.makedirs(directory)
with open('%s/index.html' % directory, 'w') as code_file:
code_file.write(code(request=None))
for s in Script.get_scripts():
if s.is_solution() and s.get_open_source():
directory = '%s/docs/solution/%s' % (kwargs['path'], s.get_tag())
print 'Writing:', directory
if not os.path.exists(directory): os.makedirs(directory)
with open('%s/index.html' % directory, 'w') as solution_file:
solution_file.write(solution(request=None, tag=s.get_tag()))
def get_json(self, credentials=True):
return Script.get_json(
self.uid(), self.get_project_identifier(),
self.get_credentials_user() if credentials else '',
self.get_credentials_service() if credentials else '',
self.timezone, self.get_days(), self.get_hours(),
self.get_values())
def solution(request, tag):
script = Script(tag)
# if open source then request will be null
if request:
return render(request, "website/solution.html", { 'script':script })
else:
return render_to_string('website/solution.html', { 'script':script, "external":True })
def render(self, context):
scripts = sorted(Script.get_scripts(ui=True),
key=lambda x: x.get_name())
if self.filter == 'SOME':
scripts = [
s for s in scripts
if s.is_manual() == context.get('manual', False)
]
if context.get('external', False):
scripts = [s for s in scripts if s.get_open_source()]
context['scripts'] = scripts
context['agos'] = sorted(
set(chain.from_iterable([s.get_released_ago() for s in scripts])))
categories = {}
for s in scripts:
for f in s.get_from():
categories.setdefault(f, [])
categories[f].extend(s.get_to())
for f in categories.keys():
categories[f] = sorted(set(categories[f]))
context['categories'] = sorted(categories.items())
return ''
def solution(request, tag):
script = Script(tag)
if request:
return render(request, "website/solution.html", {'script': script})
else:
return render_to_string('website/solution.html', {
'script': script,
"external": True
})
def get_scripts(self):
for script in Script.get_scripts():
yield (
script.get_name(), # solution
script.get_description().replace('"', '\''), # description
'Global', # region
', '.join(script.get_products()), # entity
', '.join(x.replace('@google.com', '') for x in script.get_authors()), # POC
'%s - current' % script.get_released().year, # year
script.get_status() or 'Live', # status
script.get_link(), # link
)
def handle(self, *args, **kwargs):
for script in Script.get_scripts():
if script.get_open_source():
print('Writing: %s_dag.py' % script.get_tag())
with open(
'%s/dags/%s_dag.py' %
(settings.UI_ROOT, script.get_tag()), 'w') as dag_file:
dag_file.write(
script_to_dag(script.get_tag(), script.get_name(),
script.get_description(),
script.get_instructions(),
script.get_tasks()))
def render(self, context):
scripts = sorted(Script.get_scripts(ui=True),
key=lambda x: x.get_name())
if self.filter == 'SOME':
scripts = [
s for s in scripts
if s.is_manual() == context.get('manual', False)
]
if context.get('external', False):
scripts = [s for s in scripts if s.get_open_source()]
context['scripts'] = scripts
context['categories'] = sorted(
set(chain.from_iterable([s.get_categories() for s in scripts])))
context['requirements'] = sorted(
set(chain.from_iterable([s.get_requirements() for s in scripts])))
context['agos'] = sorted(
set(chain.from_iterable([s.get_released_ago() for s in scripts])))
return ''
def code(request):
scripts = sorted(Script.get_scripts(), key=lambda s: s.get_name())
products = {}
for s in scripts:
if s.get_open_source():
products.setdefault(s.get_product(), [])
products[s.get_product()].append(s)
products = sorted([{
'name': k,
'scripts': v
} for k, v in products.items()],
key=lambda p: p['name'])
if request:
return render(request, "website/code.html", {'products': products})
else:
return render_to_string('website/code.html', {
'products': products,
"external": True
})
def __init__(self, recipe, account, *args, **kwargs):
post = args[0] if len(args) > 0 else None
# load scripts ( used when creating new recipe )
if 'scripts' in kwargs:
scripts = [s for s in kwargs.pop('scripts').split(',') if s]
else:
scripts = []
# fetch the instance and load intial data
self.instance = recipe or Recipe()
self.setup = SetupForm(account, post, instance=self.instance)
super(ScriptForm, self).__init__(*args, **kwargs)
# group blanks by product
self.products = {}
for s in sorted(Script.get_scripts(account.email),
key=lambda s: s.get_tag()):
self.products.setdefault(s.get_product(), [])
self.products[s.get_product()].append(
ScriptJsonForm('[BLANK]',
s, {},
prefix='%s_[BLANK]' % s.get_tag()))
self.products = sorted([{
'name': k,
'blanks': v
} for k, v in self.products.items()],
key=lambda p: p['name'])
# processing incoming form
self.forms = []
if post:
for prefix in post.getlist('form_prefix'):
tag, sequence = prefix.rsplit('_', 1)
sequence = int(sequence) - 1
s = Script(tag)
self.forms.append(
ScriptJsonForm(
sequence,
s,
self.instance.get_values()[sequence]['values']
if self.instance
and sequence < len(self.instance.get_values()) else {},
post,
prefix=prefix))
# loading from existing recipe
elif self.instance:
for sequence, script in enumerate(self.instance.get_values()):
s = Script(script['tag'])
self.forms.append(
ScriptJsonForm(sequence + 1,
s,
script['values'],
prefix='%s_%d' %
(s.get_tag(), sequence + 1)))
# starting a new recipe
else:
for sequence, script in enumerate(scripts):
s = Script(script)
self.forms.append(
ScriptJsonForm(sequence + 1,
s, [],
prefix='%s_%d' %
(s.get_tag(), sequence + 1)))
def handle(self, *args, **kwargs):
day = date.today() - timedelta(days=kwargs['days'])
email = {
'subject':
'Announcing Six New Open Source Modules For Ecosystems',
'style': {
'background': '#f2f2f2',
'foreground': '#ffffff',
'text': '#414347',
'link': '#4285f4',
'font': 'Roboto, Helvetica, Arial sans-serif;',
'align': 'left'
},
'logo':
'https://storage.googleapis.com/starthinker-ui/gTech_StarThinker.png',
'body': {
'sections': [{
'header':
'Six New Solutions For Partners To Build New Services',
'paragraph':
'In Q1, StarThinker released 6 new building blocks '
'available as Python, Airflow, Colab, and no-coding UI. '
'These building blocks are now open sourve and availbale '
'for deployment by Partners. Below is a description of '
'each solution and possible service or efficiency gain by '
'partners.',
'grid': []
}]
},
'footer': [{
'text': 'Internal UI',
'link': 'http://go/starthinker'
}, {
'text': 'Esteemed Solution Gallery',
'link': 'http://go/esteemed/#/search/StarThinker'
}, {
'text': 'GitHub Solution Gallery',
'link': 'https://google.github.io/starthinker/'
}, {
'text': 'Google3 Repository',
'link': 'http://go/starthinker-google3'
}, {
'text': 'GOB Repository ( Official )',
'link': 'http://go/starthinker-code'
}, {
'text': 'GitHub Repository',
'link': 'https://github.com/google/starthinker'
}],
'copyright':
'Copyright 2020 Google LLC'
}
odd = True
for s in Script.get_scripts():
if s.get_released() < day:
continue
print('SCRIPT: ', s.get_tag())
if not s.get_image():
continue
row = [{
'image': {
'src': s.get_image(),
'link': s.get_link_client()
}
}, {
'header': '[%s](%s)' % (s.get_name(), s.get_link_client()),
'paragraph': s.get_description()
}]
email['body']['sections'][0]['grid'].append(row)
if odd:
row.reverse()
odd = not odd
email = EmailTemplate(email)
# send or print
#if project.args.email_to and project.args.email_from:
# print('EMAILING: ', project.args.email_to)
# send_email('user', project.args.email_to, project.args.email_from, None, email.get_subject(), email.get_text(), email.get_html())
#else:
if 1:
# write to STDOUT
print(email.get_html())
print('<pre style="width:600px;margin:0px auto;">%s</pre>' %
email.get_text())
def get_scripts(self):
for value in self.get_values():
yield Script(value['tag'])
def handle(self, *args, **kwargs):
day = date.today() - timedelta(days=kwargs['days'])
email = {
"subject":
"Announcing Six New Open Source Modules For Ecosystems",
"style": {
"background": "#f2f2f2",
"foreground": "#ffffff",
"text": "#414347",
"link": "#4285f4",
"font": "Roboto, Helvetica, Arial sans-serif;",
"align": "left"
},
"logo":
"https://storage.googleapis.com/starthinker-ui/gTech_StarThinker.png",
"body": {
"sections": [{
"header":
"Six New Solutions For Partners To Build New Services",
"paragraph":
"In Q1, StarThinker released 6 new building blocks available as Python, Airflow, Colab, and no-coding UI. These building blocks are now open sourve and availbale for deployment by Partners. Below is a description of each solution and possible service or efficiency gain by partners.",
"grid": []
}]
},
"footer": [{
"text": "Internal UI",
"link": "http://go/starthinker"
}, {
"text": "Esteemed Solution Gallery",
"link": "http://go/esteemed/#/search/StarThinker"
}, {
"text": "GitHub Solution Gallery",
"link": "https://google.github.io/starthinker/"
}, {
"text": "Google3 Repository",
"link": "http://go/starthinker-google3"
}, {
"text": "GOB Repository ( Official )",
"link": "http://go/starthinker-code"
}, {
"text": "GitHub Repository",
"link": "https://github.com/google/starthinker"
}],
"copyright":
"Copyright 2020 Google LLC"
}
odd = True
for s in Script.get_scripts():
if s.get_released() < day: continue
print('SCRIPT: ', s.get_tag())
if not s.get_image(): continue
row = [{
"image": {
"src": s.get_image(),
"link": s.get_link_client()
}
}, {
"header": "[%s](%s)" % (s.get_name(), s.get_link_client()),
"paragraph": s.get_description()
}]
email['body']['sections'][0]['grid'].append(row)
if odd: row.reverse()
odd = not odd
email = EmailTemplate(email)
# send or print
#if project.args.email_to and project.args.email_from:
# print('EMAILING: ', project.args.email_to)
# send_email('user', project.args.email_to, project.args.email_from, None, email.get_subject(), email.get_text(), email.get_html())
#else:
if 1:
# write to STDOUT
print(email.get_html())
print('<pre style="width:600px;margin:0px auto;">%s</pre>' %
email.get_text())
def get_metrics():
metrics = {
'account': {}, #'account':{ 'recipe':'', 'script':'', 'author':'' }
'script': {}, # 'script':{ 'account':'', 'recipe':'' }
'author': {}, # 'author':{ 'account':'', 'recipe':'', 'script' }
}
totals = {
'account': set(),
'author': set(),
'script': set(),
'recipe': set(),
}
for account in Account.objects.all():
totals['account'].add(account.name)
for recipe in account.recipe_set.all():
totals['recipe'].add(recipe.pk)
for s in recipe.get_values():
script = Script(s['tag'])
authors = script.get_authors()
metrics['account'].setdefault(account.name, {
'recipe': set(),
'script': set(),
'author': set()
})
metrics['account'][account.name]['recipe'].add(recipe.pk)
metrics['account'][account.name]['script'].add(s['tag'])
metrics['account'][account.name]['author'].update(authors)
totals['script'].add(s['tag'])
metrics['script'].setdefault(s['tag'], {
'account': set(),
'recipe': set()
})
metrics['script'][s['tag']]['recipe'].add(recipe.pk)
metrics['script'][s['tag']]['account'].add(account.name)
for author in authors:
totals['author'].add(author)
metrics['author'].setdefault(author, {
'account': set(),
'recipe': set(),
'script': set()
})
metrics['author'][author]['account'].add(account.name)
metrics['author'][author]['recipe'].add(recipe.pk)
metrics['author'][author]['script'].add(s['tag'])
# compute totals
for dimension in iter(totals.keys()):
totals[dimension] = len(totals[dimension])
for metric_key, metric in metrics.items():
for row_key, row in metric.items():
for dimension in list(row.keys()):
row[dimension] = len(row[dimension])
row['%s_percent' % dimension] = int(
(row[dimension] * 100) / (totals[dimension] or 1))
row[metric_key] = row_key
metrics[metric_key] = metrics[metric_key].values()
return metrics