def _build_paths(config, addon_path):
direct_link_dict = {}
# if the paths file does not exists
# verify that the shellrc has a connection to paths file
for bin_path in config:
bin_path = os.path.join(addon_path, bin_path)
new_path = get_new_path(bin_path, paths.custom_bin_path, 'addon')
src_dir_path = os.path.join(addon_path, bin_path)
create_symlink(src_dir_path, new_path)
file_names = get_dir_list(src_dir_path)
for file_name in file_names:
if file_name.startswith('.'):
continue
src_full_path = os.path.join(src_dir_path, file_name)
if os.path.isdir(src_full_path):
continue
dst_full_path = os.path.join(paths.custom_bin_path, file_name)
result = direct_link_dict.setdefault(dst_full_path, [])
result.append(src_full_path)
for dst, src_list in direct_link_dict.items():
# This is a bit nasty for 2.7, 3.x can use first, *remander syntax
first = True
for src in src_list:
if first:
create_symlink(src, dst)
first = False
continue
create_directory(paths.custom_bin_conflicts_path)
epoch_time = get_epoc_time()
src_base = os.path.basename(src)
conflict_dst = '{0}_{1}'.format(epoch_time, src_base)
conflict_dst = os.path.join(paths.custom_bin_conflicts_path,
conflict_dst)
create_symlink(src, conflict_dst)
def main(args, logger):
create_directory()
ytdl = YoutubeDL({
'quiet':
True,
'prefer_insecure':
True,
'outtmpl':
os.path.join(args.output_dir, '%(title)s.wav'),
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'wav'
}],
'keepvideo':
False
})
logger.info("Fetching playlist (this can take a while)")
playlist_name, playlist = scrapper.get_playlist_details(args.playlist_url)
saved_playlist = SavedPlaylist(playlist_name.lstrip().rstrip(),
args.mpd_playlist_dir)
removed_elements, remaining_elements = saved_playlist.compare_playlists(
playlist, args.output_dir, args.keep_removed)
if (removed_elements > 0):
logger.info(
'{} elements were removed from playlist'.format(removed_elements))
new_elements = len(remaining_elements)
logger.info("Found {} new elements, downloading...".format(new_elements))
for i in range(0, new_elements):
try:
ytdl.download([
'https://youtube.com/watch?v={}'.format(
remaining_elements[i]['upload_id'])
])
except Exception as e:
logger.error(e)
saved_playlist.remove_failed(remaining_elements[i:])
saved_playlist.save()
saved_playlist.save()
if new_elements > 0:
logger.finish("{} new elements added".format(new_elements))
def main():
print('-' * 40)
print('Start mk_custom...')
create_directory(paths.custom_path)
copy_file(paths.clither_tmp_config, paths.custom_path)
create_directory(paths.custom_lib_path)
copy_file(paths.clither_run, paths.custom_path, write_over=True) # may want another lvl deep?
copy_file(paths.clither_pather, paths.custom_path, write_over=True)
copy_file(paths.clither_run_help, paths.custom_lib_path, write_over=True)
# create_directory(paths.custom_default_addon_path)
# copy_file(paths.clither_default_json, paths.custom_default_addon_path, write_over=True)
# create_directory(paths.custom_default_addon_rcs_path)
# copy_file(paths.clither_default_addons_rc, paths.custom_default_addon_rcs_path, write_over=True)
create_directory(paths.custom_configs_path)
create_directory(paths.custom_overrides_path)
create_directory(paths.custom_fallbacks_path)
create_directory(paths.custom_lockfile_path)
print('Finished mk_custom!')
# Root directory of the project
ROOT_DIR = os.path.abspath("../")
sys.path.append(ROOT_DIR)
import helpers
def get_data(path):
img = cv2.imread(path)
return img
# Training file directory
DATASET = os.path.join(ROOT_DIR, 'dataset')
helpers.create_directory("{}/isic2016gan/".format(DATASET))
NEW_DATASET_PATH = "{}/{}".format(DATASET, "isic2016gan")
NEW_DATASET_PATH
helpers.create_directory("{}/trainA".format(NEW_DATASET_PATH))
helpers.create_directory("{}/trainB".format(NEW_DATASET_PATH))
# Learn mapping from normal lesions to maligant lesions
NORMAL_FOLDER = "{}/trainA/".format(NEW_DATASET_PATH)
CANCER_FOLDER = "{}/trainB/".format(NEW_DATASET_PATH)
CANCER_FOLDER, NORMAL_FOLDER
# IMAGES PATH
TRAINING_IMAGES = os.path.join(DATASET, 'isic2016',
'ISBI2016_ISIC_Part3_Training_Data')
def construct_numpy(images, meta, fname, lname):
'''
Creates a new numpy arrays.
INPUT
IMAGES:
df:
OUTPUT
Numpy arrays
'''
# filenames and gts
filenames = meta['FILENAME'].values
gt = meta['CLASS'].values
# convert string labels to numeric values
labels = []
for s in gt:
if s == "benign" or s == 0.0:
labels.append(0)
if s == "malignant" or s == 1.0:
labels.append(1)
# all training images and labels
inp_feat = []
g_t = []
# two classes individually
cancer = []
non_cancer = []
for f, l in tqdm(zip(filenames[:], labels[:])):
f = "{}/{}.jpg".format(images, f)
img = get_data(f)
inp_feat.append(img)
g_t.append(l)
#----------------
#if l == 1:
# cancer.append(img)
#if l == 0:
# non_cancer.append(img)
#else:
# pass
#-----------------
img = None
# make nummpy arrays
inp_feat = np.array(inp_feat)
g_t = np.array(g_t)
# one hot encoded vectors
num_classes = 2
g_t = np_utils.to_categorical(g_t, num_classes)
#cancer = np.array(cancer)
#non_cancer = np.array(non_cancer)
print(inp_feat.shape, g_t.shape)
# Create directory
helpers.create_directory("{}/isic2016numpy/".format(DATASET))
# Save
np.save("{}/isic2016numpy/{}.npy".format(DATASET, fname), inp_feat)
np.save("{}/isic2016numpy/{}.npy".format(DATASET, lname), g_t)
print("Done!")
def generate(self):
config = get_config()
# Download data from Google docs etc
response = requests.get(config['riskAssessmentCsvUrl'])
risk_assessments_file = open('%s/risk-assessments.csv' % config['dataDir'], 'w')
risk_assessments_file.write(response.text)
risk_assessments_file.close()
# Load risk assessments from CSV
risk_assessments = []
with open('%s/risk-assessments.csv' % config['dataDir']) as csv_file:
assessments_csv = csv.DictReader(csv_file)
for item in assessments_csv:
risk_assessment = models.RiskAssessment()
risk_assessment.employer = csv_string(item['employer'])
risk_assessment.employer_address = csv_string(item['employer address'])
risk_assessment.employer_website_url = csv_string(item['employer website url'])
risk_assessment.workplace = csv_string(item['workplace'])
risk_assessment.workplace_address = csv_string(item['workplace address'])
risk_assessment.workplace_website_url = csv_string(item['workplace website url'])
risk_assessment.workplace_approximate_number_of_workers = csv_int(item['workplace approximate number of workers'])
risk_assessment.risk_assessment_status = csv_string(item['risk assessment status'])
risk_assessment.risk_assessment_url = csv_string(item['risk assessment url'])
risk_assessment.risk_assessment_title = csv_string(item['risk assessment title'])
risk_assessment.risk_assessment_date = csv_date(item['risk assessment date'])
risk_assessment.additional_information_url = csv_string(item['additional information url'])
risk_assessment.public_visit = csv_boolean(item['public visit'])
risk_assessment.union_presence = csv_boolean(item['union presence'])
risk_assessment.sic_codes = csv_list(item['sic codes'])
risk_assessment.company_number = csv_int(item['company number'])
if risk_assessment.is_valid():
risk_assessments.append(risk_assessment)
else:
print("Invalid risk assessment %s" % ", ".join(risk_assessment._validation_errors))
# mailchimp
mailchimp_api_key = os.environ['MAILCHIMP_API_KEY']
mailchimp_username = os.environ['MAILCHIMP_USERNAME']
mailchimp_audience_id = os.environ['MAILCHIMP_AUDIENCE_ID']
mailchimp_client = MailChimp(mc_api=mailchimp_api_key, mc_user=mailchimp_username)
mailchimp_list = mailchimp_client.lists.get(mailchimp_audience_id)
# Stats
stats = {'risk_assessments': 0, 'volunteers': 0, 'risk_assessments_public': 0, 'risk_assessments_private': 0, 'risk_assessments_onrequest': 0, 'risk_assessments_unknown': 0}
stats['risk_assessments'] = len(risk_assessments)
stats['volunteers'] = mailchimp_list['stats']['member_count']
# output directory
create_directory(config['outputDir'])
create_directory("%s/assets" % config['outputDir'])
# Risk assessments
create_directory("%s/risk-assessments" % config['outputDir'])
for risk_assessment in risk_assessments:
if risk_assessment.risk_assessment_status == 'public':
stats['risk_assessments_public'] += 1
elif risk_assessment.risk_assessment_status == 'private':
stats['risk_assessments_private'] += 1
elif risk_assessment.risk_assessment_status == 'on request':
stats['risk_assessments_onrequest'] += 1
elif risk_assessment.risk_assessment_status == None:
stats['risk_assessments_unknown'] += 1
#html
render("%s/risk-assessments/%s.html" % (config['outputDir'], risk_assessment.get_slug()), "risk-assessment.html", risk_assessment=risk_assessment)
# #json
# create_file("%s/risk-assessments/%s.json" % (config['outputDir'], risk_assessment.get_slug()), risk_assessment.to_json())
# Risk assessment index
render("%s/%s" % (config['outputDir'], "risk-assessments/index.html"), "risk-assessments.html", risk_assessments=risk_assessments, stats=stats)
# index page
render("%s/index.html" % config['outputDir'], "index.html", stats=stats)
# volunteers
create_directory("%s/monitors" % config['outputDir'])
render("%s/monitors/index.html" % config['outputDir'], "volunteer.html")
render("%s/monitors/check-email.html" % config['outputDir'], "volunteer-check-email.html")
render("%s/monitors/confirmed.html" % config['outputDir'], "volunteer-confirmed.html")
# Report
render("%s/report.html" % config['outputDir'], "report.html")
# static pages
for path in glob.glob('%s/static/*.md' % config['dataDir']):
page = frontmatter.load(path)
#replaces any config variables in markdown
parsed_content = Template(page.content)
page.content = parsed_content.render(config=get_config())
file_name = os.path.basename(path).replace('.md', '.html')
render("%s/%s" % (config['outputDir'], file_name), "static.html", page=page)
# reports and guides
create_directory("%s/reports" % config['outputDir'])
for path in glob.glob('%s/reports/*.md' % config['dataDir']):
page = frontmatter.load(path)
parsed_content = Template(page.content)
page.content = parsed_content.render(config=get_config())
file_name = os.path.basename(path).replace('.md', '.html')
template_name = "analysis.html"
if file_name == "index.html":
template_name = "static.html"
render("%s/reports/%s" % (config['outputDir'], file_name), template_name, page=page)
#assets
path = "assets/CNAME"
file_name = os.path.basename(path)
copyfile(path, "%s/CNAME" % config['outputDir'])
for path in glob.glob('%s/*.js' % config['assetsDir']):
file_name = os.path.basename(path)
copyfile(path, "%s/assets/%s" % (config['outputDir'], file_name))
for path in glob.glob('%s/*.png' % config['assetsDir']):
file_name = os.path.basename(path)
copyfile(path, "%s/assets/%s" % (config['outputDir'], file_name))
for path in glob.glob('%s/*.jpg' % config['assetsDir']):
file_name = os.path.basename(path)
copyfile(path, "%s/assets/%s" % (config['outputDir'], file_name))
for path in glob.glob('%s/*.svg' % config['assetsDir']):
file_name = os.path.basename(path)
copyfile(path, "%s/assets/%s" % (config['outputDir'], file_name))
path = "node_modules/govuk-frontend/govuk/all.js"
file_name = os.path.basename(path)
copyfile(path, "%s/assets/govuk.js" % config['outputDir'])
sass.compile(dirname=('assets', 'docs/assets'), output_style='compressed')