def download_slides(slides_remote, prefix='data'):
slides = []
for slide in slides_remote:
filename = "slides/{}".format(slide.split('/')[-1])
helper.download_file(slide, prefix + '/' + filename)
slides.append(filename)
return slides
def download_datasets(link, folder):
page = con.session.get(helper.fix_link(link))
soup = BeautifulSoup(page.text, 'html.parser')
dataset = soup.findAll('a', {
'href': re.compile('^https'),
'class': re.compile('^link-borderless')
})
if len(dataset) == 0:
sys.stdout.write(
f'{bcolors.FAIL}No dataset found!{bcolors.ENDC}\n')
return
titles = [x.text.strip() for x in dataset]
all_links = [x['href'] for x in dataset]
sys.stdout.write(
f'{bcolors.BOLD}Downloading dataset...{bcolors.ENDC}\n')
if not os.path.exists(folder):
os.mkdir(folder)
if(not os.path.exists(os.path.join(folder, 'Dataset'))):
os.mkdir(os.path.join(folder, 'Dataset'))
for link, title in zip(all_links, titles):
dir = os.path.join(folder, 'Dataset', title) + \
'.' + link.split('.')[-1]
helper.download_file(con, link, dir)
def download_slides(course_id, folder):
page = con.session.get(
'https://www.datacamp.com/courses/{}/continue'.format(course_id))
slide_links = set(
re.findall(r'(https?://s3.[/|\w|:|.|-]+[^/])&', page.text))
slide_links = slide_links.union(
set(re.findall(r'(https?://projector[/|\w|:|.|-]+[^/])&', page.text)))
if len(slide_links) == 0:
sys.stdout.write(f'{bcolors.FAIL}No slides found!{bcolors.ENDC}\n')
return
sys.stdout.write(f'{bcolors.BOLD}Downloading slides...{bcolors.ENDC}\n')
for link in slide_links:
helper.download_file(con, link,
os.path.join(folder,
link.split('/')[-1]))
def _test_photo_type_get(type, date):
temporary_folder, folder = helper.create_working_folder()
photo_name = 'photo.{}'.format(type)
photo_file = helper.get_file(photo_name)
origin = '{}/{}'.format(folder, photo_name)
if not photo_file:
photo_file = helper.download_file(photo_name, folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('{} file not downlaoded'.format(type))
# downloading for each test is costly so we save it in the working directory
file_path_save_as = helper.get_file_path(photo_name)
if os.path.isfile(photo_file):
shutil.copyfile(photo_file, file_path_save_as)
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
metadata = photo.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert(date), '{} date {}'.format(type, metadata['date_taken'])
def test_set_metadata_on_arw():
temporary_folder, folder = helper.create_working_folder()
photo_file = helper.get_file('photo.arw')
origin = '%s/photo.arw' % folder
if not photo_file:
photo_file = helper.download_file('photo.arw', folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('arw file not downlaoded')
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
origin_metadata = photo.get_metadata()
status = photo.set_location(11.1111111111, 99.9999999999)
assert status == True, status
photo_new = Photo(origin)
metadata = photo_new.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert((2007, 4, 8, 17, 41, 18, 6, 98, 0)), metadata['date_taken']
assert helper.isclose(metadata['latitude'], 11.1111111111), metadata['latitude']
assert helper.isclose(metadata['longitude'], 99.9999999999), metadata['longitude']
def test_set_metadata_on_rw2():
raise SkipTest('gh-94 Writing to RW2 images is not supported')
temporary_folder, folder = helper.create_working_folder()
photo_file = helper.get_file('photo.rw2')
origin = '%s/photo.rw2' % folder
if not photo_file:
photo_file = helper.download_file('photo.rw2', folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('rw2 file not downlaoded')
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
origin_metadata = photo.get_metadata()
status = photo.set_location(11.1111111111, 99.9999999999)
assert status == True, status
photo_new = Photo(origin)
metadata = photo_new.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert((2014, 11, 19, 23, 7, 44, 2, 323, 0)), metadata['date_taken']
assert helper.isclose(metadata['latitude'], 11.1111111111), metadata['latitude']
assert helper.isclose(metadata['longitude'], 99.9999999999), metadata['longitude']
def _test_photo_type_get(type, date):
temporary_folder, folder = helper.create_working_folder()
photo_name = 'photo.{}'.format(type)
photo_file = helper.get_file(photo_name)
origin = '{}/{}'.format(folder, photo_name)
if not photo_file:
photo_file = helper.download_file(photo_name, folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('{} file not downlaoded'.format(type))
# downloading for each test is costly so we save it in the working directory
file_path_save_as = helper.get_file_path(photo_name)
if os.path.isfile(photo_file):
shutil.copyfile(photo_file, file_path_save_as)
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
metadata = photo.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert(
date), '{} date {}'.format(type, metadata['date_taken'])
def _test_photo_type_set(type, date):
temporary_folder, folder = helper.create_working_folder()
photo_name = 'photo.{}'.format(type)
photo_file = helper.get_file(photo_name)
origin = '{}/{}'.format(folder, photo_name)
if not photo_file:
photo_file = helper.download_file(photo_name, folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('{} file not downlaoded'.format(type))
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
origin_metadata = photo.get_metadata()
status = photo.set_location(11.1111111111, 99.9999999999)
assert status == True, status
photo_new = Photo(origin)
metadata = photo_new.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert(
date), '{} date {}'.format(type, metadata['date_taken'])
assert helper.isclose(metadata['latitude'],
11.1111111111), '{} lat {}'.format(
type, metadata['latitude'])
assert helper.isclose(metadata['longitude'],
99.9999999999), '{} lon {}'.format(
type, metadata['latitude'])
def _test_photo_type_set(type, date):
temporary_folder, folder = helper.create_working_folder()
photo_name = 'photo.{}'.format(type)
photo_file = helper.get_file(photo_name)
origin = '{}/{}'.format(folder, photo_name)
if not photo_file:
photo_file = helper.download_file(photo_name, folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('{} file not downlaoded'.format(type))
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
origin_metadata = photo.get_metadata()
status = photo.set_location(11.1111111111, 99.9999999999)
assert status == True, status
photo_new = Photo(origin)
metadata = photo_new.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert(date), '{} date {}'.format(type, metadata['date_taken'])
assert helper.isclose(metadata['latitude'], 11.1111111111), '{} lat {}'.format(type, metadata['latitude'])
assert helper.isclose(metadata['longitude'], 99.9999999999), '{} lon {}'.format(type, metadata['latitude'])
async def camp(ctx, arg):
try:
chapter, stage = arg.split('-')
# SEARCH
if ctx.message.channel.name == SEARCH_CHANNEL and not ctx.message.author.bot:
return_message = ""
search_folder = get_folder_id_by_name(chapter, service, memo)
stage_ids = search_file_in_folder(search_folder, stage, service)
if stage_ids is not None:
for stage_id, file_name in stage_ids:
# return_message += f"Stage link: https://drive.google.com/file/d/{stage_id}\n"
if DB is not None:
stage_doc = DB.table(UPLOADED_STAGES).get(
User.file_id == stage_id)
if stage_doc:
return_message += f"Upload caption: {stage_doc['message']}\n"
try:
stage_file = download_file(stage_id, service)
sending_file = File(stage_file, f"{file_name}.jpg")
await ctx.send(return_message, file=sending_file)
except Exception as e:
print(e)
print(e.args)
else:
await ctx.send(f"Couldn't find it, sowwy {PEPE}")
return
if not (ctx.message.channel.name == CHANNEL_NAME
and not ctx.message.author.bot):
return
for attachment in ctx.message.attachments:
uploaded_file_id = upload_file(service, attachment.url, chapter,
ctx.message.author, stage, memo)
if uploaded_file_id is not None:
print(f"Chapter: {chapter} - Stage: {stage}")
DB.table(LAST_UPLOADED_TABLE).upsert(
{
'userId': ctx.author.id,
'file_id': uploaded_file_id,
'message': ctx.message.content,
'message_id': ctx.message.id,
'removed': False
}, User.userId == ctx.author.id)
DB.table(UPLOADED_STAGES).insert({
'userId':
ctx.author.id,
'file_id':
uploaded_file_id,
'message':
ctx.message.content.replace(f"a!camp {arg}", '')
})
print(ctx.message.content)
print(ctx.message.content.replace(f"a!camp {arg}", ''))
await ctx.message.add_reaction('👍')
else:
await ctx.message.add_reaction('👎')
except:
await ctx.message.add_reaction('👎')
def test_set_original_name():
files = ['plain.jpg', 'audio.m4a', 'photo.nef', 'video.mov']
for file in files:
ext = os.path.splitext(file)[1]
temporary_folder, folder = helper.create_working_folder()
random_file_name = '%s%s' % (helper.random_string(10), ext)
origin = '%s/%s' % (folder, random_file_name)
file_path = helper.get_file(file)
if file_path is False:
file_path = helper.download_file(file, folder)
shutil.copyfile(file_path, origin)
media = Media.get_class_by_file(origin, [Audio, Media, Photo, Video])
metadata = media.get_metadata()
media.set_original_name()
metadata_updated = media.get_metadata()
shutil.rmtree(folder)
assert metadata['original_name'] is None, metadata['original_name']
assert metadata_updated['original_name'] == random_file_name, metadata_updated['original_name']
def download_course_data(conn, link, path):
page = con.session.get(helper.embbed_link(link))
soup = BeautifulSoup(page.text, 'html.parser')
dataset = soup.findAll('a', {
'href': re.compile('^https'),
'class': re.compile('^link-borderless')
})
titles = [x.text.strip() for x in dataset]
all_links = [x['href'] for x in dataset]
sys.stdout.write(f'{bcolors.BOLD}Downloading dataset...{bcolors.ENDC}\n')
if not os.path.exists(path):
os.mkdir(path)
if (not os.path.exists(os.path.join(path, 'Dataset'))):
os.mkdir(os.path.join(path, 'Dataset'))
for link, title in zip(all_links, titles):
dir = os.path.join(path, 'Dataset', title)
dir = dir + '.' + link.split('.')[-1]
download_file(con, link, dir)
def test_get_metadata_from_arw():
temporary_folder, folder = helper.create_working_folder()
photo_file = helper.get_file('photo.arw')
origin = '%s/photo.arw' % folder
if not photo_file:
photo_file = helper.download_file('photo.arw', folder)
if not photo_file or not os.path.isfile(photo_file):
raise SkipTest('arw file not downlaoded')
# downloading for each test is costly so we save it in the working directory
file_path_save_as = helper.get_file_path('photo.arw')
if os.path.isfile(photo_file):
shutil.copyfile(photo_file, file_path_save_as)
shutil.copyfile(photo_file, origin)
photo = Photo(origin)
metadata = photo.get_metadata()
shutil.rmtree(folder)
assert metadata['date_taken'] == helper.time_convert((2007, 4, 8, 17, 41, 18, 6, 98, 0)), metadata['date_taken']
def setup(self, solution_name, settings):
log = Logger()
# check if nvda is allready installed on the system, if not, install it
standard_nvda_path = "C:\\Programme\\NVDA\\nvda.exe"
nvda_path = helper.is_process_running("nvda.exe")
if nvda_path == "":
nvda_path = standard_nvda_path
if os.path.exists(nvda_path) == False:
# nvda download url
url = "http://downloads.sourceforge.net/project/nvda/releases/2012.2.1/nvda_2012.2.1.exe"
# download nvda installer
nvda_installer = helper.download_file(url)
if nvda_installer == "":
return 200
# install nvda
rc = subprocess.call([nvda_installer, "--install"])
if rc > 0:
return 201
# if nvda runs, exit it
nvda_path = helper.is_process_running("nvda.exe")
if nvda_path != "":
rc = subprocess.call([nvda_path, "-q"])
if rc > 0:
return 202
else:
nvda_path = standard_nvda_path
# configure the program and start it
config_file = os.environ['APPDATA'] + "\\nvda\\nvda.ini"
if os.path.exists(config_file) == False:
return 203
# parse the ini config file
try:
config = ConfigObj(config_file)
except configobj.ParseError:
return 204
# apply the settings
attrs_without_value = []
for attr in self.get_solution_list()[solution_name]:
if settings.has_key(attr) == True:
value = self.convert_values(attr, settings[attr])
if value == None:
log.log_msg("NVDA: The attribute " + attr + " couldn't be converted into a NVDA specific format, skipped", "warning")
continue
if attr == "preferred-lang":
try:
config['speech']['espeak']['voice'] = value['voice']
except:
log.log_msg("NVDA: Error while changing the attribute " + attr + " in the NVDA settings file.", "warning")
if attr == "speech-rate":
try:
config['speech']['espeak']['rate'] = value['rate']
config['speech']['espeak']['rateBoost'] = value['rateBoost']
except:
log.log_msg("NVDA: Error while changing the attribute " + attr + " in the NVDA settings file.", "warning")
else:
attrs_without_value.append(attr)
# list of attributes without a corresponding value
if len(attrs_without_value) > 0:
attr_str = ""
for attr in attrs_without_value:
attr_str.join(attr)
log.log_msg("NVDA: The following supported attributes have no value in the user profile: " + attr_str, "warning")
# write back the nvda settings file
try:
config.write()
except:
return 205
# start configured nvda
rc = os.popen(nvda_path)
# rc = os.system(nvda_path + " &")
print "start nvda, rc = ", rc
return 0
def download_videos(course_id, folder):
chapters = get_course_chapters(course_id)
display_text = True
for chapter in chapters['user_chapters']:
page = con.session.get(
'https://www.datacamp.com/courses/{}/chapters/{}/continue'.format(
course_id, chapter['chapter_id']))
video_ids = set(
re.findall(r';(course_{}_[\d|\w]+)&'.format(course_id), page.text))
video_type = 1
if len(video_ids) == 0:
video_ids = set(
re.findall(r'(//videos.[/|\w|:|.|-]+[^/])&', page.text))
video_type = 2
if len(video_ids) == 0:
sys.stdout.write(f'{bcolors.FAIL}No videos found!{bcolors.ENDC}\n')
return
if display_text:
sys.stdout.write(
f'{bcolors.BOLD}Downloading videos...{bcolors.ENDC}\n')
display_text = False
for video_id in video_ids:
while True:
try:
if video_type == 1:
video_page = con.session.get(
'https://projector.datacamp.com/?projector_key=' +
video_id)
elif video_type == 2:
video_page = con.session.get(
'https://projector.datacamp.com/?video_hls=' +
video_id)
except:
helper.handle_error(con)
continue
break
soup = BeautifulSoup(video_page.text, 'html.parser')
video_url = json.loads(
soup.find("input", {"id": "videoData"})['value'])
link = video_url['video_mp4_link']
if link is None:
sys.stdout.write(
f'{bcolors.FAIL}Videos cannot be downloaded!{bcolors.ENDC}\n'
)
return
if link.endswith('mp4') and not link.startswith('http'):
link = 'https://' + link[2:]
name = link.split('/')[-1]
else:
if video_type == 1:
video_name_url = json.loads(
soup.find("input", {"id": "slideDeckData"})['value'])
link_name = video_name_url['plain_video_mp4_link']
if link_name is not None:
name = link_name.split('/')[-1]
else:
name = video_url['audio_link'].split('/')[-1].split(
'.')[0] + '.mp4'
elif video_type == 2:
link_name = video_url['video_mp4_link']
name = link_name.split('/')[-1]
if name.count('_') > 1:
name = name.split('_')[1:]
name = '_'.join(name)
file_path = os.path.join(folder, name)
if helper.file_exist(file_path):
continue
helper.download_file(con, link, file_path)
imported_modules = helper.find_modules("modules", AbstractModule)
# print the name of every found module
print "Imported modules:"
for i in range(len(imported_modules)):
print imported_modules[i]()
print"\n"
while(1):
# fetch json profile from server
# download the profile file
# the following if statement only belongs to my vm settings, if I
# start the program under the host, the url is the localhost,
# otherwise it's the given ip address
file_name = ""
if platform.node() == "scimitar":
file_name = helper.download_file("http://localhost/cloud4all/profile_json.txt", True)
else:
file_name = helper.download_file("http://10.0.2.2/cloud4all/profile_json.txt", True)
if os.path.exists(file_name) == False:
log.log_msg("Error: Download of the profile file failed.", "error")
sys.exit(1)
f = open(file_name, "r")
json_string = f.read()
# parse the string
try:
user_profile = json.loads(json_string)
except ValueError:
log.log_msg("Error: Parsing of the user profile failed.", "error")
sys.exit(2)
dev_profile = DeviceProfile()
def sync(self):
if not self.session:
raise Exception("You need to login() first.")
if not self.courses:
raise Exception("You need to get_courses() first.")
if not self.sections:
raise Exception("You need to get_sections() first.")
### Syncing all courses
for course_id, session_key, semestername, coursename in self.sections.keys(
):
print(f"Syncing {coursename}...")
for sec in self.sections[course_id, session_key, semestername,
coursename]:
sectionname = helper.clean_filename(
sec.select_one(".sectionname").get_text())
#print(f"[{datetime.now()}] Section {sectionname}")
mainsectionpath = os.path.join(self.config["basedir"],
semestername, coursename,
sectionname)
# Categories can be multiple levels deep like folders, see https://moodle.rwth-aachen.de/course/view.php?id=7053§ion=1
label_categories = sec.findAll(
"li", {
"class": [
"modtype_label", "modtype_resource", "modtype_url",
"modtype_folder", "modtype_assign", "modtype_page"
]
})
categories = []
category = None
for l in label_categories:
# Create a category for all labels if enableExperimentalCategories is set
if "modtype_label" in l['class'] and self.config[
"enableExperimentalCategories"]:
category = (helper.clean_filename(
l.findAll(text=True)[-1]), [])
categories.append(category)
else:
if category == None:
category = (None, [])
categories.append(category)
category[1].append(l)
## Download Opencast Videos directly embedded in section
helper.scan_for_opencast(sec, course_id, session_key,
mainsectionpath, self.session)
for category_name, category_soups in categories:
if category_name == None:
sectionpath = mainsectionpath
else:
sectionpath = os.path.join(mainsectionpath,
category_name)
for s in category_soups:
mod_link = s.find('a', href=True)
if not mod_link:
continue
mod_link = mod_link["href"]
## Get Resources
if "modtype_resource" in s["class"]:
# First check if the file is directly accessible:
if helper.download_file(mod_link, sectionpath,
self.session):
continue
# If no file was found, then it could be an html page with an enbedded video
response = self.session.get(mod_link,
params=self.params)
if "Content-Type" in response.headers and "text/html" in response.headers[
"Content-Type"]:
tempsoup = bs(response.text,
features="html.parser")
videojs = tempsoup.select_one(".video-js")
if videojs:
videojs = videojs.select_one("source")
if videojs and videojs.get("src"):
helper.download_file(
videojs["src"], sectionpath,
self.session,
videojs["src"].split("/")[-1])
## Get Resources in URLs
if "modtype_url" in s["class"]:
url = None
try:
response = self.session.head(
mod_link, params=self.params)
if "Location" in response.headers:
url = response.headers["Location"]
response = self.session.head(
url, params=self.params)
if "Content-Type" in response.headers and "text/html" not in response.headers[
"Content-Type"]:
# Don't download html pages
helper.download_file(
url, sectionpath, self.session)
elif "engage.streaming.rwth-aachen.de" in url:
# Maybe its a link to an OpenCast video
helper.downloadOpenCastVideos(
url, course_id, session_key,
sectionpath, self.session)
except:
# Maybe the url is down?
print(f"Error while downloading url {url}")
## Get Folders
if "modtype_folder" in s["class"]:
response = self.session.get(mod_link,
params=self.params)
soup = bs(response.text, features="html.parser")
soup_results = soup.find("a", {"title": "Folder"})
if not soup_results:
# page has no title?
continue
foldername = helper.clean_filename(
soup_results.text)
filemanager = soup.select_one(
".filemanager").findAll('a', href=True)
# Scheiß auf folder, das mach ich 1 andernmal
for file in filemanager:
link = file["href"]
filename = file.select_one(".fp-filename").text
helper.download_file(
link, os.path.join(sectionpath,
foldername),
self.session, filename)
## Get Assignments
if "modtype_assign" in s["class"]:
response = self.session.get(mod_link,
params=self.params)
soup = bs(response.text, features="html.parser")
soup_results = soup.find("a",
{"title": "Assignment"})
if not soup_results:
# page has no title?
continue
foldername = helper.clean_filename(
soup_results.text)
files = soup.select(".fileuploadsubmission")
for file in files:
link = file.find('a', href=True)["href"]
filename = file.text
helper.download_file(
link, os.path.join(sectionpath,
foldername),
self.session, filename)
## Get embedded videos in pages
if "modtype_page" in s["class"]:
response = self.session.get(mod_link,
params=self.params)
soup = bs(response.text, features="html.parser")
soup_results = soup.find("a", {"title": "Page"})
if not soup_results:
# page has no title?
continue
pagename = helper.clean_filename(soup_results.text)
path = os.path.join(sectionpath, pagename)
# Youtube videos
helper.scanAndDownloadYouTube(soup, path)
# OpenCast videos
helper.scan_for_opencast(soup, course_id,
session_key, path,
self.session)
