def main():
"""Main running function of entire script."""
reddit = bot_setup()
args = parse_args(sys.argv[1:])
print('Gathering posts...')
posts = get_posts(reddit, args.subreddit, args.count, args.sort, args.time)
post_dict = mirror_posts(reddit, args.destination, posts)
if args.comments:
progress = Bar(
'Mirroring comments for {} threads, this may take a while...'.
format(len(post_dict)),
max=len(post_dict))
progress.start()
for original, mirror in post_dict.items():
original_submission = reddit.submission(original)
mirrored_submission = reddit.submission(mirror)
original_comments = original_submission.comments
original_comments.replace_more()
mirror_comments(original_comments, mirrored_submission, None)
progress.next()
progress.finish()
print()
def draw_gazemaps(self):
"""
Writes a reduced gazemap for all user/images
Note, Gazemaps take up a lot of memory, therefore it needs to be freed after being used
:return: Nothing
"""
nb = 0
for im in self.image_list:
nb += im.nb_of_users()
if nb == 0:
nb = 1
bar = Bar('Generating Heatmaps', max=nb, stream=sys.stdout)
bar.start()
for image in self.image_list:
#if image.image_id == "1217722":
#image.generate_all_heatmaps()
#image.save_all_heatmaps()
#image.save_all_heatmaps_ln()
#image.save_all_heatmaps_by_image()
image.save_all_heatmaps_reduced(bar)
#image.remove_all_heatmaps()
gc.collect()
#for user in self.user_list:
#if user.user_id == "3425935":
# for im_id in user.image_data:
# image = user.image_data[im_id]
# image.generate_heatmap(user.user_id)
# user.save_all_heatmaps_by_user()
# for im_id in user.image_data:
# image = user.image_data[im_id]
# image.remove_heatmap(user.user_id)
# gc.collect()
bar.finish()
def maxInRows(self, f, N):
x = self.x
t = self.t
c = self.c
dt = self.dt
dx = self.dx
num1 = len(t)
num2 = len(x)
n = zeros((num2, num1))
u = zeros((num2, num1))
# Defining constant for easy interpreting
C = c * dt / dx
alpha = (2 * rho) / (H)
beta = dt**2 / m
print(
"--------------------------------------------------------------------------------"
)
print('Length of M = ', num1)
print('Length of N = ', num2)
print('C = ', C)
print('beta =', beta)
print('alpha =', alpha)
print('dx =', dx)
print('dt =', dt)
print("Time: ", Time)
"""
widgets = ['Progress: ', Percentage(), ' ', Bar(marker='0',left='[',right=']'),
' ', ETA(), ' ', FileTransferSpeed()] #see docs for other options
"""
pbar = Bar('Loading', fill='#', suffix='%(percent)d%%')
pbar.start()
# For loop for N X M matrix
for j in range(num1 - 1):
# Initial force, applied from staples.
u[0, j] = A * sin(t[j] * 2 * pi * f)
pbar.next(j + 1)
time.sleep(0.1)
for i in range(num2 - 1):
k[i] = 4e9 * exp(-4 * x[i])
# my statement consisting of n, amplitude of membrane wave
n[i,
j + 1] = beta * (u[i, j] -
k[i] * n[i, j]) + 2 * n[i, j] - n[i, j - 1]
# my statement consisting of u, pressure difference
u[i, j+1] = C**2*(u[i+1, j] - 2*u[i, j] + u[i-1, j]) + 2*u[i, j]\
- u[i, j-1] - alpha * (n[i, j+1] - 2*n[i, j] + n[i, j-1])
pbar.finish()
print(
"--------------------------------------------------------------------------------"
)
maxInRows = amax(n, axis=1)
normalized = maxInRows / maxInRows.max()
return normalized
def do_daily_iteration(self):
log("Start Daily Iteration", is_heading=True)
self.incorrect_aspect = []
self.correct_aspect = []
dailyDirectory = os.path.join(dest_directory, "DailySource")
remove_all_files(dailyDirectory)
progressBar = None
try:
urls = self.get_submissions_for_subreddit("day")
log("URLS Gathered")
totalCount = len(urls)
if self.args.show_progress:
progressBar = Bar("Downloading Daily Images",
max=totalCount,
suffix='%(index)d / %(max)d %(percent)d%%')
progressBar.start()
# get suitable image for desktop background
while len(urls) != 0:
# select specific url at random
attempt, urls = get_random_url(urls)
image_url, post_permalink = attempt
log("Process URL:", image_url, "URLS Left:", len(urls))
# save first attempt at okay file url
imageObj = RedditImage(image_url, post_permalink)
imageObj.get_image_from_url(dailyDirectory)
if not imageObj.image_downloaded():
continue
if imageObj.image_is_landscape():
self.correct_aspect.append(imageObj)
else:
self.incorrect_aspect.append(imageObj)
if progressBar is not None:
progressBar.next()
log("URL has been processed")
if progressBar is not None:
progressBar.finish()
progressBar = None
RedditDatabase().insert_images(self.correct_aspect +
self.incorrect_aspect)
if len(self.incorrect_aspect) > 0:
log("Start Image Combining Process")
ci = CombineImages(self.incorrect_aspect, dest_directory)
log("Save Resulting Image")
pathOfResult = ci.do_combine_landscape_process()
RedditDatabase().insert_combined(ci)
log("Set Image as Desktop Background")
RedditImage.set_image_to_desktop_background(pathOfResult)
else:
self.correct_aspect[0].set_to_desktop_background()
finally:
# cleanup all images that had been temporarily downloaded
if progressBar is not None:
progressBar.finish()
def editorial_publish(guides,
endpoint,
function_class,
user_agent,
nailgun_bin,
content_generator):
"""
takes care of publishing the editorial content for the guides.
"""
# init the nailgun thing for ed content generation.
nailguninit(nailgun_bin,content_generator)
searches= {}
pbar = Bar('extracting editorial content for guides:',max=len(guides)+1)
pbar.start()
error = False
for i, guide in enumerate(guides):
jsonguide = None
with open(guide,'r') as g:
jsonguide = json.load(g)
if not jsonguide:
logging.error('could not load json from {0}'.format())
error = True
continue
search = cityinfo.cityinfo(jsonguide)
uri = cityres.cityres(search,endpoint)
if not uri:
logging.error(
'no dbpedia resource was found for {0}'.format(guide))
error = True
continue
urls = urlinfer.urlinferdef([unquote(uri)])
if len(urls) < 1:
logging.error('no wikipedia/wikivoyage urls found/inferred'\
' for resource {0}'.format(uri))
error = True
continue
content = editorial_content(urls,function_class,user_agent)
if not content:
logging.error('no editorial content could be'\
' generated for {0}'.format(guide))
error = True
continue
#insert the content into the guide
jsonsert.jsonsert(content, guide)
logging.info('editorial content for {0} sucessfully'\
' inserted.'.format(guide))
pbar.next()
pbar.finish()
return error
def wait_with_progress_bar(
self,
message: str,
timeout: Optional[int] = PROGRESS_TIMEOUT) -> None:
progress_bar = Bar(message, max=self.futures_count, check_tty=False)
progress_bar.start()
for progress in self.progress(timeout=timeout):
progress_bar.goto(progress.completed)
progress_bar.finish()
def package_repo():
setup_repo()
repo_name = repo.repo_name
pkg_temp = 'packaged-repos'
pkg_dir = os.path.join(SCRIPT_DIR, pkg_temp)
web_base = os.path.dirname(os.path.abspath(repo.web_dir))
if not os.path.exists(web_base):
print("Parent directory of repo '{0}' not found at: {1}"
.format(repo_name, web_base))
return False
os.chdir(web_base)
print("Gathering '{0}' repo directory data".format(repo_name))
item_count = sum(len(d) + len(f) for _, d, f in os.walk(repo_name)) + 1
print(" {0} items to archive".format(item_count))
if not os.path.exists(pkg_dir):
os.mkdir(pkg_dir)
curdatetime = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
targz = os.path.join(pkg_dir,
"{0}-repo_{1}.tar.gz".format(repo_name, curdatetime))
if os.path.exists(targz):
os.unlink(targz)
bar = Bar('Archiving repo', fill='=', max=item_count)
def size_progress(tarinfo):
bar.next()
return tarinfo
os.chdir(web_base) # just to make sure
try:
bar.start()
with tarfile.open(targz, "w:gz") as tar:
tar.add(repo_name, filter=size_progress)
except KeyboardInterrupt:
print("\nArchiving error: keyboard interrupt; archive incomplete")
return False
except tarfile.TarError as e:
print("\nArchiving error: {0}".format(e))
return False
finally:
bar.finish()
print("Repo '{0}' archived: {1}".format(repo_name, targz))
return True
def banner(guides,
endpoint,
function_description_class,
user_agent,
nailgun_bin,
description_gen):
"""
Insert a banner picture into a guide and download it on the file system.
"""
nailguninit(nailgun_bin, description_gen)
error = False
pbar = Bar('fetching the depiction banner for the guides',max=len(guides)+1)
pbar.start()
for i, g in enumerate(guides):
url = depiction_url(g, user_agent, function_description_class,
endpoint)
if url:
guide_folder = os.path.dirname(g)
filename = url_filename(url)
error = download(guide_folder, filename, url)
error |= zip_insert(guide_folder, filename)
error |= remove_banner(guide_folder, filename)
if error:
logging.error('could not download/insert/remove {0}. There '\
'will be no banner for {1}'.format(url, g))
insert_error = jsonsert.imagesert(g, None, None)
else:
logging.info('inerting details into the guide {0}'.format(g))
insert_error = jsonsert.imagesert(g, filename, url)
else:
logging.error('could not find a depiction image for {0} so there will be no banner'.format(g))
insert_error = jsonsert.imagesert(g, None, None)
error = True
if insert_error:
logging.error("problem inserting the image into {0}".format(g))
error = True
pbar.next()
pbar.finish()
return error
def draw_scanpaths(self):
"""
Draws all the scanpaths for the user/image pairs
:return:
"""
nb = 0
for im in self.image_list:
nb += im.nb_of_users()
if nb == 0:
nb = 1
bar = Bar('Generating Scanpaths', max=nb, stream=sys.stdout)
bar.start()
for image in self.image_list:
image.save_all_scanpath(bar)
bar.finish()
def draw_raw(self):
"""
Draws all raw point visualisations for the user/image pairs.
:return:
"""
nb = 0
for im in self.image_list:
nb += im.nb_of_users()
if nb == 0:
nb = 1
bar = Bar('Generating raw points images', max=nb, stream=sys.stdout)
bar.start()
for image in self.image_list:
image.save_all_raw(bar)
bar.finish()
def filter_poi(guides, f):
"""
Remove certain POIS based on a filter function. Filter function should
return True if the poi should be REMOVED.
"""
nbr_guides = len(guides)
bar = Bar('filtering the guides poi with a function.',max=nbr_guides)
Error = False
bar.start()
for g in guides:
cur_content = None
with open(g,'r') as file_guide:
cur_content = json.load(
file_guide,
object_pairs_hook=collections.OrderedDict)
if not cur_content:
logging.error('could not load content for:{}. POI not filtered'.format(g))
else:
# get the POI.
guide_pois = None
try:
guide_pois = cur_content['Cities'][0]['pois']
except:
logging.error('{} did not contain any POI. They will not be filtered'.format(g))
bar.next()
continue
new_pois = [p for p in guide_pois if not f(p)]
cur_content['Cities'][0]['pois'] = new_pois
# reserialize the guide content.
with open(g,'w') as file_guide:
json.dump(cur_content, file_guide)
bar.next()
bar.finish()
return Error
def remove_street_picture(guides):
"""
Remove pictures from the poi when the subcategory is street.
"""
removed_pic_name = []
Error = False
bar = Bar('removing street pics',max=len(guides))
bar.start()
for guide in guides:
content = guide_content(guide)
if not content:
continue
else:
# get the pois
pois = None
try:
pois = content['Cities'][0]['pois']
except Exception as e:
logging.error('guide {} did not contain pois. Street picture'\
' will not be removed.'.format(guide))
bar.next()
continue
for p in pois:
sub = p.get('subcategory')
if sub == 'street':
try:
pic = p['picture']['picture']
removed_pic_name.append(pic)
p.pop('picture')
except Exception as e:
continue
# reserialize the content.
with open(guide, 'w') as file_guide:
json.dump(content,file_guide)
#remove_from_zip(guide, removed_pic_name)
bar.next()
bar.finish()
return Error
def update_plugin():
setup_repo()
if args.zip_name.lower() == 'all':
zips = [
z for z in os.listdir(repo.upload_dir)
if (os.path.isfile(os.path.join(repo.upload_dir, z))
and z.lower().endswith('.zip'))
]
else:
zips = [args.zip_name]
if not zips:
if args.zip_name.lower() == 'all':
print('No plugins archives found in uploads directory')
else:
print('No plugin archive name defined')
return False
repo.output = False # nix qgis_repo output, since using progress bar
up_bar = Bar("Updating plugins in '{0}'".format(repo.repo_name),
fill='=',
max=len(zips))
up_bar.start()
for i in up_bar.iter(range(0, len(zips))):
try:
repo.update_plugin(zips[i],
name_suffix=args.name_suffix,
auth=args.auth,
auth_role=args.auth_role,
git_hash=args.git_hash,
versions=args.versions,
keep_zip=args.keep_zip,
untrusted=args.untrusted,
invalid_fields=args.invalid_fields)
except KeyboardInterrupt:
return False
if args.sort_xml:
print("Sorting repo plugins.xml")
post_sort = QgisPluginTree.plugins_sorted_by_name(
repo.plugins_tree.plugins())
repo.plugins_tree.set_plugins(post_sort)
return True
def add_parse_address(guides):
"""
Uses reverse geocoding to try and add a parsed version of the address.
"""
Error = False
bar = Bar('adding parsed address',max=len(guides))
bar.start()
for guide in guides:
content = guide_content(guide)
if not content:
continue
else:
# get the pois
pois = None
try:
pois = content['Cities'][0]['pois']
except Exception as e:
logging.error('guide {} did not contain pois. Street picture'\
' will not be removed.'.format(guide))
bar.next()
continue
for p in pois:
latitude = get_in(p, "location", "latitude")
longitude = get_in(p, "location", "longitude")
try:
coords = ", ".join([str(latitude),str(longitude)])
parsed = reversegeo.reverse_geocode(coords)
p['address']['parsed'] = parsed
except:
Error=True
logging.error('could not add parsed address to a poi ...')
# reserialize the content.
with open(guide, 'w') as file_guide:
json.dump(content,file_guide)
bar.next()
bar.finish()
return Error
def update_plugin():
setup_repo()
if args.zip_name.lower() == 'all':
zips = [z for z in os.listdir(repo.upload_dir)
if (os.path.isfile(os.path.join(repo.upload_dir, z))
and z.lower().endswith('.zip'))]
else:
zips = [args.zip_name]
if not zips:
if args.zip_name.lower() == 'all':
print('No plugins archives found in uploads directory')
else:
print('No plugin archive name defined')
return False
repo.output = False # nix qgis_repo output, since using progress bar
up_bar = Bar("Updating plugins in '{0}'".format(repo.repo_name),
fill='=', max=len(zips))
up_bar.start()
for i in up_bar.iter(range(0, len(zips))):
try:
repo.update_plugin(
zips[i],
name_suffix=args.name_suffix,
auth=args.auth,
auth_role=args.auth_role,
git_hash=args.git_hash,
versions=args.versions,
keep_zip=args.keep_zip,
untrusted=args.untrusted,
invalid_fields=args.invalid_fields
)
except KeyboardInterrupt:
return False
if args.sort_xml:
print("Sorting repo plugins.xml")
post_sort = QgisPluginTree.plugins_sorted_by_name(
repo.plugins_tree.plugins())
repo.plugins_tree.set_plugins(post_sort)
return True
def remove_homepage_from_domains(guides,domains):
"""
for all the guides, will remove the homepage of the poi that match a
given domain.
"""
pbar = Bar('removing bad homepages from guides',max=len(guides)+1)
pbar.start()
error = False
for i,g in enumerate(guides):
remove_error = remove_homepage_guide(g,domains)
if remove_error:
logging.error("could not remove the bad homepage'\
' from {0}".format(g))
error |= True
pbar.next()
pbar.finish()
return error
def draw_all_timelines(self):
"""
Draws all the timelines
:return: None
"""
self.draw_timeline(key_dates=[(
"24/04/2017",
"White Exam"), ("12/06/2017",
"Exam"), ("21/08/2017", "2nd Sess. Exam")])
bar = Bar('Generating Timelines',
max=len(self.user_list),
stream=sys.stdout)
bar.start()
for u in self.user_list:
u_id = u.user_id
self.draw_timeline(key_dates=[("24/04/2017", "White Exam"),
("12/06/2017", "Exam"),
("21/08/2017", "2nd Sess. Exam")],
user_id=u_id)
bar.next()
bar.finish()
def retrieve_data_from(category, year_from, year_to):
print('\nSearching arXiv papers from %s category...' % (category))
for year in range(year_from, year_to + 1):
print('Retrieving papers from year: %i...' % year)
# Build a simple seach query with 1 paper request to retrieve info from the feed metadata
query = (ARXIV_SEARCH_QUERY_URI % (year, year, category, 0, 1))
# Perform a GET request and retrieve from the feed the total amount of data avaliable in the year
response = urllib.request.urlopen(query)
feed = feedparser.parse(response)
total_results = int(feed.feed.opensearch_totalresults)
print("Total: %i papers." % total_results)
# Create a graph to keep the data
g = nx.Graph()
start_indexes = range(0, total_results, MAX_RESULTS_PER_REQUEST)
bar = Bar('Pulling data',
max=len(start_indexes),
suffix='%(percent)d%%') # set up progress bar
bar.start()
for start_from in start_indexes:
query = (
ARXIV_SEARCH_QUERY_URI %
(year, year, category, start_from, MAX_RESULTS_PER_REQUEST))
response = urllib.request.urlopen(query)
feed = feedparser.parse(response)
parse_feed(feed, g)
time.sleep(WAIT_TIME) # Avoid flooding the arXiv server
# Update progress
bar.next()
bar.finish()
print("Saving data to file...")
save_as_csv(g, category, year)
print('-' * 40)
def zipclean(path, guide_name, frequency=1):
"""
remove zip code from US postal addresses when they are found in mtrip
guide.
"""
logging.info('zipcode cleaning from directory {0} started'.format(path))
guide_filenames = list_guide(path, guide_name)
if len(guide_filenames) == 0:
msg ='there was not guide with filename {0}'\
'found under {1} so no zipcode cleaning was'\
' performed'.format(guide_name, path)
logging.warning(msg)
die(msg)
error = False
pbar = Bar('filtering the guides poi with a function.',max=len(guide_filenames))
pbar.start()
for i,g in enumerate(guide_filenames):
guide_data = None
with open(g,'r') as guide:
guide_data = json.load(guide)
if not guide_data:
logging.error("could not load content from {0}".format(g))
error = True
continue
clean_guide_data = clean_guide(guide_data, frequency)
with open(g, 'w') as guide:
json.dump(clean_guide_data, guide)
logging.info('zipcode cleaning for {0} done'.format(g))
pbar.next()
logging.info('zipcode cleaning from directory {0} finished'.format(path))
return error
def load_raw_data(data_dir):
"""Load all the results
"""
data_list = sorted(glob.glob(data_dir + "*.txt"))
x, y, u_tmp, v_tmp, mask = openpiv.tools.load_vectors(data_list[0])
u = np.empty([len(data_list), x.shape[0], x.shape[1]])
v = np.empty_like(u)
mask = np.empty_like(u)
u[0, :, :] = u_tmp
v[0, :, :] = v_tmp
# load the rest of the data
pbar = Bar("Importing data from {}".format(data_dir), max=len(data_list))
pbar.start()
for i in range(1, len(data_list)):
pbar.next()
x, y, u[i, :, :], v[i, :, :], mask[
i, :, :] = openpiv.tools.load_vectors(data_list[i])
pbar.finish()
return (x, y, u, v, mask)
def mirror_posts(reddit, destination, posts):
"""Mirror (crosspost) posts from one subreddit to another
Arguments:
reddit: A `praw.Reddit` instance
destination: A Subreddit model to post to
posts: A `praw.models.ListingGenerator` containing the posts to
crosspost
"""
post_dict = {}
posts = list(posts)
if reddit.user.me().name in reddit.subreddit(destination).moderator():
progress = Bar('Crossposting...', max=len(posts))
progress.start()
for post in posts:
crosspost = post.crosspost(destination, send_replies=False)
post_dict[post.id] = crosspost.id
progress.next()
progress.finish()
else:
raise NotModeratorError("You are not a moderator of this subreddit.")
return post_dict
def mirror_repo():
setup_repo()
mirror_temp = 'mirror-temp'
mirror_dir = os.path.join(SCRIPT_DIR, mirror_temp)
merge_xml = 'merged.xml'
if args.only_download and args.skip_download:
print('Both --only-download and --skip-download specified! '
'Choose either, but not both.')
return False
if args.skip_download:
tree = QgisPluginTree(os.path.join(mirror_dir, merge_xml))
else:
xml_url = args.plugins_xml_url
if not xml_url or not xml_url.lower().endswith('.xml'):
print('Missing plugins.xml or URL does not end with .xml')
return False
url_parts = urlparse(xml_url)
b_name = '{0}_{1}'.format(
url_parts.hostname.replace('.', '-'),
os.path.splitext(os.path.basename(xml_url))[0])
if not os.path.exists(mirror_dir):
os.mkdir(mirror_dir)
repo.remove_dir_contents(mirror_dir, strict=False)
q_vers = args.qgis_versions.replace(' ', '').split(',') \
if args.qgis_versions is not None else None
if q_vers is None:
urls = [xml_url]
names = ['{0}.xml'.format(b_name)]
else:
urls = ['{0}?qgis={1}'.format(xml_url, v)
for v in q_vers]
names = ['{0}_{1}.xml'.format(b_name, v.replace('.', '-'))
for v in q_vers]
tree = QgisPluginTree()
dl_bar = Bar('Downloading/merging xml', fill='=', max=len(urls))
dl_bar.start()
try:
for i in dl_bar.iter(range(0, len(urls))):
out_xml = os.path.join(mirror_dir, names[i])
download(urls[i], out=out_xml, bar=None)
tree.merge_plugins(out_xml)
except KeyboardInterrupt:
return False
print("Sorting merged plugins")
name_sort = QgisPluginTree.plugins_sorted_by_name(tree.plugins())
tree.set_plugins(name_sort)
xml = tree.to_xml()
print("Writing merged plugins to '{0}/{1}'".format(mirror_temp,
merge_xml))
with open(os.path.join(mirror_dir, merge_xml), 'w') as f:
f.write(xml)
if args.only_xmls:
return True
downloads = {}
elements = {}
for p in tree.plugins():
dl_url = p.findtext("download_url")
file_name = p.findtext("file_name")
if all([file_name, dl_url, dl_url not in downloads]):
downloads[file_name] = dl_url
elements[file_name] = p
# for testing against plugins.qgis.org
# if len(downloads) == 10:
# break
if not args.skip_download:
repo.remove_dir_contents(repo.upload_dir)
dl_bar = Bar('Downloading plugins', fill='=', max=len(downloads))
dl_bar.start()
try:
for f_name, dl_url in dl_bar.iter(downloads.iteritems()):
out_dl = os.path.join(repo.upload_dir, f_name)
download(dl_url, out=out_dl, bar=None)
except KeyboardInterrupt:
return False
if args.only_download:
print("Downloads complete, exiting since --only-download specified")
return True
zips = [z for z in os.listdir(repo.upload_dir)
if (os.path.isfile(os.path.join(repo.upload_dir, z))
and z.lower().endswith('.zip'))]
if not zips:
print('No plugins archives found in uploads directory')
return False
repo.output = False # nix qgis_repo output, since using progress bar
up_bar = Bar("Adding plugins to '{0}'".format(repo.repo_name),
fill='=', max=len(downloads))
up_bar.start()
try:
for zip_name in up_bar.iter(downloads.iterkeys()):
repo.update_plugin(
zip_name,
name_suffix=args.name_suffix,
auth=args.auth,
auth_role=args.auth_role,
# don't remove existing or just-added plugins when mirroring
versions='none',
untrusted=True,
invalid_fields=(not args.validate_fields)
)
# plugins are 'untrusted,' until overwritten with mirrored repo data
except KeyboardInterrupt:
return False
print("Sort plugins in '{0}'".format(repo.repo_name))
# Sorting is the right thing to do here, plus...
# Helps ensure 'startswith' finding of plugins will find earliest occurrance
# of a partial version, e.g. plugin.1.0 is found before plugin.1.0.1
init_sort = QgisPluginTree.plugins_sorted_by_name(
repo.plugins_tree.plugins())
repo.plugins_tree.set_plugins(init_sort)
up_bar = Bar("Updating '{0}' plugins with mirrored repo data"
.format(repo.repo_name),
fill='=', max=len(elements))
up_bar.start()
cp_tags = ['about', 'average_vote', 'author_name', 'create_date',
'deprecated', 'description', 'downloads', 'experimental',
'external_dependencies', 'homepage', 'rating_votes',
'repository', 'tags', 'tracker', 'trusted', 'update_date',
'uploaded_by']
maybe_missing = []
needs_resorted = False
try:
for file_name, el in up_bar.iter(elements.iteritems()):
nam, _ = os.path.splitext(file_name)
p = repo.plugins_tree.find_plugin_by_package_name(nam,
starts_with=True)
if not p: # maybe the base version has been adjusted, try again
temp_nam = re.sub(r'((\d+\.)?(\d+\.)?(\d+))', r'.\1', nam)
p = repo.plugins_tree.find_plugin_by_package_name(
temp_nam, starts_with=True)
if not p:
maybe_missing.append(file_name)
continue
else:
p = p[0]
# print("Updating '{0}'...".format(p[0].get('name')))
for tag in cp_tags:
tag_el = el.find(tag)
tag_p = p.find(tag)
if tag_el is not None and tag_p is not None:
txt = tag_el.text
# print(" {0}: {1} <- {2}".format(tag, tag_p.text, txt))
if tag in QgisPlugin.metadata_types('cdata'):
if tag_el.text is not None:
txt = etree.CDATA(tag_el.text)
tag_p.text = txt
# update plugin name
ns = args.name_suffix if args.name_suffix is not None \
else repo.plugin_name_suffix
if el.get('name') is not None:
el_name = u"{0}{1}".format(el.get('name'), ns)
if p.get('name') != el_name:
needs_resorted = True
p.set('name', el_name)
except KeyboardInterrupt:
return False
if needs_resorted:
print("Re-sorting plugins in '{0}'".format(repo.repo_name))
re_sort = QgisPluginTree.plugins_sorted_by_name(
repo.plugins_tree.plugins())
repo.plugins_tree.set_plugins(re_sort)
print("Writing '{0}' {1}".format(repo.repo_name, repo.plugins_xml_name))
repo.write_plugins_xml(repo.plugins_tree_xml())
print('\nDone mirroring...')
print("Plugin results:\n attempted: {0}\n mirrored: {1}"
.format(len(tree.plugins()), len(repo.plugins_tree.plugins())))
if maybe_missing:
print('\nWARNING (version conflicts): plugins downloaded but MAY not '
'be in XML after update:\n {0}\n'
.format(', '.join(maybe_missing)))
return True
def description_publish(guides,
user_agent,
function_class,
nailgun_bin,
description_gen):
"""
Publish the description content for the guides.
"""
# start the nailgun thing for usage with decription_generation.
nailguninit(nailgun_bin, description_gen)
sources_domain = {'wikipedia','wikivoyage'}
error = False
for g in guides:
jsonguide = None
with open(g,'r') as guide:
jsonguide = json.load(guide)
if not jsonguide:
logging.error('could not load json from {0}'.format(g))
error = True
continue
# notice the 0 index here. This is ok because there is only one city
# per guide. Maybe that will not be the case in the future.
pois = jsonguide['Cities'][0]['pois']
pbar = Bar('extracting description for the poi(s) in {0}:'.format(g),max=len(pois)+1)
pbar.start()
for i,p in enumerate(pois):
desc = p['descriptions']
for k, v in desc.items():
try:
url = v['source'].get('url')
except:
logging.error("source did not contain a dictionary"\
" for {0}".format(p['name']['name']))
continue
hostname = urlparse(url).hostname
if hostname:
tldn = hostname.split('.')[-2]
else:
continue
if tldn in sources_domain:
content_raw = description_content(
[url],
function_class,
user_agent)
c_list = json.loads(content_raw)
content = c_list[0] if len(c_list) > 0 else None
if not content:
poi_name = p['name']['name']
logging.error(
'failed to generate descriptive content'\
' for {0} using url {1}'.format(poi_name,
url))
error = True
else:
v['text'] = content.get('article',None)
v['source']['url'] = content.get('url',None)
pbar.next()
# redump the guide into the file
pbar.finish()
with open(g,'w') as guide:
json.dump(jsonguide, guide)
logging.info('description content succesfully inserted in all guides')
return error
bytes1 = int(n21)
bytes2 = bytes1 - 128
#Set Difficulty / IV
n = int(sys.argv[1])
n1 = str(n)
bar = Bar('Processing', max=n);
info1 = str("Keys: "+ str(a)+ ":" + str(b))
info1 = colored(info1, 'yellow')
Diff = colored(str(n), 'red')
info = colored(str("Hash Difficulty: "), 'white')
#print(info1)
#print(info, Diff);
bar.start();
h = int(finEnc);
while(n>2):
c=a+b;
a=b;
b=c;
n=n-1;
bar.next()
#Second Round Preparation
j = c
p = n *n
g07 = int(c * 2)
bar.next()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-s", "--serial", default="/dev/ttyACM0", help="Path to USB serial device"
)
parser.add_argument(
"-f", "--file", type=argparse.FileType("rb"), help="Path to .bin image file"
)
parser.add_argument(
"-r",
"--reboot",
action="store_true",
help="Reset device without flashing any image",
)
parser.add_argument(
"-p",
"--printsn",
action="store_true",
help="Print the serial number or an empty string if no device detected",
)
parser.add_argument(
"-a",
"--argument",
type=int,
default=-1,
help="User argument (u32) passed to application after reset",
)
parser.add_argument(
"-d", "--dummy", default=False, action="store_true", help="Dummy mode",
)
args = parser.parse_args()
if args.file is not None and args.reboot:
print("-f and -r can not both be specified at the same time")
exit(1)
if args.file is None and not args.reboot and not args.printsn:
print("Either -f or -r or -p required")
exit(1)
if not args.dummy:
ser = serial.Serial(args.serial)
tty.setraw(ser.fd)
# flush the serial port buffer, if any
ser.timeout = 0.1
ser.read(4096)
ser.timeout = 2
else:
ser = DummySerial()
if args.printsn:
sn = read_serialNumber(ser)
ret = 0
if sn == None:
sn = ""
ret = 1
print(sn)
exit(ret)
if not detect_dfu(ser):
print("Device not in DFU mode")
print(
"Please enter DFU mode through menu CONFIG -> DFU or hold down "
"the JOG LEFT button and power cycle the device."
)
exit(1)
if args.file is not None:
print(f"Uploading firmware file {args.file.name}")
# set the flash address where the firmware is to be written
ser.write(send_write_addr(0x08004000))
size = os.stat(args.file.name).st_size
pbar = Bar(message=os.path.basename(args.file.name), max=size)
pbar.start()
# write the firmware
for data in block_reader(args.file, blocksize=1200):
if data:
ser.write(send_bytes(data))
pbar.next(len(data))
ser.read(1) # wait for block write to complete
pbar.finish()
# set user argument
if args.argument >= 0:
print("Sending user argument")
cmd = b"\x22\xe8" + int.to_bytes(args.argument, 4, "little")
ser.write(cmd)
print("Resetting")
# reboot device
ser.write([0x20, 0xEF, 0x5E])
from progress.bar import Bar
# Secure access to domo
import secret
# General functions for RR
import rr_fun
import boto3 #AWS
import botocore
s3 = boto3.client('s3')
BUCKET_NAME = 'fabiano-crm-consys'
bar = Bar('Processing', max=3)
bar.start()
def employee(dispo):
if pd.isna(dispo) == False:
if any(x in dispo for x in [
'apset', 'ows', 'lms', 'apconfirmed', 'gotrid', 'cbk',
'aprschld', 'cant', 'email', 'notenough', 'nas', 'awc', 'mbf',
'clsrnoap', 'nit', 'moveto', 'dnc', 'custnoap', 'dna',
'custnoap_rsch', 'wrong', 'moveto', 'asm',
'mtg not qualified (no fraud)', 'notwknumb', 'spanish',
'30 days cbk', '60 days cbk', '90 days cbk', 'allbusy'
]):
return "Fronter"
elif any(x in dispo for x in [
'nosalecb', 'sale', 'ccbk'
SELECTOR = ("//*[not(contains(@style,'display:none')"
" or contains(@class,'hidden'))]"
"/*/a[contains(@href,'{0}')"
" or starts-with(@href,'/')]").format(host)
SELENIUM_SELECTOR = ("//*[not(contains(@style,'display:none')"
" or contains(@class,'hidden'))]"
"/*/a[contains(@href,'{0}')]")
childs = get_page_childs(initial_url)
CACHE = []
new_pages_count = len(childs)
bar = Bar('Processing', max=len(childs))
bar.start()
recursion = 0
while new_pages_count > 0:
queue = Queue()
results_queue = Queue()
new_pages_count = 0
for page in childs:
if page['link'] not in CACHE:
CACHE.append(page['link'])
queue.put(page, timeout=10)
# woraround for queue.put:
time.sleep(0.01)
def do_weekly_iteration(self):
if RedditAPI.should_run_weekly():
log("Start Weekly Iteration", is_heading=True)
initialSource = os.path.join(base_directory, "PictureSource",
"LockScreenSource")
finalSource = os.path.join(base_directory, "PictureSource",
"LockScreen")
remove_all_files(initialSource)
remove_all_files(finalSource)
progressBar = None
try:
weekly_urls = self.get_submissions_for_subreddit("week")
log("URLS Gathered")
totalCount = len(weekly_urls)
landscape = []
portrait = []
if self.args.show_progress:
progressBar = Bar(
"Downloading Weekly Images",
max=totalCount,
suffix='%(index)d / %(max)d %(percent)d%%')
progressBar.start()
with open(
os.path.join(base_directory, "PictureSource",
"lockScreenStat.txt"), "w") as f:
while len(weekly_urls) > 0:
# select specific url at random
attempt, urls = get_random_url(weekly_urls)
image_url, post_permalink = attempt
log("Process URL:", image_url, "URLS Left:", len(urls))
# get image and save if able
imageObj = RedditImage(image_url, post_permalink)
imageObj.get_image_from_url(initialSource)
if imageObj.image_is_landscape():
landscape.append(imageObj)
else:
portrait.append(imageObj)
if imageObj.image_downloaded():
f.write(str(imageObj) + "\n")
if progressBar is not None:
progressBar.next()
log("URL has been processed")
if progressBar is not None:
progressBar.finish()
for imageObj in landscape:
imageObj.move_to_folder(finalSource)
print("did the copy")
RedditDatabase().insert_images(landscape + portrait)
# iterate through creating landscape photos
resulting = CombineImages.iterate_combine_landscape(
portrait, finalSource)
RedditDatabase().insert_all_combined(resulting)
self.set_weekly_run_file()
finally:
if progressBar is not None:
progressBar.finish()
