def getVideoUrl(content):
fmtre = re.search('(?<=fmt_stream_map=).*', content)
grps = fmtre.group(0).split('&')
vurls = urllib2.unquote(grps[0])
videoUrl = None
for vurl in vurls.split(','):
print urllib2.url2pathname(vurl[4:]),"\n"
if vurl.find('itag=5') > 0:
continue
return urllib2.url2pathname(vurl[4:])
exit()
return None
def downloadLatest(d, location='downloads\\'):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@return the path to the downloaded file.
"""
try:
name = d['name']
version = getWebVersion(d)
downurl = getDownloadURL(d)
furl = urllib2.urlopen(downurl)
filecontents = furl.read()
furl.close()
parsed = urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
newfileloc = location + name + '---' + version + '---' + filename
with open(newfileloc, "wb") as f:
f.write(filecontents)
except IOError as (errno, strerror):
print 'could not open file, I/O error({0}): {1}'.format(
errno, strerror)
print 'when calling downloadLatest(%s, %s)' % (d, location)
def _download_url(self, scheme, url, tmpdir):
# Determine download filename
#
name = filter(None,urlparse.urlparse(url)[2].split('/'))
if name:
name = name[-1]
while '..' in name:
name = name.replace('..','.').replace('\\','_')
else:
name = "__downloaded__" # default if URL has no path contents
if name.endswith('.egg.zip'):
name = name[:-4] # strip the extra .zip before download
filename = os.path.join(tmpdir,name)
# Download the file
#
if scheme=='svn' or scheme.startswith('svn+'):
return self._download_svn(url, filename)
elif scheme=='file':
return urllib2.url2pathname(urlparse.urlparse(url)[2])
else:
self.url_ok(url, True) # raises error if not allowed
return self._attempt_download(url, filename)
def add_paths(self, paths):
"""Add the given URLs to the control
paths - a sequence of URLs
"""
uid = uuid.uuid4()
npaths = len(paths)
for i, path in enumerate(paths):
if i % 100 == 0:
cellprofiler.preferences.report_progress(
uid, float(i) / npaths,
"Loading %s into UI" % path)
folder, filename = self.splitpath(path)
display_name = urllib2.url2pathname(filename)
width, _ = self.GetTextExtent(display_name)
idx = bisect.bisect_left(self.folder_names, folder)
if idx >= len(self.folder_names) or self.folder_names[idx] != folder:
folder_item = self.FolderItem(self, folder)
self.folder_names.insert(idx, folder)
self.folder_items.insert(idx, folder_item)
else:
folder_item = self.folder_items[idx]
fp = folder_item.filenames
pidx = bisect.bisect_left(fp, filename)
if pidx >= len(fp) or fp[pidx] != filename:
fp.insert(pidx, filename)
folder_item.widths.insert(pidx, width)
folder_item.file_display_names.insert(pidx, display_name)
folder_item.enabled.insert(pidx, True)
if len(paths) > 0:
cellprofiler.preferences.report_progress(uid, 1, "Done")
self.schmutzy = True
self.Refresh(eraseBackground=False)
def add_paths(self, paths):
'''Add the given URLs to the control
paths - a sequence of URLs
'''
uid = uuid.uuid4()
npaths = len(paths)
for i, path in enumerate(paths):
if i % 100 == 0:
cpprefs.report_progress(uid,
float(i) / npaths,
"Loading %s into UI" % path)
folder, filename = self.splitpath(path)
display_name = urllib2.url2pathname(filename)
width, _ = self.GetTextExtent(display_name)
idx = bisect.bisect_left(self.folder_names, folder)
if idx >= len(
self.folder_names) or self.folder_names[idx] != folder:
folder_item = self.FolderItem(self, folder)
self.folder_names.insert(idx, folder)
self.folder_items.insert(idx, folder_item)
else:
folder_item = self.folder_items[idx]
fp = folder_item.filenames
pidx = bisect.bisect_left(fp, filename)
if pidx >= len(fp) or fp[pidx] != filename:
fp.insert(pidx, filename)
folder_item.widths.insert(pidx, width)
folder_item.file_display_names.insert(pidx, display_name)
folder_item.enabled.insert(pidx, True)
if len(paths) > 0:
cpprefs.report_progress(uid, 1, "Done")
self.schmutzy = True
self.Refresh(eraseBackground=False)
def get_bearer_token(context, user_role, token_regexp='access_token=\w+'):
'''
Wrapping up authentication on OAUTH2 Implicit Grant
:param user_role:
:param token_regexp: we gather this from a Location header. It was tricky becase there is a redirrection on this request
:return: {'Authorization': 'Bearer tteDnkzjFhTxDicNir1Pn0gKFCxYGN'})
'''
erase_last_response(context)
if user_role in USERS.keys():
cookies={'sessionid': get_session_id(context,user_role)}
url=urlparse.urljoin(OAUTH_URL,"/o/authorize/?client_id={id}&response_type=token"\
.format(id=CLIENT_IDS[context.config.userdata['target_env']]))
r=requests.head(url, cookies=cookies, verify=True, allow_redirects=True)
u=r.history[0].headers['Location']
# 'https://alpha-hub.dev.blippar.com/dashboard/#access_token=co0y0a19e72aUlcvLZF46rNzuzv7V3&token_type=Bearer&expires_in=36000&scope=read+everything+write'
encoded_url=urllib2.url2pathname(u)
print("Location URL:", encoded_url)
exp = re.compile(token_regexp)
search = exp.search(encoded_url)
if search:
token = search.group(0)
# we receive 'access_token=co0y0a19e72aUlcvLZF46rNzuzv7V3'
context.config.userdata['bake_api_token'] = {'Authorization': 'Bearer ' + token.replace('access_token=', "")}
print("Resulting OAUTH Bearer Token", context.config.userdata['bake_api_token'])
else:
assert False, "ERROR: Cannot extract token"
else:
assert False, "Error, wrong user role {role}. Valid roles are: {roles}".format(role=user_role,roles = USERS.keys())
def downloadLatest(d, location='downloads\\'):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@return the path to the downloaded file.
"""
try:
name = d['name']
version = getWebVersion(d)
downurl = getDownloadURL(d)
furl = urllib2.urlopen(downurl)
filecontents = furl.read()
furl.close()
parsed=urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
newfileloc = location + name + '---' + version + '---' + filename
with open(newfileloc, "wb") as f:
f.write(filecontents)
except IOError as (errno, strerror):
print 'could not open file, I/O error({0}): {1}'.format(errno, strerror)
print 'when calling downloadLatest(%s, %s)' %(d, location)
def open_local_file(self, req):
import email.Utils
host = req.get_host()
file = req.get_selector()
localfile = url2pathname(file)
stats = os.stat(localfile)
size = stats.st_size
modified = email.Utils.formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(file)[0]
headers = mimetools.Message(StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
if host:
host, port = splitport(host)
if not host or \
(not port and socket.gethostbyname(host) in self.get_names()):
try:
file_list = dircache.listdir(localfile)
s = StringIO()
s.write('<html><head><base href="%s"/></head><body>' % ('file:' + file))
s.write('<p>Directory Content:</p>')
for f in file_list:
s.write('<p><a href="%s">%s</a></p>\n' % (urllib.quote(f), f))
s.write('</body></html>')
s.seek(0)
headers = mimetools.Message(StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
('text/html', size, modified)))
return addinfourl(s, headers, 'file:' + file)
except OSError:
return addinfourl(open(localfile, 'rb'),
headers, 'file:'+file)
raise URLError('file not on local host')
def _download_url(self, scheme, url, tmpdir):
# Determine download filename
#
name = filter(None,urlparse.urlparse(url)[2].split('/'))
if name:
name = name[-1]
while '..' in name:
name = name.replace('..','.').replace('\\','_')
else:
name = "__downloaded__" # default if URL has no path contents
if name.endswith('.egg.zip'):
name = name[:-4] # strip the extra .zip before download
filename = os.path.join(tmpdir,name)
# Download the file
#
if scheme=='svn' or scheme.startswith('svn+'):
return self._download_svn(url, filename)
elif scheme=='file':
return urllib2.url2pathname(urlparse.urlparse(url)[2])
else:
headers = self.retry_sf_download(url, filename)
if 'html' in headers['content-type'].lower():
return self._download_html(url, headers, filename, tmpdir)
else:
return filename
def open_local_file(self, req):
host = req.get_host()
file = req.get_selector()
localfile = urllib2.url2pathname(file)
stats = os.stat(localfile)
size = stats.st_size
modified = formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(file)[0]
headers = mimetools.Message(cStringIO.StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
if host:
host, port = urllib.splitport(host)
if not host or \
(not port and socket.gethostbyname(host) in self.get_names()):
try:
file_list = dircache.listdir(localfile)
s = cStringIO.StringIO()
s.write('<html><head><base href="%s"/></head><body>' % ('file:' + file))
s.write('<p>Directory Content:</p>')
for f in file_list:
s.write('<p><a href="%s">%s</a></p>\n' % (urllib.quote(f), f))
s.write('</body></html>')
s.seek(0)
headers = mimetools.Message(cStringIO.StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
('text/html', size, modified)))
return urllib2.addinfourl(s, headers, 'file:' + file)
except OSError:
return urllib2.addinfourl(open(localfile, 'rb'),
headers, 'file:'+file)
raise urllib2.URLError('file not on local host')
def set_album_art(self, button) :
tburl = self.webview.get_main_frame().get_title()
if not tburl:
return
print "Url: " + tburl
filep = tburl.split('/')
filepath = filep[len(filep)-1]
(filen, filextn) = os.path.splitext(filepath)
request = urllib2.Request(tburl)
opener = urllib2.build_opener()
image = None
try:
image = opener.open(request).read()
except:
print "Failed to download image"
if(self.mode == self.MODE_RHYTHM):
filename = os.environ['HOME']+"/.cache/rhythmbox/covers/" + self.current_artist + " - " + self.current_album + ".jpg"
else:
location_path_improper = urllib2.url2pathname(self.current_location)
location_path_arr = location_path_improper.split("//")
location_path = location_path_arr[1]
filename = location_path.rsplit("/",1)[0] + "/" + "folder.jpg"
output = open(filename, 'w')
output.write(image)
output.close()
def local_open(url):
"""Read a local path, with special support for directories"""
scheme, server, path, param, query, frag = urlparse.urlparse(url)
filename = urllib2.url2pathname(path)
if os.path.isfile(filename):
return urllib2.urlopen(url)
elif path.endswith('/') and os.path.isdir(filename):
files = []
for f in os.listdir(filename):
if f == 'index.html':
body = open(os.path.join(filename, f), 'rb').read()
break
elif os.path.isdir(os.path.join(filename, f)):
f += '/'
files.append("<a href=%r>%s</a>" % (f, f))
else:
body = ("<html><head><title>%s</title>" % url) + \
"</head><body>%s</body></html>" % '\n'.join(files)
status, message = 200, "OK"
else:
status, message, body = 404, "Path not found", "Not found"
return urllib2.HTTPError(url, status, message,
{'content-type': 'text/html'},
cStringIO.StringIO(body))
def local_open(url):
"""Read a local path, with special support for directories"""
scheme, server, path, param, query, frag = urlparse.urlparse(url)
filename = urllib2.url2pathname(path)
if os.path.isfile(filename):
return urllib2.urlopen(url)
elif path.endswith('/') and os.path.isdir(filename):
files = []
for f in os.listdir(filename):
if f=='index.html':
fp = open(os.path.join(filename,f),'rb')
body = fp.read()
fp.close()
break
elif os.path.isdir(os.path.join(filename,f)):
f+='/'
files.append("<a href=%r>%s</a>" % (f,f))
else:
body = ("<html><head><title>%s</title>" % url) + \
"</head><body>%s</body></html>" % '\n'.join(files)
status, message = 200, "OK"
else:
status, message, body = 404, "Path not found", "Not found"
return urllib2.HTTPError(url, status, message,
{'content-type':'text/html'}, cStringIO.StringIO(body))
def _download_url(self, scheme, url, tmpdir):
# Determine download filename
#
name = filter(None, urlparse.urlparse(url)[2].split('/'))
if name:
name = name[-1]
while '..' in name:
name = name.replace('..', '.').replace('\\', '_')
else:
name = "__downloaded__" # default if URL has no path contents
if name.endswith('.egg.zip'):
name = name[:-4] # strip the extra .zip before download
filename = os.path.join(tmpdir, name)
# Download the file
#
if scheme == 'svn' or scheme.startswith('svn+'):
return self._download_svn(url, filename)
elif scheme == 'file':
return urllib2.url2pathname(urlparse.urlparse(url)[2])
else:
self.url_ok(url, True) # raises error if not allowed
return self._attempt_download(url, filename)
def open(self, request):
"""
Open a file or url
If request.url can't be identified as a url, it will
return the content in a file-like object
@param request: A suds Request
@type Request: suds.transport.Request
@return: A file-like object
@rtype: file
"""
log.debug('opening: (%s)', request.url)
fp = None
location = request.url.lstrip()
if location.startswith('<?'):
log.debug('returning url (%s) as StringIO file')
fp = cStringIO.StringIO(location)
else:
parsed = urlparse(request.url)
if parsed.scheme == 'file':
# the path component of the urlparse output is not automatically a valid filesystem path!
filepath = u2.url2pathname(parsed.path)
log.debug('opening file (%s) with open', filepath)
try:
fp = open(filepath)
except Exception, e:
raise TransportError(str(e), 500, None)
else:
def open(self, request):
"""
Open a file or url
If request.url can't be identified as a url, it will
return the content in a file-like object
@param request: A suds Request
@type Request: suds.transport.Request
@return: A file-like object
@rtype: file
"""
log.debug('opening: (%s)', request.url)
fp = None
location = request.url.lstrip()
if location.startswith('<?'):
log.debug('returning url (%s) as StringIO file')
fp = cStringIO.StringIO(location)
else:
parsed = urlparse(request.url)
if parsed.scheme == 'file':
# the path component of the urlparse output is not automatically a valid filesystem path!
filepath = u2.url2pathname(parsed.path)
log.debug('opening file (%s) with open', filepath)
try:
fp = open(filepath)
except Exception, e:
raise TransportError(str(e), 500, None)
else:
def texthandler(parseString=None):
# s = re.sub(r'%0A', r'\n', parseString)
# s = re.sub(r'\n+', r'\\n\\n', parseString)
s = urllib2.url2pathname(parseString)
s = re.sub(r'\n+', r'\\n\\n', s)
print s
return render_template('spritz.html', text=s, filename="", titleText="Highlighted Text")
def _raise_open_error(self, url, message):
if url[:7].lower() == "file://":
what = "file"
ident = urllib2.url2pathname(url[7:])
else:
what = "URL"
ident = url
raise ZConfig.ConfigurationError("error opening %s %s: %s" % (what, ident, message), url)
def texthandler(parseString=None):
# s = re.sub(r'%0A', r'\n', parseString)
# s = re.sub(r'\n+', r'\\n\\n', parseString)
s = urllib2.url2pathname(parseString)
s = re.sub(r'\n+', r'\\n\\n', s)
print s
return render_template('spritz.html', text=s, filename="", titleText="Highlighted Text")
def _url_to_local_path(url, path):
"""Mirror a url path in a local destination (keeping folder structure)"""
destination = urlparse.urlparse(url).path
# First char should be '/', and it needs to be discarded
if len(destination) < 2 or destination[0] != '/':
raise ValueError('Invalid URL')
destination = os.path.join(path, urllib2.url2pathname(destination)[1:])
return destination
def _url_to_local_path(url, path):
"""Mirror a url path in a local destination (keeping folder structure)"""
destination = urlparse.urlparse(url).path
# First char should be '/', and it needs to be discarded
if len(destination) < 2 or destination[0] != '/':
raise ValueError('Invalid URL')
destination = os.path.join(path, urllib2.url2pathname(destination)[1:])
return destination
def url_or_fname_to_fname(url_or_fname):
""" Assert that is_local(url_or_fname) then if it is a "file:" url, parse it and run url2pathname on it, else just return it. """
assert is_local(url_or_fname)
mo = URL_SCHEME(url_or_fname)
if mo:
return urllib2.url2pathname(urlparse.urlparse(url)[2])
else:
return url_or_fname
def url_or_fname_to_fname(url_or_fname):
""" Assert that is_local(url_or_fname) then if it is a "file:" url, parse it and run url2pathname on it, else just return it. """
assert is_local(url_or_fname)
mo = URL_SCHEME(url_or_fname)
if mo:
return urllib2.url2pathname(urlparse.urlparse(url)[2])
else:
return url_or_fname
def _raise_open_error(self, url, message):
if url[:7].lower() == "file://":
what = "file"
ident = urllib2.url2pathname(url[7:])
else:
what = "URL"
ident = url
raise ZConfig.ConfigurationError(
"error opening %s %s: %s" % (what, ident, message), url)
def load_fragment(fragment_url, mime_type=None):
"""Load fragment
Load a fragment (e.g. time series data) from a given URL
"""
parsed_url = windows_compatible(up.urlparse)(fragment_url)
scheme = parsed_url.scheme
server_loc = parsed_url.netloc
path = parsed_url.path
fragment_string = parsed_url.fragment
# reform path for windows files
if scheme == "file" and os.name == "nt":
path = ur.url2pathname(r"\\" + server_loc + path)
if mime_type is None:
mimetypes.init()
mime_type, _ = mimetypes.guess_type(path)
if mime_type is None:
raise TypeError(
'load_fragment: impossible to guess MIME type from url, '
'please specify the type manually as argument: %s' % path)
if scheme == 'file' and os.name == 'nt':
file_handle = open(path, 'r')
elif scheme == 'file':
file_handle = open(os.path.join(server_loc, path), 'r')
if 'text/' in mime_type:
import numpy
if fragment_string == '':
cols = None
else:
import re
match = re.match("col=([0-9]+)", fragment_string)
if match is None or len(match.groups()) != 1:
raise TypeError(
"load_fragment: don't understand url fragment %s" %
fragment_string)
else:
cols = int(match.groups()[0]) - 1
# Unfortunately we need this if statement
# Setting usecols to None throws an error in the loadtxt call
if cols is not None:
fragment_content = numpy.loadtxt(file_handle, usecols=[cols])
else:
fragment_content = numpy.loadtxt(file_handle)
return fragment_content
else:
raise TypeError('load_fragment: unknown mime type %s' % mime_type)
def dispatch_selected(*args):
"""BGG BSG Function to dispatch a single user hand"""
document = XSCRIPTCONTEXT.getDocument()
maindir = urllib2.url2pathname(os.path.dirname(document.Location.replace("file://","")))
logfile = os.path.join(maindir, 'bsg-dispatcher-debug.log')
sys.stdout = open(logfile, "w", 0) # unbuffered
# Useful variables so we don't need to do a lot of typing
worksheet = document.getSheets().getByName('Game State')
dispatcherinfo = document.getSheets().getByName('Posting Templates')
# Find the selected char
selected_char = worksheet.DrawPage.Forms.getByName('formGameState').getByName('lstPlayers').CurrentValue
selected_player = ''
if not selected_char:
MessageBox(document, "Error: no player selected", "Invalid player", "warningbox")
return False
# Find out which player we're looking at
for i in range(7):
charname = worksheet.getCellByPosition(4, 2+i).getString() # Character name on Game State
if charname == selected_char:
selected_player = worksheet.getCellByPosition(1, 2+i).getString() # Player name on Game State
player_id = i
break
else:
MessageBox(document, "Error: player not found, maybe a bug?", "Invalid player", "warningbox")
return False
# Verify if file exists
playerfile = os.path.join(maindir, selected_char + '.txt')
if not os.path.exists(playerfile):
MessageBox(document, "Error: file '%s' not found (use the 'Create Hand Lists' first)" % (selected_char + '.txt'), "File not found", "warningbox")
return False
# Verify if we already sent this file
old_md5 = dispatcherinfo.getCellByPosition(player_id+4, 31).getString()
current_md5 = get_md5(playerfile)
if old_md5 == current_md5: # We DID send this already!!!
confirm = MessageBox(document, "It seems we've already sent this file. Send again?", "File already sent", "querybox", YES_NO)
if confirm == 3: # Pressed "No"
return False
# Now we finally try to send our files
try:
gm = GeekMail(workdir=maindir)
gm.dispatch(selected_player, playerfile)
# Set the current MD5 on the spreadsheet (so that we only send it again after something is changed)
dispatcherinfo.getCellByPosition(player_id+4, 31).setString(current_md5)
except Exception as e:
MessageBox(document, e.message, "Alert!", "warningbox")
return False
MessageBox(document, "Successfully sent file to %s" % selected_player, "Success!", "infobox")
def import_new_db(self, filename, callback):
"""
Attempt to import the provided file into a new database.
A new database will only be created if an appropriate importer was
found.
@return: A tuple of (new_path, name) for the new database
or (None, None) if no import was performed.
"""
pmgr = BasePluginManager.get_instance()
# check to see if it isn't a filename directly:
if not os.path.isfile(filename):
# Allow URL names here; make temp file if necessary
url = urlparse.urlparse(filename)
if url.scheme != "":
if url.scheme == "file":
filename = urllib2.url2pathname(filename[7:])
else:
url_fp = urllib2.urlopen(filename) # open URL
# make a temp local file:
ext = os.path.splitext(url.path)[1]
fd, filename = tempfile.mkstemp(suffix=ext)
temp_fp = os.fdopen(fd, "w")
# read from URL:
data = url_fp.read()
# write locally:
temp_fp.write(data)
url_fp.close()
temp_fp.close()
(name, ext) = os.path.splitext(os.path.basename(filename))
format = ext[1:].lower()
for plugin in pmgr.get_import_plugins():
if format == plugin.get_extension():
new_path, name = self._create_new_db(name)
# Create a new database
self.__start_cursor(_("Importing data..."))
dbclass = gen.db.DbBsddb
dbase = dbclass()
dbase.write_version(new_path)
dbase.load(new_path, callback)
import_function = plugin.get_import_function()
import_function(dbase, filename, callback)
# finish up
self.__end_cursor()
dbase.close()
return new_path, name
return None, None
def dispatcher_call(*args):
"""BGG BSG Function to dispatch a generic message via GeekMail"""
document = XSCRIPTCONTEXT.getDocument()
maindir = urllib2.url2pathname(
os.path.dirname(document.Location.replace("file://", "")))
logfile = os.path.join(maindir, 'bsg-dispatcher-debug.log')
sys.stdout = open(logfile, "a", 0) # unbuffered
# Useful variables so we don't need to do a lot of typing
worksheet = document.getSheets().getByName('Game State')
dispatcherinfo = document.getSheets().getByName('Posting Templates')
dispatchersheet = document.getSheets().getByName('Dispatcher')
playername = dispatchersheet.getCellByPosition(2, 1).getString()
subject = dispatchersheet.getCellByPosition(2, 3).getString()
body = dispatchersheet.getCellByPosition(2, 4).getString()
oldhash = dispatchersheet.getCellByPosition(2, 5).getString()
if not playername:
MessageBox(document, "Error: Username can't be empty", "No username!",
'warningbox')
return False
if not subject:
MessageBox(document, "Error: Subject not defined",
"Subject not defined!", 'warningbox')
return False
if not body:
MessageBox(document, "Error: Body is empty", "Empty body!",
'warningbox')
return False
hashish = calculate_md5("%s%s%s" % (playername, subject, body))
if oldhash == hashish:
confirm = MessageBox(
document, "It seems we've already sent this data. Send again?",
"Data already sent", "querybox", YES_NO)
if confirm == 3: # Pressed "No"
return False
try:
gm = GeekMail(workdir=maindir)
gm.dispatch_text(playername, subject, body)
# Set the current MD5 on the spreadsheet (so that we only send it again after something is changed)
dispatchersheet.getCellByPosition(2, 5).setString(hashish)
except IOError:
MessageBox(document, e.message, "Alert!", "warningbox")
return False
MessageBox(document,
"Successfully sent the following message:\n\n%s" % (subject),
"Success!", "infobox")
def scrape_bizSource(self):
if self.bizSource != None:
try:
self.bizSoup = BS(self.bizSource)
except:
print 'Incompatible with soup: this url: ' + self.bizUrl.encode('utf-8')
print 'given on this page: ' + self.path
return
comments = self.bizSoup.findAll (text=lambda text:isinstance(text, Comment))
[comment.extract() for comment in comments]
bizContacts = self.bizSoup.findAll('a')
regex_contact = re.compile (r'.*contact.*', re.IGNORECASE)
for link in bizContacts:
result = re.search (regex_contact, str(link))
if result != None:
try:
self.bizContactLink = urljoin ( self.bizUrl, urllib2.url2pathname(link['href']) )
break
except:
continue
if self.bizContactLink != None:
try:
self.bizContactSource = urllib2.urlopen (self.bizContactLink, timeout=300)
try:
if 'text/html' in self.bizContactSource.info()['content-type']:
try:
self.bizContactSource = self.bizContactSource.read() #now just HTML source
except:
print 'Failed to read contact-us source for: ' + self.bizContactLink.encode('utf-8')
self.bozContactSource = None
else:
print 'Contact Us page in NOT HTML for: ' + self.bizContactLink.encode('utf-8')
self.bizContactSource = None
except:
self.bizContactSource = None
except:
print 'unable to open the contact us link: ' + self.bizContactLink.encode('utf-8')
self.bizContactSource = None
if self.bizContactSource != None:
self.scrape_bizContactSource()
self.scrape_bizEmail (str(self.bizSoup))
def do_GET(self):
"""Serve a GET request."""
command = self.path.split('/')
command.remove('')
try:
if self.path == "/start":
print "intra ffs"
buttons_list = BROWSER.start()
print buttons_list
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=None)
elif self.path == "/next":
buttons_list = BROWSER.next()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=None)
elif self.path == "/prev":
buttons_list = BROWSER.prev()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=None)
elif self.path == "/close":
buttons_list = BROWSER.close()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=None)
elif self.path == "/":
buttons_list = BROWSER.home()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=None)
elif self.path.startswith('/expand/'):
path = urllib2.url2pathname(self.path[len('/expand'):])
BROWSER.click_expand_collapse(path)
buttons_list, fields_list = BROWSER.get_current_page_options()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=fields_list)
elif command[0] == "get-current-options":
buttons_list, fields_list = BROWSER.get_current_page_options()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=fields_list)
else:
BROWSER.navigate_to(self.path)
buttons_list, fields_list = BROWSER.get_current_page_options()
response = Response(code=200, current_page=BROWSER.current_url,
buttons=buttons_list, fields=fields_list)
except:
buttons_list, fields_list = BROWSER.get_current_page_options()
response = Response(code=405, message="Not allowed",
current_page=BROWSER.current_url,
buttons=buttons_list, fields=fields_list)
finally:
self.send_response(response)
def _get_content_packaged_path(self, _new_url=None):
"""
Return the path of the content data if it is packaged, else None.
"""
#If `_new_url` is provided, it is used instead of the current
#`content_url` (useful when changing URL).
url = _new_url or self.content_url
if not url or url[:9] != "packaged:":
return None
p_root = self._owner.get_meta(PACKAGED_ROOT, None)
if p_root is None:
return None
return join(p_root, url2pathname(url[10:]))
def on_folder_changed(self,fchooser,data=None):
url = fchooser.get_current_folder_uri( )
if not url:
return
pname = url2pathname(url)[7:]
model = self.get_model( )
model.clear( )
self.master_list = [ ]
self.depth = len( pname.split(os.path.sep) )
if self.recurse_widget.get_active( ):
self.recurse = 1
else:
self.recurse = 0
self.my_walker(pname,model)
self.okbtn.set_sensitive(False)
def dispatch_all(*args):
"""BGG BSG Function to dispatch all player hands"""
document = XSCRIPTCONTEXT.getDocument()
maindir = urllib2.url2pathname(os.path.dirname(document.Location.replace("file://","")))
logfile = os.path.join(maindir, 'bsg-dispatcher-debug.log')
sys.stdout = open(logfile, "w", 0) # unbuffered
# Useful variables so we don't need to do a lot of typing
worksheet = document.getSheets().getByName('Game State')
dispatcherinfo = document.getSheets().getByName('Posting Templates')
to_send = []
# Maximum of 7 players
for i in range(7):
playername = worksheet.getCellByPosition(1, 2+i).getString() # Player name on Game State
charname = worksheet.getCellByPosition(4, 2+i).getString() # Character name on Game State
if not playername: # If there isn't a player for this number, skip it
continue
# Verify if file exists
playerfile = os.path.join(maindir, charname + '.txt')
if not os.path.exists(playerfile):
MessageBox(document, "Error: file '%s' not found (use the 'Create Hand Lists' first)" % (charname + '.txt'), "File not found", "warningbox")
return False
# Let's see if this file was modified
old_md5 = dispatcherinfo.getCellByPosition(i+4, 31).getString()
current_md5 = get_md5(playerfile)
if old_md5 != current_md5: # File was modified. Set up to send it
to_send.append({'player': playername, 'character': charname, 'playerfile': playerfile, 'md5': current_md5, 'player_id': i})
if not to_send:
MessageBox(document, "Nothing new to send. Maybe you forgot to use Create Hand Lists?", "No files modified!", "infobox")
else:
for p in to_send:
# Now we finally try to send our files
try:
gm = GeekMail(workdir=maindir)
gm.dispatch(p['player'], p['playerfile'])
# Set the current MD5 on the spreadsheet (so that we only send it again after something is changed)
dispatcherinfo.getCellByPosition(p['player_id']+4, 31).setString(p['md5'])
except Exception as e:
MessageBox(document, e.message, "Alert!", "warningbox")
MessageBox(document, "Successfully sent the updated hands to: %s" % (", ".join([e['player'] for e in to_send])), "Success!", "infobox")
def saveTo(self, path):
response = connection.opener.open(self.url)
http_headers = response.info()
data = response.read()
if 'content-disposition' in http_headers.keys():
# The server is answering with a file
cd = http_headers['content-disposition']
fName = re.search('filename="(.*)"', cd).group(1)
else:
# We got a workaround page, Extract real resource url
resourceUrl = Resource.getResourceUrl(data)
fName = basename(resourceUrl) # Get resource name
data = connection.getUrlData(resourceUrl) # Get resource
fName = url2pathname(fName)
print 'Saving ', fName
scraptools.saveResource(data, fName, path) # Save file
def get_style(style='abstract-art', root_path='../Data/Paintings/'):
"""Given a style of painting, downloads all the thumbnails from wikiart
and saves it in a folder"""
domain = "http://www.wikiart.org"
current_link = domain + "/en/paintings-by-style/" + style
ths = crawl_style(current_link)
#new_size = (64,64)
directory = root_path + style
if not os.path.exists(directory):
os.makedirs(directory)
for i in ths:
im = link_to_PILim(i)
if im is None:
continue
file_name = (re.sub(u"[\\\\,\:,\/]", '_', urllib.url2pathname(i)))[31:]
# Optional resize im.resize(new_size)
im.save(directory + '/' + file_name)
print 'Saved ', len(ths), ' files into dir:', style
def saveTo(self, path):
response = connection.opener.open(self.url)
http_headers = response.info()
data = response.read()
if "content-disposition" in http_headers.keys():
# The server is answering with a file
cd = http_headers["content-disposition"]
fName = re.search('filename="(.*)"', cd).group(1)
else:
# We got a workaround page, Extract real resource url
resourceUrl = Resource.getResourceUrl(data)
fName = basename(resourceUrl) # Get resource name
data = connection.getUrlData(resourceUrl) # Get resource
fName = url2pathname(fName)
print "Saving ", fName
scraptools.saveResource(data, fName, path) # Save file
def get_style(style='abstract-art',
root_path='../Data/Paintings/'):
"""Given a style of painting, downloads all the thumbnails from wikiart
and saves it in a folder"""
domain = "http://www.wikiart.org"
current_link = domain + "/en/paintings-by-style/" + style
ths =crawl_style(current_link)
#new_size = (64,64)
directory = root_path + style
if not os.path.exists(directory):
os.makedirs(directory)
for i in ths:
im = link_to_PILim(i)
if im is None:
continue
file_name = (re.sub(u"[\\\\,\:,\/]", '_', urllib.url2pathname(i)))[31:]
# Optional resize im.resize(new_size)
im.save(directory + '/' + file_name)
print 'Saved ', len(ths), ' files into dir:', style
def downloadLatest(d, location='downloads\\', overwrite=False):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@param overwrite Boolean enabling overwriting of a file if it exists.
@return the path to the downloaded file.
"""
try:
name = d['name']
version = getWebVersion(d)
downurl = getDownloadURL(d)
opener.addheaders = userAgent + [('Referer', downurl)]
furl = opener.open(downurl)
parsed = urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
newfileloc = location + name + '---' + version + '---' + filename
print "Downloading: %s" % newfileloc
# if the file doesn't exist or we allow overwriteing write the file
if overwrite or not os.path.exists(newfileloc):
#XXX This needs to be modified to be done in blocks.
filecontents = furl.read()
with open(newfileloc, "wb") as f:
f.write(filecontents)
else:
print 'File already exists and overwriting was not enabled'
print 'when calling downloadLatest(%s, %s, %s)' % (d, location,
overwrite)
furl.close()
except IOError as (errno, strerror):
print 'could not open file, I/O error({0}): {1}'.format(
errno, strerror)
print 'when calling downloadLatest(%s, %s, %s)' % (d, location,
overwrite)
def dispatcher_call(*args):
"""BGG BSG Function to dispatch a generic message via GeekMail"""
document = XSCRIPTCONTEXT.getDocument()
maindir = urllib2.url2pathname(os.path.dirname(document.Location.replace("file://","")))
logfile = os.path.join(maindir, 'bsg-dispatcher-debug.log')
sys.stdout = open(logfile, "a", 0) # unbuffered
# Useful variables so we don't need to do a lot of typing
worksheet = document.getSheets().getByName('Game State')
dispatcherinfo = document.getSheets().getByName('Posting Templates')
dispatchersheet = document.getSheets().getByName('Dispatcher')
playername = dispatchersheet.getCellByPosition(2, 1).getString()
subject = dispatchersheet.getCellByPosition(2, 3).getString()
body = dispatchersheet.getCellByPosition(2, 4).getString()
oldhash = dispatchersheet.getCellByPosition(2, 5).getString()
if not playername:
MessageBox(document, "Error: Username can't be empty", "No username!", 'warningbox')
return False
if not subject:
MessageBox(document, "Error: Subject not defined", "Subject not defined!", 'warningbox')
return False
if not body:
MessageBox(document, "Error: Body is empty", "Empty body!", 'warningbox')
return False
hashish = calculate_md5("%s%s%s" % (playername, subject, body))
if oldhash == hashish:
confirm = MessageBox(document, "It seems we've already sent this data. Send again?", "Data already sent", "querybox", YES_NO)
if confirm == 3: # Pressed "No"
return False
try:
gm = GeekMail(workdir=maindir)
gm.dispatch_text(playername, subject, body)
# Set the current MD5 on the spreadsheet (so that we only send it again after something is changed)
dispatchersheet.getCellByPosition(2, 5).setString(hashish)
except IOError:
MessageBox(document, e.message, "Alert!", "warningbox")
return False
MessageBox(document, "Successfully sent the following message:\n\n%s" % (subject), "Success!", "infobox")
def set_album_art(self, button):
tburl = self.webview.get_main_frame().get_title()
if not tburl:
return
print "Url: " + tburl
filep = tburl.split('/')
filepath = filep[len(filep) - 1]
(filen, filextn) = os.path.splitext(filepath)
request = urllib2.Request(tburl)
opener = urllib2.build_opener()
image = None
try:
image = opener.open(request).read()
except:
print "Failed to download image"
save_filename = self.current_artist + " - " + self.current_album + ".jpg"
if (self.mode == self.MODE_PICTURES):
covers_folder = glib.get_user_special_dir(
glib.USER_DIRECTORY_PICTURES)
filename = covers_folder + "/" + save_filename
if (self.mode == self.MODE_RHYTHM):
covers_folder = os.environ['HOME'] + "/.cache/rhythmbox/covers/"
filename = covers_folder + save_filename
if (os.path.isdir(covers_folder) == False):
os.mkdir(covers_folder)
if (self.mode == self.MODE_FOLDER):
location_path_improper = urllib2.url2pathname(
self.current_location)
location_path_arr = location_path_improper.split("//")
location_path = location_path_arr[1]
filename = location_path.rsplit("/",
1)[0] + "/cover" #+ save_filename
output = open(filename, 'w')
output.write(image)
output.close()
def drag_data_received(self, widget, context, x, y, sel_data, info, time):
"""
Handle the standard gtk interface for drag_data_received.
If the selection data is define, extract the value from sel_data.data,
and decide if this is a move or a reorder.
The only data we accept on mediaview is dropping a file, so URI_LIST.
We assume this is what we obtain
"""
if not sel_data:
return
#modern file managers provide URI_LIST. For Windows split sel_data.data
files = sel_data.get_uris()
for file in files:
if win():
clean_string = conv_to_unicode(
file.replace('\0', ' ').replace("\r", " ").strip(), None)
else:
clean_string = file
protocol, site, mfile, j, k, l = urlparse(clean_string)
if protocol == "file":
name = url2pathname(mfile)
mime = get_type(name)
if not is_valid_type(mime):
return
photo = MediaObject()
self.uistate.set_busy_cursor(True)
photo.set_checksum(create_checksum(name))
self.uistate.set_busy_cursor(False)
base_dir = cuni(media_path(self.dbstate.db))
if os.path.exists(base_dir):
name = relative_path(name, base_dir)
photo.set_path(name)
photo.set_mime_type(mime)
basename = os.path.basename(name)
(root, ext) = os.path.splitext(basename)
photo.set_description(root)
with DbTxn(_("Drag Media Object"), self.dbstate.db) as trans:
self.dbstate.db.add_object(photo, trans)
widget.emit_stop_by_name('drag_data_received')
def downloadLatest(d, location='downloads\\', overwrite=False):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@param overwrite Boolean enabling overwriting of a file if it exists.
@return the path to the downloaded file.
"""
try:
name = d['name']
version = getWebVersion(d)
downurl = getDownloadURL(d)
opener.addheaders=userAgent + [('Referer', downurl)]
furl = opener.open(downurl)
parsed=urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
newfileloc = location + name + '---' + version + '---' + filename
print "Downloading: %s" % newfileloc
# if the file doesn't exist or we allow overwriteing write the file
if overwrite or not os.path.exists(newfileloc):
#XXX This needs to be modified to be done in blocks.
filecontents = furl.read()
with open(newfileloc, "wb") as f:
f.write(filecontents)
else:
print 'File already exists and overwriting was not enabled'
print 'when calling downloadLatest(%s, %s, %s)' %(d, location, overwrite)
furl.close()
except IOError as (errno, strerror):
print 'could not open file, I/O error({0}): {1}'.format(errno, strerror)
print 'when calling downloadLatest(%s, %s, %s)' %(d, location, overwrite)
def set_album_art(self, button) :
tburl = self.webview.get_main_frame().get_title()
if not tburl:
return
print "Url: " + tburl
filep = tburl.split('/')
filepath = filep[len(filep)-1]
(filen, filextn) = os.path.splitext(filepath)
request = urllib2.Request(tburl)
opener = urllib2.build_opener()
image = None
try:
image = opener.open(request).read()
except:
print "Failed to download image"
save_filename=self.current_artist + " - " + self.current_album + ".jpg"
if(self.mode == self.MODE_PICTURES):
covers_folder = glib.get_user_special_dir(glib.USER_DIRECTORY_PICTURES)
filename = covers_folder + "/" + save_filename
if(self.mode == self.MODE_RHYTHM):
covers_folder=os.environ['HOME']+"/.cache/rhythmbox/covers/"
filename = covers_folder + save_filename
if (os.path.isdir(covers_folder) == False):
os.mkdir(covers_folder)
if(self.mode == self.MODE_FOLDER):
location_path_improper = urllib2.url2pathname(self.current_location)
location_path_arr = location_path_improper.split("//")
location_path = location_path_arr[1]
filename = location_path.rsplit("/",1)[0] + "/cover" #+ save_filename
output = open(filename, 'w')
output.write(image)
output.close()
def scrape_bizUrl(self):
global count
self.bizUrl = self.soup.find (id='bizUrl')
if self.bizUrl != None:
count = count + 1
self.bizUrl = self.bizUrl.find('a')
if self.bizUrl != None:
regex = re.compile(r'url=.*src_bizid=')
self.bizUrl = re.findall(regex, self.bizUrl['href'])[0]
self.bizUrl = urllib2.url2pathname ( self.bizUrl[4:-11] )
try:
self.bizSource = urllib2.urlopen( self.bizUrl, timeout=300 )
try:
if 'text/html' in self.bizSource.info()['content-type']:
try:
self.bizSource = self.bizSource.read() #now its just html string
except:
print 'Unable to read the source of url: ' + self.bizUrl.encode('utf-8')
print 'given on this page: ' + self.path
self.bizSource = None
else:
print 'Case 2: url is NOT html, given on this page: ' + self.path
self.bizSource = None
except:
print 'Case 1: url is NOT html, given on this page: ' + self.path
self.bizSource = None
except:
print 'unable to open given link: ' + self.bizUrl.encode('utf-8')
print 'page: ' + self.path
self.bizSource = None
def tidy(self, p1, p2=None):
if not p2:
return url2pathname(p1)
else:
return url2pathname(os.path.join(p1, p2))
def downloadLatest(d, location='downloads\\', overwrite=False):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@param overwrite Boolean enabling overwriting of a file if it exists.
@return the path to the downloaded file.
"""
try:
name = d['name']
logger.info('getting %s web version' % d['name'])
version = getWebVersion(d)
downurl = getDownloadURL(d)
#Sourceforge Antibot Hack
if "sourceforge" in downurl:
opener.addheaders = []
else:
opener.addheaders = userAgent + [('Referer', downurl)]
furl = opener.open(downurl)
furl = opener.open(furl.geturl())
parsed = urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
#create Downloads folder if doesn't exist
if not os.path.exists(location):
os.mkdir(location)
newfileloc = location + name + '---' + version + '---' + filename
logger.info("Downloading: %s" % newfileloc)
# if the file doesn't exist or we allow overwriting write the file
if overwrite or not os.path.exists(newfileloc):
file_size_dl = 0
block_sz = 8192
with open(newfileloc, "wb") as f:
if "Content-Length" in furl.headers:
logger.info("File Size:" + furl.headers["Content-Length"])
while True:
#read in blocks until no more blocks
buffer = furl.read(block_sz)
if not buffer:
break
#add blocks read to blocks downloaded
file_size_dl += len(buffer)
f.write(buffer)
#print progress
p = float(file_size_dl /
float(furl.headers["Content-Length"]))
status = r"{0} [{1:.2%}]".format(file_size_dl, p)
status = status + chr(8) * (len(status) + 1)
sys.stdout.write(status)
f.close()
else:
logger.warning(
'File already exists and overwriting was not enabled \
when calling downloadLatest(%s, %s, %s)' %
(d['name'], location, overwrite))
furl.close()
except TypeError as strerror:
logger.error(
'TypeError: %s, location may not be a string when calling downloadLatest(%s, %s, %s)'
% (strerror, d['name'], location, overwrite))
except urllib2.URLError:
logger.error(
'Could not connect to and read from %s, when calling downloadLatest(%s, %s, %s)'
% (downurl, d['name'], location, overwrite))
except KeyError:
logger.error(
'd did not contain a "name" entry when calling downloadLatest(%s, %s, %s)'
% (d, location, overwrite))
except Exception, e:
logger.error('%s while running downloadLatest(%s, %s, %s)' %
(e, d['name'], location, overwrite))
raise
def get_object(self):
obj_name = urlparse.urlparse(self.urlpath).path
return Object(urllib2.url2pathname(obj_name), None, 0, None)
def make_playlist(playlist, track_db, rootfolder, share_music_files, verbose, dry_run, fname_len, nicer_names):
import urllib2
playlist_folder = os.path.join(rootfolder, sanitize_filename(playlist['Name']))
if verbose: print 'playlist folder:', playlist['Name']
playlist_filename = '%s.m3u' % sanitize_filename(playlist['Name'])
if verbose: print 'playlist file:', playlist_filename
playlist_file = os.path.join(playlist_folder, playlist_filename)
f = None
if dry_run:
print 'Dry run: creating playlist file', playlist_file
else:
if verbose: print 'Creating playlist file', playlist_file
create_filepath(playlist_file)
try:
f = open(playlist_file, 'wb')
s = '#EXTM3U\n'
f.write(s.encode('UTF-8'))
except:
print 'Warning: error opening', playlist_file, 'skipping this playlist'
if f: f.close()
return
for id in playlist['Song IDs']:
if not id in track_db.keys():
print '\tWarning: song', id, 'not found in track_db. Skipping.'
continue
track = track_db[id]
old_filepath = urllib2.url2pathname(track['Location']).replace('file://','')
if not os.path.isfile(old_filepath):
print '\tWarning:', old_filepath, 'not found. Skipping.'
continue;
old_filesize = os.stat(old_filepath).st_size
_, file_extension = os.path.splitext(old_filepath)
if nicer_names:
new_filename = make_a_nicer_filename(old_filepath, track, fname_len)
else:
new_filename = make_a_nice_filename(old_filepath, track, fname_len)
new_filepath = os.path.join(playlist_folder, new_filename.decode('utf-8'))
if os.path.isfile(new_filepath) and os.stat(new_filepath).st_size == old_filesize:
print '\tSkipping:', new_filepath, 'already exists'
else:
if dry_run:
print '\tDry run: copying', old_filepath, ' to ', new_filepath
else:
print '\tCopying', old_filepath, ' to ', new_filepath
try:
shutil.copyfile(old_filepath.decode('utf-8'), new_filepath)
except:
print '\tWarning: ', new_filepath, ' copy failed.'
continue
if f:
try:
s = u'#EXTINF:%d,%s - %s\n' % (int(track['Total Time'])/1000, track['Name'], track['Artist'])
f.write(s.encode('UTF-8'))
s = '%s\n' % new_filename.decode('utf-8')
f.write(s.encode('utf-8'))
except:
print '\tWarning: error writing to playlist file, skipping this playlist'
if f: f.close()
return
if f: f.close()
def dispatch_all(*args):
"""BGG BSG Function to dispatch all player hands"""
document = XSCRIPTCONTEXT.getDocument()
maindir = urllib2.url2pathname(
os.path.dirname(document.Location.replace("file://", "")))
logfile = os.path.join(maindir, 'bsg-dispatcher-debug.log')
sys.stdout = open(logfile, "a", 0) # unbuffered
# Useful variables so we don't need to do a lot of typing
worksheet = document.getSheets().getByName('Game State')
dispatcherinfo = document.getSheets().getByName('Posting Templates')
to_send = []
# Maximum of 7 players
for i in range(7):
playername = worksheet.getCellByPosition(
1, 2 + i).getString() # Player name on Game State
charname = worksheet.getCellByPosition(
4, 2 + i).getString() # Character name on Game State
if not playername: # If there isn't a player for this number, skip it
continue
# Verify if file exists
playerfile = os.path.join(maindir, charname + '.txt')
if not os.path.exists(playerfile):
MessageBox(
document,
"Error: file '%s' not found (use the 'Create Hand Lists' first)"
% (charname + '.txt'), "File not found", "warningbox")
return False
# Let's see if this file was modified
old_md5 = dispatcherinfo.getCellByPosition(i + 4, 31).getString()
current_md5 = get_md5(playerfile)
if old_md5 != current_md5: # File was modified. Set up to send it
to_send.append({
'player': playername,
'character': charname,
'playerfile': playerfile,
'md5': current_md5,
'player_id': i
})
if not to_send:
MessageBox(
document,
"Nothing new to send. Maybe you forgot to use Create Hand Lists?",
"No files modified!", "infobox")
else:
for p in to_send:
# Now we finally try to send our files
try:
gm = GeekMail(workdir=maindir)
gm.dispatch_file(p['player'], p['playerfile'])
# Set the current MD5 on the spreadsheet (so that we only send it again after something is changed)
dispatcherinfo.getCellByPosition(p['player_id'] + 4,
31).setString(p['md5'])
except Exception as e:
MessageBox(document, e.message, "Alert!", "warningbox")
MessageBox(
document, "Successfully sent the updated hands to: %s" %
(", ".join([e['player'] for e in to_send])), "Success!", "infobox")
def do_GET(self):
Log.Debug('Proxy request %s' % self.path)
params = dict([
p.split('=')
for p in self.path[self.path.index('?') + 1:].split('&')
])
if 'd_url' in params and params['d_url']:
url = urllib2.url2pathname(params['d_url'])
else:
try:
info = JSON.ObjectFromURL(urllib2.url2pathname(params['url']),
cacheTime=5)
except:
info = None
if not info:
self.send_error(403)
self.headers['Sisa'] = 'keep-alive'
return None
if 'videos' in info:
info = info['videos']
else:
ext_meta = GetExternalMeta(info)
Log.Debug(ext_meta)
if ext_meta and ext_meta['videos']:
info = ext_meta['videos'].values()
else:
self.send_error(403)
return None
url = None
key = params['key'] + 'p'
for item in info:
if item['key'] == key:
url = item['url']
break
if not url:
url = info[len(info) - 1]['url']
Log.Debug('Start processing %s' % url)
headers = {
'User-Agent': MAILRU_USER_AGENT,
}
for key in ('Range', 'Accept', 'Connection'):
val = self.headers.get(key, None)
if val:
headers[key] = val
try:
val = HTTP.CookiesForURL(url)
if val:
headers['Cookie'] = val
except:
pass
ph = urllib2.urlopen(urllib2.Request(url, headers=headers))
self.protocol_version = 'HTTP/1.1'
self.send_response(ph.getcode())
self.wfile.write('%s' % ph.info())
self.end_headers()
self.copyfile(ph, self.wfile)
else:
# No filePathListArg from the arg parser? Try selected file(s) from nautilus environment variables:
# $NAUTILUS_SCRIPT_SELECTED_FILE_PATHS (only for local storage)
# $NAUTILUS_SCRIPT_SELECTED_URIS
if opt_gui == 'gnome':
# Try to get file(s) provided by nautilus
filePathListEnv = os.environ.get('NAUTILUS_SCRIPT_SELECTED_URIS')
if filePathListEnv != None:
# Check file(s) type and validity
for filePath in filePathListEnv.splitlines():
# Work a little bit of magic (Make sure we have a clean and absolute path, even from an URI)
filePath = os.path.abspath(os.path.basename(filePath))
if sys.version_info >= (3, 0):
filePath = urllib.request.url2pathname(filePath)
else: # python2
filePath = urllib2.url2pathname(filePath)
if checkFile(filePath):
videoPathList.append(filePath)
# ==== Instances dispatcher
# If videoPathList is empty, abort!
if len(videoPathList) == 0:
parser.print_help()
sys.exit(1)
# The first video file will be processed by this instance
videoPath = videoPathList[0]
videoPathList.pop(0)
# The remaining file(s) are dispatched to new instance(s) of this script
def downloadLatest(d, location='downloads\\', overwrite=False):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@param overwrite Boolean enabling overwriting of a file if it exists.
@return the path to the downloaded file.
"""
try:
name = d['name']
version = getWebVersion(d)
downurl = getDownloadURL(d)
#Sourceforge Antibot Hack
if "sourceforge" in downurl:
opener.addheaders=[]
else:
opener.addheaders=userAgent + [('Referer', downurl)]
furl = opener.open(downurl)
furl=opener.open(furl.geturl())
parsed=urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
newfileloc = location + name + '---' + version + '---' + filename
print "Downloading: %s" % newfileloc
# if the file doesn't exist or we allow overwriteing write the file
if overwrite or not os.path.exists(newfileloc):
#XXX This needs to be modified to be done in blocks.
with open(newfileloc, "wb") as f:
block=furl.read(2**20)
written=0.0
percent=0.01
if "Content-Length" in furl.headers:
print "File Size:"+furl.headers["Content-Length"]
while block:
f.write(block)
written+=len(block)
if "Content-Length" in furl.headers:
try:
x=written/float(furl.headers["Content-Length"])
if x>percent:
percent=round(x,2)+.01
sys.stdout.write("..."+str((percent-0.01)*100)+"%...")
except Exception as e:
print e
block=furl.read(2**20)
print ""
else:
print 'File already exists and overwriting was not enabled'
print 'when calling downloadLatest(%s, %s, %s)' %(d, location, overwrite)
furl.close()
except IOError as (errno, strerror):
print 'could not open file, I/O error({0}): {1}'.format(errno, strerror)
print 'when calling downloadLatest(%s, %s, %s)' %(d, location, overwrite)
def read_playlist(infile, absolute_paths=True):
"""Iterates over media files in playlist.
Extended M3U directives are skipped.
:param infile: filename of playlist
:type infile: str
:param absolute_paths: converts relative path names to absolute ones
if set to `True`.
:type absolute_paths: bool
:returns: iterator over `PlaylistEntry` objects.
:rtype: iterable of `PlaylistEntry`
"""
root = os.getcwd()
if infile == '-':
finfile = sys.stdin
else:
finfile = file(infile, "r")
root = os.path.join(root, os.path.dirname(infile))
log.debug("root dir: %s", root)
for filename in finfile.read().strip().split("\n"):
log.debug("entry: %s", filename)
if len(filename
) == 0 or filename[0] == "#": # skip empty and comment lines
log.debug("ignore %s", filename)
continue
# convert thinks like %20 in file name
filename = urllib2.url2pathname(filename)
if filename[:7] == "file://":
filename = filename[7:]
if factory.is_media_file(filename):
log.debug("found %s", filename)
if filename[0] != "/":
full_name = os.path.join(root, filename)
else:
full_name = filename
log.debug("full_name: %s", full_name)
if not os.path.isfile(full_name):
log.info("ignore %s do not exist", full_name)
continue
if absolute_paths:
full_name = os.path.abspath(full_name)
else:
l = len(
os.path.commonprefix((root, full_name)).rpartition("/")[0])
if l > 0:
full_name = full_name[l + 1:]
log.debug("post full_name: %s", full_name)
yield factory.create_entry(full_name)
elif factory.is_media_url(filename):
yield factory.create_entry(filename)
else:
log.info("ignore %s", filename)
finfile.close()
def import_new_db(self, filename, user):
"""
Attempt to import the provided file into a new database.
A new database will only be created if an appropriate importer was
found.
:param filename: a fully-qualified path, filename, and
extension to open.
:param user: a :class:`.cli.user.User` or :class:`.gui.user.User`
instance for managing user interaction.
:returns: A tuple of (new_path, name) for the new database
or (None, None) if no import was performed.
"""
pmgr = BasePluginManager.get_instance()
# check to see if it isn't a filename directly:
if not os.path.isfile(filename):
# Allow URL names here; make temp file if necessary
url = urlparse(filename)
if url.scheme != "":
if url.scheme == "file":
filename = url2pathname(filename[7:])
else:
url_fp = urlopen(filename) # open URL
# make a temp local file:
ext = os.path.splitext(url.path)[1]
fd, filename = tempfile.mkstemp(suffix=ext)
temp_fp = os.fdopen(fd, "w")
# read from URL:
data = url_fp.read()
# write locally:
temp_fp.write(data)
url_fp.close()
from gen.db.dbconst import BDBVERSFN
versionpath = os.path.join(name, BDBVERSFN)
_LOG.debug("Write bsddb version %s" % str(dbase.version()))
with open(versionpath, "w") as version_file:
version_file.write(str(dbase.version()))
temp_fp.close()
(name, ext) = os.path.splitext(os.path.basename(filename))
format = ext[1:].lower()
for plugin in pmgr.get_import_plugins():
if format == plugin.get_extension():
new_path, name = self._create_new_db(name)
# Create a new database
self.__start_cursor(_("Importing data..."))
dbclass = DbBsddb
dbase = dbclass()
dbase.load(new_path, user.callback)
import_function = plugin.get_import_function()
import_function(dbase, filename, user)
# finish up
self.__end_cursor()
dbase.close()
return new_path, name
return None, None
def sortFiles(indexname):
print indexname
#get pathname corresponding to indexname
#check to make sure it is a directory
#get list of filenames in that directory
p_files = url2pathname(test_paths.get(indexname))
#Use this for TDrive original files
p_source = url2pathname(paths.get(indexname))
#Use this for local copy of test files
#p_source = test_paths.get(indexname)
if os.path.isdir(p_source):
filenames = os.listdir(p_source)
print 'Total number of files to be sorted:' + str(len(filenames))
else:
return
#set path and filenames for storing index files
g = indexname + '_good.csv'
b = indexname + '_bad.csv'
#store the index files in the same directory as original files
p_g = url2pathname(os.path.join(p_files, g))
p_b = url2pathname(os.path.join(p_files, b))
#open index files
goodfile = open(p_g, 'w+')
badfile = open(p_b, 'w+')
#create csv writers for index files
goodwriter = csv.writer(goodfile, dialect=csv.excel)
badwriter = csv.writer(badfile, dialect=csv.excel)
#create temporary lists to hold good and bad filenames
tmp_good = list()
tmp_bad = list()
#create an instance of msword
word = win32com.client.Dispatch('Word.Application')
word.Visible = False
#--------Sampler - remove for production------------------
#filenames = shuffle(filenames)
i = 0
#--------End of Sampler - remove for production------------------
#start iterating through filenames
for aFile in filenames[10:20]:
#if this is not a word document or is a temporary file, move on
if aFile.find('.doc') < 0 or aFile.find('~') >= 0: continue
try:
#set docname to filepath + filename
#url2pathname avoids %20
docname = url2pathname(os.path.join(p_files, aFile))
# open the document
doc = word.Documents.Open(docname, False, False, False)
result = scanForNHSIds(doc.Content.Text)
if len(result) == 12:
result = result.replace(' ', '') #get rid of spaces
pdfname = result + '.pdf'
pdfpath = url2pathname(os.path.join(p_files, 'good', pdfname))
#if the file already exists, save with timestamp
#this is an unlikely scenario
if os.path.exists(pdfpath):
pdfname = result + '_multifile_' + strftime(
'%H_%M_%S', gmtime()) + '.pdf'
pdfpath = url2pathname(
os.path.join(p_files, 'good', pdfname))
#FileFormat 17 is pdf
doc.SaveAs(pdfpath, FileFormat=17)
tmp_good.append([aFile, result])
else:
docpath = url2pathname(
os.path.join(p_files, 'bad', result + '-' + aFile))
doc.SaveAs(docpath)
tmp_bad.append([aFile, result])
#close the word document if it has been opened
if doc: doc.Close()
#export count
except:
result = 'rd_err'
tmp_bad.append([aFile, result])
i += 1
if i > 1000: break
print 'Successful PDF exports to good folder: ' + str(len(tmp_good))
print 'Files with errors copied to bad folder: ' + str(len(tmp_bad))
#save list to index file
for a in tmp_good:
goodwriter.writerow(a)
for a in tmp_bad:
badwriter.writerow(a)
goodfile.close()
badfile.close()
del doc
del word
def downloadLatest(d, location='downloads\\', overwrite=False):
"""Download the latest version of the package d.
Use the information specified in the package d to download the latest
version of the package from the web. The default download location is
'./downloads'
@param d The dictionary entry for a package, containing at least a 'name',
as well as a 'version', and 'download' dict containing 'url', 'regex', and
'regexpos'.
@param location The location to download the file to.
@param overwrite Boolean enabling overwriting of a file if it exists.
@return the path to the downloaded file.
"""
try:
name = d['name']
logger.info('getting %s web version' %d['name'])
version = getWebVersion(d)
downurl = getDownloadURL(d)
#Sourceforge Antibot Hack
if "sourceforge" in downurl:
opener.addheaders=[]
else:
opener.addheaders=userAgent + [('Referer', downurl)]
furl = opener.open(downurl)
furl=opener.open(furl.geturl())
parsed=urllib2.urlparse.urlparse(furl.geturl())
pathname = urllib2.url2pathname(parsed.path)
filename = pathname.split("\\")[-1]
#create Downloads folder if doesn't exist
if not os.path.exists(location):
os.mkdir(location)
newfileloc = location + name + '---' + version + '---' + filename
logger.info( "Downloading: %s" % newfileloc)
# if the file doesn't exist or we allow overwriting write the file
if overwrite or not os.path.exists(newfileloc):
file_size_dl = 0
block_sz = 8192
with open(newfileloc, "wb") as f:
if "Content-Length" in furl.headers:
logger.info( "File Size:"+furl.headers["Content-Length"])
while True:
#read in blocks until no more blocks
buffer = furl.read(block_sz)
if not buffer:
break
#add blocks read to blocks downloaded
file_size_dl += len(buffer)
f.write(buffer)
#print progress
p = float(file_size_dl / float(furl.headers["Content-Length"]))
status = r"{0} [{1:.2%}]".format(file_size_dl,p)
status = status + chr(8)*(len(status)+1)
sys.stdout.write(status)
f.close()
else:
logger.warning( 'File already exists and overwriting was not enabled \
when calling downloadLatest(%s, %s, %s)' %(d['name'], location, overwrite))
furl.close()
except TypeError as strerror:
logger.error( 'TypeError: %s, location may not be a string when calling downloadLatest(%s, %s, %s)' %(strerror,d['name'], location, overwrite))
except urllib2.URLError:
logger.error( 'Could not connect to and read from %s, when calling downloadLatest(%s, %s, %s)' %(downurl,d['name'], location, overwrite))
except KeyError:
logger.error( 'd did not contain a "name" entry when calling downloadLatest(%s, %s, %s)' %(d, location, overwrite))
except Exception, e:
logger.error( '%s while running downloadLatest(%s, %s, %s)' %(e,d['name'], location, overwrite))
raise
else:
# No filePathListArg from the arg parser? Try selected file(s) from nautilus environment variables:
# $NAUTILUS_SCRIPT_SELECTED_FILE_PATHS (only for local storage)
# $NAUTILUS_SCRIPT_SELECTED_URIS
if gui == 'gnome':
# Try to get file(s) provided by nautilus
filePathListEnv = os.environ.get('NAUTILUS_SCRIPT_SELECTED_URIS')
if filePathListEnv != None:
# Check file(s) type and validity
for filePath in filePathListEnv.splitlines():
# Work a little bit of magic (Make sure we have a clean and absolute path, even from an URI)
filePath = os.path.abspath(os.path.basename(filePath))
if sys.version_info >= (3,0):
filePath = urllib.request.url2pathname(filePath)
else: # python2
filePath = urllib2.url2pathname(filePath)
if checkFile(filePath):
videoPathList.append(filePath)
# ==== Instances dispatcher
# If videoPathList is empty, abort!
if len(videoPathList) == 0:
parser.print_help()
sys.exit(1)
# The first video file will be processed by this instance
videoPath = videoPathList[0]
videoPathList.pop(0)
# The remaining file(s) are dispatched to new instance(s) of this script
def dispatch_selected(*args):
"""BGG BSG Function to dispatch a single user hand"""
document = XSCRIPTCONTEXT.getDocument()
maindir = urllib2.url2pathname(
os.path.dirname(document.Location.replace("file://", "")))
logfile = os.path.join(maindir, 'bsg-dispatcher-debug.log')
sys.stdout = open(logfile, "a", 0) # unbuffered
# Useful variables so we don't need to do a lot of typing
worksheet = document.getSheets().getByName('Game State')
dispatcherinfo = document.getSheets().getByName('Posting Templates')
# Find the selected char
selected_char = worksheet.DrawPage.Forms.getByName(
'formGameState').getByName('lstPlayers').CurrentValue
selected_player = ''
if not selected_char:
MessageBox(document, "Error: no player selected", "Invalid player",
"warningbox")
return False
# Find out which player we're looking at
for i in range(7):
charname = worksheet.getCellByPosition(
4, 2 + i).getString() # Character name on Game State
if charname == selected_char:
selected_player = worksheet.getCellByPosition(
1, 2 + i).getString() # Player name on Game State
player_id = i
break
else:
MessageBox(document, "Error: player not found, maybe a bug?",
"Invalid player", "warningbox")
return False
# Verify if file exists
playerfile = os.path.join(maindir, selected_char + '.txt')
if not os.path.exists(playerfile):
MessageBox(
document,
"Error: file '%s' not found (use the 'Create Hand Lists' first)" %
(selected_char + '.txt'), "File not found", "warningbox")
return False
# Verify if we already sent this file
old_md5 = dispatcherinfo.getCellByPosition(player_id + 4, 31).getString()
current_md5 = get_md5(playerfile)
if old_md5 == current_md5: # We DID send this already!!!
confirm = MessageBox(
document, "It seems we've already sent this file. Send again?",
"File already sent", "querybox", YES_NO)
if confirm == 3: # Pressed "No"
return False
# Now we finally try to send our files
try:
gm = GeekMail(workdir=maindir)
gm.dispatch_file(selected_player, playerfile)
# Set the current MD5 on the spreadsheet (so that we only send it again after something is changed)
dispatcherinfo.getCellByPosition(player_id + 4,
31).setString(current_md5)
except Exception as e:
MessageBox(document, e.message, "Alert!", "warningbox")
return False
MessageBox(document, "Successfully sent file to %s" % selected_player,
"Success!", "infobox")
def __makepath(self, path):
# returns os absolute path from relative path
pathparts=urllib2.url2pathname(path).split(os.path.sep)
return os.path.join(self._base, *pathparts)
def __makepath(self, path):
# returns os absolute path from relative path
pathparts=urllib2.url2pathname(path).split(os.path.sep)
return os.path.join(self._base, *pathparts)
