def get_html_string(url):
'''
This method takes a url (string) of a webpage, and connects to the URL,
then downloads, htmldecodes, and returns the contents of that page as
an html string.
This method will throw an WebException or IOException if anything goes wrong,
including if the response code is not valid (i.e. 200).
'''
try:
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12
request = WebRequest.Create(url)
request.UserAgent = "[ComicVineScraper, version " + \
Resources.SCRIPT_VERSION + "]"
response = request.GetResponse()
# if the response code is not "OK", throw a web exception immediately.
# this stops red-herring errors later on as we try to parse bad results.
# usually this only happens if the CV server is temporarily down.
if response.StatusCode != HttpStatusCode.OK:
raise WebException("server response code " +
sstr(int(response.StatusCode)) + " (" +
sstr(response.StatusCode) + ")")
responseStream = response.GetResponseStream()
reader = StreamReader(responseStream, Encoding.UTF8)
page = reader.ReadToEnd()
with StringWriter() as writer:
HttpUtility.HtmlDecode(page, writer)
page = writer.ToString()
return page
finally:
if 'reader' in vars(): reader.Close()
if 'responseStream' in vars(): responseStream.Close()
if 'response' in vars(): response.Close()
def download_tts(file_path, text, settings):
with WebClient() as wc:
url = "https://translate.google.com/translate_tts?ie=UTF-8&tl={1}&client=tw-ob&q={0}".format(
HttpUtility.UrlEncode(text), settings["lang"])
wc.Headers["Referer"] = "http://translate.google.com/"
wc.Headers["User-Agent"] = "stagefright/1.2 (Linux;Android 5.0)"
wc.DownloadFile(url, file_path)
def _query_series_ids_dom(API_KEY, searchterm_s, page_n=1):
'''
Performs a query that will obtain a dom containing all the comic book series
from ComicVine that match a given search string. You can also provide a
second argument that specifies the page of the results (each page contains
100 results) to display. This is useful, because this query will not
necessarily return all available results.
This method doesn't return null, but it may throw Exceptions.
'''
# {0} is the search string, {1} is the page number of the results we want
QUERY = 'http://comicvine.com/api/search/?api_key=' + API_KEY + \
__CLIENTID + '&format=xml&limit=100&resources=volume' + \
'&field_list=name,start_year,publisher,id,image,count_of_issues' + \
'&query={0}'
# leave "page=1" off of query to fix a bug, e.g. search for 'bprd vampire'
PAGE = "" if page_n == 1 else "&page={0}".format(page_n)
if searchterm_s is None or searchterm_s == '' or page_n < 0:
raise ValueError('bad parameters')
searchterm_s = " AND ".join(re.split(r'\s+', searchterm_s))
# issue 349
return __get_dom(
QUERY.format(HttpUtility.UrlPathEncode(searchterm_s)) + PAGE)
def getTimesheetsFromZendesk(templateQueryContext):
# --- setup zendesk settings ---
serverURL = "https://yourZendeskHost"
userName = "******"
token = "yourZendeskToken"
# --- setup zendesk settings ---
clr.AddReference("System.Web")
clr.AddReference("System.Net.Http")
clr.AddReference("Newtonsoft.Json")
from System.Text import Encoding
from Newtonsoft.Json.Linq import JObject
from System.Collections.Generic import List
from System.Web import HttpUtility
from System.Net.Http import HttpClient
from System.Net.Http.Headers import AuthenticationHeaderValue
user = Context.SelectSingleWithParams({
"Query": "From U In UserDetail Where U.UserDetailUuid = @UserDetailUuid Select U",
"@UserDetailUuid": templateQueryContext.UserDetailUuid
})
query = HttpUtility.UrlEncode("type:ticket updated:" + templateQueryContext.BeginTime.ToString("yyyy-MM-dd") + " assignee:" + user.Username)
queryURL = serverURL + "/api/v2/search.json?query=" + query
credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(userName + "/token:"+ token));
modelEntity = ModelEntity({ "Name": "Result" })
modelEntity.Properties.Add(TextProperty({ "Name": "Description" }))
modelEntity.Properties.Add(TextProperty({ "Name": "ResultTitle" }))
modelEntity.Properties.Add(TextProperty({ "Name": "ResultSortOrder" }))
result = List[EntityObject]()
with HttpClient() as client:
client.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue("Basic", credentials)
response = client.GetAsync(queryURL).Result
response.EnsureSuccessStatusCode()
text = response.Content.ReadAsStringAsync().Result
json = JObject.Parse(text)
for item in json.results:
id = str(item.id)
name = str(item.via.source["from"].name)
subject = str(item.subject)
status = str(item.status)
entity = modelEntity.CreateEntityObject()
entity.Description = "#" + id + ": " + subject + " (" + name + ", " + status + ")"
entity.ResultTitle = entity.Description
entity.ResultSortOrder = entity.Description
result.Add(entity)
templateQueryContext.Templates = result
def _query_issue_id_dom(API_KEY, seriesid_s, issue_num_s):
'''
Performs a query that will obtain a dom containing the issue ID for the
given issue number in the given series id.
This method doesn't return null, but it may throw Exceptions.
'''
# {0} is the series ID, an integer, and {1} is issue number, a string
QUERY = 'http://comicvine.com/api/issues/?api_key=' + API_KEY + \
__CLIENTID + '&format=xml&field_list=name,issue_number,id,image' + \
'&filter=volume:{0},issue_number:{1}'
if not seriesid_s or not issue_num_s:
raise ValueError('bad parameters')
return __get_dom(
QUERY.format(sstr(seriesid_s),
HttpUtility.UrlPathEncode(sstr(issue_num_s))))
def _query_issue_id_dom(API_KEY, seriesid_s, issue_num_s):
'''
Performs a query that will obtain a dom containing the issue ID for the
given issue number in the given series id.
This method doesn't return null, but it may throw Exceptions.
'''
# {0} is the series ID, an integer, and {1} is issue number, a string
QUERY = 'http://comicvine.gamespot.com/api/issues/?api_key=' + API_KEY + \
__CLIENTID + '&format=xml&field_list=name,issue_number,id,image' + \
'&filter=volume:{0},issue_number:{1}'
# cv does not play well with leading zeros in issue nums. see issue #403.
issue_num_s = sstr(issue_num_s).strip()
if len(issue_num_s) > 0: # fix issue 411
issue_num_s = issue_num_s.lstrip('0').strip()
issue_num_s = issue_num_s if len(issue_num_s) > 0 else '0'
if not seriesid_s or not issue_num_s:
raise ValueError('bad parameters')
return __get_dom(
QUERY.format(sstr(seriesid_s),
HttpUtility.UrlPathEncode(sstr(issue_num_s))))
def __str__(self):
''' create string respresentation of individual request '''
# default to json format for get queries
if self.method == OdataRequest.Method.GET and "format" not in self.query.keys(
):
self.query["format"] = "json"
# request
requestUrl = self.url
for key in self.query.keys():
if self.query.keys()[0] == key: # is first
requestUrl += "?"
requestUrl += "${key}={value}".format(key=key,
value=HttpUtility.UrlEncode(
self.query[key]))
if self.query.keys()[-1] != key: # not last
requestUrl += "&"
request = " ".join([self.method, requestUrl, "HTTP/1.1"])
# headers
if self.method != OdataRequest.Method.GET:
request += "\nContent-Type: " + self.contentType
if self.contentId is not None:
request += "\nContent-ID: " + self.contentId
if self.body is not None:
request += "\nContent-Length: " + str(len(self.body))
if self.method != OdataRequest.Method.GET:
request += "\nAccept: " + self.accept
# body
if self.body is not None:
request += "\n\n" + self.body
else:
request += "\n"
return request
def get_html_string(url):
'''
This method takes a url (string) of a webpage, and connects to the URL,
then downloads, htmldecodes, and returns the contents of that page as
an html string.
This method will throw an WebException or IOException if anything goes wrong,
including if the response code is not valid (i.e. 200).
'''
try:
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12
request = WebRequest.Create(url)
# latest version of comicvine api insists on a user agent (see bug #484).
# previously, it didn't like my non-standard user agent, I think (see bug #471).
# now, I have a standard user agent, which comicvine api seems to like.
request.UserAgent = "ComicVineScraper/" + \
Resources.SCRIPT_VERSION + " (https://github.com/cbanack/comic-vine-scraper/)"
response = request.GetResponse()
# if the response code is not "OK", throw a web exception immediately.
# this stops red-herring errors later on as we try to parse bad results.
# usually this only happens if the CV server is temporarily down.
if response.StatusCode != HttpStatusCode.OK:
raise WebException("server response code " +
sstr(int(response.StatusCode))+" ("+sstr(response.StatusCode)+")" )
responseStream = response.GetResponseStream()
reader = StreamReader(responseStream, Encoding.UTF8)
page = reader.ReadToEnd()
with StringWriter() as writer:
HttpUtility.HtmlDecode(page, writer)
page = writer.ToString()
return page
finally:
if 'reader' in vars(): reader.Close()
if 'responseStream' in vars(): responseStream.Close()
if 'response' in vars(): response.Close()
def search(self, jql):
clr.AddReference("System.Web")
from System.Web import HttpUtility
from System.Net import HttpWebRequest
from System.IO import StreamReader
clr.AddReference("Newtonsoft.Json")
from Newtonsoft.Json import JsonTextReader
from Newtonsoft.Json.Linq import JObject
from System import Decimal
import Newtonsoft.Json
clr.ImportExtensions(Newtonsoft.Json.Linq)
usernamepw = Convert.ToBase64String(
Encoding.UTF8.GetBytes(
String.Format("{0}:{1}", self.username, self.password)))
fieldsparam = String.Join(",", self.requestedFields)
requestUri = String.Format(
"{0}rest/api/2/search?jql={1}&fields={2}",
self.repository.AbsoluteUri, HttpUtility.UrlEncode(jql),
fieldsparam)
Logger.Write(LogLevel.Verbose, "Jira.Search: {0}", requestUri)
request = HttpWebRequest.Create(requestUri)
request.ContentType = "application/json"
request.Headers.Add("Authorization", "Basic " + usernamepw)
request.Method = "GET"
with request.GetResponse() as response:
with StreamReader(response.GetResponseStream()) as sr:
with JsonTextReader(sr) as jr:
result = JObject.Load(jr)
issues = result["issues"]
items = list()
for issue in issues:
item = Issue()
item.Key = Newtonsoft.Json.Linq.Extensions.Value[
String](issue["key"])
fields = issue["fields"]
item.Updated = Newtonsoft.Json.Linq.Extensions.Value[
DateTime](fields["updated"])
# transform seconds to hours
estimate = Newtonsoft.Json.Linq.Extensions.Value[
System.Object](fields["timeoriginalestimate"])
if estimate is not None:
estimate = Newtonsoft.Json.Linq.Extensions.Value[
Decimal](fields["timeoriginalestimate"])
estimate = estimate / (60.0 * 60.0)
item.TimeOriginalEstimate = estimate
status = fields["status"]
item.Status = Newtonsoft.Json.Linq.Extensions.Value[
String](status["name"])
item.Summary = Newtonsoft.Json.Linq.Extensions.Value[
String](fields["summary"])
type = fields["issuetype"]
item.Type = Newtonsoft.Json.Linq.Extensions.Value[
String](type["name"])
item.Link = self.repository.ToString(
) + "browse/" + item.Key
subTasks = fields["subtasks"]
item.SubTaskKeys = System.Linq.Enumerable.Cast[
JObject](subTasks).Select(
lambda t: Newtonsoft.Json.Linq.Extensions.
Value[String](t["key"])).ToArray[String]()
items.Add(item)
return items
def encode(self):
return HttpUtility.UrlEncodeUnicode(self.str).Replace('\'', '%27').Replace('%2f', '/').Replace('+', '%20')