def open(self, url, data=None, max_tries=None):
"""
Ouvrir une url
@param url
@param data les data (POST)
@param max_tries le nombre maximum d'essais, si non préciser la valeur donnée lors de l'initialisation sera prise
@return un stream
@throw ExceptionMaxTries quand le nombre maximum de tentatives est atteind
@throw ExceptionUrlForbid quand le robot n'a pas le droit de visiter l'url
"""
if not max_tries:
max_tries = self.max_tries
else:
self.max_tries = max_tries
if not self.robot or self.robot.can_fetch(url):
for _ in range(max_tries):
try:
stream = FancyURLopener.open(self, url, data)
except Exception as ex:
error = get_traceback()+"\n"+str(ex)
else:
return stream
else:
raise ExceptionMaxTries("max tries %s : %s" % (url, error))
else:
raise ExceptionUrlForbid("robots can't access to %s" % url)
def ach5():
z = t7.get("1.0", END)
print(f)
print(k)
print(g)
sql1 = "SELECT mobile FROM customer_register WHERE email = '" + g + "' "
sql = "UPDATE complaints SET solution ='" + z + "' WHERE cid = '" + g + "' AND comp_heading = '" + k + "' "
db = pymysql.connect("localhost", "root", "", "cms")
c = db.cursor()
c.execute(sql)
try:
c.execute(sql1)
achaa = c.fetchone()
db.commit()
print(achaa)
tinn = FancyURLopener()
DL = []
for i in achaa:
DL.append(i)
ok = achaa[0]
phone = ok
yourmsg = "Hi customer, The solution of the complain '" + k + "' on '" + g + "' has been provided to your registered account. Kindly check it. \nThank you \nCMS "
page = tinn.open(
'http://5.189.169.241:5012/api/SendSMS?api_id=API245772015763&api_password=12345678&sms_type=T&encoding=T&sender_id=BLKSMS&phonenumber='
+ str(phone) + '&textmessage="' + yourmsg +
'"')
except:
print("Invalid No.")
db.close()
tech1()
def getFeed(url):
try:
opener = FancyURLopener()
page = opener.open(url)
contents = page.read().decode('utf-8')
feed = feedparser.parse(contents)
return feed
except:
return None
def getFeed(url):
try:
opener = FancyURLopener()
page = opener.open(url)
contents = page.read().decode('utf-8')
feed=feedparser.parse(contents)
return feed
except:
return None
def getBashorg(): #Получить цитату с баша
FancyURLopener.version = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0'
myopener = FancyURLopener()
page = myopener.open('https://bash.im/forweb/')
s = page.read().decode("cp1251").replace("<\' + \'br>", "\\n").replace(
""", "\"").replace("<", "<").replace(">", ">")
page.close()
strt = s.index('id="b_q_t" style="padding: 1em 0;">') + len(
'id="b_q_t" style="padding: 1em 0;">')
s = s[strt:]
return s[:s.index("<\'")]
def download(who, url):
tmp = FancyURLopener()
url = tmp.open(url)
if 2 < sys.version_info.major:
path = url.info().get_filename()
else:
path = url.info().get('Content-Disposition').split('=')[1].strip('"')
path = os.path.basename(path)
with open(path, 'wb') as f:
while True:
tmp = url.read(4096)
if 0 == len(tmp):
break
f.write(tmp)
f.close()
return path
def urlopen(url):
charset = None
tmp = FancyURLopener()
tmp = tmp.open(url)
if 2 < sys.version_info.major:
charset = tmp.info().get_content_charset()
else:
charset = 'windows-1251'
tmp = tmp.read().decode(charset)
if str != type(tmp):
tmp = str(tmp.encode('utf-8'))
return tmp
def run(self):
opener = FancyURLopener()
try:
remotefile = opener.open(self.url)
except IOError:
logic.logger.new("Unable to connect to internet", 'ERROR')
return
if remotefile.getcode() == 404:
logic.logger.new("PDB file not found on pdb.org", 'ERROR')
return
elif remotefile.getcode() >= 500:
logic.logger.new("PDB.org is currently unavailable", 'ERROR')
return
localfile = open(self.pdbFullPath, 'wb')
localfile.write(remotefile.read())
localfile.close()
remotefile.close()
def file_download(self, url):
urlx = 'http:' + url
imgname = url.split('/')[-1]
if imgname.split('.')[-1] == 'gif':
imgPath = os.path.join(PATH, 'gif', imgname)
else:
imgPath = os.path.join(PATH, 'pic', imgname)
if not os.path.lexists(imgPath):
# urllib.request.urlretrieve(urlx, imgPath)
opener = FancyURLopener()
opener.addheaders.clear()
opener = self.insert_header(opener, urlx, imgPath)
with open(imgPath, 'wb+') as f:
while True:
chunk = opener.open(urlx, self._chunk_size)
if not chunk: break
f.write(chunk)
def gen_qrcode(self):
url = "https://zxing.org/w/chart?cht=qr&chs=350x350&chld=L&choe=UTF-8&chl=" + self.mecard(
)
webbrowser.open(url)
png_file = "{}.png".format(self.fname)
# urlretrieve(url, png_file)
# with urlopen(url) as response, open("test.png", 'wb') as out_file:
# data = response.read() # a `bytes` object
# out_file.write(data)
# with urlopen(url) as response, open(png_file, 'wb') as out_file:
# shutil.copyfileobj(response, out_file)
# credit: http://stackoverflow.com/questions/7243750/download-file-from-web-in-python-3
# credit: http://stackoverflow.com/questions/31893207/python-3-4-http-error-505-retrieving-json-from-url
url_request = FancyURLopener({})
with url_request.open(url) as response, open(png_file,
'wb') as out_file:
shutil.copyfileobj(response, out_file)
def __init__(self, url, level):
super(myParser, self).__init__()
sleep(0.1)
print('Checking URL', url)
self.__level = level
self.__done = False
self.__currentlyParsingDeadATag = False
self.__currentlyParsingTitleTag = False
self.__url = url
self.linkWasDead = False
parsedURLs.append(self.__url)
try:
opener = FancyURLopener({})
f = opener.open(self.__url)
data = f.read()
except ssl.SSLError:
return
except OSError:
return
except ValueError:
if not self.__url in deadURLs:
print()
print('Found a dead link:', self.__url)
deadURLs.append(self.__url)
self.linkWasDead = True
self.__done = True
return
try:
text = data.decode(errors='replace')
except UnicodeDecodeError:
pass
#print('This is a binary file:', self.__url)
else:
try:
self.feed(text)
except ValueError:
pass
except ssl.SSLError:
pass
def main():
import argparse
parser = argparse.ArgumentParser(description='Train model.')
parser.add_argument('data_file',
metavar='data_file',
type=str,
nargs=1,
help='file with count data')
parser.add_argument("-s", "--settings", default=None)
parser.add_argument("-j", "--jobs", type=int, default=None)
parser.add_argument("-p",
"--preprocessing",
choices=['normalize_sf'],
default=None)
parser.add_argument("-n", "--nosubset", action='store_true')
args = parser.parse_args()
if args.settings is not None:
settings = h.load_settings(args.settings)
else:
import outpyr.settings as settings
if validate_web_url(args.data_file[0]):
if args.data_file[0].endswith('/'):
data_file = args.data_file[0][:-1].rsplit('/', 1)[1]
else:
data_file = args.data_file[0].rsplit('/', 1)[1]
if os.path.isfile(data_file):
print(
'It seems that the provided URL was already downloaded to file %s. Skipping downloading.'
% data_file)
else:
print('Downloading URL', args.data_file[0], '...')
opener = FancyURLopener({})
with opener.open(args.data_file[0]) as f:
text = f.read().decode('utf-8')
print('Finished!')
with open('tmp.txt', 'w', encoding='utf-8') as f:
f.write(text)
df = h.csv_to_df('tmp.txt')
if not args.nosubset and (list(
df.index)) == helpers_kremer.INDEX_FULL and (list(
df.columns)) == helpers_kremer.COLUMNS:
print('Kremer dataset recognized, filtering genes...')
df = df.loc[helpers_kremer.INDEX_FILTERED, :]
print('Done!')
h.save_df_to_csv(df, data_file)
os.remove('tmp.txt')
else:
data_file = args.data_file[0]
base_name, ext = os.path.splitext(os.path.basename(data_file))
dir_ = os.path.abspath(os.path.dirname(data_file))
pvalues_file = os.path.join(dir_, base_name + '-pvalues.csv')
pvalues_std_file = os.path.join(dir_, base_name + '-pvalues-std.csv')
pvalues_sample_adjusted_file = os.path.join(
dir_, base_name + '-pvalues-adjusted.csv')
pvalues_gene_adjusted_file = os.path.join(
dir_, base_name + '-pvalues-gene-adjusted.csv')
zscores_file = os.path.join(dir_, base_name + '-zscores.csv')
if os.path.isfile(data_file):
print('Running OutPyR on', data_file, '...')
if args.jobs is None:
gene_subset = None
output_dir = outpyr.train_cpu_singleprocess.run(
data_file, settings, args.preprocessing, gene_subset,
'p_j_r_j_numba')
else:
output_dir = outpyr.train_cpu_multiprocess.run(
data_file, settings, args.preprocessing, args.jobs)
dir_abs = os.path.join(dir_, output_dir)
from outpyr import helpers_tensorflow as htf
ti = htf.TraceInspector(dir_abs)
if 'p_values_mean' not in ti.v:
print('Post-sampling: calculating p-values...')
ti.set_final_values_from_trace()
print('Done!')
print('Saving p-values inside of the trace directory...')
with open(os.path.join(ti.trace_data_folder, 'variables.pickle'),
'wb') as f_variables:
pickle.dump(ti.v, f_variables)
print('Finished!')
print('Saving scores as CSV files...')
h.save_df_to_csv(
pd.DataFrame(ti.get_p_value_matrix(),
index=ti.df.index,
columns=ti.df.columns), pvalues_file)
h.save_df_to_csv(
pd.DataFrame(np.sqrt(ti.v['p_values_var']),
index=ti.df.index,
columns=ti.df.columns), pvalues_std_file)
else:
parser.error("The file (%s) you provided does not exist." % data_file)
def main():
import argparse
parser = argparse.ArgumentParser(description='Train model.')
parser.add_argument('data_file',
metavar='data_file',
type=str,
nargs=1,
help='file with count data')
parser.add_argument("-s", "--settings", default=None)
parser.add_argument("-p",
"--preprocessing",
choices=['normalize_sf'],
default=None)
parser.add_argument("-n", "--nosubset", action='store_true')
args = parser.parse_args()
if args.settings is not None:
settings = h.load_settings(args.settings)
else:
import outpyr.settings as settings
if validate_web_url(args.data_file[0]):
if args.data_file[0].endswith('/'):
data_file = args.data_file[0][:-1].rsplit('/', 1)[1]
else:
data_file = args.data_file[0].rsplit('/', 1)[1]
if os.path.isfile(data_file):
print(
'It seems that the provided URL was already downloaded to file %s. Skipping downloading.'
% data_file)
else:
print('Downloading URL', args.data_file[0], '...')
opener = FancyURLopener({})
with opener.open(args.data_file[0]) as f:
text = f.read().decode('utf-8')
print('Finished!')
with open('tmp.txt', 'w', encoding='utf-8') as f:
f.write(text)
df = h.csv_to_df('tmp.txt')
if not args.nosubset and (list(
df.index)) == helpers_kremer.INDEX_FULL and (list(
df.columns)) == helpers_kremer.COLUMNS:
print('Kremer dataset recognized, filtering genes...')
df = df.loc[helpers_kremer.INDEX_FILTERED, :]
print('Done!')
h.save_df_to_csv(df, data_file)
os.remove('tmp.txt')
else:
data_file = args.data_file[0]
base_name, ext = os.path.splitext(os.path.basename(data_file))
dir_ = os.path.abspath(os.path.dirname(data_file))
pvalues_file = os.path.join(dir_, base_name + '-opx-pv.csv')
if os.path.isfile(data_file):
print('Running OutPyRX on', data_file, '...')
gene_subset = None
output_dir = outpyr.train_cpu_singleprocess.run(
data_file, settings, args.preprocessing, gene_subset,
'p_j_r_j_numba')
dir_abs = os.path.join(dir_, output_dir)
from outpyr import helpers_tensorflow as htf
ti = htf.TraceInspector(dir_abs)
if 'p_values_mean' not in ti.v:
print('Post-sampling: calculating p-values...')
ti.set_final_values_from_trace()
print('Done!')
print('Saving p-values inside of the trace directory...')
with open(os.path.join(ti.trace_data_folder, 'variables.pickle'),
'wb') as f_variables:
pickle.dump(ti.v, f_variables)
print('Finished!')
print('Saving scores as CSV files...')
delta__ = ti.v['r_avg_diff_matrix'] / (ti.v['iteration'] + 1)
zs__ = np.empty_like(delta__)
for j in range(delta__.shape[0]):
row = delta__[j, :]
mu = row.mean()
std = row.std()
if std == 0:
std = 0.000000000000001
zs_row_ = (row - mu) / std
_tmp = np.copy(zs_row_)
_tmp[np.isinf(_tmp)] = 0
_tmp[np.isneginf(_tmp)] = 0
zs_row_[np.isinf(zs_row_)] = max(7, np.abs(_tmp).max())
zs_row_[np.isneginf(zs_row_)] = min(-7, -np.abs(_tmp).max())
zs__[j, :] = zs_row_
pv__ = 2 * scipy.stats.norm.cdf(-np.abs(zs__))
h.save_df_to_csv(
pd.DataFrame(pv__, index=ti.df.index, columns=ti.df.columns),
pvalues_file)
else:
parser.error("The file (%s) you provided does not exist." % data_file)
def openit(self, url):
self.numTries = 0
return FancyURLopener.open(self, url)
def download(self):
bean = self.bean
update = self.update
if not bean or not bean.path:
return None
opener = FancyURLopener()
remote = opener.open(bean.path)
remote_size = 0
if "Content-Length" in remote.headers:
remote_size = int(remote.headers["Content-Length"])
bean.size = size2text(remote_size)
block_size = 4096
block_count = 0
ext = get_file_extension(bean.path)
path = FC().online_save_to_folder
if not os.path.isdir(path):
os.makedirs(path)
if bean.save_to:
to_file = os.path.join(bean.save_to, bean.text + ".mp3")
else:
to_file = get_bean_download_path(bean, FC().online_save_to_folder)
if not os.path.exists(os.path.dirname(to_file)):
os.makedirs(os.path.dirname(to_file))
to_file_tmp = to_file + ".tmp"
if os.path.exists(to_file_tmp):
bean.status = DOWNLOAD_STATUS_INACTIVE
bean.to_file = to_file
update(bean)
return None
if os.path.exists(to_file):
bean.status = DOWNLOAD_STATUS_COMPLETED
bean.to_file = to_file
update(bean)
return None
bean.save_to = to_file
with open(to_file_tmp, "wb") as tmp_file:
data = True
"""begin download"""
self.bean.status = DOWNLOAD_STATUS_DOWNLOADING
self.bean.path = to_file
self.update(self.bean)
while data:
data = remote.read(block_size)
if data:
block_count += 1
tmp_file.write(data)
#time.sleep(0.1)
persent = block_count * block_size * 100.0 / remote_size
if block_count % 50 == 0:
bean.persent = persent
update(bean)
time.sleep(0.5)
"""update file info on finish"""
logging.debug("rename %s - %s" % (to_file_tmp, to_file))
os.rename(to_file_tmp, to_file)
bean.status = DOWNLOAD_STATUS_COMPLETED
bean.to_file = to_file
bean.persent = 100
update(bean)
#!/usr/bin/python3 from urllib.request import FancyURLopener import re email = '' # @gmail.com can be left out password = '' url = 'https://%s:%[email protected]/mail/feed/atom' % (email, password) opener = FancyURLopener() page = opener.open(url) contents = page.read().decode('utf-8') ifrom = contents.index('<fullcount>') + 11 ito = contents.index('</fullcount>') titles = contents.split('<title>') fullcount = contents[ifrom:ito] print(fullcount + ' new messages\n') for mess in titles[2:]: mess = (re.sub(r'(<modified>)[\w|\-|:]*', '', mess)) mess = (re.sub(r'\<[^>]*\>(tag:)*', ' ', mess)) mess =(re.sub(r'(gmail.google.com,2004:)\d*', '', mess)) mess = (re.sub(r'\&[^&]*\;','', mess)) print(mess[:45]) page.close()
def get_month_data(month, cookie, token):
params = 'date={}&_token={}'.format(month, token)
opener = FancyURLopener()
opener.addheader('Cookie', cookie)
stream = opener.open(APIURL, params)
return stream.read().decode('utf-8')
#!/usr/bin/env python from urllib.request import FancyURLopener email = 'your email' # @gmail.com can be left out password = '******' url = 'https://%s:%[email protected]/mail/feed/atom' % (email, password) opener = FancyURLopener() page = opener.open(url) contents = page.read().decode('utf-8') ifrom = contents.index('<fullcount>') + 11 ito = contents.index('</fullcount>') fullcount = contents[ifrom:ito] print(fullcount + ' new')
def auth():
'''The method to do HTTPBasicAuthentication'''
opener = FancyURLopener()
f = opener.open(_URL)
feed = f.read()
return feed
