def __update():
global weathers
download(DATA_URL, TMP_FILE)
xml = etree.parse(TMP_FILE)
root = xml.getroot()
for node in root.iter('{urn:cwb:gov:tw:cwbcommon:0.1}location'):
name = node.find(
'{urn:cwb:gov:tw:cwbcommon:0.1}locationName').text #城市名稱
weathers[name] = {}
for weather in node.iter(
'{urn:cwb:gov:tw:cwbcommon:0.1}weatherElement'):
weather_type = weather.find(
'{urn:cwb:gov:tw:cwbcommon:0.1}elementName').text
weathers[name][weather_type] = []
for weather_value in weather.iter(
'{urn:cwb:gov:tw:cwbcommon:0.1}parameterName'):
weathers[name][weather_type].append(weather_value.text)
for weather_time in ['startTime', 'endTime']:
weathers[name][weather_time] = []
for t in weather.iter('{urn:cwb:gov:tw:cwbcommon:0.1}%s' %
weather_time):
if len(weathers[name][weather_time]) > 3:
break
weathers[name][weather_time].append(t.text)
def update_pythings(version):
files = apost(api='/pythings/get/',
data={
'version': version,
'list': True,
'platform': platform
})['content']
path = env.root + '/' + version
try:
os.mkdir(path)
except OSError as e:
pass
for file_name in files:
if file_name != 'version.py':
download(file_name=file_name,
version=version,
platform=platform,
dest='{}/{}'.format(path, file_name),
what='pythings')
for file_name in files:
if file_name == 'version.py':
download(file_name=file_name,
version=version,
platform=platform,
dest='{}/{}'.format(path, file_name),
what='pythings')
def main():
"""
Example:
Pornhub/Youjizz/melodysheep/mixturecloud video link (don't forget http://): http://www.*****.com/view_video.php?viewkey=******
Getting link...
Found:
name_here
http://link_to_/video.mp4
Downloading video... this may take a while.
Download size = 17735 ko, packet: 2166/2166
File name = name_here.mp4
ETA = 45 second(s)
md5 checksum = 3fa1d9682acc09934d1e3ff0404eaf8e
File size = 17735 kb
Saved in: /home/you/pyorn
Downloaded!
Press enter to continue...
"""
try:
if ("http://" in argv[1]):
link = argv[1]
else:
print("This is not a valid link don't forget the 'http://'")
return 1
except (IndexError):
link = raw_input("Enter mixturecloud/pornhub/youjizz/Bandcamp link (don't forget http://): ")
print("Getting link...")
if ("pornhub" in link):
print("pornhub detected")
Flink = GetPornhubVideoLink(link)
elif ("bandcamp" in link):
Flink = GetMp3LinkFromBandCamp(link)
elif ("mixturecloud" in link):
print("Mixturecloud detected")
Flink = GetMixtureDirectLink(link)
elif ("youjizz" in link):
print("youjizz detected")
Flink = GetYoujizzVideoLink(link)
else:
print("Incorrect link")
if (Flink == 1):
print("Check your internet connection or your link")
return 1
del link
print("Found:\n%s\n%s\n\nDownloading file... this may take a while." % (Flink[0][:40], Flink[1]))
try:
download(Flink[1], Flink[0])
except (IOError):
return 1
return 0
def main():
"""
Example:
Pornhub/Youjizz video link (don't forget http://): http://www.*****.com/view_video.php?viewkey=******
Getting link...
Found:
name_here
http://link_to_/video.mp4
Downloading video... this may take a while.
Download size = 17735 ko, packet: 2166/2166
File name = name_here.mp4
ETA = 45 second(s)
md5 checksum = 3fa1d9682acc09934d1e3ff0404eaf8e
File size = 17735 kb
Saved in: /home/you/pyorn
Downloaded!
Press enter to continue...
"""
try:
if ("http://" in argv[1]):
link = argv[1]
else:
print("This is not a valid link don't forget the 'http://'")
return 1
except (IndexError):
link = raw_input("Enter pornhub/youjizz/melodysheep link (don't forget http://): ")
print("Getting link...")
if ("pornhub" in link):
print("pornhub detected")
Flink = GetPornhubVideoLink(link)
elif ("melodysheep" in link):
Flink = GetMp3LinkFromMelodySheep(link)
elif ("youjizz" in link):
print("youjizz detected")
Flink = GetYoujizzVideoLink(link)
else:
print("Incorrect link")
if (Flink == 1):
print("Check your internet connection or your link")
return 1
del link
print("Found:\n%s\n%s\n\nDownloading file... this may take a while." % (Flink[0][:40], Flink[1]))
try:
download(Flink[1], Flink[0])
except (IOError):
return 1
return 0
def download(self, event):
try:
link = self.xinput.GetValue()
if ("pornhub" in link):
Flink = GetPornhubVideoLink(link)
elif ("youjizz" in link):
Flink = GetYoujizzVideoLink(link)
elif ("melodysheep" in link):
Flink = GetMp3LinkFromMelodySheep(link)
else:
print("Incorrect link")
download(Flink[1], Flink[0])
self.output.SetLabel("Downloaded! : %s" % Flink[0])
except:
self.output.SetLabel("Error, check the link or the internet connection")
event.Skip()
def get(self, iid, filename):
"""Download the specified file"""
r = download(iid, filename, headers=request.headers)
return Response(r.iter_content(chunk_size=1024),
headers=dict(r.headers),
status=r.status_code,
mimetype=mimetype(filename))
def get(self, iid, filename):
"""Download the specified file"""
r = download(iid, filename, headers=request.headers)
return Response(r.iter_content(chunk_size=1024),
headers=dict(r.headers),
status=r.status_code,
mimetype=mimetype(filename))
def download(self, event):
try:
link = self.xinput.GetValue()
if ("pornhub" in link):
Flink = GetPornhubVideoLink(link)
elif ("youjizz" in link):
Flink = GetYoujizzVideoLink(link)
elif ("bandcamp" in link):
Flink = GetMp3LinkFromBandCamp(link)
elif ("mixturecloud" in link):
Flink = GetMixtureDirectLink(link)
else:
print("Incorrect link")
download(Flink[1], Flink[0])
self.output.SetLabel("Downloaded! : %s" % Flink[0])
except:
self.output.SetLabel(
"Error, check the link or the internet connection")
event.Skip()
def test_download_match_not_found(self, _client):
resp_mock = mock.MagicMock()
resp_mock.status_code = 404
_client.return_value = resp_mock
resp, status = download(match_id='1')
_client.assert_called_once_with('1')
self.assertEqual(resp, '1')
self.assertEqual(status, 404)
def test_download_match(self, _client):
resp_mock = mock.MagicMock()
resp_mock.status_code = 200
resp_mock.json.return_value = {'id': 0}
_client.return_value = resp_mock
resp, status = download(match_id='0')
_client.assert_called_once_with('0')
self.assertEqual(resp, {'id': 0})
self.assertEqual(status, 200)
def get(self, iid):
""" Return a torrent adjusted correctly for webtorrents
"""
fs = download(iid, iid + '_archive.torrent').content
t = better_bencode.loads(fs)
# print(t)
t[b'url-list'] = [b'https://api.archivelab.org/v2/webtorrents/files/']
t[b'announce'] = b'wss://tracker.webtorrent.io'
t[b'announce-list'] = [[b'wss://tracker.webtorrent.io'],
[b'wss://tracker.btorrent.xyz'],
[b'wss://tracker.fastcast.nz'],
[b'wss://tracker.openwebtorrent.com']]
return Response(better_bencode.dumps(t),
mimetype=mimetype(iid + '_archive.torrent'))
def update_app(version):
files = apost(api='/apps/get/', data={
'version': version,
'list': True
})['content']
# First of all remove /app.py to say that there is no valid App yet (download is in progress)
import os
try:
os.remove(env.root + '/app.py')
except:
pass
for file_name in files:
if file_name in ['worker_task.py', 'management_task.py']:
download(file_name=file_name,
version=version,
dest='{}/{}'.format(env.root, file_name),
what='apps',
platform=platform)
else:
logger.info(
'NOT downloading "{}" as in forbidden list'.format(file_name))
with open(env.root + '/app.py', 'w') as f:
f.write('\nversion=\'{}\''.format(version))
logger.info('Got new, updated app')
async def message_handler(self, text, message_id):
if not api.validate(text):
logging.debug('invalid url received: {0}'.format(text))
await self.reply(replies.INVALID_URL, message_id)
return
await self.reply(replies.UPLOAD_STARTED, message_id)
video_data = api.download(text, config['tmp_path'])
video_vk_url = api.upload_video(vk_api, video_data.path,
video_data.name)
os.remove(video_data.path)
logging.info("uploaded: {0} -> {1}".format(text, video_vk_url))
await self.reply(replies.UPLOADED_VIDEO.format(video_vk_url),
message_id)
def get(self, iid):
""" Return a torrent adjusted correctly for webtorrents
"""
fs = download(iid, iid + '_archive.torrent').content
t = better_bencode.loads(fs)
# print(t)
t[b'url-list'] = [b'https://api.archivelab.org/v2/webtorrents/files/']
t[b'announce'] = b'wss://tracker.webtorrent.io'
t[b'announce-list'] = [[b'wss://tracker.webtorrent.io'],
[b'wss://tracker.btorrent.xyz'],
[b'wss://tracker.fastcast.nz'],
[b'wss://tracker.openwebtorrent.com']]
return Response(
better_bencode.dumps(t),
mimetype=mimetype(iid + '_archive.torrent')
)
def download_file(self, file_id, dest):
"""
Download a file to local disk.
:param dest: a path or a ``file`` object
"""
f = self.getFile(file_id)
try:
d = dest if _isfile(dest) else open(dest, 'wb')
r = api.download((self._token, f['file_path']),
preload_content=False)
while 1:
data = r.read(self._file_chunk_size)
if not data:
break
d.write(data)
finally:
if not _isfile(dest) and 'd' in locals():
d.close()
if 'r' in locals():
r.release_conn()
def transactions():
"""
DataFrame Update 1 - Stock information pulled from Robinhood CSV
DataFrame Update 2 - Options information pulled from Robinhood
"""
feather = featherfile('transactions')
cache_exists = os.path.exists(feather)
T = None
if cache_exists:
T = pd.read_parquet(feather)
return T
symbols = api.symbols(remove='expired')
with ShadyBar('%48s' % 'Building Transactions',
max=len(symbols) + 6) as bar:
# 1. Download Stock Transactions from Robinhood...
feather = featherfile('transactions')
imported = '/tmp/.cached/stocks.csv'
if not os.path.exists(imported):
# The `ignore_cache' flag here is eaten up by the cachier decorator
imported = api.download('stocks', ignore_cache=True)
bar.next()
# 2. Import Stock Transaction data to Pandas DataFrame
header = pd.read_csv(imported, comment='#', nrows=0).columns
df = pd.read_csv(imported,
comment='#',
converters=dict(
date=util.datetime.dtp.parse,
fees=F,
quantity=F,
average_price=F,
))
bar.next()
# 3. Update Stocks Transaction history with position-altering Options Contracts...
# (stocks bought/sold via the options market (assigments and excercises)
for ticker in symbols:
for se in api.events(ticker):
for ec in se['equity_components']:
df.append(dict(
symbol=ticker,
date=se['updated_at'],
order_type=se['type'],
side=ec['side'],
fees=0,
quantity=ec['quantity'],
average_price=ec['price'],
),
ignore_index=True)
bar.next()
# 4. Sort
df.sort_values(by='date', inplace=True)
bar.next()
# 5. Reset the Index
df.reset_index(inplace=True, drop=True)
bar.next()
cbtally = defaultdict(list)
def _costbasis(row):
if api.blacklisted(row.symbol, full=True):
return pd.Series([NaN, NaN, NaN, NaN])
data = cbtally[row.symbol]
signum = {'buy': -1, 'sell': +1}[row.side]
datum = [
'trade',
F(signum) * F(row.quantity),
F(row.average_price), row.date
]
if len(data) > 0:
_, _, _, d0 = data[-1]
_, _, _, d2 = datum
splits = api.splits(row.symbol)
for split in splits:
d1 = util.datetime.parse(split['exDate'])
if not d0 < d1 < d2:
continue
for existing in data:
existing[1] *= F(split['toFactor'])
existing[1] /= F(split['fromFactor'])
existing[2] *= F(split['fromFactor'])
existing[2] /= F(split['toFactor'])
data.append(datum)
_, _, price, _ = datum
bought = -sum(qty for cls, qty, pps, date in data if qty < 0)
sold = +sum(qty for cls, qty, pps, date in data if qty > 0)
held = bought - sold
return pd.Series([
F(bought),
F(sold),
F(held),
F((sum(qty * pps
for cls, qty, pps, date in data) + held * price) /
bought) if bought > 0 else 0
])
# 6. Cost Basis
df[['bought', 'sold', 'held', 'cbps']] = df.apply(_costbasis, axis=1)
bar.next()
# 7. Dump
if 'test' not in feather:
df.to_parquet(feather, index=False)
bar.next()
return df