def load(self, loadUrl):
o = urlparse(loadUrl)
print o.scheme
print o.port
print o
o = urlsplit(loadUrl)
print o.geturl()
reader = Fetch()
htmlFile = reader.get(loadUrl)
soup = BeautifulSoup(htmlFile)
form = soup.find('form')
print form
print form['action']
print form['method']
#print form['onsubmit']
if 'action' in form:
print form['action']
if 'method' in form:
print form['method']
if 'onsubmit' in form:
print form['onsubmit']
data = {"username": "******", "password": "******"}
response = reader.post('https://210.154.183.61:4443/mantis/login.php',
data)
print response
def create_socket():
# Get info about our authentication
this.id = 1234
this.token = 1234
parent_this = this
parent_this.callbacks = 2
def callback():
parent_this.callbacks -= 1
if parent_this.callbacks == 0:
parent_this.socket = MessageSocket(lambda parent_this: parent_this.handle_message(this),
parent_this.token, parent_this.id)
def data_callback(data):
parent_this.id = data['id']
callback()
def auth_callback(data):
parent_this.token = data['token']
callback()
Fetch.get("data/") \
.then(lambda response: response.json()) \
.then(lambda data: data_callback(data))
Fetch.get("auth/") \
.then(lambda response: response.json()) \
.then(lambda data: auth_callback(data))
def __init__(self, auth:str, logger=logging.getLogger()):
self.auth = auth
self.logger = logger
self.fetch = Fetch(auth, logger)
self.user = self.fetch.fetch("/v1/me")
self.userId = self.user["id"]
self.imageUrl = self.user["images"][0]["url"]
self.market = self.user["country"]
logger.debug("Initialized Spotify: {0} @ {1}".format(self.userId, self.market))
self.fetch.fetch("/v1/me/tracks", limit=1, market=self.market)
def __init__(self):
self.config = {
"max_result": 1000,
"topic_score": 10.0,
"threshold": 1.0,
"base_url": "http://export.arxiv.org/api/query?",
}
self.config = Bunch(self.config)
self.fetch = Fetch(self.config)
def subscribe_to_feed(self, feed_url):
feed_id = db.get_feed_id(feed_url)
if feed_id < 0:
feed_fetch = Fetch(feed_url)
feed_id = db.create_new_feed(feed_fetch.get_feed_details())
db.add_new_item(feed_fetch.get_entries())
user_feed_id = db.add_user_to_feed(self.uid, feed_id)
return feed_id
def subscribe_to_feed(self,feed_url):
feed_id = db.get_feed_id(feed_url)
if feed_id < 0 :
feed_fetch = Fetch(feed_url)
feed_id = db.create_new_feed(feed_fetch.get_feed_details())
db.add_new_item(feed_fetch.get_entries())
user_feed_id = db.add_user_to_feed(self.uid,feed_id)
return feed_id
class FetchTestCase(unittest.TestCase):
def setUp(self):
self.fetch = Fetch(username='******', pw='liumengchao')
def test_get(self):
resp = self.fetch.get('http://www.douban.com/update/')
print resp
resp = self.fetch.get('http://www.douban.com/fdsafdsafdsa/')
print resp
resp = self.fetch.get('dajfdsoajfeowaof')
print resp
def handle(self, deviceJobId):
cameraFolder = os.listdir(configCamera['CAMERA_FROM_FOLDER'])
# make sure it's not empty
if cameraFolder:
hashname = self.compressFiles()
checksum = self.checksum(configCamera['CAMERA_TEMP_FOLDER'] +
hashname + ".7z")
self.uploadFiles(hashname, checksum)
self.cleanFolders(cameraFolder, hashname)
Fetch.patch(config['API_URL'] + "/api/device/" + str(deviceJobId) +
"/jobs/update?status=done")
def execute():
"""Run script every X minutes.
"""
# Get google api credentials.
my_creds = Creds()
# Construct a service for interacting with sheets api.
service = build('sheets', 'v4', credentials=my_creds.creds)
print(CONSOLE_COLOR_START_DONE + f'{datetime.now()} running script' +
CONSOLE_ENDC)
main_start = timer()
# Create new spreadsheet.
start = timer()
print('creating spreadsheet...')
sheet = Spreadsheet(service, f'everyday_sheet{datetime.now()}')
print(CONSOLE_COLOR +
f'new sheet created in {round(timer() - start, 3)}s' + CONSOLE_ENDC)
# Fetch data from all data sources defined in json file.
fetcher = Fetch('reddit_inputs.json')
start = timer()
print('fetching data...')
data = fetcher.fetch_all()
print(CONSOLE_COLOR + f'data fetched in {round(timer() - start, 3)}s' +
CONSOLE_ENDC)
# Convert data into format needed for spreadsheet.
sheet.convert_data(data)
# Insert data into spreadsheet.
start = timer()
print('inserting data into spreadsheet...')
sheet.insert_data()
print(CONSOLE_COLOR +
f'data inserted into a sheet in {round(timer() - start, 3)}s' +
CONSOLE_ENDC)
# Adjust columns size.
sheet.set_col_size()
print(CONSOLE_COLOR +
f'elapsed time since start {round(timer() - main_start, 3)}s' +
CONSOLE_ENDC)
print('added ', datetime.now())
print(CONSOLE_COLOR_START_DONE + f'{datetime.now()} script finished' +
CONSOLE_ENDC)
return f'https://docs.google.com/spreadsheets/d/{sheet.sheet_id}/edit#gid=0'
def fetchListPages(self, listtype="html"):
print "Start to fetch and parse List"
urls = self.listRule.getListUrls()
for url in urls:
print "Fetching list page: ", url, "charset:", safestr(self.seed["charset"]), "timeout:", safestr(self.seed["timeout"])
f = Fetch(url, charset = self.seed["charset"], timeout = self.seed["timeout"])
if f.isReady():
doc = f.read()
if listtype == "html":
self.parseListPage(f, doc, url)
elif listtype == "json":
self.parseJsonPage(f, doc, url)
print "List has finished parsing. It has %s docs." % ansicolor.red(self.__len__())
class ClusterTopic:
def __init__(self):
self.config = {
"max_result": 1000,
"topic_score": 10.0,
"threshold": 1.0,
"base_url": "http://export.arxiv.org/api/query?",
}
self.config = Bunch(self.config)
self.fetch = Fetch(self.config)
@staticmethod
def format_json(papers):
json_out = {"payload": []}
for topic, paper_data in papers.items():
topic_data = []
_paper_dict = []
for paper, score in topic_data:
_paper_dict.append({
"title": paper.title,
"url": paper.id,
"summary": paper.summary,
"author": "",
"score": score,
})
json_out["payload"].append({topic: _paper_dict})
return json_out
def __call__(self, profile):
papers = self.fetch._get_clusterd_papers(profile)
papers_json = self.format_json(papers)
return papers_json
def fetchMedia(self):
f = Fetch(self.mediaUrl)
if f.connected:
self.media = f.site
self.mediaData = self.media.read()
self.urlinfo = self.media.info()
self.fetched = True
def start_backup(self, info, deviceJobId):
self.clean_up()
# Only execute if there has been changes in one fo the folders
if info[0] is not 0:
Fetch.patch(config['API_URL'] + "/api/device/" + str(deviceJobId) +
"/jobs/update?status=inprogress")
self.compress_files(info[0])
if info[2] is 'usb':
self.copy_to_usb()
if info[2] is 'ftp':
self.upload_files()
self.clean_up()
# Save updated folder sizes to pickle file for next run
folders_with_stat = self.get_folder_stats()
self.pickle_dump(info[1], folders_with_stat)
def get_one_page(self, keyword, page):
task_manager = TaskManager()
task_manager.task_begin(keyword, page)
res = Fetch.get_one_page_by_keyword(keyword, page)
if res:
task_manager.task_end(keyword, page)
else:
task_manager.task_failed(keyword, page)
def __init__(self):
self.__reset()
self.__read_info()
self._fetch = Fetch(username='******', pw='liumengchao')
self._tasks_url = 'http://localhost:8080/id/books/'
self._url = 'https://api.douban.com/v2/book/user/%s/collections?count=%d&start=%d'
self._upload_url = 'http://localhost:8080/upload/'
logging.basicConfig(filename='user_books_error.log', filemod='a+', level=logging.ERROR)
def __init__(self):
self.__reset()
self.__read_info()
self._fetch = Fetch(username='******', pw='liumengchao')
self._tasks_url = 'http://localhost:8080/id/followed/'
self._url = 'http://www.douban.com/people/%s/contacts'
self._upload_url = 'http://localhost:8080/upload/'
logging.basicConfig(filename='followed_error.log', filemod='a+', level=logging.ERROR)
def parseFeed(self):
print "Start to fetch and parse Feed list"
seed = self.seed
f = Fetch(seed.prefixurl, seed.charset, self.seed.timeout)
if f.isReady():
feed = feedparser.parse(f.read())
items = feed["entries"]
if len(items) > 0:
for item in items:
_item = Item({"url": item["link"], "type": self.seed_type})
if self.guid_rule is None:
self.guid_rule = "url"
guid = self.getItemGUID(item)
self.items[guid] = _item
print "List has finished parsing. It has %s docs." % ansicolor.red(
self.__len__())
def fetchListPages(self, listtype="html"):
print "Start to fetch and parse List"
urls = self.listRule.getListUrls()
for url in urls:
print "Fetching list page: ", url, "charset:", safestr(
self.seed["charset"]), "timeout:", safestr(
self.seed["timeout"])
f = Fetch(url,
charset=self.seed["charset"],
timeout=self.seed["timeout"])
if f.isReady():
doc = f.read()
if listtype == "html":
self.parseListPage(f, doc, url)
elif listtype == "json":
self.parseJsonPage(f, doc, url)
print "List has finished parsing. It has %s docs." % ansicolor.red(
self.__len__())
def parseFeed(self):
print "Start to fetch and parse Feed list"
seed = self.seed
f = Fetch(seed.prefixurl, seed.charset, self.seed.timeout);
if f.isReady():
feed = feedparser.parse(f.read())
items = feed["entries"]
if len(items) > 0:
for item in items:
_item = Item({
"url" : item["link"],
"type" : self.seed_type
})
if self.guid_rule is None:
self.guid_rule = "url"
guid = self.getItemGUID(item)
self.items[guid] = _item
print "List has finished parsing. It has %s docs." % ansicolor.red(self.__len__())
def load(self, loadUrl):
o = urlparse(loadUrl)
print o.scheme
print o.port
print o
o = urlsplit(loadUrl)
print o.geturl()
reader = Fetch()
htmlFile = reader.get(loadUrl)
soup = BeautifulSoup(htmlFile)
form = soup.find("form")
print form
print form["action"]
print form["method"]
# print form['onsubmit']
if "action" in form:
print form["action"]
if "method" in form:
print form["method"]
if "onsubmit" in form:
print form["onsubmit"]
def load(self, loadUrl):
o = urlparse(loadUrl)
print o.scheme
print o.port
print o
o = urlsplit(loadUrl)
print o.geturl()
reader = Fetch()
htmlFile = reader.get(loadUrl)
soup = BeautifulSoup(htmlFile)
form = soup.find('form')
print form
print form['action']
print form['method']
#print form['onsubmit']
if 'action' in form:
print form['action']
if 'method' in form:
print form['method']
if 'onsubmit' in form:
print form['onsubmit']
def uploadFiles(self, hashname, checksum):
files = {
'file':
open(configCamera['CAMERA_TEMP_FOLDER'] + hashname + '.7z', 'rb')
}
data = {
'device_id': config['CLIENT_ID'],
'type': 'camera',
'filename': hashname + '.7z',
'checksum': checksum,
'status': 'done'
}
response = Fetch.post(config['API_URL'] + "/api/files/store", files,
data)
def set_wallpaper(self, img_path):
if system == "Linux":
if "~" in img_path:
img_path = img_path.replace("~", HOME)
if img_path == ".":
img_path = start_path
if os.path.isfile(img_path):
self.linux_wallpaper(img_path)
elif os.path.isfile(join(start_path, img_path)):
self.linux_wallpaper(join(start_path, img_path))
elif os.path.isdir(img_path):
Fetch().custom_folder([img_path])
elif os.path.isdir(join(start_path, img_path)):
Fetch().custom_folder([join(start_path, img_path)])
else:
print(
f"{colors['red']}Error, file path not recognized{colors['normal']}"
)
else:
print(
f"{colors['red']}Sorry, your system is not supported yet.{colors['normal']}"
)
def __init__(self, deviceJobId):
# Small arguement resets active pickle. (In case uploads were aborted somehow)
if len(sys.argv) > 1 and sys.argv[1] == 'reset':
print('Reset pickle...')
self.pickle_dump(configBackup["PICKLE_ACTIVE_FILEPATH"], 0)
active_script = self.pickle_load(
configBackup["PICKLE_ACTIVE_FILEPATH"])
if active_script is 0:
# Backup is now active. Prevent it from running again
self.pickle_dump(configBackup["PICKLE_ACTIVE_FILEPATH"], 1)
#FTP backup
if self.check_conditions('ftp'):
backup_info = self.check_filesizes(
configBackup["PICKLE_FTP_FILEPATH"], 'ftp')
self.start_backup(backup_info, deviceJobId)
#USB backup
if self.check_conditions('usb'):
backup_info = self.check_filesizes(
configBackup["PICKLE_USB_FILEPATH"], 'usb')
self.start_backup(backup_info, deviceJobId)
if self.check_conditions('ftp') or self.check_conditions('usb'):
Fetch.patch(config['API_URL'] + "/api/device/" +
str(deviceJobId) + "/jobs/update?status=done")
files = {}
data = {'value': time.strftime('%Y-%m-%d %H:%M:%S')}
Fetch.post(
config['API_URL'] + "/api/device/" +
str(config['CLIENT_ID']) +
"/settings/backup_last_synced/update", files, data)
self.pickle_dump(configBackup["PICKLE_ACTIVE_FILEPATH"], 0)
print('All done!')
def load(self, loadUrl):
o = urlparse(loadUrl)
print o.scheme
print o.port
print o
o = urlsplit(loadUrl)
print o.geturl()
reader = Fetch()
htmlFile = reader.get(loadUrl)
soup = BeautifulSoup(htmlFile)
form = soup.find("form")
print form
print form["action"]
print form["method"]
# print form['onsubmit']
if "action" in form:
print form["action"]
if "method" in form:
print form["method"]
if "onsubmit" in form:
print form["onsubmit"]
data = {"username": "******", "password": "******"}
response = reader.post("https://210.154.183.61:4443/mantis/login.php", data)
print response
def sequetial(self, go_back):
try:
with open(wall_data_file, encoding="utf-8") as data:
saved_walls = json.load(data)
except (FileNotFoundError, ValueError):
Fetch().wall_dl()
"""chooses the wallpaper in the order in which they were downloaded"""
with open(wall_data_file, "r") as data:
saved_walls = json.load(data)
with open("point.pickle", "rb+") as wall_point:
# selection_point stores the value of the current wallpaper
# it is necessary so that wallpapers don't repeat
selection_point = pickle.load(wall_point)
if selection_point > len(saved_walls):
selection_point = 1
elif selection_point == len(saved_walls) and go_back == 1:
selection_point -= 1
elif selection_point == len(saved_walls) and go_back == 0:
selection_point = 1
elif (selection_point < len(saved_walls) and selection_point != 1
and go_back == 1):
selection_point -= 1
elif selection_point < len(saved_walls) and go_back == 0:
selection_point += 1
elif (selection_point < len(saved_walls) and selection_point == 1
and go_back == 0):
selection_point += 1
elif (selection_point < len(saved_walls) and selection_point == 1
and go_back == 1):
selection_point = len(saved_walls)
elif (selection_point < len(saved_walls) and selection_point == 0
and go_back == 0):
selection_point = 1
elif (selection_point < len(saved_walls) and selection_point == 0
and go_back == 1):
selection_point = len(saved_walls)
img_name = str(saved_walls.get(str(selection_point)))
# the new value of selection point is stored for the next run
print(f"selection point is {selection_point}")
with open("point.pickle", "wb") as point:
pickle.dump(selection_point, point)
return join(pictures, str(img_name))
def main_menu(self, message=message):
refresh(message)
choice = input(f"""{colors['green']}
Welcome to Redpaper. This is a TUI used to
control the underlying Redpaper program.
Select an option:\n{colors['normal']}
{colors['red']} 1 {colors['normal']}: {colors['blue']} Download wallpapers {colors['normal']} \n
{colors['red']} 2 {colors['normal']}: {colors['blue']} Next wallpaper{colors['normal']}\n
{colors['red']} 3 {colors['normal']}: {colors['blue']} Previous wallpaper{colors['normal']}\n
{colors['red']} 4 {colors['normal']}: {colors['blue']} Settings{colors['normal']}\n
{colors['red']} 5 {colors['normal']}: {colors['blue']} Help {colors['normal']}\n
{colors['red']} x {colors['normal']}: {colors['blue']} exit {colors['normal']}\n
>>> """)
if choice == "1":
refresh(message)
Fetch().wall_dl()
elif choice == "2":
message = f"{colors['green']} Changed wallpaper {colors['normal']}\n"
refresh(message)
img_path = WallSet().sequetial(0)
WallSet().set_wallpaper(img_path)
self.main_menu()
elif choice == "3":
message = f"{colors['green']} Changed wallpaper {colors['normal']}\n"
refresh(message)
img_path = WallSet().sequetial(1)
WallSet().set_wallpaper(img_path)
self.main_menu()
elif choice == "4":
message = ""
Settings().main_settings()
elif choice == "5":
# TODO: create a help page
message = "HELP\n"
refresh(message)
print(f"""
{colors['green']}You can check the wiki for help:
https://github.com/keystroke3/redpaper/wiki{colors['normal']}"""
)
self.main_menu()
elif choice == "x" or choice == "X":
clear()
else:
Home().main_menu()
async def on_message(self, message):
channel = message.channel
channel.id = 842475416160698379
if message.content.startswith('tag everyone'):
self.members_to_ping = Fetch().return_data()
print(self.members_to_ping)
self.ids, self.members_to_tag = [], []
for member in self.members:
if member[2] in self.members_to_ping or member[
0] in self.members_to_ping:
self.ids.append(member[1])
self.members_to_tag.append(member[2] if member[2] in self.
members_to_ping else member[0])
print(self.members_to_tag)
s = ''
for members in self.ids:
s += '<@{}>'.format(
members) if members == self.ids[-1] else '<@{}>,'.format(
members)
await channel.send("{} Please report for your shift".format(s))
def __init__(self, item, seed):
'''
document base url
'''
self.url = item["url"]
self.data = item
self.seed = seed
item["tags"] = ",".join(self.seed.tags)
#文章采集规则
self.articleRule = seed.getRule().getArticleRule()
print "Document %s is fetcing" % ansicolor.green(self.url)
firstContent = Fetch(self.url,
charset=seed["charset"],
timeout=seed["timeout"]).read()
if firstContent:
self.parseDocument(firstContent)
async def on_ready(self):
self.members = []
self.required_members = []
# for guild in self.guilds:
# for channel in guild.channels:
# print(channel.id,channel.name)
for guild in self.guilds:
data = guild.members
for x in data:
if x.name == 'REAPER':
print(x.name, x.id, x.nick)
self.members.append([x.name, x.id, x.nick])
print('Logged on as {0}!'.format(self.user))
# await asyncio.sleep(60 * 50 * 2)
self.members_to_ping = Fetch().return_data()
print(self.members_to_ping)
self.ids, self.members_to_tag = [], []
for member in self.members:
if member[2] in self.members_to_ping or member[
1] in self.members_to_ping:
self.ids.append(member[1])
self.members_to_tag.append(member[2])
from network import Network
from fetch import Fetch
co = Network()
co.send("Test\n")
num = co.receive()
dataFetching = Fetch(num)
data = co.receive()
print(dataFetching.fetchMessage(data))
lst.write('E', cur.regE)
lst.write('M', cur.regM)
lst.write('W', cur, regW)
mem = Memory()
InsCode = {}
Init(InsCode, mem)
reg = Register()
pipereg = PipeRegister()
tmp_pipereg = PipeRegister()
CC = ConditionCode()
Stat = Status()
PC = 0
while Stat.stat == 'AOK':
print 'Current Time:', PC
tmp_pipereg = PipeRegister()
Fetch(tmp_pipereg, InsCode[hex(PC)], PC)
Decode(pipereg, tmp_pipereg, reg)
Execute(pipereg, tmp_pipereg)
Memory(pipereg, tmp_pipereg)
WriteBack(pipereg, tmp_pipereg)
PC = pipereg.regF['predPC']
Update(cur=tmp_pipereg, lst=pipireg)
print 'RegF:', reg.regF
print 'RegD:', reg.regD
print 'RegE:', reg.regE
print 'RegM:', reg.regM
print 'RegW:', reg.regW
def main(args=None):
parser = argparse.ArgumentParser(description='Simple training script.')
parser.add_argument('--cls_id', help='class id', type=int)
parser.add_argument('--version', help='model version', type=float)
parser.add_argument('--gamma', help='gamma for the SoftL1Loss', type=float, default=9.0)
parser.add_argument('--lr', help='lr for optimization', type=float, default=1e-4)
parser.add_argument('--epoches', help='num of epoches for optimization', type=int, default=4)
parser.add_argument('--resume_epoch', help='trained model for resume', type=int, default=0)
parser.add_argument('--batch_size', help='batch size for optimization', type=int, default=10)
parser.add_argument('--checkpoints', help='checkpoints path', type=str, default='voc_checkpoints')
parser = parser.parse_args(args)
cls_name = classes[parser.cls_id]
parser.checkpoints = '_'.join([parser.checkpoints,cls_name])
if not os.path.isdir(parser.checkpoints):
os.mkdir(parser.checkpoints)
print('will save checkpoints in '+parser.checkpoints)
cls_dir = "../context_profile/voc_detection_{:s}_p10/"\
.format(cls_name)
batch_size = parser.batch_size
print('[data prepare]....')
dataloader_train = DataLoader(Fetch('train_benign', root_dir=cls_dir), batch_size=batch_size, num_workers=2, shuffle=True)
print('[model prepare]....')
use_gpu = torch.cuda.device_count()>0
model = AutoEncoder(parser.gamma)
if use_gpu:
model = torch.nn.DataParallel(model).cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=parser.lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=2, verbose=True)
if parser.resume_epoch > 0 :
checkpoint_name = os.path.join(parser.checkpoints, 'model_{:1.1f}_epoch{:d}.pt'.format(parser.version, parser.resume_epoch))
if not os.path.isfile(checkpoint_name):
raise ValueError('No checkpoint file {:s}'.format(checkpoint_name))
model.load_state_dict(torch.load(checkpoint_name))
print('model loaded from {:s}'.format(checkpoint_name))
print('[model training]...')
loss_hist = []
epoch_loss = []
num_iter = len(dataloader_train)
for epoch_num in range(parser.resume_epoch, parser.epoches):
model.train()
for iter_num, sample in enumerate(dataloader_train):
if True:#try:
optimizer.zero_grad()
if use_gpu:
data = sample['data'].cuda().float()
else:
data = sample['data'].float()
loss = model(data).mean()
if bool(loss==0):
continue
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
optimizer.step()
epoch_loss.append(float(loss))
loss_hist.append(float(loss))
if iter_num % 30 == 0:
print('Epoch {:d}/{:d} | Iteration: {:d}/{:d} | loss: {:1.5f}'.format(
epoch_num+1, parser.epoches, iter_num+1, num_iter, float(loss)))
if iter_num % 3000 == 0:
scheduler.step(np.mean(epoch_loss))
epoch_loss = []
if epoch_num < 1:
continue
checkpoint_name = os.path.join(parser.checkpoints, 'model_{:1.1f}_epoch{:d}.pt'.format(parser.version, epoch_num+1))
torch.save(model.state_dict(), checkpoint_name)
print('Model saved as {:s}'.format(checkpoint_name))
np.save('loss_hist.npy', loss_hist)
import rospy
import time
import math
import cv2
import tf
from sensor_msgs.msg import Image
from ar_track_alvar_msgs.msg import AlvarMarkers
from geometry_msgs.msg import PoseStamped
from tf.transformations import euler_from_quaternion, quaternion_from_euler
from fetch import Fetch
if ((__name__ == "__main__") and (not rospy.is_shutdown())):
rospy.init_node("fetch_builder",anonymous=True)
Fetch_Robot = Fetch()
rospy.loginfo("Initialization")
Fetch_Robot.Head.look_at(0.7,0,0.5,"base_link")
rospy.loginfo("Till head")
Fetch_Robot.Gripper.Open()
rospy.loginfo("Gripper Open")
Fetch_Robot.Arm.Tuck()
rospy.loginfo("Tuck Arm")
rospy.sleep(rospy.Duration(2))
#Take images
RGB_image = Fetch_Robot.GetRGBImage()
rospy.loginfo("Get RGB image")
class BooksTask:
''' get books '''
def __init__(self):
self.__reset()
self.__read_info()
self._fetch = Fetch(username='******', pw='liumengchao')
self._tasks_url = 'http://localhost:8080/id/books/'
self._url = 'https://api.douban.com/v2/book/user/%s/collections?count=%d&start=%d'
self._upload_url = 'http://localhost:8080/upload/'
logging.basicConfig(filename='user_books_error.log', filemod='a+', level=logging.ERROR)
def __del__(self):
self.__save_info()
def __reset(self):
self._status = 'free'
self._free_tasks = set()
self._done_tasks = set()
def __read_info(self):
if os.path.exists('books_task_config.cfg'):
with open('books_task_config.cfg', 'r') as f:
cfg = json.loads(f.read())
self._status = cfg.get('status')
self._free_tasks = set(cfg.get('free_tasks'))
self._done_tasks = set(cfg.get('done_tasks'))
def __save_info(self):
with open('books_task_config.cfg', 'w') as f:
cfg = {}
cfg['status'] = self._status
cfg['free_tasks'] = list(self._free_tasks)
cfg['done_tasks'] = list(self._done_tasks)
f.write(json.dumps(cfg))
def __get_tasks(self):
if self._status == 'free':
print 'getting task....'
resp = requests.get(self._tasks_url)
js = resp.json()
for t in js.get('tasks'):
self._free_tasks.add(t)
self._status = 'running'
def __do_tasks(self):
with open('books.txt', 'a') as f:
for t in self._free_tasks:
if t not in self._done_tasks:
print 'fetch %s....' % t
books = self.__get_books(t)
obj = {'_id':t, 'books':books}
f.write(json.dumps(obj) + '\n')
self._done_tasks.add(t)
def __get_books(self, user):
books = []
count = 100
for i in range(20):
url = self._url % (user, count, i*count)
content = self._fetch.get(url, sleeptime=6)
js = json.loads(content.decode('utf-8', 'ignore'))
books.extend(js.get('collections'))
if (i+1)*count >= js.get('total'):
break
return books
def __upload_tasks(self):
tasks = {'type':'books', 'data':[]}
with open('books.txt', 'r') as f:
for line in f:
obj = json.loads(line.rstrip('\n'))
tasks['data'].append(obj)
while True:
print 'uploading task ....'
data = json.dumps(tasks)
headers = {'Content-type':'application/json; charset=utf8'}
resp = requests.put(self._upload_url, data=data, headers=headers)
js = resp.json()
if js.get('code') == 200:
self.__reset()
os.remove('books.txt')
break
def run(self):
while True:
try:
self.__get_tasks()
self.__do_tasks()
self.__upload_tasks()
except KeyboardInterrupt:
break
except Exception, e:
logging.error(repr(e))
def newsapi(topic):
"""Return newsapi json data according to topic.
"""
fetcher = Fetch()
data = fetcher.fetch_newsapi(topic)
return jsonify(data)
def coinpaprika():
"""Return coinpaprika json data.
"""
fetcher = Fetch()
data = fetcher.fetch_coinpaprika()
return jsonify(data)
data[i]['date'] = time.mktime(item['date'].timetuple())
return jsonResponse(data=data, extra_data=[{'total': total}])
@app.route('/api/news/<int:id>')
def news(id):
news = News(news_id=id)
data = news.get()
if data is None:
return jsonResponse(type='404')
return jsonResponse(data=data)
@app.route('/rss')
def rss():
news = News()
data = news.sort()
response = make_response(render_template('rss.xml', data=data, site_url=site_url))
response.headers['Content-Type'] = 'application/atom+xml; charset=utf-8'
return response
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def catch_all(path):
return render_template('index.html')
if __name__ == '__main__':
fetch = Fetch()
fetch.start()
app.run()
body = Column(String)
title = Column(String)
image_source = Column(String)
image = Column(String)
thumbnail = Column(String)
date = Column(DateTime)
# def create_conf_file():
# access_key = input('qiniu access key')
# secrect_key = input('qiniu secrect key')
# bucket_name = input('qiniu bucket name')
# bucket_host = input('qiniu bucket host')
# site_url = input('site url include http')
# conf_file = open(os.path.dirname(__file__) + 'conf.py', 'w+')
# data = '''
# access_key="%s"\n
# secrect_key="%s"\n
# qiniu_bucket_name=%s\n
# qiniu_bucket_host=%s\n
# site_url=%s
# ''' %(access_key, secrect_key, bucket_name, qiniu_bucket_host, site_url)
# conf_file.write(data)
# conf_file.close()
if __name__ == '__main__':
#create_conf_file()
Base.metadata.create_all(engine)
fetch = Fetch()
fetch.init_fetch()
class FollowedTask:
''' get douban user followed'''
def __init__(self):
self.__reset()
self.__read_info()
self._fetch = Fetch(username='******', pw='liumengchao')
self._tasks_url = 'http://localhost:8080/id/followed/'
self._url = 'http://www.douban.com/people/%s/contacts'
self._upload_url = 'http://localhost:8080/upload/'
logging.basicConfig(filename='followed_error.log', filemod='a+', level=logging.ERROR)
def __del__(self):
self.__save_info()
def __reset(self):
self._status = 'free'
self._free_tasks = set()
self._done_tasks = set()
def __read_info(self):
if os.path.exists('user_followed_config.cfg'):
with open('user_followed_config.cfg', 'r') as f:
cfg = json.loads(f.read())
self._status = cfg.get('status')
self._free_tasks = set(cfg.get('free_tasks'))
self._done_tasks = set(cfg.get('done_tasks'))
def __save_info(self):
with open('user_followed_config.cfg', 'w') as f:
cfg = {}
cfg['status'] = self._status
cfg['free_tasks'] = list(self._free_tasks)
cfg['done_tasks'] = list(self._done_tasks)
f.write(json.dumps(cfg))
def __get_followed(self, user):
page = self._fetch.get(self._url % user, sleeptime=2.1)
followed = user_followed_parser(page)
return followed
def __get_tasks(self):
if self._status == 'free':
print 'get tasks.....'
resp = requests.get(self._tasks_url)
js = resp.json()
self._free_tasks = js.get('tasks')
self._status = 'running'
def __do_tasks(self):
with open('followed.txt', 'a') as f:
for t in self._free_tasks:
if t not in self._done_tasks:
print 'fetching %s' % t
obj = {'_id':t}
obj['followed'] = self.__get_followed(t)
f.write(json.dumps(obj) + '\n')
self._done_tasks.add(t)
def __upload_tasks(self):
tasks = {'type':'followed', 'data':[]}
with open('followed.txt', 'r') as f:
for line in f:
obj = json.loads(line.rstrip('\n'))
tasks['data'].append(obj)
data = json.dumps(tasks)
headers = {'Content-type':'application/json; charset=utf8'}
while True:
print 'uploading '
resp = requests.put(self._upload_url, data=data, headers=headers)
js = resp.json()
if js.get('code') == 200:
os.remove('followed.txt')
self.__reset()
break
def run(self):
while True:
try:
self.__get_tasks()
self.__do_tasks()
self.__upload_tasks()
except KeyboardInterrupt:
break
except Exception, e:
logging.error(repr(e))
import datetime as dt
from fetch import Fetch
from display import Display
import pprint
if __name__ == '__main__':
# Performs queries:
fetchy = Fetch()
#fetchy.fetchTweets("prayforsyria", 1000, begindate=dt.date(2011,1,21), enddate=dt.date.today())
# Defines words that we want to use in the visualization. All other words will be ignored:
chosenWords = [
"Syria", "people", "you", "all", "no", "we", "world", "will", "about",
"innocent", "Allah", "paz", "just", "pray", "heart", "Syrian", "mundo",
"children", "who", "one", "don't", "please", "help", "Pray", "more",
"like", "apoyo", "oración", "#Aleppo", "going", "war", "God", "uno",
"need", "prayers", "ayuno", "happening", "killed", "being", "peace",
"sad", "NO", "know", "now", "because", "stop", "many", "everyone",
"live", "unimos", "really", "lives", "orar", "much", "love",
"suffering", "protect", "guerra", "morning", "even", "todos", "hope",
"country", "#LLAP", "forget", "never", "over", "every", "still",
"brothers", "think", "llamado", "ayunar", "human", "time", "feel",
"praying", "keep", "bomb", "bombing", "kids", "Virgen", "today",
"Assad", "better", "diligencia", "killing", "breaks", "trabajo",
"video", "life", "firmemente", "rezar", "where", "lost"
"kill", "pidiendo", "humanity", "deserve", "always", "gente", "little",
"take", "safe", "end", "say", "sisters", "doesn't", "any", "dying",
"dead", "after", "things", "good", "Trump", "some", "child", "needs",
"heartbreaking", "other", "Dios", "care", "Syrians", "US", "personas",
"news", "civilians", "não", "against", "inocentes", "unidos", "media",
#import sys
#sys.path.append('/home/ubuntu/projects/financial')
from fetch import Fetch
fetch = Fetch()
params = [('sc', 431)]
stocklist = fetch.fetch_stocks(params)
print stocklist
def setUp(self):
self.fetch = Fetch(username='******', pw='liumengchao')
