def aborted(self):
# remove partial file
try:
gnomevfs.unlink(self.output_filename)
except:
log("cannot delete: '%s'" % beautify_uri(self.output_filename))
return
def _clear_watched_items( clear_type ):
utils.log( "_clear_watched_items( %s )" % ( clear_type ), xbmc.LOGNOTICE )
# initialize base_path
base_paths = []
# clear trivia or trailers
if ( clear_type == "ClearWatchedTrailers" ):
# handle AMT db special
sys.path.append( os.path.join( BASE_RESOURCE_PATH, "lib", "scrapers") )
from amt_database import scraper as scraper
Scraper = scraper.Main()
# update trailers
Scraper.clear_watched()
# set base watched file path
base_paths += [ os.path.join( BASE_CURRENT_SOURCE_PATH, "amt_current_watched.txt" ) ]
base_paths += [ os.path.join( BASE_CURRENT_SOURCE_PATH, "local_watched.txt" ) ]
else:
# set base watched file path
base_paths = [ os.path.join( BASE_CURRENT_SOURCE_PATH, "trivia_watched.txt" ) ]
try:
# set proper message
message = ( 32531, 32541, )[ sys.argv[ 1 ] == "ClearWatchedTrailers" ]
# remove watched status file(s)
for base_path in base_paths:
# remove file if it exists
if ( xbmcvfs.exists( base_path ) ):
xbmcvfs.delete( base_path )
except:
# set proper message
message = ( 32532, 32542, )[ sys.argv[ 1 ] == "ClearWatchedTrailers" ]
# inform user of result
ok = xbmcgui.Dialog().ok( __language__( 32000 ), __language__( message ) )
def initialize(context):
# hook into the Control Panel
global translation_service
# allow for disabling PTS entirely by setting a environment variable.
if bool(os.getenv('DISABLE_PTS')):
log('Disabled by environment variable "DISABLE_PTS".', logging.WARNING)
return
cp = context._ProductContext__app.Control_Panel # argh
if cp_id in cp.objectIds():
cp_ts = getattr(cp, cp_id)
# use the ts in the acquisition context of the control panel
# translation_service = translation_service.__of__(cp)
translation_service = PTSWrapper(cp_ts)
else:
cp_ts = make_translation_service(cp)
# don't touch - this is the last version that didn't have the
# attribute (0.4)
instance_version = getattr(cp_ts, '_instance_version', (0, 4, 0, 0))
if instance_version[3] > 99:
log('development mode: translation service recreated',
detail = '(found %s.%s.%s.%s)\n' % instance_version)
cp._delObject(cp_id)
cp_ts = make_translation_service(cp)
if instance_version < PlacelessTranslationService._class_version:
log('outdated translation service found, recreating',
detail = '(found %s.%s.%s.%s)\n' % instance_version)
cp._delObject(cp_id)
purgeMoFileCache()
cp_ts = make_translation_service(cp)
# sweep products
log('products: %r' % get_products(), logging.DEBUG)
for prod in get_products():
# prod is a tuple in the form:
# (priority, dir_name, index, base_dir) for each Product directory
cp_ts._load_i18n_dir(os.path.join(prod[3], prod[1], 'i18n'))
cp_ts._load_locales_dir(os.path.join(prod[3], prod[1], 'locales'))
# sweep the i18n directory for local catalogs
instance_i18n = os.path.join(INSTANCE_HOME, 'i18n')
if os.path.isdir(instance_i18n):
cp_ts._load_i18n_dir(instance_i18n)
instance_locales = os.path.join(INSTANCE_HOME, 'locales')
if os.path.isdir(instance_locales):
cp_ts._load_locales_dir(instance_locales)
# didn't found any catalogs
if not cp_ts.objectIds():
log('no translations found!', logging.DEBUG)
# set ZPT's translation service
# NOTE: since this registry is a global var we can't register the
# persistent service itself (zodb connection) therefore a
# wrapper is created around it
setGlobalTranslationService(PTSWrapper(cp_ts))
def on_task_finished(self, task):
task.sound_file.progress = 1.0
if task.error:
debug("error in task, skipping rename:", task.output_filename)
if vfs_exists(task.output_filename):
vfs_unlink(task.output_filename)
self.errors.append(task.error)
self.error_count += 1
return
duration = task.get_duration()
if duration:
self.duration_processed += duration
# rename temporary file
newname = self.window.prefs.generate_filename(task.sound_file)
log(beautify_uri(task.output_filename), "->", beautify_uri(newname))
# safe mode. generate a filename until we find a free one
p, e = os.path.splitext(newname)
p = p.replace("%", "%%")
p = p + " (%d)" + e
i = 1
while vfs_exists(newname):
newname = p % i
i += 1
task.error = vfs_rename(task.output_filename, newname)
if task.error:
self.errors.append(task.error)
self.error_count += 1
def answer_from_div(div):
a = Answer()
a.author = div.xpath('.//a[@class="author-link"]')[0].text
log('author,', a)
content = div.xpath('.//div[@class="zm-editable-content clearfix"]/text()')
a.content = '\n'.join(content)
return a
def main():
if not len(sys.argv) == 2:
usage()
if not config.enabled.lower() == "true":
fatal('ebsmount is not enabled (%s)' % config.CONF_FILE)
action = sys.argv[1]
DEVNAME = os.getenv('DEVNAME', None)
PHYSDEVPATH = os.getenv('PHYSDEVPATH', None)
if not action in ('add', 'remove'):
usage('action must be one of: add, remove')
if not DEVNAME:
usage('DEVNAME is required')
if not PHYSDEVPATH:
usage('PHYSDEVPATH is required')
if not PHYSDEVPATH.startswith('/devices/xen/vbd-'):
usage('PHYSDEVPATH is not of the expected structure')
# log trigger
log(DEVNAME, "received %s trigger" % action)
mountdir = os.path.join(config.mountdir, os.path.basename(PHYSDEVPATH))
func = getattr(ebsmount, 'ebsmount_' + action)
func(DEVNAME, mountdir)
def getImagesMjpegInterlace(self, camera, url, control, prefix):
""" Update camera position with interlaced mjpeg frames """
try:
stream = requests.get(url, stream=True, timeout=TIMEOUT).raw
except requests.RequestException as e:
utils.log(3, e)
control[0].setImage(_error, useCache=False)
return
x = 0
while not monitor.abortRequested() and self.isRunning:
filename = os.path.join(_datapath, "%s_%s.%d.jpg") % (prefix, camera.number, x)
filename_exists = utils.get_mjpeg_frame(stream, filename)
if filename_exists:
if x % 2 == 0: # Interlacing for flicker reduction/elimination
control[0].setImage(filename, useCache=False)
else:
control[1].setImage(filename, useCache=False)
xbmcvfs.delete(os.path.join(_datapath, "%s_%s.%d.jpg") % (prefix, camera.number, x - 2))
x += 1
else:
utils.log(3, "Camera %s - Error on MJPEG" % camera.number)
control[0].setImage(_error, useCache=False)
return
def delete_msgs_wrapper(self, service, msgIds=[], user_id=''):
utils.log( 'Delete ' + str(len(msgIds)) + ' messages')
for msgId in msgIds:
self.delete_msg(service, user_id, msgId)
print '*',
utils.log('Done')
def ebsmount_add(devname, mountdir):
"""ebs device attached"""
matching_devices = []
for device in udevdb.query():
if device.name.startswith(basename(devname)):
matching_devices.append(device)
for device in matching_devices:
devpath = join('/dev', device.name)
mountpath = join(mountdir, device.env.get('ID_FS_UUID', devpath[-1])[:6])
mountoptions = ",".join(config.mountoptions.split())
filesystem = device.env.get('ID_FS_TYPE', None)
if not filesystem:
log(devname, "could not identify filesystem: %s" % devpath)
continue
if not filesystem in config.filesystems.split():
log(devname, "filesystem (%s) not supported: %s" % (filesystem,devpath))
continue
if is_mounted(devpath):
log(devname, "already mounted: %s" % devpath)
continue
mount(devpath, mountpath, mountoptions)
if exists(config.postmountscript):
log(devname, "Executing: %s -m %s -d %s" % (config.postmountscript, mountpath, devname))
res = system("exec '%s' -m %s -d %s" % (config.postmountscript, mountpath, devname))
else:
log(devname, "Script does not exist at %s" % config.postmountscript)
def import_into_db(self, full_path):
temp_path = os.path.join(xbmc.translatePath("special://profile"), 'temp_import_%s.csv' % (int(time.time())))
utils.log('Copying import file from: |%s| to |%s|' % (full_path, temp_path), xbmc.LOGDEBUG)
if not xbmcvfs.copy(full_path, temp_path):
raise Exception('Import: Copy from |%s| to |%s| failed' % (full_path, temp_path))
try:
with open(temp_path, 'r') as f:
reader = csv.reader(f)
mode = ''
_ = f.readline() # read header
for line in reader:
if CSV_MARKERS.FAVORITES in line[0] or CSV_MARKERS.SUBSCRIPTIONS in line[0] or CSV_MARKERS.BOOKMARKS in line[0] or CSV_MARKERS.EXT_SUBS in line[0]:
mode = line[0]
continue
elif mode == CSV_MARKERS.FAVORITES:
try:
self.save_favorite(line[0], line[1], line[2], line[3])
except: pass # save_favorite throws exception on dupe
elif mode == CSV_MARKERS.SUBSCRIPTIONS:
# don't allow import of days with values other than 0-6
if line[5].translate(None, '0123456'): line[5] = '0123456'
self.add_subscription(line[0], line[1], line[2], line[3], line[4], line[5])
elif mode == CSV_MARKERS.BOOKMARKS:
self.set_bookmark(line[0], line[1])
elif mode == CSV_MARKERS.EXT_SUBS:
self.add_ext_sub(line[0], line[1], line[2], line[3])
else:
raise Exception('CSV line found while in no mode')
finally:
if not xbmcvfs.delete(temp_path):
raise Exception('Import: Delete of %s failed.' % (temp_path))
def delete_temp_files():
log('Deleting temporary files...')
os.remove(settings.BROKEN_CSV_FILE_NAME)
os.remove(settings.FIXED_CSV_FILE_NAME)
log('Done')
def main():
parser = argparse.ArgumentParser(description="Retrieve petitions from We The People")
parser.add_argument(
"-m",
"--max",
metavar="INTEGER",
dest="max",
type=int,
default=None,
help="maximum number of petitions to retrieve",
)
parser.add_argument(
"-s",
"--start",
metavar="INTEGER",
dest="start",
type=int,
default=1,
help="starting page, 20 per page, default is 1",
)
args = parser.parse_args()
if args.max is not None and args.max < 1:
parser.error("How can I scrape less than one petition? You make no sense! --max must be one or greater.")
if args.start < 1:
parser.error("--start must be one or greater.")
log("Found %i petitions" % (petitions(args.start, args.max)))
# write log
scrapelog["end"] = datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
write(json.dumps(scrapelog, indent=2), "log-wh-" + scrapelog["begin"] + ".json", log_dir())
def export_from_db(self, full_path):
temp_path = os.path.join(xbmc.translatePath("special://profile"), 'temp_export_%s.csv' % (int(time.time())))
with open(temp_path, 'w') as f:
writer = csv.writer(f)
f.write('***VERSION: %s***\n' % self.__get_db_version())
if self.__table_exists('favorites'):
f.write(CSV_MARKERS.FAVORITES + '\n')
for fav in self.get_favorites():
writer.writerow(fav)
if self.__table_exists('subscriptions'):
f.write(CSV_MARKERS.SUBSCRIPTIONS + '\n')
for sub in self.get_subscriptions():
writer.writerow(sub)
if self.__table_exists('new_bkmark'):
f.write(CSV_MARKERS.BOOKMARKS + '\n')
for bookmark in self.get_bookmarks():
writer.writerow(bookmark)
if self.__table_exists('external_subs'):
f.write(CSV_MARKERS.EXT_SUBS + '\n')
for sub in self.get_external_subs():
writer.writerow(sub)
utils.log('Copying export file from: |%s| to |%s|' % (temp_path, full_path), xbmc.LOGDEBUG)
if not xbmcvfs.copy(temp_path, full_path):
raise Exception('Export: Copy from |%s| to |%s| failed' % (temp_path, full_path))
if not xbmcvfs.delete(temp_path):
raise Exception('Export: Delete of %s failed.' % (temp_path))
def parse(queue):
utils.log('start...')
request = queue.get()
success, response = page_hybridflightresults_aspx(request)
print success, response
utils.log('done')
queue.put({ 'success': success, 'flags':['no_range'], 'response': response })
def main():
if not len(sys.argv) == 2:
usage()
if not config.enabled.lower() == "true":
fatal('ebsmount is not enabled (%s)' % config.CONF_FILE)
action = sys.argv[1]
devname = os.getenv('DEVNAME', None)
devpath = os.getenv('PHYSDEVPATH', os.getenv('DEVPATH', None))
if not action in ('add', 'remove'):
usage('action must be one of: add, remove')
if not devname:
usage('DEVNAME is required')
if not devpath:
usage('PHYSDEVPATH or DEVPATH is required')
if not _expected_devpath(devpath, config.devpaths.split()):
usage('PHYSDEVPATH/DEVPATH is not of the expected structure')
# log trigger
log(devname, "received %s trigger" % action)
func = getattr(ebsmount, 'ebsmount_' + action)
func(devname, config.mountdir)
def build_xsl_reports(
locate_root_dir
, tag
, expected_results_file
, failures_markup_file
, comment_file
, results_dir
, result_file_prefix
, dont_collect_logs = 0
, reports = report_types
, warnings = []
, user = None
, upload = False
):
( run_date ) = time.strftime( '%Y-%m-%dT%H:%M:%SZ', time.gmtime() )
root_paths.append( locate_root_dir )
root_paths.append( results_dir )
bin_boost_dir = os.path.join( locate_root_dir, 'bin', 'boost' )
output_dir = os.path.join( results_dir, result_file_prefix )
utils.makedirs( output_dir )
if expected_results_file != '':
expected_results_file = os.path.abspath( expected_results_file )
else:
expected_results_file = os.path.abspath( map_path( 'empty_expected_results.xml' ) )
extended_test_results = os.path.join( output_dir, 'extended_test_results.xml' )
execute_tasks(
tag
, user
, run_date
, comment_file
, results_dir
, output_dir
, reports
, warnings
, extended_test_results
, dont_collect_logs
, expected_results_file
, failures_markup_file
)
if upload:
upload_dir = 'regression-logs/'
utils.log( 'Uploading results into "%s" [connecting as %s]...' % ( upload_dir, user ) )
archive_name = '%s.tar.gz' % result_file_prefix
utils.tar(
os.path.join( results_dir, result_file_prefix )
, archive_name
)
utils.sourceforge.upload( os.path.join( results_dir, archive_name ), upload_dir, user )
utils.sourceforge.untar( os.path.join( upload_dir, archive_name ), user, background = True )
def _get_totals(self):
utils.log('calculating totals')
df = self.df[self.independent_vars].sum().to_frame('total')
df = df.set_value('tds_0','total',None)
df = df.set_value('tds_1','total',None)
df = df.reset_index().rename(columns={'index':'independent_variable'})
return df
def save(self):
show_name = None
if self.rageid is not None:
show_name = self.rageid_show_name(self.rageid)
if show_name is not None:
show_name = unicode(show_name, 'utf-8')
if show_name is None:
manual_name = unikeyboard(self.nzbname, 'Enter show name')
if manual_name is None:
log("Tvshow: save: did not recieve a name for the TV-show")
return
#show_name = manual_name.decode("utf_8").encode("raw_unicode_escape")
show_name = unicode(manual_name, 'utf-8').replace('\n','')
strm_path_show = utils.join(self.strm_path, os.path.join(remove_disallowed_filename_chars(show_name),''))
# Check if showname folder exist in path, if not create it.
if not utils.exists(strm_path_show):
try:
utils.mkdir(strm_path_show)
except:
log("Tvshow: save: failed to create TV-show folder %s" % strm_path_show)
return
# Check if tvshow.nfo is there, if not, create it.
tv_nfo = self.info
tv_nfo.path(strm_path_show)
# The Episode name has to be picked up by XBMC
# regexps
episode_name = self.check_episode_name(self.nzbname)
if not self.save_nfo_type == "disabled":
if self.save_nfo_type == "minimal":
tv_nfo.mini()
if not utils.exists(os.path.join(strm_path_show, 'tvshow.nfo')):
tv_nfo.save_tvshow(show_name)
# now, save the episodename.nfo
tv_nfo.save_episode(episode_name)
strm.StrmFile(strm_path_show, episode_name, self.nzb).save()
def ftp_task( site, site_path , destination ):
__log__ = 1
utils.log( '' )
utils.log( 'ftp_task: "ftp://%s/%s" -> %s' % ( site, site_path, destination ) )
utils.log( ' logging on ftp site %s' % site )
f = ftplib.FTP( site )
f.login()
utils.log( ' cwd to "%s"' % site_path )
f.cwd( site_path )
source_content = list_ftp( f )
source_content = [ x for x in source_content if re.match( r'.+[.](?<!log[.])zip', x.name ) and x.name.lower() != 'boostbook.zip' ]
destination_content = list_dir( destination )
d = diff( source_content, destination_content )
def synchronize():
for source in d[0]:
utils.log( 'Copying "%s"' % source )
result = open( os.path.join( destination, source ), 'wb' )
f.retrbinary( 'RETR %s' % source, result.write )
result.close()
mod_date = find_by_name( source_content, source ).date
m = time.mktime( mod_date )
os.utime( os.path.join( destination, source ), ( m, m ) )
for obsolete in d[1]:
utils.log( 'Deleting "%s"' % obsolete )
os.unlink( os.path.join( destination, obsolete ) )
utils.log( " Synchronizing..." )
__log__ = 2
synchronize()
f.quit()
def the_end_dialog(params, **kwargs):
log("the_end_dialog: params: %s kwargs: %s" %(params, kwargs))
dialog = xbmcgui.Dialog()
if 'is_stopped' in kwargs:
is_stopped = kwargs['is_stopped']
else:
is_stopped = False
if 'progressing' in kwargs:
progressing = kwargs['progressing']
else:
progressing = False
if progressing:
options = ['Delete', 'Just download']
if is_stopped:
heading = 'Downloading, what do you want to do?'
else:
heading = 'Still downloading, what do you want to do?'
else:
heading = 'Download finished, what do you want to do?'
options = ['Delete', 'Repair']
ret = dialog.select(heading, options)
if ret == 0:
delete(params)
if ret == 1 and progressing:
just_download(params)
elif ret == 1 and not progressing:
repair(params)
return
def save_strm(nzbname, url):
log("save_strm: nzbname: %s url: %s" % (nzbname, url))
strm2lib.save_strm(__settings__, nzbname, url)
if SAVE_NZB and utils.exists(NZB_CACHE):
nzb_path = utils.join(NZB_CACHE, '%s%s' % (nzbname, '.nzb'))
log("save_strm: nzb_path: %s" % nzb_path)
m_nzb.save(url, nzb_path)
def connect_to_sql(self, sql_connect, db_name="", force_reconnect=False, create_db=True):
"""
Connect to SQL database or create the database and connect
:param sql_connect: the variable to set
:param db_name: the name of the database
:param force_reconnect: force the database connection
:param create_db: create the database
:return the created SQL connection
"""
print self
if sql_connect is None or force_reconnect:
try:
sql_connect = MySQLdb.connect(host=config.SQL_HOST, user=config.SQL_USERNAME, passwd=config.SQL_PASSWORD, db=db_name)
return sql_connect
except Exception, e:
# Create the database
if e[0] and create_db and db_name != "":
if sql_connect is None:
sql_connect = MySQLdb.connect(host=config.SQL_HOST, user=config.SQL_USERNAME, passwd=config.SQL_PASSWORD)
utils.log("Creating database " + db_name)
cur = sql_connect.cursor()
cur.execute("CREATE DATABASE " + db_name)
sql_connect.commit()
sql_connect.select_db(db_name)
return sql_connect
else:
utils.log("Could not connect to MySQL: %s" % e)
return None
def ensure_mongo_indexes():
"""
Runs commands on the mongo indexes to ensure that they are set
:return: None
"""
utils.log("Setting index for countries")
cmd = config.MONGO_PATH + "mongo --quiet " + config.MONGO_HOST + "/logs --eval \"db.clickstream.ensureIndex({country:1})\""
def _run(self):
utils.log("[%s] pulling from source '%s'" % (self.name, self._source.name))
self._source.startProducing()
self.processQueue(self._source._output)
self._source.join()
self._output.put(StopIteration)
def crossValidate(opts):
"""
Starts the tree cross validation.
The subset tree has a depth defined by the the division-depth opttion.
Each dataset will be split into a number of subsets defined by the division-factor option.
@type opts: Options
@param opts: Options object
"""
totaltime = time.time()
opts.training_file = scale(opts, opts.training_file)
log("Scaling input data")
while opts.division_depth >= 0:
starttime = time.time()
log("Range: C={0}; gamma={1}".format(opts.c_range,opts.g_range))
res = crossValidationRound(opts)
cstart,cstop,cstep = opts.c_range
gstart,gstop,gstep = opts.g_range
cmin,cmax,gmin,gmax = calcNewRange(res, opts)
opts.division_depth = opts.division_depth -1
opts.c_range = cmin,cmax,cstep
opts.g_range = gmin,gmax,gstep
endtime = time.time()
log("Time for this round: {0}".format(time.strftime("%H:%M:%S", time.gmtime(endtime-starttime))))
log("Remaining division steps: {0}".format(opts.division_depth+1))
log("Total time: {}".format(time.strftime("%H:%M:%S", time.gmtime(time.time()-totaltime))))
def get_latest_version():
"""
Sort the list, and get the latest version
"""
versions = get_versions()
utils.log('Version check found versions: %s' % versions)
return sorted(versions, reverse=True)[0]
def upload_log():
"""
Upload our full XBMC log as a GitHub gist
"""
try:
log_content = get_xbmc_log()
except Exception as e:
utils.log("Failed to read log: %s" % e)
return None
utils.log("Uploading xbmc.log")
try:
response = urllib2.urlopen(make_request(config.GIST_API_URL), json.dumps({
"files": {
"xbmc.log": {
"content": log_content
}
}
}))
except urllib2.HTTPError as e:
print e
utils.log("Failed to save log: HTTPError %s" % e.code)
return False
except urllib2.URLError as e:
print e
utils.log("Failed to save log: URLError %s" % e.reason)
return False
try:
return json.load(response)["html_url"]
except:
utils.log("Failed to parse API response: %s" % response.read())
def process(self, message):
end = message.get('end')
start = message.get('start')
startDoy = message.get('startDoy')
startYear = message.get('startYear')
layer = message.get('layer')
region = message.get('region')
productName = message.get('productName').replace('.','_')
outputFilename = "_".join([productName, layer['name'], startYear + startDoy , region]) + '.tif'
outputFilepath = os.path.join(self.module_path, outputFilename)
utils.removeFileIfExist(outputFilepath)
utils.log(self.name, 'Generating', outputFilename, 'merging', str(len(layer['files'])), 'files')
gdal_utils.mosaic(layer['files'], outputFilepath, layer['nodata'])
tmpFiles = message.get('tmpFiles')
tmpFiles += layer['files']
message.set('tmpFiles', tmpFiles)
del layer['files']
layer['file'] = outputFilepath
message.set('layer', layer)
utils.log(self.name, 'Forward message (', outputFilename, ')')
self.publish(message)
def setup(self):
if(APP_KEY == '' or APP_SECRET == ''):
xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30058),utils.getString(30059))
return
user_token_key,user_token_secret = self.getToken()
sess = session.DropboxSession(APP_KEY,APP_SECRET,"app_folder")
if(user_token_key == '' and user_token_secret == ''):
token = sess.obtain_request_token()
url = sess.build_authorize_url(token)
#print url in log
utils.log("Authorize URL: " + url)
xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30056),utils.getString(30057))
#if user authorized this will work
user_token = sess.obtain_access_token(token)
self.setToken(user_token.key,user_token.secret)
else:
sess.set_token(user_token_key,user_token_secret)
self.client = client.DropboxClient(sess)
try:
utils.log(str(self.client.account_info()))
except:
#this didn't work, delete the token file
self.deleteToken()
def playlist_item(play_list, rar_file_list, folder, sab_nzo_id, sab_nzo_id_history):
log("playlist_item: play_list: %s rar_file_list: %s folder: %s sab_nzo_id: %s sab_nzo_id_history: %s" %\
(play_list, rar_file_list, folder, sab_nzo_id, sab_nzo_id_history))
new_play_list = play_list[:]
for arch_rar, movie_file in zip(play_list[0::2], play_list[1::2]):
info = nfo.ReadNfoLabels(folder)
xurl = "%s?mode=%s" % (sys.argv[0],MODE_LIST_PLAY)
url = (xurl + "&nzoid=" + str(sab_nzo_id) + "&nzoidhistory=" + str(sab_nzo_id_history)) +\
"&play_list=" + utils.quote_plus(';'.join(new_play_list)) + "&folder=" + utils.quote_plus(folder) +\
"&file_list=" + utils.quote_plus(';'.join(rar_file_list))
new_play_list.remove(arch_rar)
new_play_list.remove(movie_file)
item = xbmcgui.ListItem(movie_file, iconImage='DefaultVideo.png', thumbnailImage=info.thumbnail)
item.setInfo(type="Video", infoLabels=info.info_labels)
item.setProperty("Fanart_Image", info.fanart)
item.setPath(url)
isfolder = False
# item.setProperty("IsPlayable", "true")
cm = []
if sab_nzo_id_history:
cm_url_repair = sys.argv[0] + '?' + "mode=repair" + "&nzoidhistory=" + str(sab_nzo_id_history) + "&folder=" + utils.quote_plus(folder)
cm.append(("Repair" , "XBMC.RunPlugin(%s)" % (cm_url_repair)))
cm_url_delete = sys.argv[0] + '?' + "mode=delete" + "&nzoid=" + str(sab_nzo_id) + "&nzoidhistory=" + str(sab_nzo_id_history) + "&folder=" + utils.quote_plus(folder)
cm.append(("Delete" , "XBMC.RunPlugin(%s)" % (cm_url_delete)))
item.addContextMenuItems(cm, replaceItems=True)
xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=url, listitem=item, isFolder=isfolder)
xbmcplugin.setContent(HANDLE, 'movies')
xbmcplugin.endOfDirectory(HANDLE, succeeded=True, cacheToDisc=True)
return
def exception_dialog(self, error):
if debug.get() == False:
log(error, xbmc.LOGERROR)
dialog.error(error)
def comment_delete(request):
comment_id = int(request.query['id'])
Comment.delete(comment_id)
log('删除的评论id', comment_id)
d = dict(message="成功删除 comment")
return json_response(d)
import utils
import conexion
import cliente
log = utils.log("INIT")
log.info("inicio del programa")
lstCliente = []
lstTipoPago = []
lstEmpresa = []
lstProductos = []
def cargarObjetos():
conn = conexion.conexionBDD(1)
query = "select idCliente, nombreCliente as Nombre, nroIdentCliente as ID, direccionCliente as Direccion from cliente;"
resconn = conn.consultarBDD(query)
for row in resconn:
cliente = cliente.cliente(row[0], row[1], row[2], row[3])
lstClientes.append(cliente)
for obj in lstCliente:
print(obj.nombreCliente)
input("continuar")
def mantenimientoCliente():
dicMenuCliente = {
"\t- Buscar Cliente Todos": 1,
"\t- Buscar Cliente por DNI": 2,
"\t- Modificar Cliente por DNI": 3,
"\t- Crear Cliente": 4,
def parse_feed(self, key, value, i):
if self.source == "AP" or self.source == "Reuters":
try:
news_url = self.url % key
if self.source == "Reuters":
news_url += "?since=" + str(
int((time.mktime(datetime.utcnow().timetuple()))) *
10000000000)
feed = requests.get(news_url).json(
) # we use AP's API to download their news, it's epic and it uses JSON
except:
return i
elif self.source == "AFP_French" or self.source == "ANP":
feed = feedparser.parse(self.url)
elif self.source == "ANSA" and value == "italy":
feed = feedparser.parse(self.sourceinfo["url2"] % (key, key))
elif self.source == "ANSA":
feed = feedparser.parse(self.url % (key, key))
else:
feed = feedparser.parse(self.url % key)
j = 0
if self.source == "AP":
entries = feed["cards"]
elif self.source == "Reuters":
entries = feed["wireitems"]
elif self.source == "AFP_French":
entries = feed.entries + feedparser.parse(
self.sourceinfo["url2"]).entries
else:
entries = feed.entries
for entry in entries:
try:
if self.source == "AP":
try:
entry = entry["contents"][0]
except:
continue
elif self.source == "Reuters":
try:
_ = entry["templates"][1]["story"]["hed"]
entry = entry["templates"][1]
except:
continue
current_time = int(
(time.mktime(datetime.utcnow().timetuple()) - 946684800) /
60)
if self.source == "AP":
update = time.strptime(entry["updated"],
"%Y-%m-%d %H:%M:%S")
elif self.source == "Reuters":
update = time.strptime(entry["story"]["updated_at"],
"%Y-%m-%dT%H:%M:%SZ")
else:
update = entry["updated_parsed"]
updated_time = int((time.mktime(update) - 946684800) / 60)
if self.source == "AFP_French" and current_time - updated_time < 0:
updated_time -= 180
if (current_time - updated_time <
60): # if it's a new article since the last hour
i += 1
j += 1
if (
i > 25
): # in case we have too many articles, we don't want the news file to get too big, there's a limit
break
if self.source == "AFP_French" or self.source == "ANP_Dutch":
if key not in entry["link"]:
continue
elif self.source == "AFP" and "SID" in entry["description"]:
self.source = "SID"
elif self.source == "NU.nl" and entry["author"] == "ANP":
self.source = "ANP"
if self.source == "AP":
title = entry["headline"]
elif self.source == "Reuters":
title = entry["story"]["hed"]
else:
title = entry["title"]
print(title)
if self.source == "AP":
entry_url = entry["gcsUrl"]
elif self.source == "Reuters":
entry_url = self.url[:30] + entry["template_action"][
"api_path"]
else:
entry_url = entry["link"]
downloaded_news = Parse(entry_url, self.source,
updated_time, title,
self.language).get_news()
if downloaded_news:
self.newsdata[value + str(j)] = downloaded_news
except Exception as e:
ex = "Failed to parse feed - line {}: {}".format(
sys.exc_info()[-1].tb_lineno, str(e))
print(ex)
log(ex, "INFO")
continue
return i
def locations_download(
language_code, data
): # using Google Maps API is so much better than the crap Nintendo used for say, AP news.
locations = {}
gmaps = googlemaps.Client(key=config["google_maps_api_key"])
languages = { # corresponds to the Wii's language codes
0: "ja",
1: "en",
2: "de",
3: "fr",
4: "es",
5: "it",
6: "nl",
}
for keys, values in list(data.items()):
location = values[7]
if location and location != "":
if location not in locations:
locations[location] = [None, None, []]
locations[location][2].append(keys)
for name in list(locations.keys()):
if name == "":
continue
uni_name = (
name if languages[language_code] == "ja" else unidecode(name)
) # if using unidecode with Japanese, it'll translate all the characters to English
print(uni_name)
coordinates = None
if name not in cities:
try:
read = gmaps.geocode(uni_name,
language=languages[language_code])
loc_name = read[0]["address_components"][0]["long_name"]
if languages[language_code] == "ja":
loc_name = enc(loc_name)
else:
loc_name = enc(unidecode(loc_name))
"""Not doing anything with these."""
country = u8(0)
region = u8(0)
location = u16(0)
zoom_factor = u32_littleendian(
6
) # Nintendo used the value of 3 for states and countries but we probably don't have any articles that are just states or countries
coordinates = (
s16(
int(read[0]["geometry"]["location"]["lat"] /
(360 / 65536))) + s16(
int(read[0]["geometry"]["location"]["lng"] /
(360 / 65536))) + country + region +
location + zoom_factor
) # latitude and longitude is divided by the value of 360 (degrees of a full circle) divided by the max int for a 16-bit int
except Exception as e:
ex = "There was a error downloading the location data - line {}: {}".format(
sys.exc_info()[-1].tb_lineno, str(e))
print(ex)
log(ex, "INFO")
else:
coordinates = binascii.unhexlify(cities[name][0] +
"0000000006000000")
loc_name = enc(cities[name][1])
if locations[name][0] is None and coordinates is not None:
locations[name][0] = coordinates
else:
del locations[name]
continue
if locations[name][1] is None:
locations[name][1] = loc_name
return locations
def log(msg):
utils.log(utils.log_level_debug, TAG, msg)
def start(bot, update):
bot.send_message(chat_id=update.message.chat_id, text="Test if it works")
try:
log(update.message.text.split(' ', 1)[1], 'info')
except IndexError:
log('No input given.', 'info')
def update( self ):
try:
utils.log( ' Unzipping "%s" ... into "%s"' % ( shorten( self.source_ ), os.path.dirname( self.file_path_ ) ) )
self.unzip_func_( self.source_, os.path.dirname( self.file_path_ ) )
except Exception, msg:
utils.log( ' Skipping "%s" due to errors (%s)' % ( self.source_, msg ) )
def run( self ):
utils.log( "%s: run" % shorten( self.file_path_ ) )
__log__ = 2
for dependency in self.dependencies_:
if not os.path.exists( dependency ):
utils.log( "%s doesn't exists, removing target" % shorten( dependency ) )
self.clean()
return
if not os.path.exists( self.file_path_ ):
utils.log( "target doesn't exists, building" )
self.update()
return
dst_timestamp = _modtime_timestamp( self.file_path_ )
utils.log( " target: %s [%s]" % ( shorten( self.file_path_ ), dst_timestamp ) )
needs_updating = 0
utils.log( " dependencies:" )
for dependency in self.dependencies_:
dm = _modtime_timestamp( dependency )
update_mark = ""
if dm > dst_timestamp:
needs_updating = 1
utils.log( ' %s [%s] %s' % ( shorten( dependency ), dm, update_mark ) )
if needs_updating:
utils.log( "target needs updating, rebuilding" )
self.update()
return
else:
utils.log( "target is up-to-date" )
def build_reports(
locate_root_dir
, tag
, expected_results_file
, failures_markup_file
, comment_file
, results_dir
, result_file_prefix
, dont_collect_logs = 0
, reports = report_types
, report_executable = None
, warnings = []
, user = None
, upload = False
, filter_runners = None
):
( run_date ) = time.strftime( '%Y-%m-%dT%H:%M:%SZ', time.gmtime() )
root_paths.append( locate_root_dir )
root_paths.append( results_dir )
bin_boost_dir = os.path.join( locate_root_dir, 'bin', 'boost' )
output_dir = os.path.join( results_dir, result_file_prefix )
utils.makedirs( output_dir )
if expected_results_file != '':
expected_results_file = os.path.abspath( expected_results_file )
else:
expected_results_file = os.path.abspath( map_path( 'empty_expected_results.xml' ) )
extended_test_results = os.path.join( output_dir, 'extended_test_results.xml' )
if filter_runners == None:
if default_filter_runners.has_key(tag):
filter_runners = default_filter_runners[tag]
execute_tasks(
tag
, user
, run_date
, comment_file
, results_dir
, output_dir
, reports
, warnings
, extended_test_results
, dont_collect_logs
, expected_results_file
, failures_markup_file
, report_executable
, filter_runners
)
if upload:
upload_dir = 'regression-logs/'
utils.log( 'Uploading results into "%s" [connecting as %s]...' % ( upload_dir, user ) )
archive_name = '%s.tar.gz' % result_file_prefix
utils.tar(
os.path.join( results_dir, result_file_prefix )
, archive_name
)
utils.sourceforge.upload( os.path.join( results_dir, archive_name ), upload_dir, user )
utils.sourceforge.untar( os.path.join( upload_dir, archive_name ), user, background = True )
def image(filename):
log('images path', filename)
return send_from_directory('images', filename)
def clean( self ):
to_unlink = self.other_results_ + [ self.file_path_ ]
for result in to_unlink:
utils.log( ' Deleting obsolete "%s"' % shorten( result ) )
if os.path.exists( result ):
os.unlink( result )
def cross_validate_model_fold(chunk_input: WorkerInput) -> ModelResult:
log("Execution fold", level=2)
timer = Timer()
classifier = chunk_input['classifier']
X_train = chunk_input['X_train']
y_train = chunk_input['y_train']
X_test = chunk_input['X_test']
return_model = chunk_input['return_model']
if get_log_level() == 1:
print(".")
feature_names = \
chunk_input['feature_names'] if \
('feature_names' in chunk_input and chunk_input['feature_names'] is not None) \
else list(X_train.columns)
classifier.fit(X_train, y_train, **chunk_input['fit_kwargs'])
y_predict = Series(classifier.predict(X_test), index=X_test.index)
y_train_predict = Series(classifier.predict(X_train), index=X_train.index)
try:
y_predict_probabilities_raw = classifier.predict_proba(X_test)
y_train_predict_probabilities_raw = classifier.predict_proba(X_train)
except AttributeError:
y_predict_probabilities = y_predict
y_train_predict_probabilities = y_train_predict
else:
probability_columns = [
f'y_predict_probabilities_{i}'
for i in range(y_predict_probabilities_raw.shape[1])
]
y_predict_probabilities = DataFrame(y_predict_probabilities_raw,
index=X_test.index,
columns=probability_columns)
y_train_predict_probabilities = DataFrame(
y_train_predict_probabilities_raw,
index=X_train.index,
columns=probability_columns)
if y_predict.dtype == np.float:
y_predict = y_predict \
.map(lambda v: 0 if v < 0 else v) \
.map(lambda v: 1 if v > 1 else v) \
.map(lambda v: round(v))
try:
feature_importance = Series(
classifier[-1].feature_importances_,
index=feature_names,
)
except (TypeError, AttributeError):
try:
classifier[-1].coef_
except AttributeError:
feature_importance = None
logging.debug("No feature importance in the result")
else:
feature_importance = None
# feature_importance = Series(classifier[-1].coef_[0], index=feature_names)
if not return_model:
try:
classifier[-1].get_booster().__del__()
except AttributeError:
pass
return ModelResult(y_test_score=y_predict_probabilities,
y_test_predict=y_predict,
y_train_predict=y_train_predict,
y_train_score=y_train_predict_probabilities,
feature_importance=feature_importance,
model=classifier[-1] if return_model else None,
elapsed=timer.elapsed_cpu())
File Name： main.py
Description : 运行主函数
Author : Sam
date： 2017/7/21
-------------------------------------------------
"""
__author__ = 'Sam'
import logging
import os
import sys
import utils
from scrapy import cmdline
if __name__ == '__main__':
reload(sys)
sys.setdefaultencoding('utf-8')
if not os.path.exists('log'):
os.makedirs('log')
logging.basicConfig(filename='log/item.log',
format='%(levelname)s %(asctime)s: %(message)s',
level=logging.DEBUG)
utils.log('*******************run spider start...*******************')
cmdline.execute('scrapy crawl category_urls'.split())
#cmdline.execute('scrapy crawl item_list'.split())
if __name__ == '__main__':
# Step 1. Configuration file
config = parsing_configurations()
# Create logger file
training_loss_file = None
evaluate_stat_file = None
if config['LOG_FOLDER'] != '':
if not os.path.exists(config['LOG_FOLDER']):
os.makedirs(config['LOG_FOLDER'])
training_loss_file = open(config['LOG_FOLDER'] + '/training_loss.txt',
'w')
evaluate_stat_file = open(config['LOG_FOLDER'] + '/evaluate_stat.txt',
'w')
log(config, training_loss_file)
log(config, evaluate_stat_file)
# Step 2. Create dataset loader
train_dataloader, test_dataloader, val_dataloader = create_dataset_loader(
config)
# Step 3. Create cnn
cnn = create_network(config)
if config['CKPT_PATH'] is not '':
print('Loading checkpoint from %s' % config['CKPT_PATH'])
cnn.load_state_dict(torch.load(config['CKPT_PATH']))
# Step 4. Create optimizer
optimizer = None
nExemplars=opt.shot, # num training examples per novel category
nTestNovel=opt.query *
opt.way, # num test examples for all the novel categories
nTestBase=0, # num test examples for all the base categories
batch_size=1,
num_workers=1,
epoch_size=opt.episode, # num of batches per epoch
)
# if opt.aws == 1:
# set_gpu(opt.gpu)
test_loader = dloader_test.get_dataloader()
log_file_path = os.path.join(os.path.dirname(opt.load), "test_log.txt")
log(log_file_path, str(vars(opt)))
# Define the models
(embedding_net, cls_head) = get_model(opt)
if opt.test_fusion_module == 1:
fusion_mod = fuse_score(dc_factor=dc_factor,
weight_learn=opt.learn_fusion_weight).cuda()
elif opt.test_fusion_module == 2:
fusion_mod = fuse_prob(dc_factor=dc_factor,
weight_learn=opt.learn_fusion_weight).cuda()
else:
fusion_mod = fuse_score(dc_factor=dc_factor,
weight_learn=opt.learn_fusion_weight).cuda()
# Load saved model checkpoints
def save(data, path):
s = json.dumps(data, indent=2, ensure_ascii=False)
with open(path, 'w+', encoding='utf-8') as f:
log('save', path, s, data)
f.write(s)
def compute_sequence(epoch, model, loss_fn, loader, optimizer=None, mode='eval', fh=None, backprop_batch_size=None, tolog=[],return_preds=False):
global cuda
#Pdb().set_trace()
if backprop_batch_size is None:
backprop_batch_size = loader.batch_sampler.batch_size
t1 = time.time()
if mode == 'train':
model.train()
else:
model.eval()
last_print = 0
count = 0
num_words = 0
cum_loss = 0
ypred_cum = []
y_cum = []
#idx_mask = []
#variables to write output to a file in correct order
if return_preds:
#all_y= [None for i in range(len(loader.dataset))]
#all_x = [None for i in range(len(loader.dataset))]
all_pred = [None for i in range(len(loader.dataset))]
if mode == 'train':
this_backprop_count = 0
optimizer.zero_grad()
backprop_loss = 0
for x, y, idx in loader:
# break
count += len(idx)
num_words += y.shape[0]*y.shape[1]
# print(len(idx))
#
volatile = True
if mode == 'train':
this_backprop_count += len(idx)
volatile = False
x, y = Variable(x, volatile=volatile), Variable(
y.long(), volatile=volatile)
if cuda:
x, y = x.cuda(), y.cuda()
#
loss,pred = loss_fn(x,y,model)
#ypred = model(x)
#loss = criterion(ypred.transpose(1,2), y)
if mode == 'train':
backprop_loss += loss
if this_backprop_count >= backprop_batch_size:
#utils.log("backproping now: {0}".format(this_backprop_count))
backprop_loss.backward()
# loss.backward()
optimizer.step()
this_backprop_count = 0
backprop_loss = 0
optimizer.zero_grad()
#
y = y.data.cpu().numpy()
x = x.data.cpu().numpy()
pred = pred.cpu().numpy()
y_cum.extend(y.flatten())
ypred_cum.extend(pred.flatten())
cum_loss = cum_loss + loss.data[0]*y.shape[0]*y.shape[1]
if return_preds:
for i in range(len(idx)):
#all_y[idx[i]] = y[i,:]
all_pred[idx[i]] = pred[i,:]
#all_x[idx[i]] = x[i,:]
if (count - last_print) >= 20000:
last_print = count
rec = [epoch, mode, 1.0 * cum_loss / num_words,
count, num_words, time.time() - t1] + tolog + stats_to_log(ypred_cum, y_cum)
utils.log(','.join([str(round(x, 5)) if isinstance(
x, float) else str(x) for x in rec]))
rec = [epoch, mode, 1.0 * cum_loss / num_words,
count, num_words, time.time() - t1] + tolog + stats_to_log(ypred_cum, y_cum)
utils.log(','.join([str(round(x, 5)) if isinstance(
x, float) else str(x) for x in rec]), file=fh)
if return_preds:
return(rec,-1,all_pred)
else:
return (rec,-1)
def load(path):
with open(path, 'r', encoding='utf-8') as f:
s = f.read()
log('load', s)
return json.loads(s)
def handlerproxy(dispatcher):
for c in Commands:
handler_proxy = CommandHandler(c.value, getattr(commands, c.value))
dispatcher.add_handler(handler_proxy)
log('added: ' + c.value, 'info')
from scipy.interpolate import InterpolatedUnivariateSpline as spline
from scipy.optimize import curve_fit, bisect, brentq, least_squares
from scipy.interpolate import splrep, splev
from numpy import log, sqrt, exp, inf, nan, pi
from numpy.ctypeslib import ndpointer
from scipy.optimize import minimize
from scipy.stats import norm
from scipy import stats
from time import sleep
import concurrent.futures
import numpy as np
import ctypes
import utils
import os
cout = utils.log(__file__, __name__, disp=False)
N, n = norm.cdf, norm.pdf
np.seterr(divide='ignore')
np.warnings.filterwarnings('ignore')
base = 253
#------------------------------------------------------------------------------------------------
curr_dir = os.path.dirname(os.path.realpath(__file__))
lib = ctypes.cdll.LoadLibrary(f"{curr_dir}/KimIntegral.dll")
class KimIntegral:
def __init__(self, R, S, K, T, s, r, q, H0, H1, Kh, M=12):
R = 1 if R == "C" or R == "Call" or R == 1 else 0
lib.KI.argtypes = [
ctypes.c_int, ctypes.c_int, ctypes.c_double, ctypes.c_double,
def all(cls):
path = cls.db_path()
models = load(path)
log('models in all', models)
ms = [cls.new(m) for m in models]
return ms
def checkModels():
log("checkModels(): %s" % 1)
stanza.download('en')
def params_test(policy,
env,
n_steps=args.n_steps_test,
ppo_epochs=args.ppo_epoch_test,
clip_params=args.clip_param_test,
gammas=args.gamma_test,
lambdas=args.gae_lambda_test,
loss_coefs=args.value_loss_coef_test,
entropy_coefs=args.entropy_coef_test,
lrs=args.lr_test):
i = 0
for n_step in n_steps: #[512]
for ppo_epoch in ppo_epochs: #[4]
for clip_param in clip_params: #[0.1, 0.2]
for gamma in gammas: #[0.99, 0.9997]
for lb in lambdas: #[0.95]
for value_loss_coef in loss_coefs: #[0.5]
for entropy_coef in entropy_coefs: #[0.01, 0.001]
for lr in lrs: #[2.5e-4, 7e-4]
i += 1
# if i in [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 15, 16]: #Falta el 6 y el 14 por hacer completo
# pass
# else:
# if i == 15:
# print('Continue Training')
# trained_model = "/home/home/Data/Carmen/py_workspace/ObstacleTower_v3/python_scripts/Obstacle_Tower_Carmen_Raposo/results/June-17-2020_01_46AM/model/__15__study_0000350000.zip"
# model = PPO2.load(trained_model, env=env, tensorboard_log="/home/home/Data/Carmen/py_workspace/ObstacleTower_v3/python_scripts/Obstacle_Tower_Carmen_Raposo/results/June-17-2020_01_46AM/tensorboard/15/")
# t = 375000
# GLOBAL_PATH = "/home/home/Data/Carmen/py_workspace/ObstacleTower_v3/python_scripts/Obstacle_Tower_Carmen_Raposo/results/June-17-2020_01_46AM/model/"
# filename = 'argsparams' + str(i) + '.txt'
# os.makedirs(args.results_dir, exist_ok=True)
#
#
# else:
print(
'Start Training: \n n_step: %f \n ppo_epoch: %f \n clip_param: %f \n gamma: %f'
'\n lambda: %f \n value_loss_coef: %f \n entropy_coef: %f \n learning_rate : %f'
% (n_step, ppo_epoch, clip_param,
gamma, lb, value_loss_coef,
entropy_coef, lr))
#Fixed seed
seed = random.seed(0)
#env.seed(5)
if args.use_gae_test:
model = PPO2(
policy,
env,
n_steps=n_step,
verbose=1,
tensorboard_log=args.
tensorboard_logdir + str(i) + '/',
cliprange=clip_param,
learning_rate=lr,
ent_coef=entropy_coef,
vf_coef=value_loss_coef,
max_grad_norm=args.max_grad_norm,
gamma=gamma,
lam=lb,
noptepochs=ppo_epoch,
seed=seed)
else:
model = PPO2(
policy,
env,
n_steps=n_step,
verbose=1,
tensorboard_log=args.
tensorboard_logdir + str(i) + '/',
cliprange=clip_param,
learning_rate=lr,
ent_coef=entropy_coef,
vf_coef=value_loss_coef,
max_grad_norm=args.max_grad_norm,
gamma=gamma,
noptepochs=ppo_epoch,
seed=seed)
#Save the values of the configured parameters
filename = 'argsparams' + str(i) + '.txt'
os.makedirs(args.results_dir,
exist_ok=True)
myfile = open(args.results_dir + filename,
'w+')
myfile.write(
'n_step: %f \n ppo_epoch: %f \n clip_param: %f \n gamma: %f \n lambda: %f '
'\n value_loss_coef: %f \n entropy_coef: %f \n learning_rate : %f'
% (n_step, ppo_epoch, clip_param,
gamma, lb, value_loss_coef,
entropy_coef, lr))
myfile.close()
t = 0
#t = 0
while t < args.num_env_steps_test:
# TRAIN MODEL
try:
if t == 0:
model.learn(
total_timesteps=args.
eval_interval)
else:
model.learn(
total_timesteps=args.
eval_interval,
reset_num_timesteps=False)
os.makedirs(GLOBAL_PATH,
exist_ok=True)
print("Saving in '" + GLOBAL_PATH +
"'")
model.save(GLOBAL_PATH + '__' +
str(i) + '__' +
args.training_name +
"_" +
str(int(t)).zfill(10))
avg_reward, avg_floor = test(
t,
model,
env=env,
global_path=GLOBAL_PATH +
'__' + str(i),
i=i) # Test
log('T = ' + str(t) + ' / ' +
str(args.num_env_steps_test) +
' | Avg. reward: ' +
str(avg_reward) +
' | Avg. floor: ' +
str(avg_floor))
t += args.eval_interval
except Exception as e:
env.close()
myfile = open(
GLOBAL_PATH + filename, 'a')
myfile.write(
'\n An exception %s has occured at step %f'
% (e, t))
myfile.close()
del model
from obstacle_tower_env import ObstacleTowerEnv
env = ObstacleTowerEnv(
'/home/home/Data/Carmen/py_workspace/ObstacleTower_v3/ObstacleTower-v3.1/obstacletower.x86_64',
retro=args.retro,
realtime_mode=args.test,
timeout_wait=6000)
break
env.reset()
break
del model
def all_json(cls):
ms = cls.all()
# 要转换为 dict 格式才行
js = [t.json() for t in ms]
log('all json is:', js)
return js
'model': model,
'train_loss': train_loss,
'valid_loss': valid_loss
}, opt.model_filename + '.model')
torch.save(optimizer, opt.model_filename + '.optim')
model.intype("gpu")
log_string = (
'iter: {:d}, train_loss: {:0.6f}, valid_loss: {:0.6f}, best_valid_loss: {:0.6f}, lr: {:0.5f}'
).format((i + 1) * opt.epoch_size, train_loss[-1], valid_loss[-1],
best_valid_loss, opt.lrt)
print(log_string)
utils.log(opt.model_filename + '.log', log_string)
if __name__ == '__main__':
numpy.random.seed(opt.seed)
torch.manual_seed(opt.seed)
# build the model
opt.n_in = opt.ncond * opt.nc
opt.n_out = opt.npred * opt.nc
model = models.BaselineModel3Layer(opt).cuda()
optimizer = optim.Adam(model.parameters(), opt.lrt)
if opt.loss == 'l1':
criterion = nn.L1Loss().cuda()
elif opt.loss == 'l2':
criterion = nn.MSELoss().cuda()
print('training...')
utils.log(opt.model_filename + '.log', '[training]')
train(500)
def fetchStarship(self, starship):
#Return a specific starship
url = self.base_url + 'starships/?search=' + starship
utils.log(url)
r = requests.get(url)
return r
def parseInput(input):
log("parseInput(): input: %s" % input)
nlp = stanza.Pipeline(lang='en', processors='tokenize,mwt,pos')
doc = nlp(input)
log(*[
f'word: {word.text}\tupos: {word.upos}\txpos: {word.xpos}\tfeats: {word.feats if word.feats else "_"}'
for sent in doc.sentences for word in sent.words
],
sep='\n')
parsed = {"nouns": []}
for sent in doc.sentences:
for word in sent.words:
if word.upos == POS_NOUN:
parsed["nouns"].append(word)
log("parseInput(): done parsing input, parsed: \n%s" % parsed)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='later to be descripted...')
parser.add_argument('-text',
action='store',
type=str,
help='The text to parse.')
args = parser.parse_args()
log("app.py: input from command line: %s" % repr(args))
checkModels()
parseInput(args.text)
def fetchSpeciesById(self, id):
#Return the species information for a specific species id
url = self.base_url + 'species/?search=' + id
utils.log(url)
r = requests.get(url)
return r
def fetchStarships(self):
#return all starships
url = self.base_url + 'starships/'
utils.log(url)
r = requests.get(url)
return r
